Experimental AutocompleteProvider for zerosuggest.

chrome/browser/autocomplete/zero_suggest_provider.cc

+// Copyright (c) 2012 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "chrome/browser/autocomplete/zero_suggest_provider.h"
+
+#include "base/callback.h"
+#include "base/i18n/icu_string_conversions.h"
+#include "base/json/json_string_value_serializer.h"
+#include "base/string16.h"
+#include "base/string_util.h"
+#include "base/time.h"
+#include "base/utf_string_conversions.h"
+#include "chrome/browser/autocomplete/autocomplete_input.h"
+#include "chrome/browser/autocomplete/autocomplete_match.h"
+#include "chrome/browser/autocomplete/autocomplete_provider_listener.h"
+#include "chrome/browser/profiles/profile.h"
+#include "chrome/browser/search_engines/template_url_service.h"
+#include "chrome/browser/search_engines/template_url_service_factory.h"
+#include "chrome/common/url_constants.h"
+#include "googleurl/src/gurl.h"
+#include "net/base/load_flags.h"
+#include "net/http/http_response_headers.h"
+#include "net/url_request/url_fetcher.h"
+#include "net/url_request/url_request_status.h"
+
+ZeroSuggestProvider::ZeroSuggestProvider(
+  AutocompleteProviderListener* listener,
+  Profile* profile,
+  const std::string& url_prefix)
+    : AutocompleteProvider(listener, profile, "ZeroSuggest"),
+      url_prefix_(url_prefix),
+      template_url_service_(TemplateURLServiceFactory::GetForProfile(profile)) {
+}
+
+void ZeroSuggestProvider::Start(const AutocompleteInput& input,
+                                bool /*minimal_changes*/) {
+  UpdateMatches(input.text());
+}
+
+void ZeroSuggestProvider::StartZeroSuggest(
+    const GURL& url,

[Peter Kasting, 2012/08/14 19:20:02]

Nit: Can fit on previous line, then indent next arg even

[Jered, 2012/08/14 22:27:11]

Done.

+    const string16& user_text) {
+  const std::string& spec = url.possibly_invalid_spec();

[Peter Kasting, 2012/08/14 19:20:02]

Shouldn't |url| always be valid here?  Seems like you can DCHECK is_valid() and
then just get the spec.

[Jered, 2012/08/14 22:27:11]

It may be empty. But invalid URLs will not be worth looking up either. I've
changed this to check is_valid before getting the spec.

+  if (spec.empty() || url.scheme() != chrome::kHttpScheme)

[Peter Kasting, 2012/08/14 19:20:02]

How can the spec be empty?  If it can, then my comment about validity above is
wrong.

Nit: {}, or move comment above conditional (3 places)

[Jered, 2012/08/14 22:27:11]

It's empty on initial focus. Revised.
Done.

[Peter Kasting, 2012/08/15 05:30:39]

Wow, really?  That seems odd.  The caller passes in PermanentURL(), right? 
Shouldn't that be chrome://newtab/ or something?  Maybe it's passing in a
display URL instead of the true underlying URL, which should always be
non-empty.  I'm not sure which is actually more correct.  It depends whether you
want "what the user sees" or "where the user is".  Certainly I would imagine
there are likely useful zerosuggestions if the user is currently on the new tab
page -- but knowing what they are may not be a job for a server-side request,
and may instead require querying the sorts of backends that give us the NTP
thumbnails, recently closed tabs, etc.

What is your plan regarding locally-sourced zerosuggestions?  It this something
you're looking at?

[Jered, 2012/08/15 16:32:12]

Yep.
 
The value is permanent_text_ from the omnibox edit model, which says it is "the
URL of the currently displayed page". It comes from ToolbarModel::GetText()
which sets it to blank if ToolbarModel::ShouldDisplayURL() is false, which
happens for extension pages and for WebUIs that set HideURL().
We want the latter, where the user is. This seems to be PermanentURL() in all
cases except those where the URL is set to blank.
Yeah quite possibly.
We haven't looked at this. We'd have to think carefully about how this would
work given other ntp changes we're looking at.

[Peter Kasting, 2012/08/15 17:35:31]

You're never at a blank URL, because that's not actually possible.  (Even
about:blank is not a blank URL.)  The address of the New Tab Page is
chrome://newtab/, not the empty string.

PermanentURL() is seemingly giving you the display URL, not the true URL.  Since
you say you want the latter, you should fix the caller to provide that, then
change the code here to DCHECK(is_valid()).
Consider adding TODOs or NOTEs if there are places in this code where you'd want
to look at locally-sourced suggestions, and/or filing a bug to cover.

[Jered, 2012/08/15 18:15:21]

Done.
I added a TODO here.

+    // Do not query empty URLs nor non-http URLs. There will be no useful
+    // suggestions for https or chrome URLs.
+    return;
+  matches_.clear();
+  done_ = false;
+  user_text_ = user_text;
+  current_query_ = spec;
+  current_query_text_ = ASCIIToUTF16(spec);
+  Run();
+}
+
+void ZeroSuggestProvider::Stop(bool clear_cached_results) {
+  fetcher_.reset();
+  done_ = true;
+  if (clear_cached_results) {
+    results_.clear();
+    current_query_.clear();
+    current_query_text_.clear();
+  }
+}
+
+void ZeroSuggestProvider::OnURLFetchComplete(const net::URLFetcher* source) {
+  const net::HttpResponseHeaders* const response_headers =
+      source->GetResponseHeaders();
+  std::string json_data;
+  source->GetResponseAsString(&json_data);
+  if (response_headers) {
+    std::string charset;
+    if (response_headers->GetCharset(&charset)) {
+      string16 data_16;
+      if (base::CodepageToUTF16(json_data, charset.c_str(),

[Peter Kasting, 2012/08/14 19:20:02]

Why do charset -> utf16 -> utf8 instead of just charset -> utf8?

Will this even ever be non-UTF8 to begin with?  I thought most Google services
spoke UTF8 exclusively.

[Jered, 2012/08/14 22:27:11]

This code comes from SearchProvider. Our test service certainly uses UTF-8, so
I've removed this for now.

+                                base::OnStringConversionError::FAIL,
+                                &data_16))
+        json_data = UTF16ToUTF8(data_16);
+    }
+  }
+  const bool request_succeeded =
+      source->GetStatus().is_success() && source->GetResponseCode() == 200;
+
+  bool results_updated = false;
+  if (request_succeeded) {
+    JSONStringValueSerializer deserializer(json_data);
+    deserializer.set_allow_trailing_comma(true);
+    scoped_ptr<Value> data(deserializer.Deserialize(NULL, NULL));
+    results_updated = data.get() && ParseSuggestResults(data.get());
+  }
+  done_ = true;
+
+  ConvertResultsToAutocompleteMatches();
+  if (results_updated)
+    listener_->OnProviderUpdate(results_updated);
+}
+
+ZeroSuggestProvider::~ZeroSuggestProvider() {
+}
+
+void ZeroSuggestProvider::Run() {
+  const int kFetcherID = 1;
+  fetcher_.reset(
+      net::URLFetcher::Create(kFetcherID,
+          GURL(url_prefix_ + current_query_),
+          net::URLFetcher::GET, this));
+  fetcher_->SetRequestContext(profile_->GetRequestContext());
+  fetcher_->SetLoadFlags(net::LOAD_DO_NOT_SAVE_COOKIES);
+  fetcher_->Start();
+}
+
+void ZeroSuggestProvider::UpdateMatches(const string16& user_text) {
+  user_text_ = user_text;
+  const size_t prev_num_matches = matches_.size();
+  ConvertResultsToAutocompleteMatches();
+  if (matches_.size() != prev_num_matches)
+    listener_->OnProviderUpdate(true);
+}
+
+bool ZeroSuggestProvider::ParseSuggestResults(Value* root_val) {
+  std::string query;
+  ListValue* root_list = NULL;
+  ListValue* results = NULL;
+  if (!root_val->GetAsList(&root_list) || !root_list->GetString(0, &query) ||
+      (query != current_query_) || !root_list->GetList(1, &results))
+    return false;
+
+  results_.clear();
+  ListValue* one_result = NULL;
+  for (size_t index = 0; results->GetList(index, &one_result); ++index) {
+    string16 result;
+    one_result->GetString(0, &result);
+    if (result.empty())
+      continue;
+    results_.push_back(result);
+  }
+
+  return true;
+}
+
+void ZeroSuggestProvider::ConvertResultsToAutocompleteMatches() {
+  const TemplateURL* search_provider =
+      template_url_service_->GetDefaultSearchProvider();
+  if (search_provider == NULL || !search_provider->SupportsReplacement())
+    // Fail if we can't set the clickthrough URL for query suggestions.
+    return;
+  matches_.clear();
+  if (results_.empty())
+    // Do not add anything if there are no results for this URL.
+    return;
+  AddMatchForCurrentURL();
+  for (size_t i = 0; i < results_.size(); ++i)
+    AddMatchForResult(search_provider, i, results_[i]);
+}
+
+// TODO(jered): Rip this out once the first match is decoupled from the current
+// typing in the omnibox.
+void ZeroSuggestProvider::AddMatchForCurrentURL() {
+  // If the user has typed something besides the current url, they probably
+  // don't intend to refresh it.
+  const bool user_text_is_url = user_text_ == current_query_text_;
+  if (user_text_.empty() || user_text_is_url) {
+    // The placeholder suggestion for the current URL should have the max
+    // relevance score when shown so that it is above other suggestions.
+    const int kMaxRelevance = 2000;
+    AutocompleteMatch match(this, kMaxRelevance, false,

[Peter Kasting, 2012/08/14 19:20:02]

The relevance scores you use here and below mean that if these zerosuggest
matches stick around as the user starts typing, they'll outrank everything from
every other provider.  That seems wrong and will lead to a really bad UX (e.g.
you'll completely disable inline autocompletion).

It seems like you should either have zerosuggest results disappear entirely once
the user starts typing, or be ranked very low so everything else outranks them,
or something.

[Jered, 2012/08/14 22:27:11]

This placeholder nav suggestion will get dropped when the user starts typing. I
changed zero suggestions to have really low relevance after the user starts
typing (but not so low that they disappear)

+                            AutocompleteMatch::SEARCH_ZEROSUGGEST_URL);
+    match.destination_url = GURL(current_query_);
+    match.contents = current_query_text_;
+    if (!user_text_is_url) {
+      match.fill_into_edit = current_query_text_;
+      match.inline_autocomplete_offset = 0;
+    }
+    AutocompleteMatch::ClassifyLocationInString(0, current_query_.size(),
+        match.contents.length(), ACMatchClassification::URL,
+        &match.contents_class);
+    matches_.push_back(match);
+  }
+}
+
+void ZeroSuggestProvider::AddMatchForResult(
+    const TemplateURL* search_provider,
+    size_t result_index,
+    const string16& result) {
+  // TODO(jered): Rip out user_text_is_url logic when AddMatchForCurrentURL
+  // goes away.
+  const bool user_text_is_url = user_text_ == current_query_text_;
+  const bool kCaseInsensitve = false;
+  if (!user_text_.empty() && !user_text_is_url &&
+      !StartsWith(result, user_text_, kCaseInsensitve))
+    // This suggestion isn't relevant for the current prefix.
+    return;
+  // This bogus relevance is just below max so that these suggestions are
+  // beneath the current URL match from AddMatchForCurrentURL().
+  // TODO(jered): Use real scores from the suggestion server.
+  const int kRelevance = 1999;
+  AutocompleteMatch match(this, kRelevance, false,
+      AutocompleteMatch::SEARCH_ZEROSUGGEST);
+  match.contents = result;
+  match.fill_into_edit = result;
+  if (!user_text_is_url && user_text_ != result)
+    match.inline_autocomplete_offset = user_text_.length();
+
+  // Build a URL for this query using the default search provider.
+  const TemplateURLRef& search_url = search_provider->url_ref();
+  DCHECK(search_url.SupportsReplacement());
+  match.search_terms_args.reset(
+      new TemplateURLRef::SearchTermsArgs(result));
+  match.search_terms_args->original_query = string16();
+  match.search_terms_args->accepted_suggestion = result_index;
+  match.destination_url =
+      GURL(search_url.ReplaceSearchTerms(*match.search_terms_args.get()));
+
+  if (user_text_.empty() || user_text_is_url || user_text_ == result) {
+    match.contents_class.push_back(
+        ACMatchClassification(0, ACMatchClassification::NONE));
+  } else {
+    // Style to look like normal search suggestions.
+    match.contents_class.push_back(
+       ACMatchClassification(0, ACMatchClassification::DIM));
+    match.contents_class.push_back(
+       ACMatchClassification(user_text_.length(), ACMatchClassification::NONE));
+  }
+  match.transition = content::PAGE_TRANSITION_GENERATED;
+
+  matches_.push_back(match);
+}