Move language detection to chrome/common/.

chrome/renderer/translate/translate_helper.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "chrome/renderer/translate/translate_helper.h"
 
 #include "base/bind.h"
 #include "base/compiler_specific.h"
 #include "base/logging.h"
 #include "base/message_loop.h"
 #include "base/strings/string16.h"
-#include "base/strings/string_split.h"
 #include "base/strings/string_util.h"
 #include "base/strings/utf_string_conversions.h"
 #include "chrome/common/chrome_constants.h"
 #include "chrome/common/render_messages.h"
-#include "chrome/common/translate/translate_util.h"
-#include "chrome/renderer/translate/translate_helper_metrics.h"
+#include "chrome/common/translate/language_detection_util.h"
+#include "chrome/common/translate/translate_common_metrics.h"
 #include "content/public/renderer/render_view.h"
 #include "third_party/WebKit/public/web/WebDocument.h"
 #include "third_party/WebKit/public/web/WebElement.h"
 #include "third_party/WebKit/public/web/WebFrame.h"
 #include "third_party/WebKit/public/web/WebNode.h"
 #include "third_party/WebKit/public/web/WebNodeList.h"
 #include "third_party/WebKit/public/web/WebScriptSource.h"
 #include "third_party/WebKit/public/web/WebView.h"
 #include "v8/include/v8.h"
 
-#if defined(ENABLE_LANGUAGE_DETECTION)
-#include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h"
-#endif
-
 using WebKit::WebDocument;
 using WebKit::WebElement;
 using WebKit::WebFrame;
 using WebKit::WebNode;
 using WebKit::WebNodeList;
 using WebKit::WebScriptSource;
 using WebKit::WebString;
 using WebKit::WebView;
 
 namespace {
 
 // The delay in milliseconds that we'll wait before checking to see if the
 // translate library injected in the page is ready.
 const int kTranslateInitCheckDelayMs = 150;
 
 // The maximum number of times we'll check to see if the translate library
 // injected in the page is ready.
 const int kMaxTranslateInitCheckAttempts = 5;
 
 // The delay we wait in milliseconds before checking whether the translation has
 // finished.
 const int kTranslateStatusCheckDelayMs = 400;
 
 // Language name passed to the Translate element for it to detect the language.
 const char kAutoDetectionLanguage[] = "auto";
 
-// Similar language code list. Some languages are very similar and difficult
-// for CLD to distinguish.
-struct SimilarLanguageCode {
-  const char* const code;
-  int group;
-};
-
-const SimilarLanguageCode kSimilarLanguageCodes[] = {
-  {"bs", 1},
-  {"hr", 1},
-  {"hi", 2},
-  {"ne", 2},
-};
-
-// Checks |kSimilarLanguageCodes| and returns group code.
-int GetSimilarLanguageGroupCode(const std::string& language) {
-  for (size_t i = 0; i < arraysize(kSimilarLanguageCodes); ++i) {
-    if (language.find(kSimilarLanguageCodes[i].code) != 0)
-      continue;
-    return kSimilarLanguageCodes[i].group;
-  }
-  return 0;
-}
-
-// Well-known languages which often have wrong server configuration of
-// Content-Language: en.
-// TODO(toyoshim): Remove these static tables and caller functions to
-// chrome/common/translate, and implement them as std::set<>.
-const char* kWellKnownCodesOnWrongConfiguration[] = {
-  "es", "pt", "ja", "ru", "de", "zh-CN", "zh-TW", "ar", "id", "fr", "it", "th"
-};
-
 }  // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, public:
 //
 TranslateHelper::TranslateHelper(content::RenderView* render_view)
     : content::RenderViewObserver(render_view),
       page_id_(-1),
       translation_pending_(false),
       weak_method_factory_(this) {
@@ skipping 20 matching lines @@
   WebDocument document = main_frame->document();
   std::string content_language = document.contentLanguage().utf8();
   WebElement html_element = document.documentElement();
   std::string html_lang;
   // |html_element| can be null element, e.g. in
   // BrowserTest.WindowOpenClose.
   if (!html_element.isNull())
     html_lang = html_element.getAttribute("lang").utf8();
   std::string cld_language;
   bool is_cld_reliable;
-  std::string language = DeterminePageLanguage(
+  std::string language = LanguageDetectionUtil::DeterminePageLanguage(
       content_language, html_lang, contents, &cld_language, &is_cld_reliable);
 
   if (language.empty())
     return;
 
   language_determined_time_ = base::TimeTicks::Now();
 
   GURL url(document.url());
   LanguageDetectionDetails details;
   details.time = base::Time::Now();
@@ skipping 14 matching lines @@
       IsTranslationAllowed(&document) && !language.empty()));
 }
 
 void TranslateHelper::CancelPendingTranslation() {
   weak_method_factory_.InvalidateWeakPtrs();
   translation_pending_ = false;
   source_lang_.clear();
   target_lang_.clear();
 }
 
-#if defined(ENABLE_LANGUAGE_DETECTION)
-// static
-std::string TranslateHelper::DetermineTextLanguage(const string16& text,
-                                                   bool* is_cld_reliable) {
-  std::string language = chrome::kUnknownLanguageCode;
-  int num_languages = 0;
-  int text_bytes = 0;
-  bool is_reliable = false;
-  Language cld_language =
-      DetectLanguageOfUnicodeText(NULL, text.c_str(), true, &is_reliable,
-                                  &num_languages, NULL, &text_bytes);
-  if (is_cld_reliable != NULL)
-    *is_cld_reliable = is_reliable;
-
-  // We don't trust the result if the CLD reports that the detection is not
-  // reliable, or if the actual text used to detect the language was less than
-  // 100 bytes (short texts can often lead to wrong results).
-  // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that
-  // the determined language code is correct with 50% confidence. Chrome should
-  // handle the real confidence value to judge.
-  if (is_reliable && text_bytes >= 100 && cld_language != NUM_LANGUAGES &&
-      cld_language != UNKNOWN_LANGUAGE && cld_language != TG_UNKNOWN_LANGUAGE) {
-    // We should not use LanguageCode_ISO_639_1 because it does not cover all
-    // the languages CLD can detect. As a result, it'll return the invalid
-    // language code for tradtional Chinese among others.
-    // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and
-    // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN
-    // for Simplified Chinese.
-    language = LanguageCodeWithDialects(cld_language);
-  }
-  VLOG(9) << "Detected lang_id: " << language << ", from Text:\n" << text
-          << "\n*************************************\n";
-  return language;
-}
-#endif  // defined(ENABLE_LANGUAGE_DETECTION)
-
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, protected:
 //
 bool TranslateHelper::IsTranslateLibAvailable() {
   return ExecuteScriptAndGetBoolResult(
       "typeof cr != 'undefined' && typeof cr.googleTranslate != 'undefined' && "
       "typeof cr.googleTranslate.translate == 'function'", false);
 }
 
 bool TranslateHelper::IsTranslateLibReady() {
@@ skipping 84 matching lines @@
     NOTREACHED();
     return 0.0;
   }
 
   return v->NumberValue();
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 // TranslateHelper, private:
 //
-// static
-void TranslateHelper::CorrectLanguageCodeTypo(std::string* code) {
-  DCHECK(code);
-
-  size_t coma_index = code->find(',');
-  if (coma_index != std::string::npos) {
-    // There are more than 1 language specified, just keep the first one.
-    *code = code->substr(0, coma_index);
-  }
-  TrimWhitespaceASCII(*code, TRIM_ALL, code);
-
-  // An underscore instead of a dash is a frequent mistake.
-  size_t underscore_index = code->find('_');
-  if (underscore_index != std::string::npos)
-    (*code)[underscore_index] = '-';
-
-  // Change everything up to a dash to lower-case and everything after to upper.
-  size_t dash_index = code->find('-');
-  if (dash_index != std::string::npos) {
-    *code = StringToLowerASCII(code->substr(0, dash_index)) +
-        StringToUpperASCII(code->substr(dash_index));
-  } else {
-    *code = StringToLowerASCII(*code);
-  }
-}
-
-// static
-bool TranslateHelper::IsValidLanguageCode(const std::string& code) {
-  // Roughly check if the language code follows /[a-zA-Z]{2,3}(-[a-zA-Z]{2})?/.
-  // TODO(hajimehoshi): How about es-419, which is used as an Accept language?
-  std::vector<std::string> chunks;
-  base::SplitString(code, '-', &chunks);
-
-  if (chunks.size() < 1 || 2 < chunks.size())
-    return false;
-
-  const std::string& main_code = chunks[0];
-
-  if (main_code.size() < 1 || 3 < main_code.size())
-    return false;
-
-  for (std::string::const_iterator it = main_code.begin();
-       it != main_code.end(); ++it) {
-    if (!IsAsciiAlpha(*it))
-      return false;
-  }
-
-  if (chunks.size() == 1)
-    return true;
-
-  const std::string& sub_code = chunks[1];
-
-  if (sub_code.size() != 2)
-    return false;
-
-  for (std::string::const_iterator it = sub_code.begin();
-       it != sub_code.end(); ++it) {
-    if (!IsAsciiAlpha(*it))
-      return false;
-  }
-
-  return true;
-}
-
-// static
-void TranslateHelper::ApplyLanguageCodeCorrection(std::string* code) {
-  // Correct well-known format errors.
-  CorrectLanguageCodeTypo(code);
-
-  if (!IsValidLanguageCode(*code)) {
-    *code = std::string();
-    return;
-  }
-
-  TranslateUtil::ToTranslateLanguageSynonym(code);
-}
-
-// static
-bool TranslateHelper::IsSameOrSimilarLanguages(
-    const std::string& page_language, const std::string& cld_language) {
-  // Language code part of |page_language| is matched to one of |cld_language|.
-  // Country code is ignored here.
-  if (page_language.size() >= 2 &&
-      cld_language.find(page_language.c_str(), 0, 2) == 0) {
-    // Languages are matched strictly. Reports false to metrics, but returns
-    // true.
-    TranslateHelperMetrics::ReportSimilarLanguageMatch(false);
-    return true;
-  }
-
-  // Check if |page_language| and |cld_language| are in the similar language
-  // list and belong to the same language group.
-  int page_code = GetSimilarLanguageGroupCode(page_language);
-  bool match = page_code != 0 &&
-               page_code == GetSimilarLanguageGroupCode(cld_language);
-
-  TranslateHelperMetrics::ReportSimilarLanguageMatch(match);
-  return match;
-}
-
-// static
-bool TranslateHelper::MaybeServerWrongConfiguration(
-    const std::string& page_language, const std::string& cld_language) {
-  // If |page_language| is not "en-*", respect it and just return false here.
-  if (!StartsWithASCII(page_language, "en", false))
-    return false;
-
-  // A server provides a language meta information representing "en-*". But it
-  // might be just a default value due to missing user configuration.
-  // Let's trust |cld_language| if the determined language is not difficult to
-  // distinguish from English, and the language is one of well-known languages
-  // which often provide "en-*" meta information mistakenly.
-  for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) {
-    if (cld_language == kWellKnownCodesOnWrongConfiguration[i])
-      return true;
-  }
-  return false;
-}
-
-// static
-bool TranslateHelper::CanCLDComplementSubCode(
-    const std::string& page_language, const std::string& cld_language) {
-  // Translate server cannot treat general Chinese. If Content-Language and
-  // CLD agree that the language is Chinese and Content-Language doesn't know
-  // which dialect is used, CLD language has priority.
-  // TODO(hajimehoshi): How about the other dialects like zh-MO?
-  return page_language == "zh" && StartsWithASCII(cld_language, "zh-", false);
-}
-
-// static
-std::string TranslateHelper::DeterminePageLanguage(const std::string& code,
-                                                   const std::string& html_lang,
-                                                   const string16& contents,
-                                                   std::string* cld_language_p,
-                                                   bool* is_cld_reliable_p) {
-#if defined(ENABLE_LANGUAGE_DETECTION)
-  base::TimeTicks begin_time = base::TimeTicks::Now();
-  bool is_cld_reliable;
-  std::string cld_language = DetermineTextLanguage(contents, &is_cld_reliable);
-  TranslateHelperMetrics::ReportLanguageDetectionTime(begin_time,
-                                                      base::TimeTicks::Now());
-
-  if (cld_language_p != NULL)
-    *cld_language_p = cld_language;
-  if (is_cld_reliable_p != NULL)
-    *is_cld_reliable_p = is_cld_reliable;
-  TranslateUtil::ToTranslateLanguageSynonym(&cld_language);
-#endif  // defined(ENABLE_LANGUAGE_DETECTION)
-
-  // Check if html lang attribute is valid.
-  std::string modified_html_lang;
-  if (!html_lang.empty()) {
-    modified_html_lang = html_lang;
-    ApplyLanguageCodeCorrection(&modified_html_lang);
-    TranslateHelperMetrics::ReportHtmlLang(html_lang, modified_html_lang);
-    VLOG(9) << "html lang based language code: " << modified_html_lang;
-  }
-
-  // Check if Content-Language is valid.
-  std::string modified_code;
-  if (!code.empty()) {
-    modified_code = code;
-    ApplyLanguageCodeCorrection(&modified_code);
-    TranslateHelperMetrics::ReportContentLanguage(code, modified_code);
-  }
-
-  // Adopt |modified_html_lang| if it is valid. Otherwise, adopt
-  // |modified_code|.
-  std::string language = modified_html_lang.empty() ? modified_code :
-                                                      modified_html_lang;
-
-#if defined(ENABLE_LANGUAGE_DETECTION)
-  // If |language| is empty, just use CLD result even though it might be
-  // chrome::kUnknownLanguageCode.
-  if (language.empty()) {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_CLD_ONLY);
-    return cld_language;
-  }
-
-  if (cld_language == chrome::kUnknownLanguageCode) {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_UNKNOWN);
-    return language;
-  } else if (IsSameOrSimilarLanguages(language, cld_language)) {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_CLD_AGREE);
-    return language;
-  } else if (MaybeServerWrongConfiguration(language, cld_language)) {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_TRUST_CLD);
-    return cld_language;
-  } else if (CanCLDComplementSubCode(language, cld_language)) {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_CLD_COMPLEMENT_SUB_CODE);
-    return cld_language;
-  } else {
-    TranslateHelperMetrics::ReportLanguageVerification(
-        TranslateHelperMetrics::LANGUAGE_VERIFICATION_CLD_DISAGREE);
-    // Content-Language value might be wrong because CLD says that this page
-    // is written in another language with confidence.
-    // In this case, Chrome doesn't rely on any of the language codes, and
-    // gives up suggesting a translation.
-    return std::string(chrome::kUnknownLanguageCode);
-  }
-#else  // defined(ENABLE_LANGUAGE_DETECTION)
-  TranslateHelperMetrics::ReportLanguageVerification(
-      TranslateHelperMetrics::LANGUAGE_VERIFICATION_CLD_DISABLED);
-#endif  // defined(ENABLE_LANGUAGE_DETECTION)
-
-  return language;
-}
 
 // static
 bool TranslateHelper::IsTranslationAllowed(WebDocument* document) {
   WebElement head = document->head();
   if (head.isNull() || !head.hasChildNodes())
     return true;
 
   const WebString meta(ASCIIToUTF16("meta"));
   const WebString name(ASCIIToUTF16("name"));
   const WebString google(ASCIIToUTF16("google"));
@@ skipping 54 matching lines @@
 
   // Set our states.
   translation_pending_ = true;
 
   // If the source language is undetermined, we'll let the translate element
   // detect it.
   source_lang_ = (source_lang != chrome::kUnknownLanguageCode) ?
                   source_lang : kAutoDetectionLanguage;
   target_lang_ = target_lang;
 
-  TranslateHelperMetrics::ReportUserActionDuration(language_determined_time_,
+  TranslateCommonMetrics::ReportUserActionDuration(language_determined_time_,
                                                    base::TimeTicks::Now());
 
   GURL url(main_frame->document().url());
-  TranslateHelperMetrics::ReportPageScheme(url.scheme());
+  TranslateCommonMetrics::ReportPageScheme(url.scheme());
 
   if (!IsTranslateLibAvailable()) {
     // Evaluate the script to add the translation related method to the global
     // context of the page.
     ExecuteScript(translate_script);
     DCHECK(IsTranslateLibAvailable());
   }
 
   TranslatePageImpl(0);
 }
@@ skipping 42 matching lines @@
     }
 
     if (!translation_pending_) {
       NOTREACHED();
       return;
     }
 
     translation_pending_ = false;
 
     // Check JavaScript performance counters for UMA reports.
-    TranslateHelperMetrics::ReportTimeToTranslate(
+    TranslateCommonMetrics::ReportTimeToTranslate(
         ExecuteScriptAndGetDoubleResult("cr.googleTranslate.translationTime"));
 
     // Notify the browser we are done.
     render_view()->Send(new ChromeViewHostMsg_PageTranslated(
         render_view()->GetRoutingID(), render_view()->GetPageId(),
         actual_source_lang, target_lang_, TranslateErrors::NONE));
     return;
   }
 
   // The translation is still pending, check again later.
@@ skipping 20 matching lines @@
         FROM_HERE,
         base::Bind(&TranslateHelper::TranslatePageImpl,
                    weak_method_factory_.GetWeakPtr(),
                    count),
         AdjustDelay(count * kTranslateInitCheckDelayMs));
     return;
   }
 
   // The library is loaded, and ready for translation now.
   // Check JavaScript performance counters for UMA reports.
-  TranslateHelperMetrics::ReportTimeToBeReady(
+  TranslateCommonMetrics::ReportTimeToBeReady(
       ExecuteScriptAndGetDoubleResult("cr.googleTranslate.readyTime"));
-  TranslateHelperMetrics::ReportTimeToLoad(
+  TranslateCommonMetrics::ReportTimeToLoad(
       ExecuteScriptAndGetDoubleResult("cr.googleTranslate.loadTime"));
 
   if (!StartTranslation()) {
     NotifyBrowserTranslationFailed(TranslateErrors::TRANSLATION_ERROR);
     return;
   }
   // Check the status of the translation.
   base::MessageLoop::current()->PostDelayedTask(
       FROM_HERE,
       base::Bind(&TranslateHelper::CheckTranslateStatus,
@@ skipping 12 matching lines @@
 
 WebFrame* TranslateHelper::GetMainFrame() {
   WebView* web_view = render_view()->GetWebView();
 
   // When the tab is going to be closed, the web_view can be NULL.
   if (!web_view)
     return NULL;
 
   return web_view->mainFrame();
 }