| OLD | NEW |
| 1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
| 2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
| 3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
| 4 | 4 |
| 5 #include "components/translate/core/language_detection/language_detection_util.h
" | 5 #include "components/translate/core/language_detection/language_detection_util.h
" |
| 6 | 6 |
| 7 #include "base/logging.h" | 7 #include "base/logging.h" |
| 8 #include "base/metrics/field_trial.h" | |
| 9 #include "base/strings/string_split.h" | 8 #include "base/strings/string_split.h" |
| 10 #include "base/strings/string_util.h" | 9 #include "base/strings/string_util.h" |
| 11 #include "base/strings/utf_string_conversions.h" | 10 #include "base/strings/utf_string_conversions.h" |
| 12 #include "base/time/time.h" | 11 #include "base/time/time.h" |
| 13 #include "components/translate/core/common/translate_constants.h" | 12 #include "components/translate/core/common/translate_constants.h" |
| 14 #include "components/translate/core/common/translate_metrics.h" | 13 #include "components/translate/core/common/translate_metrics.h" |
| 15 #include "components/translate/core/common/translate_util.h" | 14 #include "components/translate/core/common/translate_util.h" |
| 16 | 15 |
| 17 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 16 #if CLD_VERSION==1 |
| 18 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h" | 17 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h" |
| 19 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h" | 18 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h" |
| 20 #endif | 19 #endif |
| 21 | 20 |
| 22 #if !defined(CLD_VERSION) || CLD_VERSION==2 | 21 #if CLD_VERSION==2 |
| 23 #include "third_party/cld_2/src/public/compact_lang_det.h" | 22 #include "third_party/cld_2/src/public/compact_lang_det.h" |
| 24 #endif | 23 #endif |
| 25 | 24 |
| 26 namespace { | 25 namespace { |
| 27 | 26 |
| 28 // Similar language code list. Some languages are very similar and difficult | 27 // Similar language code list. Some languages are very similar and difficult |
| 29 // for CLD to distinguish. | 28 // for CLD to distinguish. |
| 30 struct SimilarLanguageCode { | 29 struct SimilarLanguageCode { |
| 31 const char* const code; | 30 const char* const code; |
| 32 int group; | 31 int group; |
| (...skipping 30 matching lines...) Expand all Loading... |
| 63 translate::CorrectLanguageCodeTypo(code); | 62 translate::CorrectLanguageCodeTypo(code); |
| 64 | 63 |
| 65 if (!translate::IsValidLanguageCode(*code)) { | 64 if (!translate::IsValidLanguageCode(*code)) { |
| 66 *code = std::string(); | 65 *code = std::string(); |
| 67 return; | 66 return; |
| 68 } | 67 } |
| 69 | 68 |
| 70 translate::ToTranslateLanguageSynonym(code); | 69 translate::ToTranslateLanguageSynonym(code); |
| 71 } | 70 } |
| 72 | 71 |
| 73 int GetCLDMajorVersion() { | |
| 74 #if !defined(CLD_VERSION) | |
| 75 std::string group_name = base::FieldTrialList::FindFullName("CLD1VsCLD2"); | |
| 76 if (group_name == "CLD2") | |
| 77 return 2; | |
| 78 else | |
| 79 return 1; | |
| 80 #else | |
| 81 return CLD_VERSION; | |
| 82 #endif | |
| 83 } | |
| 84 | |
| 85 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it | 72 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it |
| 86 // failed. | 73 // failed. |
| 87 // |is_cld_reliable| will be set as true if CLD says the detection is reliable. | 74 // |is_cld_reliable| will be set as true if CLD says the detection is reliable. |
| 88 std::string DetermineTextLanguage(const base::string16& text, | 75 std::string DetermineTextLanguage(const base::string16& text, |
| 89 bool* is_cld_reliable) { | 76 bool* is_cld_reliable) { |
| 90 std::string language = translate::kUnknownLanguageCode; | 77 std::string language = translate::kUnknownLanguageCode; |
| 91 int num_bytes_evaluated = 0; | 78 int num_bytes_evaluated = 0; |
| 92 bool is_reliable = false; | 79 bool is_reliable = false; |
| 93 const bool is_plain_text = true; | 80 const bool is_plain_text = true; |
| 94 | 81 |
| 95 // Language or CLD2::Language | 82 // Language or CLD2::Language |
| 96 int cld_language = 0; | 83 int cld_language = 0; |
| 97 bool is_valid_language = false; | 84 bool is_valid_language = false; |
| 98 | 85 |
| 99 switch (GetCLDMajorVersion()) { | 86 #if CLD_VERSION==1 |
| 100 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 87 int num_languages = 0; |
| 101 case 1: { | 88 cld_language = DetectLanguageOfUnicodeText( |
| 102 int num_languages = 0; | 89 NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL, |
| 103 cld_language = DetectLanguageOfUnicodeText( | 90 &num_bytes_evaluated); |
| 104 NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL, | 91 is_valid_language = cld_language != NUM_LANGUAGES && |
| 105 &num_bytes_evaluated); | 92 cld_language != UNKNOWN_LANGUAGE && |
| 106 is_valid_language = cld_language != NUM_LANGUAGES && | 93 cld_language != TG_UNKNOWN_LANGUAGE; |
| 107 cld_language != UNKNOWN_LANGUAGE && | 94 #elif CLD_VERSION==2 |
| 108 cld_language != TG_UNKNOWN_LANGUAGE; | 95 const std::string utf8_text(base::UTF16ToUTF8(text)); |
| 109 break; | 96 const int num_utf8_bytes = static_cast<int>(utf8_text.size()); |
| 110 } | 97 const char* raw_utf8_bytes = utf8_text.c_str(); |
| 98 cld_language = CLD2::DetectLanguageCheckUTF8( |
| 99 raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable, |
| 100 &num_bytes_evaluated); |
| 101 |
| 102 if (num_bytes_evaluated < num_utf8_bytes && |
| 103 cld_language == CLD2::UNKNOWN_LANGUAGE) { |
| 104 // Invalid UTF8 encountered, see bug http://crbug.com/444258. |
| 105 // Retry using only the valid characters. This time the check for valid |
| 106 // UTF8 can be skipped since the precise number of valid bytes is known. |
| 107 cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated, |
| 108 is_plain_text, &is_reliable); |
| 109 } |
| 110 is_valid_language = cld_language != CLD2::NUM_LANGUAGES && |
| 111 cld_language != CLD2::UNKNOWN_LANGUAGE && |
| 112 cld_language != CLD2::TG_UNKNOWN_LANGUAGE; |
| 113 #else |
| 114 # error "CLD_VERSION must be 1 or 2" |
| 111 #endif | 115 #endif |
| 112 #if !defined(CLD_VERSION) || CLD_VERSION==2 | |
| 113 case 2: { | |
| 114 const std::string utf8_text(base::UTF16ToUTF8(text)); | |
| 115 const int num_utf8_bytes = static_cast<int>(utf8_text.size()); | |
| 116 const char* raw_utf8_bytes = utf8_text.c_str(); | |
| 117 cld_language = CLD2::DetectLanguageCheckUTF8( | |
| 118 raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable, | |
| 119 &num_bytes_evaluated); | |
| 120 | |
| 121 if (num_bytes_evaluated < num_utf8_bytes && | |
| 122 cld_language == CLD2::UNKNOWN_LANGUAGE) { | |
| 123 // Invalid UTF8 encountered, see bug http://crbug.com/444258. | |
| 124 // Retry using only the valid characters. This time the check for valid | |
| 125 // UTF8 can be skipped since the precise number of valid bytes is known. | |
| 126 cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated, | |
| 127 is_plain_text, &is_reliable); | |
| 128 } | |
| 129 is_valid_language = cld_language != CLD2::NUM_LANGUAGES && | |
| 130 cld_language != CLD2::UNKNOWN_LANGUAGE && | |
| 131 cld_language != CLD2::TG_UNKNOWN_LANGUAGE; | |
| 132 break; | |
| 133 } | |
| 134 #endif | |
| 135 default: | |
| 136 NOTREACHED(); | |
| 137 } | |
| 138 | 116 |
| 139 if (is_cld_reliable != NULL) | 117 if (is_cld_reliable != NULL) |
| 140 *is_cld_reliable = is_reliable; | 118 *is_cld_reliable = is_reliable; |
| 141 | 119 |
| 142 // We don't trust the result if the CLD reports that the detection is not | 120 // We don't trust the result if the CLD reports that the detection is not |
| 143 // reliable, or if the actual text used to detect the language was less than | 121 // reliable, or if the actual text used to detect the language was less than |
| 144 // 100 bytes (short texts can often lead to wrong results). | 122 // 100 bytes (short texts can often lead to wrong results). |
| 145 // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that | 123 // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that |
| 146 // the determined language code is correct with 50% confidence. Chrome should | 124 // the determined language code is correct with 50% confidence. Chrome should |
| 147 // handle the real confidence value to judge. | 125 // handle the real confidence value to judge. |
| 148 if (is_reliable && num_bytes_evaluated >= 100 && is_valid_language) { | 126 if (is_reliable && num_bytes_evaluated >= 100 && is_valid_language) { |
| 149 // We should not use LanguageCode_ISO_639_1 because it does not cover all | 127 // We should not use LanguageCode_ISO_639_1 because it does not cover all |
| 150 // the languages CLD can detect. As a result, it'll return the invalid | 128 // the languages CLD can detect. As a result, it'll return the invalid |
| 151 // language code for tradtional Chinese among others. | 129 // language code for tradtional Chinese among others. |
| 152 // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and | 130 // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and |
| 153 // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN | 131 // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN |
| 154 // for Simplified Chinese. | 132 // for Simplified Chinese. |
| 155 switch (GetCLDMajorVersion()) { | 133 #if CLD_VERSION==1 |
| 156 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 134 language = LanguageCodeWithDialects(static_cast<Language>(cld_language)); |
| 157 case 1: | 135 #elif CLD_VERSION==2 |
| 158 language = | 136 // (1) CLD2's LanguageCode returns general Chinese 'zh' for |
| 159 LanguageCodeWithDialects(static_cast<Language>(cld_language)); | 137 // CLD2::CHINESE, but Translate server doesn't accept it. This is |
| 160 break; | 138 // converted to 'zh-CN' in the same way as CLD1's |
| 139 // LanguageCodeWithDialects. |
| 140 // |
| 141 // (2) CLD2's LanguageCode returns zh-Hant instead of zh-TW for |
| 142 // CLD2::CHINESE_T. This is technically more precise for the language |
| 143 // code of traditional Chinese, while Translate server hasn't accepted |
| 144 // zh-Hant yet. |
| 145 if (cld_language == CLD2::CHINESE) |
| 146 language = "zh-CN"; |
| 147 else if (cld_language == CLD2::CHINESE_T) |
| 148 language = "zh-TW"; |
| 149 else |
| 150 language = CLD2::LanguageCode(static_cast<CLD2::Language>(cld_language)); |
| 151 #else |
| 152 # error "CLD_VERSION must be 1 or 2" |
| 161 #endif | 153 #endif |
| 162 #if !defined(CLD_VERSION) || CLD_VERSION==2 | |
| 163 case 2: | |
| 164 // (1) CLD2's LanguageCode returns general Chinese 'zh' for | |
| 165 // CLD2::CHINESE, but Translate server doesn't accept it. This is | |
| 166 // converted to 'zh-CN' in the same way as CLD1's | |
| 167 // LanguageCodeWithDialects. | |
| 168 // | |
| 169 // (2) CLD2's LanguageCode returns zh-Hant instead of zh-TW for | |
| 170 // CLD2::CHINESE_T. This is technically more precise for the language | |
| 171 // code of traditional Chinese, while Translate server hasn't accepted | |
| 172 // zh-Hant yet. | |
| 173 if (cld_language == CLD2::CHINESE) { | |
| 174 language = "zh-CN"; | |
| 175 } else if (cld_language == CLD2::CHINESE_T) { | |
| 176 language = "zh-TW"; | |
| 177 } else { | |
| 178 language = | |
| 179 CLD2::LanguageCode(static_cast<CLD2::Language>(cld_language)); | |
| 180 } | |
| 181 break; | |
| 182 #endif | |
| 183 default: | |
| 184 NOTREACHED(); | |
| 185 } | |
| 186 } | 154 } |
| 187 VLOG(9) << "Detected lang_id: " << language << ", from Text:\n" << text | 155 VLOG(9) << "Detected lang_id: " << language << ", from Text:\n" << text |
| 188 << "\n*************************************\n"; | 156 << "\n*************************************\n"; |
| 189 return language; | 157 return language; |
| 190 } | 158 } |
| 191 | 159 |
| 192 // Checks if CLD can complement a sub code when the page language doesn't know | 160 // Checks if CLD can complement a sub code when the page language doesn't know |
| 193 // the sub code. | 161 // the sub code. |
| 194 bool CanCLDComplementSubCode( | 162 bool CanCLDComplementSubCode( |
| 195 const std::string& page_language, const std::string& cld_language) { | 163 const std::string& page_language, const std::string& cld_language) { |
| (...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 390 // distinguish from English, and the language is one of well-known languages | 358 // distinguish from English, and the language is one of well-known languages |
| 391 // which often provide "en-*" meta information mistakenly. | 359 // which often provide "en-*" meta information mistakenly. |
| 392 for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) { | 360 for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) { |
| 393 if (cld_language == kWellKnownCodesOnWrongConfiguration[i]) | 361 if (cld_language == kWellKnownCodesOnWrongConfiguration[i]) |
| 394 return true; | 362 return true; |
| 395 } | 363 } |
| 396 return false; | 364 return false; |
| 397 } | 365 } |
| 398 | 366 |
| 399 } // namespace translate | 367 } // namespace translate |
| OLD | NEW |