OLD | NEW |
1 // Copyright 2014 The Chromium Authors. All rights reserved. | 1 // Copyright 2014 The Chromium Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style license that can be | 2 // Use of this source code is governed by a BSD-style license that can be |
3 // found in the LICENSE file. | 3 // found in the LICENSE file. |
4 | 4 |
5 #include "components/translate/core/language_detection/language_detection_util.h
" | 5 #include "components/translate/core/language_detection/language_detection_util.h
" |
6 | 6 |
7 #include "base/logging.h" | 7 #include "base/logging.h" |
8 #include "base/metrics/field_trial.h" | |
9 #include "base/strings/string_split.h" | 8 #include "base/strings/string_split.h" |
10 #include "base/strings/string_util.h" | 9 #include "base/strings/string_util.h" |
11 #include "base/strings/utf_string_conversions.h" | 10 #include "base/strings/utf_string_conversions.h" |
12 #include "base/time/time.h" | 11 #include "base/time/time.h" |
13 #include "components/translate/core/common/translate_constants.h" | 12 #include "components/translate/core/common/translate_constants.h" |
14 #include "components/translate/core/common/translate_metrics.h" | 13 #include "components/translate/core/common/translate_metrics.h" |
15 #include "components/translate/core/common/translate_util.h" | 14 #include "components/translate/core/common/translate_util.h" |
16 | 15 |
17 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 16 #if CLD_VERSION==1 |
18 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h" | 17 #include "third_party/cld/encodings/compact_lang_det/compact_lang_det.h" |
19 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h" | 18 #include "third_party/cld/encodings/compact_lang_det/win/cld_unicodetext.h" |
20 #endif | 19 #endif |
21 | 20 |
22 #if !defined(CLD_VERSION) || CLD_VERSION==2 | 21 #if CLD_VERSION==2 |
23 #include "third_party/cld_2/src/public/compact_lang_det.h" | 22 #include "third_party/cld_2/src/public/compact_lang_det.h" |
24 #endif | 23 #endif |
25 | 24 |
26 namespace { | 25 namespace { |
27 | 26 |
28 // Similar language code list. Some languages are very similar and difficult | 27 // Similar language code list. Some languages are very similar and difficult |
29 // for CLD to distinguish. | 28 // for CLD to distinguish. |
30 struct SimilarLanguageCode { | 29 struct SimilarLanguageCode { |
31 const char* const code; | 30 const char* const code; |
32 int group; | 31 int group; |
(...skipping 30 matching lines...) Expand all Loading... |
63 translate::CorrectLanguageCodeTypo(code); | 62 translate::CorrectLanguageCodeTypo(code); |
64 | 63 |
65 if (!translate::IsValidLanguageCode(*code)) { | 64 if (!translate::IsValidLanguageCode(*code)) { |
66 *code = std::string(); | 65 *code = std::string(); |
67 return; | 66 return; |
68 } | 67 } |
69 | 68 |
70 translate::ToTranslateLanguageSynonym(code); | 69 translate::ToTranslateLanguageSynonym(code); |
71 } | 70 } |
72 | 71 |
73 int GetCLDMajorVersion() { | |
74 #if !defined(CLD_VERSION) | |
75 std::string group_name = base::FieldTrialList::FindFullName("CLD1VsCLD2"); | |
76 if (group_name == "CLD2") | |
77 return 2; | |
78 else | |
79 return 1; | |
80 #else | |
81 return CLD_VERSION; | |
82 #endif | |
83 } | |
84 | |
85 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it | 72 // Returns the ISO 639 language code of the specified |text|, or 'unknown' if it |
86 // failed. | 73 // failed. |
87 // |is_cld_reliable| will be set as true if CLD says the detection is reliable. | 74 // |is_cld_reliable| will be set as true if CLD says the detection is reliable. |
88 std::string DetermineTextLanguage(const base::string16& text, | 75 std::string DetermineTextLanguage(const base::string16& text, |
89 bool* is_cld_reliable) { | 76 bool* is_cld_reliable) { |
90 std::string language = translate::kUnknownLanguageCode; | 77 std::string language = translate::kUnknownLanguageCode; |
91 int num_bytes_evaluated = 0; | 78 int num_bytes_evaluated = 0; |
92 bool is_reliable = false; | 79 bool is_reliable = false; |
93 const bool is_plain_text = true; | 80 const bool is_plain_text = true; |
94 | 81 |
95 // Language or CLD2::Language | 82 // Language or CLD2::Language |
96 int cld_language = 0; | 83 int cld_language = 0; |
97 bool is_valid_language = false; | 84 bool is_valid_language = false; |
98 | 85 |
99 switch (GetCLDMajorVersion()) { | 86 #if CLD_VERSION==1 |
100 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 87 int num_languages = 0; |
101 case 1: { | 88 cld_language = DetectLanguageOfUnicodeText( |
102 int num_languages = 0; | 89 NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL, |
103 cld_language = DetectLanguageOfUnicodeText( | 90 &num_bytes_evaluated); |
104 NULL, text.c_str(), is_plain_text, &is_reliable, &num_languages, NULL, | 91 is_valid_language = cld_language != NUM_LANGUAGES && |
105 &num_bytes_evaluated); | 92 cld_language != UNKNOWN_LANGUAGE && |
106 is_valid_language = cld_language != NUM_LANGUAGES && | 93 cld_language != TG_UNKNOWN_LANGUAGE; |
107 cld_language != UNKNOWN_LANGUAGE && | 94 #elif CLD_VERSION==2 |
108 cld_language != TG_UNKNOWN_LANGUAGE; | 95 const std::string utf8_text(base::UTF16ToUTF8(text)); |
109 break; | 96 const int num_utf8_bytes = static_cast<int>(utf8_text.size()); |
110 } | 97 const char* raw_utf8_bytes = utf8_text.c_str(); |
| 98 cld_language = CLD2::DetectLanguageCheckUTF8( |
| 99 raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable, |
| 100 &num_bytes_evaluated); |
| 101 |
| 102 if (num_bytes_evaluated < num_utf8_bytes && |
| 103 cld_language == CLD2::UNKNOWN_LANGUAGE) { |
| 104 // Invalid UTF8 encountered, see bug http://crbug.com/444258. |
| 105 // Retry using only the valid characters. This time the check for valid |
| 106 // UTF8 can be skipped since the precise number of valid bytes is known. |
| 107 cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated, |
| 108 is_plain_text, &is_reliable); |
| 109 } |
| 110 is_valid_language = cld_language != CLD2::NUM_LANGUAGES && |
| 111 cld_language != CLD2::UNKNOWN_LANGUAGE && |
| 112 cld_language != CLD2::TG_UNKNOWN_LANGUAGE; |
| 113 #else |
| 114 # error "CLD_VERSION must be 1 or 2" |
111 #endif | 115 #endif |
112 #if !defined(CLD_VERSION) || CLD_VERSION==2 | |
113 case 2: { | |
114 const std::string utf8_text(base::UTF16ToUTF8(text)); | |
115 const int num_utf8_bytes = static_cast<int>(utf8_text.size()); | |
116 const char* raw_utf8_bytes = utf8_text.c_str(); | |
117 cld_language = CLD2::DetectLanguageCheckUTF8( | |
118 raw_utf8_bytes, num_utf8_bytes, is_plain_text, &is_reliable, | |
119 &num_bytes_evaluated); | |
120 | |
121 if (num_bytes_evaluated < num_utf8_bytes && | |
122 cld_language == CLD2::UNKNOWN_LANGUAGE) { | |
123 // Invalid UTF8 encountered, see bug http://crbug.com/444258. | |
124 // Retry using only the valid characters. This time the check for valid | |
125 // UTF8 can be skipped since the precise number of valid bytes is known. | |
126 cld_language = CLD2::DetectLanguage(raw_utf8_bytes, num_bytes_evaluated, | |
127 is_plain_text, &is_reliable); | |
128 } | |
129 is_valid_language = cld_language != CLD2::NUM_LANGUAGES && | |
130 cld_language != CLD2::UNKNOWN_LANGUAGE && | |
131 cld_language != CLD2::TG_UNKNOWN_LANGUAGE; | |
132 break; | |
133 } | |
134 #endif | |
135 default: | |
136 NOTREACHED(); | |
137 } | |
138 | 116 |
139 if (is_cld_reliable != NULL) | 117 if (is_cld_reliable != NULL) |
140 *is_cld_reliable = is_reliable; | 118 *is_cld_reliable = is_reliable; |
141 | 119 |
142 // We don't trust the result if the CLD reports that the detection is not | 120 // We don't trust the result if the CLD reports that the detection is not |
143 // reliable, or if the actual text used to detect the language was less than | 121 // reliable, or if the actual text used to detect the language was less than |
144 // 100 bytes (short texts can often lead to wrong results). | 122 // 100 bytes (short texts can often lead to wrong results). |
145 // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that | 123 // TODO(toyoshim): CLD provides |is_reliable| flag. But, it just says that |
146 // the determined language code is correct with 50% confidence. Chrome should | 124 // the determined language code is correct with 50% confidence. Chrome should |
147 // handle the real confidence value to judge. | 125 // handle the real confidence value to judge. |
148 if (is_reliable && num_bytes_evaluated >= 100 && is_valid_language) { | 126 if (is_reliable && num_bytes_evaluated >= 100 && is_valid_language) { |
149 // We should not use LanguageCode_ISO_639_1 because it does not cover all | 127 // We should not use LanguageCode_ISO_639_1 because it does not cover all |
150 // the languages CLD can detect. As a result, it'll return the invalid | 128 // the languages CLD can detect. As a result, it'll return the invalid |
151 // language code for tradtional Chinese among others. | 129 // language code for tradtional Chinese among others. |
152 // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and | 130 // |LanguageCodeWithDialect| will go through ISO 639-1, ISO-639-2 and |
153 // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN | 131 // 'other' tables to do the 'right' thing. In addition, it'll return zh-CN |
154 // for Simplified Chinese. | 132 // for Simplified Chinese. |
155 switch (GetCLDMajorVersion()) { | 133 #if CLD_VERSION==1 |
156 #if !defined(CLD_VERSION) || CLD_VERSION==1 | 134 language = LanguageCodeWithDialects(static_cast<Language>(cld_language)); |
157 case 1: | 135 #elif CLD_VERSION==2 |
158 language = | 136 // (1) CLD2's LanguageCode returns general Chinese 'zh' for |
159 LanguageCodeWithDialects(static_cast<Language>(cld_language)); | 137 // CLD2::CHINESE, but Translate server doesn't accept it. This is |
160 break; | 138 // converted to 'zh-CN' in the same way as CLD1's |
| 139 // LanguageCodeWithDialects. |
| 140 // |
| 141 // (2) CLD2's LanguageCode returns zh-Hant instead of zh-TW for |
| 142 // CLD2::CHINESE_T. This is technically more precise for the language |
| 143 // code of traditional Chinese, while Translate server hasn't accepted |
| 144 // zh-Hant yet. |
| 145 if (cld_language == CLD2::CHINESE) |
| 146 language = "zh-CN"; |
| 147 else if (cld_language == CLD2::CHINESE_T) |
| 148 language = "zh-TW"; |
| 149 else |
| 150 language = CLD2::LanguageCode(static_cast<CLD2::Language>(cld_language)); |
| 151 #else |
| 152 # error "CLD_VERSION must be 1 or 2" |
161 #endif | 153 #endif |
162 #if !defined(CLD_VERSION) || CLD_VERSION==2 | |
163 case 2: | |
164 // (1) CLD2's LanguageCode returns general Chinese 'zh' for | |
165 // CLD2::CHINESE, but Translate server doesn't accept it. This is | |
166 // converted to 'zh-CN' in the same way as CLD1's | |
167 // LanguageCodeWithDialects. | |
168 // | |
169 // (2) CLD2's LanguageCode returns zh-Hant instead of zh-TW for | |
170 // CLD2::CHINESE_T. This is technically more precise for the language | |
171 // code of traditional Chinese, while Translate server hasn't accepted | |
172 // zh-Hant yet. | |
173 if (cld_language == CLD2::CHINESE) { | |
174 language = "zh-CN"; | |
175 } else if (cld_language == CLD2::CHINESE_T) { | |
176 language = "zh-TW"; | |
177 } else { | |
178 language = | |
179 CLD2::LanguageCode(static_cast<CLD2::Language>(cld_language)); | |
180 } | |
181 break; | |
182 #endif | |
183 default: | |
184 NOTREACHED(); | |
185 } | |
186 } | 154 } |
187 VLOG(9) << "Detected lang_id: " << language << ", from Text:\n" << text | 155 VLOG(9) << "Detected lang_id: " << language << ", from Text:\n" << text |
188 << "\n*************************************\n"; | 156 << "\n*************************************\n"; |
189 return language; | 157 return language; |
190 } | 158 } |
191 | 159 |
192 // Checks if CLD can complement a sub code when the page language doesn't know | 160 // Checks if CLD can complement a sub code when the page language doesn't know |
193 // the sub code. | 161 // the sub code. |
194 bool CanCLDComplementSubCode( | 162 bool CanCLDComplementSubCode( |
195 const std::string& page_language, const std::string& cld_language) { | 163 const std::string& page_language, const std::string& cld_language) { |
(...skipping 194 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
390 // distinguish from English, and the language is one of well-known languages | 358 // distinguish from English, and the language is one of well-known languages |
391 // which often provide "en-*" meta information mistakenly. | 359 // which often provide "en-*" meta information mistakenly. |
392 for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) { | 360 for (size_t i = 0; i < arraysize(kWellKnownCodesOnWrongConfiguration); ++i) { |
393 if (cld_language == kWellKnownCodesOnWrongConfiguration[i]) | 361 if (cld_language == kWellKnownCodesOnWrongConfiguration[i]) |
394 return true; | 362 return true; |
395 } | 363 } |
396 return false; | 364 return false; |
397 } | 365 } |
398 | 366 |
399 } // namespace translate | 367 } // namespace translate |
OLD | NEW |