Chromium Code Reviews| Index: lib/i18n/bidi_utils.dart |
| =================================================================== |
| --- lib/i18n/bidi_utils.dart (revision 0) |
| +++ lib/i18n/bidi_utils.dart (revision 0) |
| @@ -0,0 +1,448 @@ |
| +// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| +// for details. All rights reserved. Use of this source code is governed by a |
| +// BSD-style license that can be found in the LICENSE file. |
| + |
| +#library('bidi_utils'); |
| + |
| +/** |
| + * Bidi stands for Bi-directional text. |
| + * According to http://en.wikipedia.org/wiki/Bi-directional_text: |
| + * Bi-directional text is text containing text in both text directionalities, |
| + * both right-to-left (RTL) and left-to-right (LTR). It generally involves text |
| + * containing different types of alphabets, but may also refer to boustrophedon, |
| + * which is changing text directionality in each row. |
| + * |
| + * This file provides some utility classes for determining directionality of |
| + * text, switching CSS layout from LTR to RTL, and other normalizing utilities |
| + * needed when switching between RTL and LTR formatting. |
| + * |
| + * It defines the TextDirection class which is used to represent directionality |
| + * of text, |
| + * In most cases, it is preferable to use bidi_formatter.dart, which provides |
| + * bidi functionality in the given directional context, instead of using |
| + * bidi_utils.dart directly. |
| + */ |
| + |
| +/** Class containing constants to represent the directionality of text. */ |
| +class TextDirection { |
| + static final LTR = const TextDirection._('LTR', 'ltr'); |
| + static final RTL = const TextDirection._('RTL', 'rtl'); |
| + static final UNKNOWN = const TextDirection._('UNKNOWN', 'ltr'); |
|
Alan Knight
2012/06/22 00:16:53
I presume the 'ltr' on UNKNOWN means that we assum
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + |
| + /** A string value representation of the directionality constant. */ |
| + final String value; |
|
Alan Knight
2012/06/22 00:16:53
Probably more informative to enumerate the values.
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + /** |
| + * A string indicating the directionality that should be indicated by the text |
| + * for explicitly specifything the direction in a span tag. |
| + */ |
|
Alan Knight
2012/06/22 00:16:53
Same as above. Also specifying is spelled wrong.
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + final String spanText; |
| + |
| + const TextDirection._(this.value, this.spanText); |
| +} |
| + |
| +class BidiUtils { |
| + /** Unicode "Left-To-Right Embedding" (LRE) character. */ |
| + static final LRE = '\u202A'; |
| + |
| + /** Unicode "Right-To-Left Embedding" (RLE) character. */ |
| + static final RLE = '\u202B'; |
| + |
| + /** Unicode "Pop Directional Formatting" (PDF) character. */ |
| + static final PDF = '\u202C'; |
| + |
| + /** Unicode "Left-To-Right Mark" (LRM) character. */ |
| + static final LRM = '\u200E'; |
| + |
| + /** Unicode "Right-To-Left Mark" (RLM) character. */ |
| + static final RLM = '\u200F'; |
| + |
| + /** Constant to define the threshold of RTL directionality. */ |
| + static num _RTL_DETECTION_THRESHOLD = 0.40; |
| + |
| + /** |
| + * Practical patterns to identify strong LTR and RTL characters, respectively. |
| + * These patterns are not completely correct according to the Unicode |
| + * standard. They are simplified for performance and small code size. |
| + */ |
| + static final String _LTR_CHARS = |
| + @'A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02B8\u0300-\u0590' |
| + @'\u0800-\u1FFF\u2C00-\uFB1C\uFDFE-\uFE6F\uFEFD-\uFFFF'; |
| + static final String _RTL_CHARS = @'\u0591-\u07FF\uFB1D-\uFDFD\uFE70-\uFEFC'; |
| + |
| + /** |
| + * Returns the input [text] with spaces instead of HTML tags or HTML escapes, |
| + * which is helpful for text directionality estimation. |
| + * Note: This function should not be used in other contexts. |
| + * It does not deal well with many things: comments, script, |
| + * elements, style elements, dir attribute,`>' in quoted attribute values, |
| + * etc. But it does handle well enough the most common use cases. |
| + * Since the worst that can happen as a result of these shortcomings is that |
| + * the wrong directionality will be estimated, we have not invested in |
| + * improving this. If [isStripNeeded] is true, this function does the |
| + * stripping, otherwise the function returns the input unchanged. |
| + */ |
| + static String stripHtmlIfNeeded(String text, bool isStripNeeded) { |
| + if (isStripNeeded) { |
| + // The regular expression is simplified for an HTML tag (opening or |
| + // closing) or an HTML escape. We might want to skip over such expressions |
| + // when estimating the text directionality. |
| + return text.replaceAll(const RegExp(@'<[^>]*>|&[^;]+;'), ' '); |
| + } else { |
| + return text; |
| + } |
| + } |
| + |
| + /** |
| + * Determines if the first character in [text] with strong directionality is |
| + * LTR. If [isHtml] is true, the text is HTML or HTML-escaped. |
| + */ |
| + static bool startsWithLtr(String text, [isHtml=false]) { |
| + return const RegExp('^[^$_RTL_CHARS]*[$_LTR_CHARS]').hasMatch( |
|
Alan Knight
2012/06/22 00:16:53
That comes out really nicely. I like the use of st
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Determines if the first character in [text] with strong directionality is |
| + * RTL. If [isHtml] is true, the text is HTML or HTML-escaped. |
| + */ |
|
Alan Knight
2012/06/22 00:16:53
I worry a bit with these about people calling many
Emily Fortuna
2012/06/25 20:25:35
As mentioned at the top of this file, bidi_formatt
Alan Knight
2012/06/25 23:29:56
OK
|
| + static bool startsWithRtl(String text, [isHtml=false]) { |
| + return const RegExp('^[^$_LTR_CHARS]*[$_RTL_CHARS]').hasMatch( |
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Determines if the exit directionality (ie, the last strongly-directional |
| + * character in [text] is LTR. If [isHtml] is true, the text is HTML or |
| + * HTML-escaped. |
| + */ |
| + static bool endsWithLtr(String text, [isHtml=false]) { |
| + return const RegExp('[$_LTR_CHARS][^$_RTL_CHARS]*\$').hasMatch( |
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Determines if the exit directionality (ie, the last strongly-directional |
| + * character in [text] is RTL. If [isHtml] is true, the text is HTML or |
| + * HTML-escaped. |
| + */ |
| + static bool endsWithRtl(String text, [isHtml=false]) { |
| + return const RegExp('[$_RTL_CHARS][^$_LTR_CHARS]*\$').hasMatch( |
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Determines if the given [text] has any LTR characters in it. |
| + * If [isHtml] is true, the text is HTML or HTML-escaped. |
| + */ |
| + static bool hasAnyLtr(String text, [isHtml=false]) { |
| + return const RegExp(@'[' '$_LTR_CHARS' @']').hasMatch( |
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Determines if the given [text] has any RTL characters in it. |
| + * If [isHtml] is true, the text is HTML or HTML-escaped. |
| + */ |
| + static bool hasAnyRtl(String text, [isHtml=false]) { |
| + return const RegExp(@'[' '$_RTL_CHARS' @']').hasMatch( |
| + BidiUtils.stripHtmlIfNeeded(text, isHtml)); |
| + } |
| + |
| + /** |
| + * Check if a BCP 47 / III [languageString] indicates an RTL language. |
| + * |
| + * i.e. either: |
| + * - a language code explicitly specifying one of the right-to-left scripts, |
| + * e.g. "az-Arab", or |
| + * - a language code specifying one of the languages normally written in a |
| + * right-to-left script, e.g. "fa" (Farsi), except ones explicitly |
| + * specifying Latin or Cyrillic script (which are the usual LTR |
| + * alternatives). |
| + * |
| + * The list of right-to-left scripts appears in the 100-199 range in |
| + * http://www.unicode.org/iso15924/iso15924-num.html, of which Arabic and |
| + * Hebrew are by far the most widely used. We also recognize Thaana, N'Ko, and |
| + * Tifinagh, which also have significant modern usage. The rest (Syriac, |
| + * Samaritan, Mandaic, etc.) seem to have extremely limited or no modern usage |
| + * and are not recognized. |
| + * The languages usually written in a right-to-left script are taken as those |
| + * with Suppress-Script: Hebr|Arab|Thaa|Nkoo|Tfng in |
| + * http://www.iana.org/assignments/language-subtag-registry, |
| + * as well as Sindhi (sd) and Uyghur (ug). |
| + * The presence of other subtags of the language code, e.g. regions like EG |
| + * (Egypt), is ignored. |
| + */ |
| + static bool isRtlLanguage(String languageString) { //TODO(case insensitive??) |
| + return const RegExp(@'^(ar|dv|he|iw|fa|nqo|ps|sd|ug|ur|yi|.*[-_]' |
| + @'(Arab|Hebr|Thaa|Nkoo|Tfng))(?!.*[-_](Latn|Cyrl)($|-|_))' |
| + @'($|-|_)', ignoreCase : true).hasMatch(languageString); |
|
Alan Knight
2012/06/22 00:16:53
Is a regular expression, especially a 3-line one,
Emily Fortuna
2012/06/25 20:25:35
The language code specification system is fairly c
Alan Knight
2012/06/25 23:29:56
Grumble, grumble, ok.
|
| + } |
| + |
| + /** |
| + * Enforce the [html] snippet in RTL directionality regardless of overall |
| + * context. If the html piece was enclosed by a tag, the direction will be |
| + * applied to existing tag, otherwise a span tag will be added as wrapper. |
| + * For this reason, if html snippet start with with tag, this tag must enclose |
| + * the whole piece. If the tag already has a direction specified, this new one |
| + * will override existing one in behavior (tested on FF and IE). |
| + */ |
|
Alan Knight
2012/06/22 00:16:53
How badly do we need these? The other things don't
Emily Fortuna
2012/06/25 20:25:35
Since this is being developed not on a Windows mac
Alan Knight
2012/06/25 23:29:56
Maybe we should change it to something like "shoul
Emily Fortuna
2012/06/26 01:04:29
Done.
|
| + static String enforceRtlInHtml(String html) { |
| + return _enforceInHtmlHelper(html, 'rtl'); |
| + } |
| + |
| + /** |
| + * Enforce RTL on both end of the given [text] using unicode BiDi formatting |
| + * characters RLE and PDF. |
| + */ |
| + static String enforceRtlInText(String text) { |
| + return '$RLE$text$PDF'; |
| + } |
| + |
| + /** |
| + * Enforce the [html] snippet in LTR directionality regardless of overall |
| + * context. If the html piece was enclosed by a tag, the direction will be |
| + * applied to existing tag, otherwise a span tag will be added as wrapper. |
| + * For this reason, if html snippet start with with tag, this tag must enclose |
| + * the whole piece. If the tag already has a direction specified, this new one |
| + * will override existing one in behavior (tested on FF and IE). |
| + */ |
| + static String enforceLtrInHtml(String html) { |
| + return _enforceInHtmlHelper(html, 'ltr'); |
| + } |
| + |
| + /** |
| + * Enforce LTR on both end of the given [text] using unicode BiDi formatting |
| + * characters LRE and PDF. |
| + */ |
| + static String enforceLtrInText(String text) { |
| + return '$LRE$text$PDF'; |
| + } |
| + |
| + /** |
| + * Enforce the [html] snippet in the desired [direction] regardless of overall |
| + * context. If the html piece was enclosed by a tag, the direction will be |
| + * applied to existing tag, otherwise a span tag will be added as wrapper. |
| + * For this reason, if html snippet start with with tag, this tag must enclose |
| + * the whole piece. If the tag already has a direction specified, this new one |
| + * will override existing one in behavior (tested on FF and IE). |
| + */ |
| + static String _enforceInHtmlHelper(String html, String direction) { |
| + if (html.startsWith('<')) { |
| + StringBuffer buffer = new StringBuffer(); |
| + var startIndex = 0; |
| + Iterator iterator = const RegExp('<\\w+').allMatches( |
| + html).iterator(); |
|
Alan Knight
2012/06/22 00:16:53
This code seems confusing. Why are we asking for a
Emily Fortuna
2012/06/25 20:25:35
Fixed. That was from a refactor that apparently di
|
| + if (iterator.hasNext()) { |
| + Match match = iterator.next(); |
| + buffer.add(html.substring( |
| + startIndex, match.end())).add(' dir=$direction'); |
| + startIndex = match.end(); |
| + } |
| + return buffer.add(html.substring(startIndex)).toString(); |
| + } |
| + // '\n' is important for FF so that it won't incorrectly merge span groups. |
| + return '\n<span dir=$direction>$html</span>'; |
| + } |
| + |
| + /** |
| + * Apply bracket guard to [str] using html span tag. This is to address the |
| + * problem of messy bracket display that frequently happens in RTL layout. |
| + * If [isRtlContext] is true, then we explicitly want to wrap in a span of RTL |
| + * directionality, regardless of the estimated directionality. |
| + */ |
| + static String guardBracketInHtml(String str, [bool isRtlContext]) { |
| + var useRtl = isRtlContext == null ? hasAnyRtl(str) : isRtlContext; |
| + RegExp regexp = |
| + const RegExp(@'(\(.*?\)+)|(\[.*?\]+)|(\{.*?\}+)|(<.*?(>)+)'); |
| + if (useRtl) { |
|
Alan Knight
2012/06/22 00:16:53
Is a regex really necessary here? Seems like strin
Emily Fortuna
2012/06/25 20:25:35
Renamed to make clearer?
Alan Knight
2012/06/25 23:29:56
Not a fan of all these regexes, but it'll do.
|
| + return _guardBracketHelper(str, regexp, '<span dir=rtl>', '</span>'); |
|
Alan Knight
2012/06/22 00:16:53
probably worth a statement to set a string to 'rtl
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + } |
| + return _guardBracketHelper(str, regexp, '<span dir=ltr>', '</span>'); |
| + } |
| + |
| + /** |
| + * Apply bracket guard to [str] using LRM and RLM. This is to address the |
| + * problem of messy bracket display that frequently happens in RTL layout. |
| + * This version works for both plain text and html, but in some cases is not |
| + * as good as guardBracketInHtml. |
| + * If [isRtlContext] is true, then we explicitly want to wrap in a span of RTL |
| + * directionality, regardless of the estimated directionality. |
| + */ |
| + static String guardBracketInText(String str, [bool isRtlContext]) { |
| + var useRtl = isRtlContext == null ? hasAnyRtl(str) : isRtlContext; |
| + var mark = useRtl ? RLM : LRM; |
| + return _guardBracketHelper(str, |
| + const RegExp(@'(\(.*?\)+)|(\[.*?\]+)|(\{.*?\}+)|(<.*?>+)'), mark, mark); |
| + } |
| + |
| + /** |
| + * (Mostly) reimplements the $& functionality of "replace" in JavaScript. |
| + * Given a [str] and the [regexp] to match with, optionally supply a string to |
| + * be inserted [before] the match and/or [after]. For example, |
| + * `_guardBracketHelper('firetruck', const RegExp('truck'), 'hydrant', '!')` |
| + * would return 'firehydrant!'. |
| + */ |
| + // TODO(efortuna): Get rid of this once this is implemented in Dart. |
| + // See Issue 2979. |
| + static String _guardBracketHelper(String str, RegExp regexp, [String before, |
| + String after]) { |
| + StringBuffer buffer = new StringBuffer(); |
| + var startIndex = 0; |
| + Iterable matches = regexp.allMatches(str); |
| + for (Match match in matches) { |
| + buffer.add(str.substring(startIndex, match.start())).add(before); |
| + buffer.add(str.substring(match.start(), match.end())).add(after); |
| + startIndex = match.end(); |
| + } |
| + return buffer.add(str.substring(startIndex)).toString(); |
| + } |
| + |
| + /** |
| + * Estimates the directionality of [text] using the best known |
| + * general-purpose method (using relative word counts). A |
| + * TextDirection.UNKNOWN return value indicates completely neutral input. |
| + * [isHtml] is true if [text] HTML or HTML-escaped. |
| + * |
| + * If the number of RTL words is above a certain percentage of the total |
| + * number of strongly directional words, returns RTL. |
| + * Otherwise, if any words are strongly or weakly LTR, returns LTR. |
| + * Otherwise, returns UNKNOWN, which is used to mean `neutral'. |
| + * Numbers and URLs are counted as weakly LTR. |
| + */ |
| + static TextDirection estimateDirection(String text, [bool isHtml=false]) { |
| + text = BidiUtils.stripHtmlIfNeeded(text, isHtml); |
| + var rtlCount = 0; |
| + var total = 0; |
| + var hasWeaklyLtr = false; |
| + // Split a string into 'words' for directionality estimation based on |
| + // relative word counts. |
| + for (String token in text.split(const RegExp(@'\s+'))) { |
| + if (BidiUtils.startsWithRtl(token)) { |
| + rtlCount++; |
| + total++; |
| + } else if (const RegExp(@'^http://').hasMatch(token)) { |
| + // Checked if token looks like something that must always be LTR even in |
| + // RTL text, such as a URL. |
| + hasWeaklyLtr = true; |
| + } else if (BidiUtils.hasAnyLtr(token)) { |
| + total++; |
| + } else if (const RegExp(@'\d').hasMatch(token)) { |
| + // Checked if token contains any numerals. |
| + hasWeaklyLtr = true; |
| + } |
| + } |
| + |
| + if (total == 0) { |
| + return hasWeaklyLtr ? TextDirection.LTR : TextDirection.UNKNOWN; |
| + } else if (rtlCount > BidiUtils._RTL_DETECTION_THRESHOLD * total) { |
| + return TextDirection.RTL; |
| + } else { |
| + return TextDirection.LTR; |
| + } |
| + } |
| + |
| + /** |
| + * Find the first index in [str] of the first closing parenthesis that does |
| + * not match an opening parenthesis. |
| + */ |
| + static int _unmatchedParenIndex(String str) { |
| + int sum = 0; |
| + int index = 0; |
| + while (sum >= 0 || index > str.length) { |
| + int char = str.charCodeAt(index); |
| + if (char == '('.charCodeAt(0)) sum++; |
| + else if (char == ')'.charCodeAt(0)) sum--; |
| + index++; |
| + } |
| + return index; |
| + } |
| + |
| + /** |
| + * Reimplementing the $num capability in the JavaScript regular expression |
| + * "replace" method. Returns a Map with keys 'one'-'five' (yes, |
| + * currently only implemented for up to five $num elements, since this |
| + * function should be a temporary measure.), where dict\['one'\] is the |
| + * equivalent of $1 in JavaScript, dict\['two'\] is $2, etc. |
| + */ |
| + static Map _backreferenceHelper(String usr_str, String regexp) { |
| + var keys = ['one', 'two', 'three', 'four', 'five']; |
| + Iterator iterator = keys.iterator(); |
| + Map dict = {}; |
| + int i = regexp.indexOf('('); |
| + Match match = new RegExp(regexp).firstMatch(usr_str); |
| + if (match == null) return null; |
| + String fullMatchedString = usr_str.substring(match.start(), match.end()); |
| + while (i > -1) { |
| + Match beforeOneMatch = new RegExp( |
| + regexp.substring(0, i)).firstMatch(fullMatchedString); |
| + // Matching parenthesis is a context free language! We can't use |
| + // regular expressions to help us here. :-( Awesome! (not really) |
| + // Also note this code should not be used un-modified for general purpose |
| + // $1, $2 support -- technically to be correct you need to match with |
| + // regexp.substring(closingParenIndex+i+1) on the fullMatchedString, and |
| + // then subtract the remainder as $1. We're able to take this shortcut |
| + // here because we know for the particular regular expressions that this |
| + // is used, this doesn't matter. |
| + // TODO(efortuna): rewrite all this silliness when we have $1, $2 support |
| + // in RegExp. See Issue 2979. |
| + int closingParenIndex = _unmatchedParenIndex(regexp.substring(i+1)); |
| + String partialString = fullMatchedString.substring(beforeOneMatch.end()); |
| + Match oneMatch = new RegExp(regexp.substring(i, |
| + closingParenIndex + i + 1)).firstMatch(partialString); |
| + dict[iterator.next()] = partialString.substring(oneMatch.start(), |
| + oneMatch.end()); |
| + i = regexp.indexOf('(', closingParenIndex + i + 1); |
| + } |
| + return dict; |
| + } |
| + |
| + /** |
| + * Replace the double and single quote directly after a Hebrew character in |
| + * [str] with GERESH and GERSHAYIM. This is most likely the user's intention. |
| + */ |
| + static String normalizeHebrewQuote(String str) { |
| + String regex1 = @'([\u0591-\u05f2])"'; |
| + String regex2 = @"([\u0591-\u05f2])'"; |
| + Map keys = _backreferenceHelper(str, regex1); |
| + Map keys2 = _backreferenceHelper(str, regex2); |
| + if (keys != null) { |
| + str = str.replaceAll(new RegExp(regex1), '${keys["one"]}\u05f4'); |
|
Alan Knight
2012/06/22 00:16:53
Does this actually work? With a hard-coded "one"?
Emily Fortuna
2012/06/25 20:25:35
Removed regular expression version.
|
| + } |
| + if (keys2 != null) { |
| + str= str.replaceAll(new RegExp(regex2), '${keys2["one"]}\u05f3'); |
| + } |
| + return str; |
| + } |
| + |
| + /** |
| + * Swap location parameters and 'left'/'right' in CSS specification in |
| + * [cssStr]. The processed string will be suited for RTL layout. Though this |
| + * function can cover most cases, there are always exceptions. It is suggested |
| + * the developer put those exceptions in separate group of CSS string. |
| + */ |
|
Alan Knight
2012/06/22 00:16:53
Do we really need to do this? Especially for CSS,
Emily Fortuna
2012/06/25 20:25:35
Done.
|
| + static String mirrorCSS(String cssStr) { |
| + var regex = ':\\s*([.\\d][.\\w]*)\\s+([.\\d][.\\w]*)\\s+([.\\d][.\\w]*)' |
| + '\\s+([.\\d][.\\w]*)'; |
| + var tempStr = '%%%%'; |
| + Map keys = _backreferenceHelper(cssStr, regex); |
| + if (keys != null) { |
| + // Reverse dimensions regex. |
| + cssStr = cssStr.replaceAll(new RegExp(regex), |
| + ':${keys["one"]} ${keys["four"]} ${keys["three"]} ${keys["two"]}'); |
| + } |
| + // Swap left and right. |
| + return cssStr. |
| + replaceAll(const RegExp('left', ignoreCase:true), tempStr). |
| + replaceAll(const RegExp('right', ignoreCase:true), 'left'). |
| + replaceAll(new RegExp(tempStr), 'right'); |
| + } |
| + |
| + /** |
| + * Check the directionality of [str], return true if the piece of |
| + * text should be laid out in RTL direction. If [isHtml] is true, the string |
| + * is HTML or HTML-escaped. |
| + */ |
| + static bool detectRtlDirectionality(String str, [bool isHtml]) { |
|
Alan Knight
2012/06/22 00:16:53
Wouldn't this be better called e.g. isRTL?
Emily Fortuna
2012/06/25 20:25:35
isRTL is reserved for knowing for certain that the
|
| + return estimateDirection(str, isHtml) == TextDirection.RTL; |
| + } |
| +} |