Introduce ENABLE_LATIN_1 compile flag

src/jsregexp.cc

 // Copyright 2012 the V8 project authors. All rights reserved.
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions are
 // met:
 //
 //     * Redistributions of source code must retain the above copyright
 //       notice, this list of conditions and the following disclaimer.
 //     * Redistributions in binary form must reproduce the above
 //       copyright notice, this list of conditions and the following
 //       disclaimer in the documentation and/or other materials provided
@@ skipping 1663 matching lines @@
                                      bool ascii_subject,
                                      unibrow::uchar* letters) {
   int length =
       isolate->jsregexp_uncanonicalize()->get(character, '\0', letters);
   // Unibrow returns 0 or 1 for characters where case independence is
   // trivial.
   if (length == 0) {
     letters[0] = character;
     length = 1;
   }
-  if (!ascii_subject || character <= String::kMaxAsciiCharCode) {
+  if (!ascii_subject || character <= String::kMaxOneByteCharCode) {
     return length;
   }
   // The standard requires that non-ASCII characters cannot have ASCII
   // character codes in their equivalence class.
   return 0;
 }
 
 
 static inline bool EmitSimpleCharacter(Isolate* isolate,
                                        RegExpCompiler* compiler,
@@ skipping 30 matching lines @@
   unibrow::uchar chars[unibrow::Ecma262UnCanonicalize::kMaxWidth];
   int length = GetCaseIndependentLetters(isolate, c, ascii, chars);
   if (length < 1) {
     // This can't match.  Must be an ASCII subject and a non-ASCII character.
     // We do not need to do anything since the ASCII pass already handled this.
     return false;  // Bounds not checked.
   }
   bool checked = false;
   // We handle the length > 1 case in a later pass.
   if (length == 1) {
-    if (ascii && c > String::kMaxAsciiCharCodeU) {
+    if (ascii && c > String::kMaxOneByteCharCodeU) {
       // Can't match - see above.
       return false;  // Bounds not checked.
     }
     if (!preloaded) {
       macro_assembler->LoadCurrentCharacter(cp_offset, on_failure, check);
       checked = check;
     }
     macro_assembler->CheckNotCharacter(c, on_failure);
   }
   return checked;
 }
 
 
 static bool ShortCutEmitCharacterPair(RegExpMacroAssembler* macro_assembler,
                                       bool ascii,
                                       uc16 c1,
                                       uc16 c2,
                                       Label* on_failure) {
   uc16 char_mask;
   if (ascii) {
-    char_mask = String::kMaxAsciiCharCode;
+    char_mask = String::kMaxOneByteCharCode;
   } else {
     char_mask = String::kMaxUtf16CodeUnit;
   }
   uc16 exor = c1 ^ c2;
   // Check whether exor has only one bit set.
   if (((exor - 1) & exor) == 0) {
     // If c1 and c2 differ only by one bit.
     // Ecma262UnCanonicalize always gives the highest number last.
     ASSERT(c2 > c1);
     uc16 mask = char_mask ^ exor;
@@ skipping 233 matching lines @@
   // encoding space can be quickly tested with a table lookup, so we don't
   // wish to do binary chop search at a smaller granularity than that.  A
   // 128-character space can take up a lot of space in the ranges array if,
   // for example, we only want to match every second character (eg. the lower
   // case characters on some Unicode pages).
   int binary_chop_index = (end_index + start_index) / 2;
   // The first test ensures that we get to the code that handles the ASCII
   // range with a single not-taken branch, speeding up this important
   // character range (even non-ASCII charset-based text has spaces and
   // punctuation).
-  if (*border - 1 > String::kMaxAsciiCharCode &&  // ASCII case.
+  if (*border - 1 > String::kMaxOneByteCharCode &&  // ASCII case.
       end_index - start_index > (*new_start_index - start_index) * 2 &&
       last - first > kSize * 2 &&
       binary_chop_index > *new_start_index &&
       ranges->at(binary_chop_index) >= first + 2 * kSize) {
     int scan_forward_for_section_border = binary_chop_index;;
     int new_border = (ranges->at(binary_chop_index) | kMask) + 1;
 
     while (scan_forward_for_section_border < end_index) {
       if (ranges->at(scan_forward_for_section_border) > new_border) {
         *new_start_index = scan_forward_for_section_border;
@@ skipping 183 matching lines @@
                           bool check_offset,
                           bool preloaded,
                           Zone* zone) {
   ZoneList<CharacterRange>* ranges = cc->ranges(zone);
   if (!CharacterRange::IsCanonical(ranges)) {
     CharacterRange::Canonicalize(ranges);
   }
 
   int max_char;
   if (ascii) {
-    max_char = String::kMaxAsciiCharCode;
+    max_char = String::kMaxOneByteCharCode;
   } else {
     max_char = String::kMaxUtf16CodeUnit;
   }
 
   int range_count = ranges->length();
 
   int last_valid_range = range_count - 1;
   while (last_valid_range >= 0) {
     CharacterRange& range = ranges->at(last_valid_range);
     if (range.from() <= max_char) {
@@ skipping 281 matching lines @@
   v |= v >> 8;
   v |= v >> 16;
   return v;
 }
 
 
 bool QuickCheckDetails::Rationalize(bool asc) {
   bool found_useful_op = false;
   uint32_t char_mask;
   if (asc) {
-    char_mask = String::kMaxAsciiCharCode;
+    char_mask = String::kMaxOneByteCharCode;
   } else {
     char_mask = String::kMaxUtf16CodeUnit;
   }
   mask_ = 0;
   value_ = 0;
   int char_shift = 0;
   for (int i = 0; i < characters_; i++) {
     Position* pos = &positions_[i];
-    if ((pos->mask & String::kMaxAsciiCharCode) != 0) {
+    if ((pos->mask & String::kMaxOneByteCharCode) != 0) {
       found_useful_op = true;
     }
     mask_ |= (pos->mask & char_mask) << char_shift;
     value_ |= (pos->value & char_mask) << char_shift;
     char_shift += asc ? 8 : 16;
   }
   return found_useful_op;
 }
 
 
@@ skipping 22 matching lines @@
   }
 
 
   bool need_mask = true;
 
   if (details->characters() == 1) {
     // If number of characters preloaded is 1 then we used a byte or 16 bit
     // load so the value is already masked down.
     uint32_t char_mask;
     if (compiler->ascii()) {
-      char_mask = String::kMaxAsciiCharCode;
+      char_mask = String::kMaxOneByteCharCode;
     } else {
       char_mask = String::kMaxUtf16CodeUnit;
     }
     if ((mask & char_mask) == char_mask) need_mask = false;
     mask &= char_mask;
   } else {
     // For 2-character preloads in ASCII mode or 1-character preloads in
     // TWO_BYTE mode we also use a 16 bit load with zero extend.
     if (details->characters() == 2 && compiler->ascii()) {
-      if ((mask & 0x7f7f) == 0x7f7f) need_mask = false;
+#ifndef ENABLE_LATIN_1
+      if ((mask & 0x7f7f) == 0xffff) need_mask = false;
+#else
+      if ((mask & 0xffff) == 0xffff) need_mask = false;
+#endif
     } else if (details->characters() == 1 && !compiler->ascii()) {
       if ((mask & 0xffff) == 0xffff) need_mask = false;
     } else {
       if (mask == 0xffffffff) need_mask = false;
     }
   }
 
   if (fall_through_on_failure) {
     if (need_mask) {
       assembler->CheckCharacterAfterAnd(value, mask, on_possible_success);
@@ skipping 21 matching lines @@
 // generating a quick check.
 void TextNode::GetQuickCheckDetails(QuickCheckDetails* details,
                                     RegExpCompiler* compiler,
                                     int characters_filled_in,
                                     bool not_at_start) {
   Isolate* isolate = Isolate::Current();
   ASSERT(characters_filled_in < details->characters());
   int characters = details->characters();
   int char_mask;
   if (compiler->ascii()) {
-    char_mask = String::kMaxAsciiCharCode;
+    char_mask = String::kMaxOneByteCharCode;
   } else {
     char_mask = String::kMaxUtf16CodeUnit;
   }
   for (int k = 0; k < elms_->length(); k++) {
     TextElement elm = elms_->at(k);
     if (elm.type == TextElement::ATOM) {
       Vector<const uc16> quarks = elm.data.u_atom->data();
       for (int i = 0; i < characters && i < quarks.length(); i++) {
         QuickCheckDetails::Position* pos =
             details->positions(characters_filled_in);
@@ skipping 196 matching lines @@
     info->visited = true;
   }
   ~VisitMarker() {
     info_->visited = false;
   }
  private:
   NodeInfo* info_;
 };
 
 
-RegExpNode* SeqRegExpNode::FilterASCII(int depth) {
+RegExpNode* SeqRegExpNode::FilterASCII(int depth, bool ignore_case) {
   if (info()->replacement_calculated) return replacement();
   if (depth < 0) return this;
   ASSERT(!info()->visited);
   VisitMarker marker(info());
-  return FilterSuccessor(depth - 1);
+  return FilterSuccessor(depth - 1, ignore_case);
 }
 
 
-RegExpNode* SeqRegExpNode::FilterSuccessor(int depth) {
-  RegExpNode* next = on_success_->FilterASCII(depth - 1);
+RegExpNode* SeqRegExpNode::FilterSuccessor(int depth, bool ignore_case) {
+  RegExpNode* next = on_success_->FilterASCII(depth - 1, ignore_case);
   if (next == NULL) return set_replacement(NULL);
   on_success_ = next;
   return set_replacement(this);
 }
 
 
-RegExpNode* TextNode::FilterASCII(int depth) {
+RegExpNode* TextNode::FilterASCII(int depth, bool ignore_case) {
   if (info()->replacement_calculated) return replacement();
   if (depth < 0) return this;
   ASSERT(!info()->visited);
   VisitMarker marker(info());
   int element_count = elms_->length();
   for (int i = 0; i < element_count; i++) {
     TextElement elm = elms_->at(i);
     if (elm.type == TextElement::ATOM) {
       Vector<const uc16> quarks = elm.data.u_atom->data();
       for (int j = 0; j < quarks.length(); j++) {
-        // We don't need special handling for case independence
-        // because of the rule that case independence cannot make
-        // a non-ASCII character match an ASCII character.
-        if (quarks[j] > String::kMaxAsciiCharCode) {
+#ifndef ENABLE_LATIN_1
+        if (quarks[j] > String::kMaxOneByteCharCode) {
           return set_replacement(NULL);
         }
+#else
+        if (quarks[j] <= String::kMaxOneByteCharCode) continue;
+        if (!ignore_case) return set_replacement(NULL);
+        // Here, we need to check for characters whose upper and lower cases
+        // are outside the Latin-1 range.
+        // TODO(dcarney): Replace this code with a simple
+        // table lookup in unibrow::Latin-1.
+        // TODO(dcarney): Test cases!.
+        unibrow::uchar result;
+        int chars;
+        chars = unibrow::ToLowercase::Convert(quarks[j], 0, &result, NULL);
+        if (chars > 1 ||
+            (chars == 1 && result <= String::kMaxOneByteCharCodeU)) {
+          continue;
+        }
+        chars = unibrow::ToUppercase::Convert(quarks[j], 0, &result, NULL);
+        if (chars > 1 ||
+            (chars == 1 && result <= String::kMaxOneByteCharCodeU)) {
+          continue;
+        }
+        // This character is definitely not in the Latin-1 range.
+        return set_replacement(NULL);
+#endif
       }
     } else {
       ASSERT(elm.type == TextElement::CHAR_CLASS);
+#ifdef ENABLE_LATIN_1
+      // TODO(dcarney): Can this be improved?
+      if (ignore_case) continue;
+#endif
       RegExpCharacterClass* cc = elm.data.u_char_class;
       ZoneList<CharacterRange>* ranges = cc->ranges(zone());
       if (!CharacterRange::IsCanonical(ranges)) {
         CharacterRange::Canonicalize(ranges);
       }
       // Now they are in order so we only need to look at the first.
       int range_count = ranges->length();
       if (cc->is_negated()) {
         if (range_count != 0 &&
             ranges->at(0).from() == 0 &&
-            ranges->at(0).to() >= String::kMaxAsciiCharCode) {
+            ranges->at(0).to() >= String::kMaxOneByteCharCode) {
           return set_replacement(NULL);
         }
       } else {
         if (range_count == 0 ||
-            ranges->at(0).from() > String::kMaxAsciiCharCode) {
+            ranges->at(0).from() > String::kMaxOneByteCharCode) {
           return set_replacement(NULL);
         }
       }
     }
   }
-  return FilterSuccessor(depth - 1);
+  return FilterSuccessor(depth - 1, ignore_case);
 }
 
 
-RegExpNode* LoopChoiceNode::FilterASCII(int depth) {
+RegExpNode* LoopChoiceNode::FilterASCII(int depth, bool ignore_case) {
   if (info()->replacement_calculated) return replacement();
   if (depth < 0) return this;
   if (info()->visited) return this;
   {
     VisitMarker marker(info());
 
-    RegExpNode* continue_replacement = continue_node_->FilterASCII(depth - 1);
+    RegExpNode* continue_replacement =
+        continue_node_->FilterASCII(depth - 1, ignore_case);
     // If we can't continue after the loop then there is no sense in doing the
     // loop.
     if (continue_replacement == NULL) return set_replacement(NULL);
   }
 
-  return ChoiceNode::FilterASCII(depth - 1);
+  return ChoiceNode::FilterASCII(depth - 1, ignore_case);
 }
 
 
-RegExpNode* ChoiceNode::FilterASCII(int depth) {
+RegExpNode* ChoiceNode::FilterASCII(int depth, bool ignore_case) {
   if (info()->replacement_calculated) return replacement();
   if (depth < 0) return this;
   if (info()->visited) return this;
   VisitMarker marker(info());
   int choice_count = alternatives_->length();
 
   for (int i = 0; i < choice_count; i++) {
     GuardedAlternative alternative = alternatives_->at(i);
     if (alternative.guards() != NULL && alternative.guards()->length() != 0) {
       set_replacement(this);
       return this;
     }
   }
 
   int surviving = 0;
   RegExpNode* survivor = NULL;
   for (int i = 0; i < choice_count; i++) {
     GuardedAlternative alternative = alternatives_->at(i);
-    RegExpNode* replacement = alternative.node()->FilterASCII(depth - 1);
+    RegExpNode* replacement =
+        alternative.node()->FilterASCII(depth - 1, ignore_case);
     ASSERT(replacement != this);  // No missing EMPTY_MATCH_CHECK.
     if (replacement != NULL) {
       alternatives_->at(i).set_node(replacement);
       surviving++;
       survivor = replacement;
     }
   }
   if (surviving < 2) return set_replacement(survivor);
 
   set_replacement(this);
   if (surviving == choice_count) {
     return this;
   }
   // Only some of the nodes survived the filtering.  We need to rebuild the
   // alternatives list.
   ZoneList<GuardedAlternative>* new_alternatives =
       new(zone()) ZoneList<GuardedAlternative>(surviving, zone());
   for (int i = 0; i < choice_count; i++) {
     RegExpNode* replacement =
-        alternatives_->at(i).node()->FilterASCII(depth - 1);
+        alternatives_->at(i).node()->FilterASCII(depth - 1, ignore_case);
     if (replacement != NULL) {
       alternatives_->at(i).set_node(replacement);
       new_alternatives->Add(alternatives_->at(i), zone());
     }
   }
   alternatives_ = new_alternatives;
   return this;
 }
 
 
-RegExpNode* NegativeLookaheadChoiceNode::FilterASCII(int depth) {
+RegExpNode* NegativeLookaheadChoiceNode::FilterASCII(int depth,
+                                                     bool ignore_case) {
   if (info()->replacement_calculated) return replacement();
   if (depth < 0) return this;
   if (info()->visited) return this;
   VisitMarker marker(info());
   // Alternative 0 is the negative lookahead, alternative 1 is what comes
   // afterwards.
   RegExpNode* node = alternatives_->at(1).node();
-  RegExpNode* replacement = node->FilterASCII(depth - 1);
+  RegExpNode* replacement = node->FilterASCII(depth - 1, ignore_case);
   if (replacement == NULL) return set_replacement(NULL);
   alternatives_->at(1).set_node(replacement);
 
   RegExpNode* neg_node = alternatives_->at(0).node();
-  RegExpNode* neg_replacement = neg_node->FilterASCII(depth - 1);
+  RegExpNode* neg_replacement = neg_node->FilterASCII(depth - 1, ignore_case);
   // If the negative lookahead is always going to fail then
   // we don't need to check it.
   if (neg_replacement == NULL) return set_replacement(replacement);
   alternatives_->at(0).set_node(neg_replacement);
   return set_replacement(this);
 }
 
 
 void LoopChoiceNode::GetQuickCheckDetails(QuickCheckDetails* details,
                                           RegExpCompiler* compiler,
@@ skipping 302 matching lines @@
     int cp_offset = trace->cp_offset() + elm.cp_offset;
     if (elm.type == TextElement::ATOM) {
       Vector<const uc16> quarks = elm.data.u_atom->data();
       for (int j = preloaded ? 0 : quarks.length() - 1; j >= 0; j--) {
         if (first_element_checked && i == 0 && j == 0) continue;
         if (DeterminedAlready(quick_check, elm.cp_offset + j)) continue;
         EmitCharacterFunction* emit_function = NULL;
         switch (pass) {
           case NON_ASCII_MATCH:
             ASSERT(ascii);
-            if (quarks[j] > String::kMaxAsciiCharCode) {
+            if (quarks[j] > String::kMaxOneByteCharCode) {
               assembler->GoTo(backtrack);
               return;
             }
             break;
           case NON_LETTER_CHARACTER_MATCH:
             emit_function = &EmitAtomNonLetter;
             break;
           case SIMPLE_CHARACTER_MATCH:
             emit_function = &EmitSimpleCharacter;
             break;
@@ skipping 178 matching lines @@
   ZoneList<CharacterRange>* ranges = node->ranges(zone());
   if (!CharacterRange::IsCanonical(ranges)) {
     CharacterRange::Canonicalize(ranges);
   }
   if (node->is_negated()) {
     return ranges->length() == 0 ? on_success() : NULL;
   }
   if (ranges->length() != 1) return NULL;
   uint32_t max_char;
   if (compiler->ascii()) {
-    max_char = String::kMaxAsciiCharCode;
+    max_char = String::kMaxOneByteCharCode;
   } else {
     max_char = String::kMaxUtf16CodeUnit;
   }
   return ranges->at(0).IsEverything(max_char) ? on_success() : NULL;
 }
 
 
 // Finds the fixed match length of a sequence of nodes that goes from
 // this alternative and back to this choice node.  If there are variable
 // length nodes or other complications in the way then return a sentinel
@@ skipping 179 matching lines @@
     for (int i = 0; i < kMapSize; i++) map_->at(i) = true;
   }
 }
 
 
 BoyerMooreLookahead::BoyerMooreLookahead(
     int length, RegExpCompiler* compiler, Zone* zone)
     : length_(length),
       compiler_(compiler) {
   if (compiler->ascii()) {
-    max_char_ = String::kMaxAsciiCharCode;
+    max_char_ = String::kMaxOneByteCharCode;
   } else {
     max_char_ = String::kMaxUtf16CodeUnit;
   }
   bitmaps_ = new(zone) ZoneList<BoyerMoorePositionInfo*>(length, zone);
   for (int i = 0; i < length; i++) {
     bitmaps_->Add(new(zone) BoyerMoorePositionInfo(zone), zone);
   }
 }
 
 
@@ skipping 1618 matching lines @@
 }
 
 
 void CharacterRange::AddCaseEquivalents(ZoneList<CharacterRange>* ranges,
                                         bool is_ascii,
                                         Zone* zone) {
   Isolate* isolate = Isolate::Current();
   uc16 bottom = from();
   uc16 top = to();
   if (is_ascii) {
-    if (bottom > String::kMaxAsciiCharCode) return;
-    if (top > String::kMaxAsciiCharCode) top = String::kMaxAsciiCharCode;
+    if (bottom > String::kMaxOneByteCharCode) return;
+    if (top > String::kMaxOneByteCharCode) top = String::kMaxOneByteCharCode;
   }
   unibrow::uchar chars[unibrow::Ecma262UnCanonicalize::kMaxWidth];
   if (top == bottom) {
     // If this is a singleton we just expand the one character.
     int length = isolate->jsregexp_uncanonicalize()->get(bottom, '\0', chars);
     for (int i = 0; i < length; i++) {
       uc32 chr = chars[i];
       if (chr != bottom) {
         ranges->Add(CharacterRange::Singleton(chars[i]), zone);
       }
@@ skipping 526 matching lines @@
         if (offset >= bm->length()) {
           if (initial_offset == 0) set_bm_info(not_at_start, bm);
           return;
         }
         uc16 character = atom->data()[j];
         if (bm->compiler()->ignore_case()) {
           unibrow::uchar chars[unibrow::Ecma262UnCanonicalize::kMaxWidth];
           int length = GetCaseIndependentLetters(
               ISOLATE,
               character,
-              bm->max_char() == String::kMaxAsciiCharCode,
+              bm->max_char() == String::kMaxOneByteCharCode,
               chars);
           for (int j = 0; j < length; j++) {
             bm->Set(offset, chars[j]);
           }
         } else {
           if (character <= max_char) bm->Set(offset, character);
         }
       }
     } else {
       ASSERT(text.type == TextElement::CHAR_CLASS);
@@ skipping 193 matching lines @@
       ChoiceNode* first_step_node = new(zone) ChoiceNode(2, zone);
       first_step_node->AddAlternative(GuardedAlternative(captured_body));
       first_step_node->AddAlternative(GuardedAlternative(
           new(zone) TextNode(new(zone) RegExpCharacterClass('*'), loop_node)));
       node = first_step_node;
     } else {
       node = loop_node;
     }
   }
   if (is_ascii) {
-    node = node->FilterASCII(RegExpCompiler::kMaxRecursion);
+    node = node->FilterASCII(RegExpCompiler::kMaxRecursion, ignore_case);
     // Do it again to propagate the new nodes to places where they were not
     // put because they had not been calculated yet.
-    if (node != NULL) node = node->FilterASCII(RegExpCompiler::kMaxRecursion);
+    if (node != NULL) {
+      node = node->FilterASCII(RegExpCompiler::kMaxRecursion, ignore_case);
+    }
   }
 
   if (node == NULL) node = new(zone) EndNode(EndNode::BACKTRACK, zone);
   data->node = node;
   Analysis analysis(ignore_case, is_ascii);
   analysis.EnsureAnalyzed(node);
   if (analysis.has_failed()) {
     const char* error_message = analysis.error_message();
     return CompilationResult(error_message);
   }
@@ skipping 43 matching lines @@
   }
 
   return compiler.Assemble(&macro_assembler,
                            node,
                            data->capture_count,
                            pattern);
 }
 
 
 }}  // namespace v8::internal