Experimental lexer generator: transfer actions across epsilon transitions correctly when constructi…

tools/lexer_generator/rule_parser.py

 # Copyright 2013 the V8 project authors. All rights reserved.
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions are
 # met:
 #
 #     * Redistributions of source code must retain the above copyright
 #       notice, this list of conditions and the following disclaimer.
 #     * Redistributions in binary form must reproduce the above
 #       copyright notice, this list of conditions and the following
 #       disclaimer in the documentation and/or other materials provided
@@ skipping 32 matching lines @@
 
   def parse(self, string):
     return RuleParser.parse(string, self)
 
 class RuleParser:
 
   tokens = RuleLexer.tokens
 
   def __init__(self):
     self.__state = None
+    self.__rule_precedence_counter = 0

[dcarney, 2013/11/07 08:08:54]

should either be class static or go in RuleParserState

[marja, 2013/11/07 08:25:41]

Done.

 
   def p_statements(self, p):
     'statements : aliases rules'
 
   def p_aliases(self, p):
     '''aliases : alias_rule aliases
                | empty'''
 
   def p_alias_rule(self, p):
     'alias_rule : IDENTIFIER EQUALS composite_regex SEMICOLON'
@@ skipping 25 matching lines @@
 
   def p_transition_rules(self, p):
     '''transition_rules : transition_rule transition_rules
                         | empty'''
 
   def p_transition_rule(self, p):
     '''transition_rule : composite_regex_or_default code action
                        | composite_regex_or_default empty action
                        | composite_regex_or_default code empty'''
     rules = self.__state.rules[self.__state.current_state]
-    rule = (p[1], p[2], p[3])
+    rule = (p[1], p[2], p[3], self.__rule_precedence_counter)
+    self.__rule_precedence_counter+=1
     if p[1] == 'default':
       assert not rules['default']
       rules['default'] = rule
     else:
       rules['regex'].append(rule)
 
   def p_action(self, p):
     'action : ACTION_OPEN IDENTIFIER ACTION_CLOSE'
     p[0] = p[2]
 
@@ skipping 78 matching lines @@
   __static_instance = None
   @staticmethod
   def parse(data, parser_state):
     if not RuleParser.__static_instance:
       RuleParser.__static_instance = RuleParser()
       RuleParser.__static_instance.build()
     parser = RuleParser.__static_instance
     parser.__state = parser_state
     parser.parser.parse(data, lexer=parser.lexer.lexer)
     parser.__state = None