Experimental lexer generator: transfer actions across epsilon transitions correctly when constructi…

tools/lexer_generator/nfa.py

Index: tools/lexer_generator/nfa.py
diff --git a/tools/lexer_generator/nfa.py b/tools/lexer_generator/nfa.py
index b3f013d983924f8a6ed948cb0a96ee42ebafe073..a1447343671aa06e80a7a9a204b8e56fb917f1fa 100644
--- a/tools/lexer_generator/nfa.py
+++ b/tools/lexer_generator/nfa.py
@@ -333,14 +333,24 @@ class Nfa:
       f = lambda acc, state: acc | state.key_matches(key)
       transitions = reduce(f, nfa_state_set, set())
       match_states = set()
-      actions = set()
+      actions = []
       for (state, action) in transitions:
         match_states.add(state)
         if action:
-          actions.add(action)
+          actions.append((action[2], action[0], action[1]))

[dcarney, 2013/11/07 08:08:54]

action should just always be (precedence, id, code), then no need for all the
reordering

[marja, 2013/11/07 08:25:41]

Done.

+
+        # Pull in actions which can be taken with an epsilon transition from the
+        # match state.

[dcarney, 2013/11/07 08:08:54]

what's with all these spaces and comments - you want this code to be readable?

+        e = TransitionKey.epsilon()
+        if e in state.transitions():
+          for e_trans in state.transitions()[e]:
+            if e_trans[1]:
+              actions.append((e_trans[1][2], e_trans[1][0], e_trans[1][1]))
+
       assert len(match_states) == len(transitions)
-      assert not actions or len(actions) == 1
-      action = iter(actions).next() if actions else None
+
+      actions.sort()
+      action = (actions[0][1], actions[0][2]) if actions else None
       transition_state = Nfa.__to_dfa(match_states, dfa_nodes, end_node)
       dfa_nodes[name]['transitions'][key] = (transition_state, action)
     return name