Validator_ragel: move checkdecoder test from Makefile to scons

src/trusted/validator_ragel/build.scons

Index: src/trusted/validator_ragel/build.scons
diff --git a/src/trusted/validator_ragel/build.scons b/src/trusted/validator_ragel/build.scons
index 4ff6cf850de0238081e2fd7292d6a3f9f6fc66b5..8710d4d732588e725bc2eef182cbd5cfbad1719a 100644
--- a/src/trusted/validator_ragel/build.scons
+++ b/src/trusted/validator_ragel/build.scons
@@ -30,13 +30,60 @@ INST_DEFS = [
     'unreviewed/nops.def'
 ]
 
+# Generate 32 and 64 bit versions of decoders and validators.  Both libraries
+# are used for command-line decoder and validator those detect specific
+# architecture of the ELF file provided.
+env.ComponentLibrary('dfa_validate_x86_32',
+                     ['gen/validator_x86_32.c'])
+env.ComponentLibrary('dfa_validate_x86_64',
+                     ['gen/validator_x86_64.c'])
+env.ComponentLibrary('dfa_decode_x86_32',
+                     ['gen/decoder_x86_32.c'])
+env.ComponentLibrary('dfa_decode_x86_64',
+                     ['gen/decoder_x86_64.c'])
+
+# Glue library called from service runtime. The source file depends on the
+# target architecture.
+caller_lib_bits = None
+if env.Bit('target_x86_32'):
+  caller_lib_bits = '32'
+if env.Bit('target_x86_64'):
+  caller_lib_bits = '64'
+if caller_lib_bits:
+  # TODO(shcherbina):
+  # interpolate env.get('TARGET_FULLARCH') / env.get('TARGET_SUBARCH') instead

[Nick Bray, 2012/08/08 18:24:44]

I think you're missing a piece of the puzzle.

env.get('TARGET_FULLARCH') returns 'x86_32', 'x86_64', etc.
env.get('TARGET_SUBARCH') will return '32' or '64'.

You can interpolate these values directly (using Python's interpolation, not
SCons') rather than calculating caller_lib_bits.  It's equivalent but simpler.

[Vlad Shcherbina, 2012/08/09 10:07:35]

Done. I used only env.get('TARGET_SUBARCH'), because env.get('TARGET_FULLARCH')
returns 'x86-32' and I need 'x86_32'.

+  caller_lib = 'dfa_validate_caller_x86_%s' % caller_lib_bits
+  env.ComponentLibrary(caller_lib,
+                       ['unreviewed/dfa_validate_%s.c' % caller_lib_bits])
+
+# Command-line decoder.
+decoder_test = env.ComponentProgram(
+    'decoder_test',
+    ['unreviewed/decoder_test.c'],
+    EXTRA_LIBS=['dfa_decode_x86_32', 'dfa_decode_x86_64'])
+
+# Command-line validator.
+validator_test_exe = env.ComponentProgram(
+    'validator_test',
+    ['unreviewed/validator_test.c'],
+    EXTRA_LIBS=['dfa_validate_x86_32', 'dfa_validate_x86_64'])
+
+# Python-based regression test. TODO(pasko): remove it when validator_tests are
+# migrated to be able to run with the new validator.
+if env.Bit('validator_ragel'):
+  test_node = env.CommandTest('validator_test_py.out',
+                              command=['${PYTHON}',
+                                       env.File('unreviewed/validator_test.py'),
+                                       validator_test_exe])
+  env.AddNodeToTestSuite(test_node, ['small_tests'], 'run_validator_test_py')
+
 # Source generation:
 #
 #   dfagen : Regenerate any autogenerated source files.
 
-generate = False
-if 'dfagen' in COMMAND_LINE_TARGETS or 'dfaclean' in COMMAND_LINE_TARGETS:
-  generate = True
+dfa_aliases = 'dfagen', 'dfaclean', 'dfacheckdecoder'
+
+generate =  any(a in COMMAND_LINE_TARGETS for a in dfa_aliases)
 
 if generate:
   if not env.Bit('host_linux'):
@@ -58,7 +105,7 @@ if generate:
       'gen_dfa',
       ['unreviewed/gen_dfa.cc'])
 
-  # Source generation step 2: Generate decoder automatas.
+  # Source generation step 2: Generate decoder automata.
   #
   # Now we are back to conditionally defining the large automata generated
   # by gen_dfa.
@@ -71,13 +118,13 @@ if generate:
     #  _consts.c file includes constants referenced by .rl file and if .rl
     #  file is not changed _consts.c is guaranteed to be the same (reverse
     #  is not true).
+
     const_file = '%s/%s_x86_%s_instruction_consts.c' % (
         val_src_dir, automaton, bits)
 
-    exe_path = '${STAGING_DIR}/${PROGPREFIX}gen_dfa${PROGSUFFIX}'
     env.Command(
         target=rl_file,
-        source=[exe_path] + INST_DEFS,
+        source=[gen_dfa] + INST_DEFS,
         action=(
             '${SOURCES[0]} -o ${TARGET} -c %s -m %s -d %s %s') % (
                 # Const file (-c): not tracked by SCONS (see above)
@@ -87,7 +134,7 @@ if generate:
                 # (-d):
                 dfa_gen_actions,
                 # pass inst defs as remaining parameters
-                ' '.join('${SOURCES[%d]}' % (i+1)
+                ' '.join('${SOURCES[%d]}' % (i + 1)
                          for i in range(len(INST_DEFS)))
         )
     )
@@ -105,20 +152,23 @@ if generate:
       target_filename = target[0].get_abspath()
       architecture = {'x86_32': 'ia32',
                       'x86_64': 'x86-64'}[target_filename[-8:-2]]
+
       with open(source_filename, 'r') as source_file:
-        with open(target_filename, 'w') as target_file:
-          target_file.write(
-"""/* native_client/%s
- * THIS FILE IS AUTO-GENERATED. DO NOT EDIT.
- * Compiled for %s mode.
- */""" % (target_filename, architecture))
-
-          comment, sep, rest = source_file.read().partition('*/')
-          if sep == '':
-            raise UserError('Generated file %s does not have '
-                            'header comment block' % source_filename)
-          target_file.write(rest)
+        comment, sep, rest = source_file.read().partition('*/')
+        if sep == '':
+          raise UserError('Generated file %s does not have '
+                          'header comment block' % source_filename)
+
+      with open(target_filename, 'w') as target_file:
+        target_file.write(
+            ('/* native_client/%s\n'
+             ' * THIS FILE IS AUTO-GENERATED. DO NOT EDIT.\n'
+             ' * Compiled for %s mode.\n'
+             ' */') % (target_filename, architecture))
+
+        target_file.write(rest)
 
+    # inject comments and place files to appropriate dir
     env.Command(
         target=c_full_filename,
         source=['%s/%s' % (rl_src_dir, c_file)],
@@ -128,75 +178,112 @@ if generate:
     return rl_file, c_file, c_full_filename
 
   decoder32 = MakeAutomaton(
-    '32', 'decoder',
-    'check_access,opcode,parse_operands_states,mark_data_fields',
-    '-T0')
+      '32', 'decoder',
+      'check_access,opcode,parse_operands_states,mark_data_fields',
+      '-T0')
   validator32 = MakeAutomaton(
-    '32', 'validator',
-    ('check_access,opcode,parse_operands,parse_operands_states,'
-     'instruction_name,mark_data_fields,nacl-forbidden,'
-     'imm_operand_action,rel_operand_action'),
-    '-G2')
+      '32', 'validator',
+      ('check_access,opcode,parse_operands,parse_operands_states,'
+       'instruction_name,mark_data_fields,nacl-forbidden,'
+       'imm_operand_action,rel_operand_action'),
+      '-G2')
   decoder64 = MakeAutomaton(
-    '64', 'decoder',
-    'check_access,opcode,parse_operands_states,mark_data_fields',
-    '-T0')
+      '64', 'decoder',
+      'check_access,opcode,parse_operands_states,mark_data_fields',
+      '-T0')
   validator64 = MakeAutomaton(
-    '64', 'validator',
-    ('opcode,instruction_name,mark_data_fields,imm_operand_action,'
-     'rel_operand_action,nacl-forbidden,parse_nonwrite_registers,'
-     'parse_x87_operands,parse_mmx_operands,parse_xmm_operands,'
-     'parse_ymm_operands,parse_relative_operands,'
-     'parse_immediate_operands,parse_operands_states,'
-     'parse_operand_positions'),
-    '-GT2')
+      '64', 'validator',
+      ('opcode,instruction_name,mark_data_fields,imm_operand_action,'
+       'rel_operand_action,nacl-forbidden,parse_nonwrite_registers,'
+       'parse_x87_operands,parse_mmx_operands,parse_xmm_operands,'
+       'parse_ymm_operands,parse_relative_operands,'
+       'parse_immediate_operands,parse_operands_states,'
+       'parse_operand_positions'),
+      '-GT2')
 
   automata = list(decoder32 + validator32 + decoder64 + validator64)
 
-  env.AlwaysBuild(env.Alias('dfagen', automata))
-  env.AlwaysBuild(env.Alias('dfaclean', action=map(Delete, automata)))
+  # Generate checkdecoder test
 
-# Generate 32 and 64 bit versions of decoders and validators.  Both libraries
-# are used for command-line decoder and validator those detect specific
-# architecture of the ELF file provided.
-env.ComponentLibrary('dfa_validate_x86_32',
-                     ['gen/validator_x86_32.c'])
-env.ComponentLibrary('dfa_validate_x86_64',
-                     ['gen/validator_x86_64.c'])
-env.ComponentLibrary('dfa_decode_x86_32',
-                     ['gen/decoder_x86_32.c'])
-env.ComponentLibrary('dfa_decode_x86_64',
-                     ['gen/decoder_x86_64.c'])
+  test_env = env.Clone()
+  test_env.Append(
+      CCFLAGS=['-g', '-Wno-unused-function'],
+      LINKFLAGS='-g',
+      CPPPATH='unreviewed')
 
-# Glue library called from service runtime. The source file depends on the
-# target architecture.
-caller_lib_bits = None
-if env.Bit('target_x86_32'):
-  caller_lib_bits = '32'
-if env.Bit('target_x86_64'):
-  caller_lib_bits = '64'
-if caller_lib_bits:
-  caller_lib = 'dfa_validate_caller_x86_%s' % caller_lib_bits
-  env.ComponentLibrary(caller_lib,
-                       ['unreviewed/dfa_validate_%s.c' % caller_lib_bits])
+  test_dfa_object = test_env.Object('unreviewed/test_dfa.c')
 
-# Command-line decoder.
-env.ComponentProgram(
-    'decoder_test',
-    ['unreviewed/decoder_test.c'],
-    EXTRA_LIBS=['dfa_decode_x86_32', 'dfa_decode_x86_64'])
+  objdump, gas = env.Command(
+      target=['objdump', 'gas'],
+      source=['obtain_binutils.py'],
+      action='python ${SOURCES[0]} ${TARGETS[0]} ${TARGETS[1]}'
+  )
 
-# Command-line validator.
-validator_test_exe = env.ComponentProgram(
-    'validator_test',
-    ['unreviewed/validator_test.c'],
-    EXTRA_LIBS=['dfa_validate_x86_32', 'dfa_validate_x86_64'])
+  check_decoders = []
 
-# Python-based regression test. TODO(pasko): remove it when validator_tests are
-# migrated to be able to run with the new validator.
-if env.Bit('validator_ragel'):
-  test_node = env.CommandTest('validator_test_py.out',
-                              command=['${PYTHON}',
-                                       env.File('unreviewed/validator_test.py'),
-                                       validator_test_exe])
-  env.AddNodeToTestSuite(test_node, ['small_tests'], 'run_validator_test_py')
+  for bits in ('32', '64'):
+    (one_valid_instr_rl,) = env.Command(
+        target='one_valid_instruction_x86_%s.rl' % bits,
+        source=[gen_dfa] + INST_DEFS,
+        action=('${SOURCES[0]} -o ${TARGET} %s '
+                '-d check_access,rex_prefix,vex_prefix,opcode '
+                '-d parse_operands_states,parse_operands,instruction_name '
+                '-m %s') % (
+               ' '.join('${SOURCES[%d]}' % (i + 1)
+                        for i in range(len(INST_DEFS))),
+               {'32': 'ia32', '64': 'amd64'}[bits])
+    )
+
+    include_dir = one_valid_instr_rl.dir.get_abspath()
+
+    (one_instr_dot,) = env.Command(
+        target='one_instruction_x86_%s.dot' % bits,
+        source=[
+            'unreviewed/one_instruction_x86_%s.rl' % bits,
+            one_valid_instr_rl],
+        action=[
+            '%s -V -I%s ${SOURCES[0]} -o ${TARGET}' %
+            (ragel_binary, include_dir)]
+    )
+
+    test_dfa_transitions = test_env.Command(
+        target='test_dfa_transitions_x86_%s.c' % bits,
+        source=['unreviewed/parse_dfa.py', one_instr_dot],
+        action='python ${SOURCES[0]} <"${SOURCES[1]}" >"${TARGET}"'
+    )
+
+    test_dfa_transitions_object = test_env.Object(test_dfa_transitions)
+
+    test_dfa = test_env.ComponentProgram(
+        'test_dfa_x86_%s' % bits,
+        [test_dfa_object, test_dfa_transitions_object])
+
+    fast_temp_for_test = '/dev/shm'
+
+    (check_decoder,) = env.Command(
+        target='objdump_test_fake_file_%s' % bits,
+        source=[
+            'unreviewed/run_objdump_test.py',
+            gas,
+            objdump,
+            decoder_test,
+            'unreviewed/decoder_test_one_file.sh',
+            test_dfa,
+        ],
+        action=[(
+            'python ${SOURCES[0]} '
+            '--gas="${SOURCES[1]} --%s" '
+            '--objdump="${SOURCES[2]}" '
+            '--decoder="${SOURCES[3]}" '
+            '--tester="${SOURCES[4]}" '
+            '-- '
+            '"${SOURCES[5]}" "%s"'
+            ) % (bits, fast_temp_for_test),
+        ])
+    check_decoders.append(check_decoder)
+
+  SideEffect(fast_temp_for_test, check_decoders)
+
+  env.AlwaysBuild(env.Alias('dfagen', automata))
+  env.AlwaysBuild(env.Alias('dfaclean', action=map(Delete, automata)))
+  env.AlwaysBuild(env.Alias('dfacheckdecoder', check_decoders))