(abandoned) Validator tests: convert hexes to TFs and run on bots (for prod. validator only)

src/trusted/validator/x86/testing/tf/val_runner.py

Index: src/trusted/validator/x86/testing/tf/val_runner.py
diff --git a/src/trusted/validator/x86/testing/tf/val_runner.py b/src/trusted/validator/x86/testing/tf/val_runner.py
new file mode 100644
index 0000000000000000000000000000000000000000..7ef69a9e91aa72192d45b0cfa8c037570755839c
--- /dev/null
+++ b/src/trusted/validator/x86/testing/tf/val_runner.py
@@ -0,0 +1,221 @@
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import collections
+import os
+import re
+import subprocess
+
+import utils
+
+
+VALIDATORS = ['nc', 'dfa']
+
+
+scons_out = '../../../../../../scons-out'
+ncval32 = os.path.join(scons_out, 'opt-linux-x86-32/staging/ncval')

[Mark Seaborn, 2012/09/10 19:38:06]

You shouldn't hard-code Scons output filenames like this...

[Vlad Shcherbina, 2012/09/11 14:26:00]

I was using it to run locally (on bots paths are always passed explicitly), but
right, it's time to get rid of it now.
Done.

+ncval64 = os.path.join(scons_out, 'opt-linux-x86-64/staging/ncval')
+rdfaval = os.path.join(scons_out, 'opt-linux-x86-32/staging/validator_test')
+
+
+def AddValidatorsOptions(option_parser):
+  option_parser.add_option('--ncval32', dest='ncval32', type='string')
+  option_parser.add_option('--ncval64', dest='ncval64', type='string')
+  option_parser.add_option('--rdfaval', dest='rdfaval', type='string')
+
+
+def ProcessValidatorsOptions(options):
+  global ncval32

[Mark Seaborn, 2012/09/10 19:38:06]

It would be cleaner if you didn't modify global variables like this.  Can you
pass around the options object instead?

[Vlad Shcherbina, 2012/09/11 14:26:00]

Well, I don't like it because it would mean passing mostly irrelevant argument
through chains of calls. I think this case does not qualify as mutable global
variables, because assignment is supposed to happen only once, so usual perils
of globals are more or less avoided. I prefer to leave it as it is unless you
really insist.

+  global ncval64
+  global rdfaval
+
+  if options.ncval32 is not None:
+    ncval32 = options.ncval32
+
+  if options.ncval64 is not None:
+    ncval64 = options.ncval64
+
+  if options.rdfaval is not None:
+    rdfaval = options.rdfaval
+
+
+def SplitToLines(content):
+  return [line.rstrip() for line in content.rstrip().split('\n')]
+
+
+def RunNcVal32(hex_name):
+  args = [
+      '--hex_text=-',
+      '--max_errors=-1',
+      '--detailed=false',
+      '--cpuid-all'
+  ]
+
+  with open(hex_name) as input:
+    result = utils.CheckOutput([ncval32] + args, stdin=input)
+
+  return SplitToLines(result)
+
+
+def RunNcVal64(hex_name):
+  args = [
+      '--hex_text=-',
+      '--max_errors=-1',
+      '--readwrite_sfi',
+      '--annotate=false',
+      '--cpuid-all',
+      '--detailed=false',
+  ]
+
+  with open(hex_name) as input:
+    result = utils.CheckOutput([ncval64] + args, stdin=input)
+
+  return SplitToLines(result)
+
+
+def ParseNcValVerdict(last_line):
+  m = re.match(r'\*\*\* <input> (is safe|IS UNSAFE) \*\*\*$', last_line)
+  assert m is not None, 'unexpected ncval output "%s"' % last_line
+  return m.group(1) == 'is safe'
+
+
+def ParseNcVal32(lines):
+  assert len(lines) > 0, 'ncval output is empty'
+
+  errors = []
+  for line in lines[:-1]:
+    line = line.strip()
+    if line == '':
+      continue
+    if re.match(r'.+ > .+ \(read overflow of .+ bytes\)', line):

[Mark Seaborn, 2012/09/10 19:38:06]

Why are you dropping this line of output?

[Vlad Shcherbina, 2012/09/11 14:26:00]

Right, this should not happen anyway. It's something very specific to prod.
validator, and it only arises in mem_overflow.hex. Removed the check.

+      continue
+    if line == 'ErrorSegmentation':

[Mark Seaborn, 2012/09/10 19:38:06]

Ditto: why drop this?

[Vlad Shcherbina, 2012/09/11 14:26:00]

Same.

+      continue
+    # Parse error message of the form
+    #   VALIDATOR: 4: Bad prefix usage
+    m = re.match(r'VALIDATOR: ([0-9a-f]+): (.*)$', line, re.IGNORECASE)
+    offset = int(m.group(1), 16)
+    message = m.group(2)
+    errors.append((offset, message))
+  safe = ParseNcValVerdict(lines[-1])
+  return safe, errors
+
+
+def ParseNcVal64(lines):
+  assert len(lines) > 0, 'ncval output is empty'
+
+  errors = []
+  for i, line in enumerate(lines[:-1]):
+    #print line

[Mark Seaborn, 2012/09/10 19:38:06]

Please remove commented-out code

[Vlad Shcherbina, 2012/09/11 14:26:00]

Done.

+    #if 'Bad basic' in line:
+    #  print 'z'*10
+    #  print line
+    #  raw_input()
+
+    if line.startswith('VALIDATOR: Checking jump targets:'):
+      continue
+    if line.startswith('VALIDATOR: Checking that basic blocks are aligned'):
+      continue
+
+    # Skip disassembler output of the form
+    #   VALIDATOR: 0000000000000003: 49 89 14 07    mov [%r15+%rax*1], %rdx
+    m = re.match(r'VALIDATOR: ([0-9a-f]+):', line, re.IGNORECASE)
+    if m is not None:
+      continue
+
+    # Parse error message of the form
+    #   VALIDATOR: ERROR: 20: Bad basic block alignment.
+    m = re.match(r'VALIDATOR: ERROR: ([0-9a-f]+): (.*)', line, re.IGNORECASE)
+    if m is not None:
+      offset = int(m.group(1), 16)
+      errors.append((offset, m.group(2)))
+      continue
+
+    # Parse two-line error messages of the form
+    #   VALIDATOR: 0000000000000003: 49 89 14 07    mov [%r15+%rax*1], %rdx
+    #   VALIDATOR: ERROR: Invalid index register in memory offset
+    m = re.match(r'VALIDATOR: ((?:ERROR|WARNING): .*)$', line, re.IGNORECASE)
+    if m is not None:
+      m2 = re.match(r'VALIDATOR: ([0-9a-f]+):', lines[i-1], re.IGNORECASE)
+      assert m2 is not None, "can't parse line '%s' preceding line '%s'" % (
+          lines[i-1],
+          line
+      )
+      offset = int(m2.group(1), 16)
+      errors.append((offset, m.group(1)))
+      continue
+
+    assert False, "can't parse line '%s'" % line

[Mark Seaborn, 2012/09/10 19:38:06]

Better style would be:
  raise AssertionError("can't parse line '%s'" % line)

[Vlad Shcherbina, 2012/09/11 14:26:00]

Done.

+
+  safe = ParseNcValVerdict(lines[-1])
+  return safe, errors
+
+
+def MakeElf(bits, data, elf_filename):
+  assert bits in [32, 64]
+
+  with utils.TempFile(mode='w') as asm_file:
+    for c in data:
+      asm_file.write('.byte %d\n' % ord(c))
+    asm_file.flush()
+
+    subprocess.check_call([
+        'as', '--%s' % bits, asm_file.name, '-o', elf_filename])
+
+
+def RunDfa(bits, data):
+  with utils.TempFile(mode='w') as elf_file:
+    with utils.TempFile(mode='w+') as out_file:
+      MakeElf(bits, data, elf_file.name)
+
+      result = subprocess.call([rdfaval, elf_file.name], stdout=out_file)
+      out_file.seek(0)
+      lines = SplitToLines(out_file.read())
+
+      return result, lines
+
+
+def ParseDfa(lines):

[Mark Seaborn, 2012/09/10 19:38:06]

Please can you take out all of the DFA-validator related stuff from this file,
and save it for a later change?

[Vlad Shcherbina, 2012/09/11 14:26:00]

Done.

+  errors = []
+  for line in lines:
+    if re.match(r"file '.*' can not be fully validated$", line):
+      continue
+    m = re.match(r'bad jump to around 0x([0-9a-f]+)$', line, re.IGNORECASE)
+    if m is not None:
+      offset = int(m.group(1), 16)
+      errors.append((offset, 'bad jump to around'))
+      continue
+
+    m = re.match(r'offset 0x([0-9a-f]+): (.*)$', line, re.IGNORECASE)
+    offset = int(m.group(1), 16)
+    message = m.group(2)
+    errors.append((offset, message))
+
+  return errors
+
+
+ValidationResults = collections.namedtuple('ValidationResults', 'safe, errors')
+
+
+def RunValidator(validator, bits, data):
+  assert validator in VALIDATORS
+  assert bits in [32, 64]
+  assert len(data) % 32 == 0
+
+  if validator == 'nc':
+    with utils.TempFile(mode='w') as hex_file:
+      hex_file.write('%s\n' % utils.DataToReadableHex(data))
+      hex_file.flush()
+
+      if bits == 32:
+        safe, errors = ParseNcVal32(RunNcVal32(hex_file.name))
+      elif bits == 64:
+        safe, errors = ParseNcVal64(RunNcVal64(hex_file.name))
+
+  elif validator == 'dfa':
+    code, output = RunDfa(bits, data)
+    safe = (code == 0)
+    errors = ParseDfa(output)
+
+  return ValidationResults(safe=safe, errors=errors)