(abandoned) Validator tests: convert hexes to TFs and run on bots (for prod. validator only)

src/trusted/validator/x86/testing/tf/tf.py

Index: src/trusted/validator/x86/testing/tf/tf.py
diff --git a/src/trusted/validator/x86/testing/tf/tf.py b/src/trusted/validator/x86/testing/tf/tf.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2ca97e9222537db880bf9ee60fafb22db8f95ee
--- /dev/null
+++ b/src/trusted/validator/x86/testing/tf/tf.py
@@ -0,0 +1,238 @@
+# Copyright (c) 2012 The Native Client Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import copy
+import re
+
+import asm
+import utils
+import val_runner
+
+
+class Instruction(object):
+  __slots__ = [
+      'offset',
+      'asm',
+      'hex',
+      'data',
+      'outs',  # {validator: [message, ...], ...}
+  ]
+
+  def __init__(self):
+    self.offset = None
+    self.hex = None
+    self.asm = None
+    self.outs = dict((v, []) for v in val_runner.VALIDATORS)
+
+  @property
+  def size(self):
+    return len(self.data)
+
+  @property
+  def end_offset(self):
+    return self.offset + self.size
+
+  def CheckAsm(self, bits):
+    if self.asm is None:
+      return
+    asm_data = asm.Assemble(bits, self.asm)
+    assert self.data == asm_data, (utils.DataToReadableHex(asm_data),
+                                   utils.DataToReadableHex(self.data))
+
+  def __repr__(self):
+    return 'Instr(%s)@0x%x' % (self.asm or self.hex, self.offset)
+
+  _out_suffix = '_out'
+
+  @staticmethod
+  def Parse(bits, lines):
+    instr = Instruction()
+    for line in lines:
+      assert ':' in line, "can't parse line '%s" % line

[Mark Seaborn, 2012/09/10 19:38:06]

Nit: missing closing '

[Vlad Shcherbina, 2012/09/11 14:26:00]

Done.

+      field, value = line.split(':', 1)
+      field = field.strip()
+
+      if field.endswith(Instruction._out_suffix):
+        validator = field[:-len(Instruction._out_suffix)]
+        instr.outs[validator].append(value.strip())
+      else:
+        assert getattr(instr, field) is None, 'field %s is already set' % field
+        setattr(instr, field, value.strip())
+
+    if instr.hex is not None:
+      instr.data = utils.ReadableHexToData(instr.hex)
+    else:
+      instr.data = asm.Assemble(bits, instr.asm)
+      instr.hex = utils.DataToReadableHex(instr.data)
+    return instr
+
+  def ToLines(self, offset=None):
+    lines = []
+    if self.offset != offset:
+      lines.append('offset: %s' % self.offset)
+    if self.asm is not None:
+      lines.append('asm: %s' % self.asm)
+    if self.hex is not None:
+      lines.append('hex: %s' % self.hex)
+
+    for validator in val_runner.VALIDATORS:
+      for msg in self.outs[validator]:
+        lines.append('%s%s: %s' % (validator, Instruction._out_suffix, msg))
+
+    return lines
+
+
+class Test(object):
+  __slots__ = [
+      'bits',
+      'separators',
+      'sections',
+      'instructions',
+      'safe',
+  ]
+
+  # Each separator/section is a list of strings.
+  # Number of separators always exceeds number of sections by one, because
+  # they interleave, starting and finishing with (possibly empty) separator.
+
+  @staticmethod
+  def Parse(lines):
+    test = Test()
+    test.safe = None
+    test.separators = [[]]
+    test.sections = []
+
+    # Parser state: whether we are currently parsing
+    # separator (comments/whitelines) or section.
+    in_section = False
+
+    for line in lines:
+      line = line.strip()
+      is_sep = line == '' or line.startswith('#')
+
+      if is_sep:
+        if in_section:
+          test.separators.append([line])
+        else:
+          test.separators[-1].append(line)
+      else:
+        if in_section:
+          test.sections[-1].append(line)
+        else:
+          test.sections.append([line])
+
+      in_section = not is_sep
+
+    if in_section:
+      test.separators.append([])
+
+    assert len(test.separators) == len(test.sections) + 1
+
+    # header section is required; it specifies BITS and OUTCOME
+    assert len(test.sections) >= 1
+
+    for line in test.sections[0]:
+      m = re.match(r'(.*):\s*(.*)$', line)
+      field, value = m.groups()
+      if field == 'BITS':
+        test.bits = int(value)
+      elif field == 'OUTCOME':
+        assert value in ['valid', 'invalid']
+        test.safe = value == 'valid'
+      else:
+        assert False, 'Unrecognized field %s in special section' % field
+
+    test.instructions = []
+    offset = 0
+    for section in test.sections[1:]:
+      instr = Instruction.Parse(test.bits, section)
+
+      if instr.hex is None:
+        code = asm.Assemble(test.bits, instr.asm)
+        instr.hex = utils.DataToReadableHex(code)
+
+      if instr.offset is not None:
+        instr.offset = int(instr.offset)
+      else:
+        instr.offset = offset
+      test.instructions.append(instr)
+      offset = instr.end_offset
+
+    return test
+
+  def Print(self, fout):
+    self.sections[0] = ['BITS: %s' % self.bits]
+    if self.safe is not None:
+      self.sections[0].append(
+          'OUTCOME: valid' if self.safe else 'OUTCOME: invalid')
+
+    offset = 0
+    for i, instr in enumerate(self.instructions):
+      self.sections[i+1] = instr.ToLines(offset)
+      offset = instr.end_offset
+
+    assert len(self.separators) == len(self.sections) + 1
+    groups = []
+    for sep, sec in zip(self.separators, self.sections):
+      groups.append(sep)
+      groups.append(sec)
+    groups.append(self.separators[-1])
+
+    for group in groups:
+      for line in group:
+        fout.write('%s\n' % line)
+
+  def PrepareCode(self):
+    code_size = max(i.end_offset for i in self.instructions)
+    code_size = ((code_size - 1) // 32 + 1) * 32
+    code = ['\x90'] * code_size
+
+    for i in self.instructions:
+      code[i.offset : i.end_offset] = list(i.data)
+
+    return ''.join(code)
+
+  def RunValidator(self, validator):
+    assert validator in val_runner.VALIDATORS
+    if validator == 'nc':
+      return val_runner.RunValidator(validator, self.bits, self.PrepareCode())
+
+    test2 = copy.deepcopy(self)
+
+    errors = set()
+    safe = True
+
+    while True:
+      res = val_runner.RunValidator(validator, test2.bits, test2.PrepareCode())
+      safe = safe and res.safe
+      errors.update(res.errors)
+
+      patched = False
+      for loc, msg in res.errors:
+        if msg == 'DFA error in validator':
+          for i in test2.instructions:
+            if i.offset == loc:
+              nops = '\x90'*i.size

[Mark Seaborn, 2012/09/10 19:38:06]

Nit: put spaces around '*'.  Also comment that you are padding with NOPs and 90
is NOP.

[Vlad Shcherbina, 2012/09/11 14:26:00]

I think the name of the variable suggests it by itself, comment here would be
redundant. But I realized that the whole process requires some explanation, so I
added a more voluminous comment few lines above.

+              if i.data != nops:
+                i.data = nops
+                patched = True
+      if not patched:
+        break
+
+    return val_runner.ValidationResults(safe=safe, errors=sorted(errors))
+
+  def ExpectedErrors(self, validator):
+    errors = []
+    for i in self.instructions:
+      for msg in i.outs[validator]:
+        # Check if message has the form
+        #   [at +<delta>] <error message>
+        m = re.match(r'\[at \+(\d+)\]\s(.*)$', msg)
+        if m is not None:
+          delta = int(m.group(1))
+          msg = m.group(2)
+        else:
+          delta = 0
+        errors.append((i.offset + delta, msg))
+    return errors