add and enable validator tests for x86

buildbot/buildbot_spec2k.sh

 #!/bin/bash
 # Copyright (c) 2012 The Native Client Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 set -o xtrace
 set -o nounset
 set -o errexit
 
 ######################################################################
 # SCRIPT CONFIG
 ######################################################################
 
-CLOBBER=${CLOBBER:-yes}
-SCONS_TRUSTED="./scons --mode=opt-host -j8"
-SCONS_NACL="./scons --mode=opt-host,nacl -j8"
-SPEC_HARNESS=${SPEC_HARNESS:-${HOME}/cpu2000-redhat64-ia32}/
+readonly CLOBBER=${CLOBBER:-yes}
+readonly SCONS_TRUSTED="./scons --mode=opt-host -j8"
+readonly SCONS_NACL="./scons --mode=opt-host,nacl -j8"
+readonly SPEC_HARNESS=${SPEC_HARNESS:-${HOME}/cpu2000-redhat64-ia32}/
 
-TRYBOT_TESTS="176.gcc 179.art 181.mcf 197.parser 252.eon 254.gap"
-TRYBOT_TRANSLATOR_TESTS="176.gcc"
-TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS="176.gcc"
+readonly TRYBOT_TESTS="176.gcc 179.art 181.mcf 197.parser 252.eon 254.gap"
+readonly TRYBOT_TRANSLATOR_TESTS="176.gcc"
+readonly TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS="176.gcc"
 
 readonly BUILDBOT_PNACL="buildbot/buildbot_pnacl.sh"
 readonly UP_DOWN_LOAD="buildbot/file_up_down_load.sh"
 
-SPEC_BASE="tests/spec2k"
+readonly SPEC_BASE="tests/spec2k"
 readonly ARCHIVE_NAME=$(${SPEC_BASE}/run_all.sh GetTestArchiveName)
 
 readonly NAME_ARM_TRY_UPLOAD=$(${BUILDBOT_PNACL} NAME_ARM_TRY_UPLOAD)
 readonly NAME_ARM_TRY_DOWNLOAD=$(${BUILDBOT_PNACL} NAME_ARM_TRY_DOWNLOAD)
 readonly NAME_ARM_UPLOAD=$(${BUILDBOT_PNACL} NAME_ARM_UPLOAD)
 readonly NAME_ARM_DOWNLOAD=$(${BUILDBOT_PNACL} NAME_ARM_DOWNLOAD)
 
+readonly QEMU_TOOL="$(pwd)/toolchain/linux_arm-trusted/run_under_qemu_arm"
+
+# Note: the tool for updating the canned nexes lives at:
+#        tools/canned_nexe_tool.sh
+readonly CANNED_NEXE_REV=1001
+
 # If true, terminate script when first error is encountered.
-FAIL_FAST=${FAIL_FAST:-false}
+readonly FAIL_FAST=${FAIL_FAST:-false}
 RETCODE=0
 
 # Print the number of tests being run for the buildbot status output
 testcount() {
   local tests="$1"
   if [[ ${tests} == "all" ]]; then
     echo "all"
   else
     echo ${tests} | wc -w
   fi
@@ skipping 75 matching lines @@
   local tests="$1"
   local try="$2" # set to "try" if this is a try run
 
   pushd ${SPEC_BASE}
   echo "@@@BUILD_STEP spec2k archive@@@"
   ./run_all.sh PackageArmBinaries ${tests}
   popd
   echo "@@@BUILD_STEP spec2k upload@@@"
   if [[ ${try} == "try" ]]; then
     ${UP_DOWN_LOAD} UploadArmBinariesForHWBotsTry ${NAME_ARM_TRY_UPLOAD} \
-      ${ARCHIVE_NAME}
+        ${ARCHIVE_NAME}
   else
     ${UP_DOWN_LOAD} UploadArmBinariesForHWBots ${NAME_ARM_UPLOAD} \
-      ${ARCHIVE_NAME}
+        ${ARCHIVE_NAME}
   fi
 }
 
 download-test-binaries() {
   local try="$1"
   echo "@@@BUILD_STEP spec2k download@@@"
   if [[ ${try} == "try" ]]; then
     ${UP_DOWN_LOAD} DownloadArmBinariesForHWBotsTry ${NAME_ARM_TRY_DOWNLOAD} \
-      ${ARCHIVE_NAME}
+        ${ARCHIVE_NAME}
   else
     ${UP_DOWN_LOAD} DownloadArmBinariesForHWBots ${NAME_ARM_DOWNLOAD} \
-      ${ARCHIVE_NAME}
+        ${ARCHIVE_NAME}
   fi
   echo "@@@BUILD_STEP spec2k untar@@@"
   pushd ${SPEC_BASE}
   ./run_all.sh UnpackArmBinaries
   popd
 }
 
+download-validator-test-nexes() {
+  local arch="$1"
+  echo "@@@BUILD_STEP validator test download@@@"
+  ${UP_DOWN_LOAD} DownloadArchivedNexes ${CANNED_NEXE_REV} \
+      "${arch}_giant" giant_nexe.tar.bz2
+  # This generates "CannedNexes/" in the current directory
+  rm -rf CannedNexes
+  tar jxf  giant_nexe.tar.bz2
+}
+
+get-validator() {
+  local arch="$1"
+  if [[ ${arch} == "x86-32" ]] ; then
+    echo "$(pwd)/scons-out/opt-linux-x86-32/staging/ncval"
+  elif [[ ${arch} == "x86-64" ]] ; then
+    echo "$(pwd)/scons-out/opt-linux-x86-64/staging/ncval"
+  elif [[ ${arch} == "arm" ]] ; then
+    echo "$(pwd)/scons-out/opt-linux-arm/staging/arm-ncval-core"
+  else
+    echo "ERROR: unknown arch"
+  fi
+}
+
+LogTimeHelper() {
+  # This format is recognized by the buildbot system
+  echo "RESULT $1_$2: $3= $(bc) secs"
+}
+
+LogTimedRun() {
+  local graph=$1
+  local benchmark=$2
+  local variant=$3
+  shift 3
+  # S: system mode CPU-seconds used by the process
+  # U: user mode CPU-seconds  used by the process
+  # We add a plus sign inbetween so that we can pipe the output to "bc"
+  # Note: the  >() magic creates a "fake" file (think named pipe)
+  #       which passes the output of time to LogTimeHelper
+  /usr/bin/time -f "%U + %S" \
+      --output >(LogTimeHelper ${graph} ${benchmark} ${variant}) \
+      "$@"
+}
+
+build-validator() {
+  local arch="$1"
+  echo "@@@BUILD_STEP build validator [${arch}]@@@"
+  if [[ ${arch} == "arm" ]] ; then
+    # TODO(robertm): build the validator
+    echo "NYI"
+  else
+    ${SCONS_NACL} platform=${arch} ncval
+  fi
+}
+
+measure-validator-speed() {
+  local arch="$1"
+  local validator=$(get-validator ${arch})
+
+  echo "@@@BUILD_STEP validator speed test [${arch}]@@@"
+  if [[ ! -e ${validator} ]] ; then
+    echo "ERROR: missing validator executable: ${validator}"
+    handle-error
+    return
+  fi
+
+  if [[ ${arch} == "arm" && $(uname -p) != arm* ]] ; then
+    # TODO(robertm): build the validator
+    validator="${QEMU_TOOL} ${validator}"
+  fi
+
+  for nexe in CannedNexes/* ; do
+    echo "timing validation of ${nexe}"
+    ls --size --block-size=1 ${nexe}
+    LogTimedRun  "validationtime" $(basename ${nexe}) "canned" \
+        ${validator} ${nexe}
+  done
+}
+
 ######################################################################
 # NOTE: trybots only runs a subset of the the spec2k tests
 # TODO: elminate this long running bot in favor per arch sharded bots
 pnacl-trybot-arm-qemu() {
   clobber
   build-prerequisites "arm" "bitcode"
   build-tests SetupPnaclArmOpt "${TRYBOT_TESTS}" 1 1
   run-tests SetupPnaclArmOpt "${TRYBOT_TESTS}" 1 1
   build-tests SetupPnaclTranslatorFastArmOpt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   run-tests SetupPnaclTranslatorFastArmOpt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
@@ skipping 23 matching lines @@
 
 pnacl-trybot-x8632() {
   clobber
   build-prerequisites "x86-32" "bitcode"
   build-tests SetupPnaclX8632Opt "${TRYBOT_TESTS}" 1 1
   run-tests SetupPnaclX8632Opt "${TRYBOT_TESTS}" 1 1
   build-tests SetupPnaclTranslatorX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   run-tests SetupPnaclTranslatorX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   build-tests SetupPnaclTranslatorFastX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   run-tests SetupPnaclTranslatorFastX8632Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
+  build-validator x86-32
+  download-validator-test-nexes x86-32
+  measure-validator-speed x86-32
 }
 
 pnacl-x86-64-zero-based-sandbox() {
   clobber
   export NACL_ENABLE_INSECURE_ZERO_BASED_SANDBOX=1
   build-prerequisites "x86-64" "bitcode" "x86_64_zero_based_sandbox=1"
   build-tests SetupPnaclX8664ZBSOpt \
     "${TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS}" 1 1
   run-tests SetupPnaclX8664ZBSOpt \
     "${TRYBOT_X86_64_ZERO_BASED_SANDBOX_TESTS}" 1 1
 }
 
 pnacl-trybot-x8664() {
   clobber
   build-prerequisites "x86-64" "bitcode"
   build-tests SetupPnaclX8664Opt "${TRYBOT_TESTS}" 1 1
   run-tests SetupPnaclX8664Opt "${TRYBOT_TESTS}" 1 1
   build-tests SetupPnaclTranslatorX8664Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   run-tests SetupPnaclTranslatorX8664Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   build-tests SetupPnaclTranslatorFastX8664Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   run-tests SetupPnaclTranslatorFastX8664Opt "${TRYBOT_TRANSLATOR_TESTS}" 1 1
   pnacl-x86-64-zero-based-sandbox
+  build-validator x86-64
+  download-validator-test-nexes x86-64
+  measure-validator-speed x86-64
 }
 
 # We probably will not keep a qemu bot on the waterfall, but will keep this
 # to test locally
 pnacl-arm-qemu() {
   clobber
   build-prerequisites "arm" "bitcode"
   # arm takes a long time and we do not have sandboxed tests working
   build-tests SetupPnaclArmOpt all 1 1
   run-tests SetupPnaclArmOpt all 1 1
@@ skipping 21 matching lines @@
   clobber
   build-prerequisites "x86-64" "bitcode"
   local setups="SetupPnaclX8664 \
                SetupPnaclX8664Opt \
                SetupPnaclTranslatorX8664 \
                SetupPnaclTranslatorX8664Opt \
                SetupPnaclTranslatorFastX8664Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
   pnacl-x86-64-zero-based-sandbox
+  measure-validator-speed x86-64
 }
 
 pnacl-x8632() {
   clobber
   build-prerequisites "x86-32" "bitcode"
   local setups="SetupPnaclX8632 \
                 SetupPnaclX8632Opt \
                 SetupPnaclTranslatorX8632 \
                 SetupPnaclTranslatorX8632Opt \
                 SetupPnaclTranslatorFastX8632Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  build-validator x86-32
+  download-validator-test-nexes x86-32
+  measure-validator-speed x86-32
 }
 
 nacl-x8632() {
   clobber
   build-prerequisites "x86-32" ""
   local setups="SetupNaclX8632 \
                 SetupNaclX8632Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  build-validator x86-32
+  download-validator-test-nexes x86-32
+  measure-validator-speed x86-32
 }
 
 nacl-x8664() {
   clobber
   build-prerequisites "x86-64" ""
   local setups="SetupNaclX8664 \
                 SetupNaclX8664Opt"
   build-tests "${setups}" all 1 3
   run-tests "${setups}" all 1 3
+  build-validator x86-64
+  download-validator-test-nexes x86-64
+  measure-validator-speed x86-64
 }
 
 
 ######################################################################
 # Script assumed to be run in native_client/
 if [[ $(pwd) != */native_client ]]; then
   echo "ERROR: must be run in native_client!"
   exit 1
 fi
 
 
 if [[ $# -eq 0 ]] ; then
   echo "you must specify a mode on the commandline:"
   exit 1
 fi
 
 if [ "$(type -t $1)" != "function" ]; then
   Usage
   echo "ERROR: unknown mode '$1'." >&2
   exit 1
 fi
 
 "$@"
 
 if [[ ${RETCODE} != 0 ]]; then
   echo "@@@BUILD_STEP summary@@@"
   echo There were failed stages.
   exit ${RETCODE}
 fi