| OLD | NEW |
| (Empty) |
| 1 #!/bin/bash | |
| 2 # | |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
| 4 # for details. All rights reserved. Use of this source code is governed by a | |
| 5 # BSD-style license that can be found in the LICENSE file. | |
| 6 | |
| 7 # Compare the current working copy of the repository to either a clean copy | |
| 8 # or the specified revision. | |
| 9 # Along with the files created by compiler_metrics.sh, creates | |
| 10 # compiler_compare[_stats_[12]].log in the working direcetory and a | |
| 11 # out_compile_samples/ directory at the root of the repository, | |
| 12 # which are left around for later examination. | |
| 13 # These files will be destroyed on script re-run. | |
| 14 | |
| 15 APP="" | |
| 16 REV="" | |
| 17 ONE_DELTA="" | |
| 18 BASE_PATH=$(pwd) | |
| 19 SCRIPT_PATH=$(dirname $0) | |
| 20 SCRIPT_PATH=$(cd $SCRIPT_PATH; pwd) | |
| 21 RUNS=10 | |
| 22 | |
| 23 function printHelp() { | |
| 24 exitValue=${1:1} | |
| 25 echo "Compare current changes as they relate to performance with another SVN r
evision" | |
| 26 echo "" | |
| 27 echo " Usage:" | |
| 28 echo " -a=, --app= The dart app file to test (required)." | |
| 29 echo " -d=, --one-delta= The filename, relative to app, to touch in order t
o trigger a one-delta compile." | |
| 30 echo " -r=, --revision= The compiler revision to compare against (default
to current repository revision)." | |
| 31 echo " --runs= Number of runs to test with, default $RUNS" | |
| 32 echo " --stats Non-destructive display of previous stats" | |
| 33 echo " -h, --help What you see is what you get." | |
| 34 exit $exitValue | |
| 35 } | |
| 36 | |
| 37 function failTest() { | |
| 38 if [ ! $1 -eq 0 ]; then | |
| 39 echo $2 | |
| 40 exit $1 | |
| 41 fi | |
| 42 } | |
| 43 | |
| 44 if [ $# -eq 0 ]; then | |
| 45 printHelp; | |
| 46 fi | |
| 47 | |
| 48 for i in $* | |
| 49 do | |
| 50 case $i in | |
| 51 --one-delta=*|-d=*) | |
| 52 ONE_DELTA=${i#*=} | |
| 53 COMPARE_OPTIONS+="--one-delta=$ONE_DELTA ";; | |
| 54 --app=*|-a=*) | |
| 55 APP=${i#*=} | |
| 56 APP=$( cd "$( dirname "$APP" )" && pwd )/$( basename "$APP");; | |
| 57 --revision=*|-r=*) | |
| 58 REV=${i#*=};; | |
| 59 --help|-h) | |
| 60 printHelp 0;; | |
| 61 --runs=*) | |
| 62 RUNS=${i#*=};; | |
| 63 --stats) | |
| 64 calcStats | |
| 65 exit 0;; | |
| 66 *) | |
| 67 echo "Invalid parameter: $i" | |
| 68 printhelp 1;; | |
| 69 esac | |
| 70 done | |
| 71 | |
| 72 COMPARE_OPTIONS+="-r=$RUNS " | |
| 73 | |
| 74 if [ "" = "$APP" ] || [ ! -r $APP ]; then | |
| 75 echo "Required --app" | |
| 76 printHelp | |
| 77 fi | |
| 78 | |
| 79 RESPONSE[0]="Performance: Better" | |
| 80 RESPONSE[1]="Performance: No Change" | |
| 81 RESPONSE[2]="Performance: Worse" | |
| 82 | |
| 83 # Passed: MAX deviation, Changed Amount | |
| 84 # Returns: Index of RESPONSE | |
| 85 function responseValue() { | |
| 86 ABS=$2 | |
| 87 (( ABS = ABS < 0 ? ABS * -1 : ABS )) | |
| 88 if [ $ABS -gt $1 ]; then | |
| 89 if [ $2 -lt 0 ]; then | |
| 90 return 2 | |
| 91 else | |
| 92 return 0 | |
| 93 fi | |
| 94 fi | |
| 95 return 1 | |
| 96 } | |
| 97 | |
| 98 function calcStat() { | |
| 99 # Assume we're always making improvments, S2 will be presented as a larger val
ue | |
| 100 LINEMATCH_DEV="s/${1}: .*stdev: \([0-9]*\).*/\1/p" | |
| 101 LINEMATCH_VALUE="s/${1}: average: \([0-9]*\).*/\1/p" | |
| 102 S1_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT1) | |
| 103 if [ "" != "$S1_DEV" ]; then | |
| 104 S2_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT2) | |
| 105 DEV_MAX=$(( S1_DEV < S2_DEV ? S2_DEV : S1_DEV )) | |
| 106 S1_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT1) | |
| 107 S2_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT2) | |
| 108 DIFF=$(( S2_VALUE - S1_VALUE )) | |
| 109 DIFF_PERCENT=`echo "$DIFF*100/$S2_VALUE" | bc -l | sed 's/\([0-9]*\.[0-9][0-
9]\)[0-9]*/\1/'` | |
| 110 responseValue $DEV_MAX $DIFF | |
| 111 echo "$1: Before/After-ms(${S2_VALUE} / ${S1_VALUE}), stdev(${S2_DEV} / ${S1
_DEV}), Difference: ${DIFF}ms (${DIFF_PERCENT}%), " ${RESPONSE[$?]} | |
| 112 fi | |
| 113 return 0 | |
| 114 } | |
| 115 | |
| 116 function calcStats() { | |
| 117 calcStat full-compile | |
| 118 calcStat zero-delta-compile | |
| 119 calcStat one-delta-compile | |
| 120 } | |
| 121 | |
| 122 ROOT_OF_REPO=$BASE_PATH | |
| 123 TEST_DIR=$BASE_PATH | |
| 124 while true; do | |
| 125 ls -d .gclient > /dev/null 2>&1 | |
| 126 if [ $? -eq 0 ]; then | |
| 127 echo "Root found: $ROOT_OF_REPO" | |
| 128 break; | |
| 129 fi | |
| 130 if [ "$TEST_DIR" = "/" ]; then | |
| 131 failTest 1 "Hit the root directory; no .git/ found?!" | |
| 132 fi | |
| 133 ROOT_OF_REPO=$TEST_DIR | |
| 134 cd .. | |
| 135 TEST_DIR=$(pwd) | |
| 136 done | |
| 137 | |
| 138 # Make a temporary directory in the current path and checkout the revision | |
| 139 TMP_DIR=$ROOT_OF_REPO/compiler/tmp_performance_comparisons | |
| 140 mkdir -p $TMP_DIR | |
| 141 | |
| 142 LOG_FILE=$TMP_DIR/compiler_compare.log | |
| 143 STAT1=$TMP_DIR/compiler_compare_stats_1.log | |
| 144 STAT2=$TMP_DIR/compiler_compare_stats_2.log | |
| 145 COMPARE_OPTIONS+="--output=$TMP_DIR " | |
| 146 | |
| 147 # zero out files | |
| 148 echo "" > $LOG_FILE | |
| 149 echo "" > $STAT1 | |
| 150 echo "" > $STAT2 | |
| 151 | |
| 152 # Do the first test | |
| 153 echo "Compiling dartc with your changes (mode=release)" | |
| 154 cd $ROOT_OF_REPO/compiler | |
| 155 gclient runhooks >> $LOG_FILE 2>&1 | |
| 156 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
| 157 failTest $? "Error compiling your location changes, check $LOG_FILE" | |
| 158 | |
| 159 echo "Running first test against current working copy" | |
| 160 echo $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=yours --dartc=./out/Release
_ia32/dartc $COMPARE_OPTIONS --app=$APP >> $LOG_FILE | |
| 161 $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=yours --dartc=./out/Release_ia32
/dartc $COMPARE_OPTIONS --app=$APP > $STAT1 | |
| 162 failTest $? "Error collecting statistics from working copy" | |
| 163 | |
| 164 # switch to tmp for remainder of building | |
| 165 cd $TMP_DIR | |
| 166 | |
| 167 gclient config https://dart.googlecode.com/svn/branches/bleeding_edge/deps/compi
ler.deps >> $LOG_FILE 2>&1 | |
| 168 failTest $? "Error calling gclient config" | |
| 169 | |
| 170 GCLIENT_SYNC="-t " | |
| 171 if [ "" != "$REV" ]; then | |
| 172 GCLIENT_SYNC+="--revision=$REV" | |
| 173 fi | |
| 174 | |
| 175 echo "Checking out clean version of $REV; will take some time. Look at $LOG_FILE
for progress" | |
| 176 gclient sync $GCLIENT_SYNC >> $LOG_FILE 2>&1 | |
| 177 failTest $? "Error calling gclient sync" | |
| 178 | |
| 179 echo "Compiler clean version; may take some time" | |
| 180 cd compiler | |
| 181 gclient runhooks >> $LOG_FILE 2>&1 | |
| 182 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
| 183 failTest $? "Error compiling comparison revision" | |
| 184 | |
| 185 # Do the second test | |
| 186 echo "Running second test against clean copy" | |
| 187 echo $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=clean --dartc=./out/Release
_ia32/dartc $COMPARE_OPTIONS --app=$APP >> $LOG_FILE | |
| 188 $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=clean --dartc=./out/Release_ia32
/dartc $COMPARE_OPTIONS --app=$APP > $STAT2 | |
| 189 failTest $? "Error collecting statistics from clean copy" | |
| 190 | |
| 191 calcStats | |
| OLD | NEW |