| OLD | NEW |
| (Empty) |
| 1 #!/bin/bash | |
| 2 # | |
| 3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
| 4 # for details. All rights reserved. Use of this source code is governed by a | |
| 5 # BSD-style license that can be found in the LICENSE file. | |
| 6 | |
| 7 # Compares a series of compiler reivsions against a target application for | |
| 8 # statistic collection. Outputs a gnuplot consumable table. | |
| 9 | |
| 10 APP="" | |
| 11 REV="" | |
| 12 ONE_DELTA="" | |
| 13 BASE_PATH=$(pwd) | |
| 14 SCRIPT_PATH=$(dirname $0) | |
| 15 SCRIPT_PATH=$(cd $SCRIPT_PATH; pwd) | |
| 16 RUNS=10 | |
| 17 COUNT=50 | |
| 18 LOW_REV="" | |
| 19 | |
| 20 function printHelp() { | |
| 21 exitValue=${1:1} | |
| 22 echo "Compare performance of multiple compiler revisions against a given targe
t application." | |
| 23 echo "Creates a cache of pre-built compiler revisions in compiler/revs/ for la
ter comparison." | |
| 24 echo "The target output of this script is a gnuplot consumable list of stats (
for now), located" | |
| 25 echo "in tmp_performance_comparisons/compiler_plots.dat." | |
| 26 echo "" | |
| 27 echo " Usage:" | |
| 28 echo " -a=, --app= The dart app file to test (required)." | |
| 29 echo " -d=, --one-delta= The filename, relative to app, to touch in order t
o trigger a one-delta compile." | |
| 30 echo " -r=, --revision= The compiler revision to start comparing against (
default to current repository revision)." | |
| 31 echo " -c=, --count= The number of compiler revisions test against. De
fault 50." | |
| 32 echo " -l=, --low-rev= Alternative to --count; set the lowest revision to
run to" | |
| 33 echo " -n=, --runs= How many times each compiler is run against the ta
rget application." | |
| 34 echo " -h, --help What you see is what you get." | |
| 35 exit $exitValue | |
| 36 } | |
| 37 | |
| 38 function failTest() { | |
| 39 if [ ! $1 -eq 0 ]; then | |
| 40 echo $2 | |
| 41 exit $1 | |
| 42 fi | |
| 43 } | |
| 44 | |
| 45 RESPONSE[0]="Performance: Better" | |
| 46 RESPONSE[1]="Performance: No Change" | |
| 47 RESPONSE[2]="Performance: Worse" | |
| 48 | |
| 49 function calcStat() { | |
| 50 # Assume we're always making improvments, S2 will be presented as a larger val
ue | |
| 51 LINEMATCH_DEV="s/${1}: .*stdev: \([0-9]*\).*/\1/p" | |
| 52 LINEMATCH_VALUE="s/${1}: average: \([0-9]*\).*/\1/p" | |
| 53 S1_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT1) | |
| 54 if [ "" != "$S1_DEV" ]; then | |
| 55 S1_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT1) | |
| 56 echo -ne "\t"$S1_VALUE"\t"$S1_DEV >> $PLOTS | |
| 57 else | |
| 58 echo -ne "\t-1\t-1" >> $PLOTS | |
| 59 fi | |
| 60 return 0 | |
| 61 } | |
| 62 | |
| 63 function calcStats() { | |
| 64 echo -n $1 >> $PLOTS | |
| 65 calcStat full-compile | |
| 66 calcStat zero-delta-compile | |
| 67 calcStat one-delta-compile | |
| 68 echo "" >> $PLOTS | |
| 69 } | |
| 70 | |
| 71 if [ $# -eq 0 ]; then | |
| 72 printHelp; | |
| 73 fi | |
| 74 | |
| 75 for i in $* | |
| 76 do | |
| 77 case $i in | |
| 78 --one-delta=*|-d=*) | |
| 79 ONE_DELTA=${i#*=} | |
| 80 COMPARE_OPTIONS+="--one-delta=$ONE_DELTA ";; | |
| 81 --app=*|-a=*) | |
| 82 APP=${i#*=};; | |
| 83 --revision=*|-r=*) | |
| 84 REV=${i#*=};; | |
| 85 --count=*|-c=*) | |
| 86 COUNT=${i#*=} | |
| 87 LOW_REV="";; | |
| 88 --runs=*|-n=*) | |
| 89 RUNS=${i#*=};; | |
| 90 --low-rev=*|-l=*) | |
| 91 LOW_REV=${i#*=} | |
| 92 COUNT=0;; | |
| 93 --help|-h) | |
| 94 printHelp 0;; | |
| 95 *) | |
| 96 echo "Parameter $i not recognized" | |
| 97 printHelp 1;; | |
| 98 esac | |
| 99 done | |
| 100 | |
| 101 COMPARE_OPTIONS+="-r=$RUNS " | |
| 102 | |
| 103 if ((RUNS > 0)); then | |
| 104 if [ "" = "$APP" ] || [ ! -r $APP ]; then | |
| 105 echo "Required --app" " got: $APP" | |
| 106 printHelp 1 | |
| 107 fi | |
| 108 APP=$( cd "$( dirname "$APP" )" && pwd )/$( basename "$APP") | |
| 109 COMPARE_OPTIONS+="--app=$APP " | |
| 110 else | |
| 111 echo "Building up compiler cache" | |
| 112 APP="Compiler cache" | |
| 113 fi | |
| 114 | |
| 115 ROOT_OF_REPO=$BASE_PATH | |
| 116 TEST_DIR=$BASE_PATH | |
| 117 while true; do | |
| 118 ls -d .gclient > /dev/null 2>&1 | |
| 119 if [ $? -eq 0 ]; then | |
| 120 echo "Root found: $ROOT_OF_REPO" | |
| 121 break; | |
| 122 fi | |
| 123 if [ "$TEST_DIR" = "/" ]; then | |
| 124 failTest 1 "Hit the root directory; no .git/ found?!" | |
| 125 fi | |
| 126 ROOT_OF_REPO=$TEST_DIR | |
| 127 cd .. | |
| 128 TEST_DIR=$(pwd) | |
| 129 done | |
| 130 | |
| 131 # Make a temporary directory in the current path and checkout the revision | |
| 132 TMP_DIR=$ROOT_OF_REPO/compiler/tmp_performance_comparisons | |
| 133 mkdir -p $TMP_DIR | |
| 134 | |
| 135 LOG_FILE=$TMP_DIR/compiler_compare.log | |
| 136 PLOTS=$TMP_DIR/compiler_plots.dat | |
| 137 STAT1=$TMP_DIR/compiler_metrics.txt | |
| 138 COMPARE_OPTIONS+="--output=$TMP_DIR " | |
| 139 | |
| 140 # zero out files | |
| 141 echo "" > $LOG_FILE | |
| 142 | |
| 143 # switch to tmp for remainder of building | |
| 144 cd $TMP_DIR | |
| 145 gclient config https://dart.googlecode.com/svn/branches/bleeding_edge/deps/compi
ler.deps >> $LOG_FILE 2>&1 | |
| 146 failTest $? "Error calling gclient config" | |
| 147 | |
| 148 if [ "" == "$REV" ]; then | |
| 149 echo "No revision specified; checking out head for test" | |
| 150 REV=`svn info https://dart.googlecode.com/svn/branches/bleeding_edge/deps/comp
iler.deps | sed -n -e 's/Revision: \([0-9]*\)/\1/p'` | |
| 151 echo "Head revision = $REV" | |
| 152 fi | |
| 153 | |
| 154 function failStats() { | |
| 155 echo -e "$1\t-1\t0\t-1\t0\t-1\t0" >> $PLOTS | |
| 156 return 0; | |
| 157 } | |
| 158 | |
| 159 function compileRevision() { | |
| 160 REVISION=$1 | |
| 161 PREBUILT_DIR=$ROOT_OF_REPO/analyzer/revs/$REVISION/prebuilt | |
| 162 PREBUILT_BIN=$PREBUILT_DIR/analyzer/bin/dart_analyzer | |
| 163 if [ ! -x $PREBUILT_BIN ]; then | |
| 164 echo "No prebuilt, building and caching" | |
| 165 echo "Checking out clean version of $REVISION; will take some time. Look at
$LOG_FILE for progress" | |
| 166 date | |
| 167 cd $TMP_DIR | |
| 168 gclient sync -t --revision=$REVISION >> $LOG_FILE 2>&1 | |
| 169 failTest $? "Error calling gclient sync" | |
| 170 echo "Run hooks" | |
| 171 gclient runhooks >> $LOG_FILE 2>&1 | |
| 172 | |
| 173 echo "Compiling clean version of dart_analyzer; may take some time" | |
| 174 date | |
| 175 cd compiler | |
| 176 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
| 177 if [ ! $? -eq 0 ]; then | |
| 178 echo "error compiling" | |
| 179 failStats $REVISION | |
| 180 return 1; | |
| 181 fi | |
| 182 | |
| 183 # Give the metrics system a backwards compatible way of getting to the | |
| 184 # artifacts that it needs. | |
| 185 cd .. | |
| 186 mkdir -p $ROOT_OF_REPO/compiler/revs/$REVISION/prebuilt | |
| 187 cd $ROOT_OF_REPO/compiler/revs/$REVISION/prebuilt | |
| 188 COMPILER_OUTDIR=$TMP_DIR/compiler/out/Release_ia32 | |
| 189 cp -r $COMPILER_OUTDIR/compiler ./compiler | |
| 190 else | |
| 191 echo "Cached prebuilt of $REVISION!" | |
| 192 fi | |
| 193 | |
| 194 # Short circuit if we're just filling in the build cache | |
| 195 if [ $RUNS -eq 0 ]; then | |
| 196 echo "run in compile only mode, no stats generating" | |
| 197 return 0; | |
| 198 fi | |
| 199 | |
| 200 # Do the second test | |
| 201 echo "Running test with dart_analyzer $REVISION!" | |
| 202 date | |
| 203 echo $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=$REVISION --analyzer=$PRE
BUILT_DIR/analyzer/bin/dart_analyzer $COMPARE_OPTIONS >> $LOG_FILE 2>&1 | |
| 204 $SCRIPT_PATH/dart_analyzer.sh --stats-prefix=$REVISION --analyzer=$PREBUILT_DI
R/analyzer/bin/dart_analyzer $COMPARE_OPTIONS > $STAT1 | |
| 205 if [ ! $? -eq 0 ]; then | |
| 206 echo "error sampling" | |
| 207 failStats $REVISION | |
| 208 return 2; | |
| 209 fi | |
| 210 | |
| 211 # Output the reivision to the PLOTS file; newline added after stats | |
| 212 calcStats $REVISION | |
| 213 } | |
| 214 | |
| 215 echo -e "#Rev\tFull-ms\tdev\tZeroD\tdev\tOneD\tdev" > $PLOTS | |
| 216 if [ "$LOW_REV" ]; then | |
| 217 COUNT=$(( REV - LOW_REV + 1 )) | |
| 218 else | |
| 219 LOW_REV=$(( REV - COUNT + 1 )) | |
| 220 fi | |
| 221 for (( i = REV ; i >= LOW_REV ; i-- )) | |
| 222 do | |
| 223 echo "["$( basename "$APP")": "$((REV - i + 1))"/"$COUNT", rev:$i]" | |
| 224 compileRevision $i | |
| 225 done | |
| 226 | |
| OLD | NEW |