OLD | NEW |
| (Empty) |
1 #!/bin/bash | |
2 # | |
3 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
4 # for details. All rights reserved. Use of this source code is governed by a | |
5 # BSD-style license that can be found in the LICENSE file. | |
6 | |
7 # Compare the current working copy of the repository to either a clean copy | |
8 # or the specified revision. | |
9 # Along with the files created by compiler_metrics.sh, creates | |
10 # compiler_compare[_stats_[12]].log in the working direcetory and a | |
11 # out_compile_samples/ directory at the root of the repository, | |
12 # which are left around for later examination. | |
13 # These files will be destroyed on script re-run. | |
14 | |
15 APP="" | |
16 REV="" | |
17 ONE_DELTA="" | |
18 BASE_PATH=$(pwd) | |
19 SCRIPT_PATH=$(dirname $0) | |
20 SCRIPT_PATH=$(cd $SCRIPT_PATH; pwd) | |
21 RUNS=10 | |
22 | |
23 function printHelp() { | |
24 exitValue=${1:1} | |
25 echo "Compare current changes as they relate to performance with another SVN r
evision" | |
26 echo "" | |
27 echo " Usage:" | |
28 echo " -a=, --app= The dart app file to test (required)." | |
29 echo " -d=, --one-delta= The filename, relative to app, to touch in order t
o trigger a one-delta compile." | |
30 echo " -r=, --revision= The compiler revision to compare against (default
to current repository revision)." | |
31 echo " --runs= Number of runs to test with, default $RUNS" | |
32 echo " --stats Non-destructive display of previous stats" | |
33 echo " -h, --help What you see is what you get." | |
34 exit $exitValue | |
35 } | |
36 | |
37 function failTest() { | |
38 if [ ! $1 -eq 0 ]; then | |
39 echo $2 | |
40 exit $1 | |
41 fi | |
42 } | |
43 | |
44 if [ $# -eq 0 ]; then | |
45 printHelp; | |
46 fi | |
47 | |
48 for i in $* | |
49 do | |
50 case $i in | |
51 --one-delta=*|-d=*) | |
52 ONE_DELTA=${i#*=} | |
53 COMPARE_OPTIONS+="--one-delta=$ONE_DELTA ";; | |
54 --app=*|-a=*) | |
55 APP=${i#*=} | |
56 APP=$( cd "$( dirname "$APP" )" && pwd )/$( basename "$APP");; | |
57 --revision=*|-r=*) | |
58 REV=${i#*=};; | |
59 --help|-h) | |
60 printHelp 0;; | |
61 --runs=*) | |
62 RUNS=${i#*=};; | |
63 --stats) | |
64 calcStats | |
65 exit 0;; | |
66 *) | |
67 echo "Invalid parameter: $i" | |
68 printhelp 1;; | |
69 esac | |
70 done | |
71 | |
72 COMPARE_OPTIONS+="-r=$RUNS " | |
73 | |
74 if [ "" = "$APP" ] || [ ! -r $APP ]; then | |
75 echo "Required --app" | |
76 printHelp | |
77 fi | |
78 | |
79 RESPONSE[0]="Performance: Better" | |
80 RESPONSE[1]="Performance: No Change" | |
81 RESPONSE[2]="Performance: Worse" | |
82 | |
83 # Passed: MAX deviation, Changed Amount | |
84 # Returns: Index of RESPONSE | |
85 function responseValue() { | |
86 ABS=$2 | |
87 (( ABS = ABS < 0 ? ABS * -1 : ABS )) | |
88 if [ $ABS -gt $1 ]; then | |
89 if [ $2 -lt 0 ]; then | |
90 return 2 | |
91 else | |
92 return 0 | |
93 fi | |
94 fi | |
95 return 1 | |
96 } | |
97 | |
98 function calcStat() { | |
99 # Assume we're always making improvments, S2 will be presented as a larger val
ue | |
100 LINEMATCH_DEV="s/${1}: .*stdev: \([0-9]*\).*/\1/p" | |
101 LINEMATCH_VALUE="s/${1}: average: \([0-9]*\).*/\1/p" | |
102 S1_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT1) | |
103 if [ "" != "$S1_DEV" ]; then | |
104 S2_DEV=$(sed -n -e "$LINEMATCH_DEV" $STAT2) | |
105 DEV_MAX=$(( S1_DEV < S2_DEV ? S2_DEV : S1_DEV )) | |
106 S1_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT1) | |
107 S2_VALUE=$(sed -n -e "$LINEMATCH_VALUE" $STAT2) | |
108 DIFF=$(( S2_VALUE - S1_VALUE )) | |
109 DIFF_PERCENT=`echo "$DIFF*100/$S2_VALUE" | bc -l | sed 's/\([0-9]*\.[0-9][0-
9]\)[0-9]*/\1/'` | |
110 responseValue $DEV_MAX $DIFF | |
111 echo "$1: Before/After-ms(${S2_VALUE} / ${S1_VALUE}), stdev(${S2_DEV} / ${S1
_DEV}), Difference: ${DIFF}ms (${DIFF_PERCENT}%), " ${RESPONSE[$?]} | |
112 fi | |
113 return 0 | |
114 } | |
115 | |
116 function calcStats() { | |
117 calcStat full-compile | |
118 calcStat zero-delta-compile | |
119 calcStat one-delta-compile | |
120 } | |
121 | |
122 ROOT_OF_REPO=$BASE_PATH | |
123 TEST_DIR=$BASE_PATH | |
124 while true; do | |
125 ls -d .gclient > /dev/null 2>&1 | |
126 if [ $? -eq 0 ]; then | |
127 echo "Root found: $ROOT_OF_REPO" | |
128 break; | |
129 fi | |
130 if [ "$TEST_DIR" = "/" ]; then | |
131 failTest 1 "Hit the root directory; no .git/ found?!" | |
132 fi | |
133 ROOT_OF_REPO=$TEST_DIR | |
134 cd .. | |
135 TEST_DIR=$(pwd) | |
136 done | |
137 | |
138 # Make a temporary directory in the current path and checkout the revision | |
139 TMP_DIR=$ROOT_OF_REPO/compiler/tmp_performance_comparisons | |
140 mkdir -p $TMP_DIR | |
141 | |
142 LOG_FILE=$TMP_DIR/compiler_compare.log | |
143 STAT1=$TMP_DIR/compiler_compare_stats_1.log | |
144 STAT2=$TMP_DIR/compiler_compare_stats_2.log | |
145 COMPARE_OPTIONS+="--output=$TMP_DIR " | |
146 | |
147 # zero out files | |
148 echo "" > $LOG_FILE | |
149 echo "" > $STAT1 | |
150 echo "" > $STAT2 | |
151 | |
152 # Do the first test | |
153 echo "Compiling dartc with your changes (mode=release)" | |
154 cd $ROOT_OF_REPO/compiler | |
155 gclient runhooks >> $LOG_FILE 2>&1 | |
156 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
157 failTest $? "Error compiling your location changes, check $LOG_FILE" | |
158 | |
159 echo "Running first test against current working copy" | |
160 echo $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=yours --dartc=./out/Release
_ia32/dartc $COMPARE_OPTIONS --app=$APP >> $LOG_FILE | |
161 $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=yours --dartc=./out/Release_ia32
/dartc $COMPARE_OPTIONS --app=$APP > $STAT1 | |
162 failTest $? "Error collecting statistics from working copy" | |
163 | |
164 # switch to tmp for remainder of building | |
165 cd $TMP_DIR | |
166 | |
167 gclient config https://dart.googlecode.com/svn/branches/bleeding_edge/deps/compi
ler.deps >> $LOG_FILE 2>&1 | |
168 failTest $? "Error calling gclient config" | |
169 | |
170 GCLIENT_SYNC="-t " | |
171 if [ "" != "$REV" ]; then | |
172 GCLIENT_SYNC+="--revision=$REV" | |
173 fi | |
174 | |
175 echo "Checking out clean version of $REV; will take some time. Look at $LOG_FILE
for progress" | |
176 gclient sync $GCLIENT_SYNC >> $LOG_FILE 2>&1 | |
177 failTest $? "Error calling gclient sync" | |
178 | |
179 echo "Compiler clean version; may take some time" | |
180 cd compiler | |
181 gclient runhooks >> $LOG_FILE 2>&1 | |
182 ../tools/build.py --mode release >> $LOG_FILE 2>&1 | |
183 failTest $? "Error compiling comparison revision" | |
184 | |
185 # Do the second test | |
186 echo "Running second test against clean copy" | |
187 echo $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=clean --dartc=./out/Release
_ia32/dartc $COMPARE_OPTIONS --app=$APP >> $LOG_FILE | |
188 $SCRIPT_PATH/compiler_metrics.sh --stats-prefix=clean --dartc=./out/Release_ia32
/dartc $COMPARE_OPTIONS --app=$APP > $STAT2 | |
189 failTest $? "Error collecting statistics from clean copy" | |
190 | |
191 calcStats | |
OLD | NEW |