| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file | |
| 2 # for details. All rights reserved. Use of this source code is governed by a | |
| 3 # BSD-style license that can be found in the LICENSE file. | |
| 4 | |
| 5 #!/usr/bin/python2.4 | |
| 6 # | |
| 7 | |
| 8 """Uses jscoverage to instrument a web unit test and runs it. | |
| 9 | |
| 10 Uses jscoverage to instrument a web unit test and extracts coverage statistics | |
| 11 from running it in a headless webkit browser. | |
| 12 """ | |
| 13 | |
| 14 import codecs | |
| 15 import json | |
| 16 import optparse | |
| 17 import os | |
| 18 import re | |
| 19 import sourcemap # located in this same directory | |
| 20 import subprocess | |
| 21 import sys | |
| 22 | |
| 23 def InstrumentScript(script_name, src_dir, dst_dir, jscoveragecmd, verbose): | |
| 24 """ Instruments a test using jscoverage. | |
| 25 | |
| 26 Args: | |
| 27 script_name: the original javascript file containing the test. | |
| 28 src_dir: the directory where the test is found. | |
| 29 dst_dir: the directory where to place the instrumented test. | |
| 30 jscoveragecmd: the fully-qualified command to run jscoverage. | |
| 31 | |
| 32 Returns: | |
| 33 0 if no errors where found. | |
| 34 1 if jscoverage gave errors. | |
| 35 """ | |
| 36 SafeMakeDirs(dst_dir) | |
| 37 | |
| 38 PrintNewStep("instrumenting") | |
| 39 src_file = os.path.join(src_dir, script_name) | |
| 40 dst_file = os.path.join(dst_dir, script_name) | |
| 41 if not os.path.exists(src_file): | |
| 42 print "Script not found: " + src_file | |
| 43 return 1 | |
| 44 | |
| 45 if os.path.exists(dst_file): | |
| 46 if os.path.getmtime(src_file) < os.path.getmtime(dst_file): | |
| 47 # Skip running jscoverage if the input hasn't been updated | |
| 48 return 0 | |
| 49 | |
| 50 # Create the instrumented code using jscoverage: | |
| 51 status, output, err = ExecuteCommand([sys.executable, | |
| 52 'tools/jscoverage_wrapper.py', jscoveragecmd, src_file, dst_dir], verbose) | |
| 53 if status: | |
| 54 print "ERROR: jscoverage had errors: " | |
| 55 print output | |
| 56 print err | |
| 57 return 1 | |
| 58 return 0 | |
| 59 | |
| 60 def SafeMakeDirs(dirname): | |
| 61 """ Creates a directory and, if necessary its parent directories. | |
| 62 | |
| 63 This function will safely return if other concurrent jobs try to create the | |
| 64 same directory. | |
| 65 """ | |
| 66 if not os.path.exists(dirname): | |
| 67 try: | |
| 68 os.makedirs(dirname) | |
| 69 except Exception: | |
| 70 # this check allows invoking this script concurrently in many jobs | |
| 71 if not os.path.exists(dirname): | |
| 72 raise | |
| 73 | |
| 74 COVERAGE_TEST_TEMPLATE = ''' | |
| 75 <html> | |
| 76 <head> | |
| 77 <title> Coverage for %s </title> | |
| 78 </head> | |
| 79 <body> | |
| 80 <h1> Running %s </h1> | |
| 81 <script type="text/javascript" src="%s"></script> | |
| 82 <script type="text/javascript" src="%s"></script> | |
| 83 </body> | |
| 84 </html> | |
| 85 ''' | |
| 86 | |
| 87 def CollectCoverageForTest(test, controller, script, drt, src_dir, | |
| 88 dst_dir, sourcemapfile, result, verbose): | |
| 89 """ Collect test coverage information from an instrumented test. | |
| 90 Internally this function generates an html page for collecting the | |
| 91 coverage results. This page is just like the HTML that runs a test, but | |
| 92 instead of using 'test_controller.js' (which indicates whether tests pass | |
| 93 or fail), we use 'coverage_controller.js' (which prints out coverage | |
| 94 information). | |
| 95 | |
| 96 Args: | |
| 97 test: the name of the test | |
| 98 controller: path to coverage_controller.js | |
| 99 script: actual javascript test (previously instrumented with jscoverage) | |
| 100 drt: path to DumpRenderTree | |
| 101 src_dir: directory where original 'script' is found. | |
| 102 dst_dir: directory where the instrumented 'script' is found and where to | |
| 103 generate the html file. | |
| 104 sourcemapfile: sourcemap file associated with the script test. This is | |
| 105 typically the source map for the original | |
| 106 (non-instrumented code) since the jscoverage results are | |
| 107 reported with respect to the original line numbers. | |
| 108 result: a dictionary that wlll hold the results of this invocation. The | |
| 109 dictionary keys are original dart file names, the value is a | |
| 110 pair containing two sets of numbers: covered lines, and | |
| 111 non-covered but executable lines. Note, some lines are not | |
| 112 mentioned because they could be dropped during compilation (e.g. | |
| 113 empty lines, comments, etc). | |
| 114 This dictionary doesn't need to be empty, in which case this | |
| 115 function will combine the results of this test with what | |
| 116 previously was recoded in 'result'. | |
| 117 verbose: show verbose progress mesages | |
| 118 """ | |
| 119 html_contents = COVERAGE_TEST_TEMPLATE % (test, test, controller, script) | |
| 120 html_output_file = os.path.join(dst_dir, test + '_coverage.html') | |
| 121 with open(html_output_file, 'w') as f: | |
| 122 f.write(html_contents) | |
| 123 | |
| 124 PrintNewStep("running") | |
| 125 status, output, err = ExecuteCommand([drt, html_output_file], verbose) | |
| 126 lines = output.split('\n') | |
| 127 if len(lines) <= 2 or status: | |
| 128 print "ERROR: can't obtain coverage for %s%s%s" % ( | |
| 129 RED_COLOR, html_output_file, NO_COLOR) | |
| 130 return 1 if not status else status | |
| 131 | |
| 132 PrintNewStep("processing") | |
| 133 # output is json, possibly split in multiple lines. We skip the first line | |
| 134 # (DumpRenderTree shows a content-type line). | |
| 135 coverage_res = ''.join(lines[1: lines.index("#EOF")]) | |
| 136 try: | |
| 137 json_obj = json.loads(coverage_res) | |
| 138 except Exception: | |
| 139 print "ERROR: can't obtain coverage for %s%s%s" % ( | |
| 140 RED_COLOR, html_output_file, NO_COLOR) | |
| 141 return 1 | |
| 142 _processResult(json_obj, script, src_dir, sourcemapfile, result) | |
| 143 | |
| 144 | |
| 145 # Patterns used to detect declaration sites in the generated js code | |
| 146 HOIST_FUNCTION_PATTERN = re.compile("^function ") | |
| 147 MEMBER_PATTERN = re.compile( | |
| 148 "[^ ]*prototype\.[^ ]*\$member = function\([^\)]*\){$") | |
| 149 GETTER_PATTERN = re.compile( | |
| 150 "[^ ]*prototype\.[^ ]*\$getter = function\([^\)]*\){$") | |
| 151 SETTER_PATTERN = re.compile( | |
| 152 "[^ ]*prototype\.[^ ]*\$setter = function\([^\)]*\){$") | |
| 153 | |
| 154 def _processResult(json_obj, js_file, src_dir, sourcemap_file, result): | |
| 155 """ Process the result of a single test run. See 'CollectCoverageForTest' for | |
| 156 more details. | |
| 157 | |
| 158 Args: | |
| 159 json_obj: the json object that was exported by coverage_controller.js | |
| 160 js_file: name of the script file | |
| 161 src_dir: directory where the generated jsfile is located of the script | |
| 162 sourcemap_file: sourcemap file associated with the script test. | |
| 163 result: a dictionary that will hold the results of this invocation. | |
| 164 """ | |
| 165 if js_file not in json_obj or len(json_obj) != 1: | |
| 166 raise Exception("broken format in coverage result") | |
| 167 | |
| 168 lines_covered = json_obj[js_file] | |
| 169 smap = sourcemap.parse(sourcemap_file) | |
| 170 with open(os.path.join(src_dir, js_file), 'r') as f: | |
| 171 js_code = f.readlines() | |
| 172 | |
| 173 for line in range(len(lines_covered)): | |
| 174 original = smap.get_source_location(line, 1) | |
| 175 if original: | |
| 176 filename, srcline, srccolumn, srcid = original | |
| 177 if filename not in result: | |
| 178 result[filename] = (set(), set()) | |
| 179 if lines_covered[line] is not None: | |
| 180 if lines_covered[line] > 0: | |
| 181 # exclude the line if it looks like a generated class declaration or | |
| 182 # a method declaration | |
| 183 srccode = js_code[line - 1] | |
| 184 if (HOIST_FUNCTION_PATTERN.match(srccode) is None | |
| 185 and GETTER_PATTERN.match(srccode) is None | |
| 186 and SETTER_PATTERN.match(srccode) is None | |
| 187 and MEMBER_PATTERN.match(srccode) is None): | |
| 188 result[filename][0].add(srcline) | |
| 189 elif lines_covered[line] == 0: | |
| 190 result[filename][1].add(srcline) | |
| 191 | |
| 192 # Global color constants | |
| 193 GREEN_COLOR = "\033[32m" | |
| 194 YELLOW_COLOR = "\033[33m" | |
| 195 RED_COLOR = "\033[31m" | |
| 196 NO_COLOR = "\033[0m" | |
| 197 | |
| 198 # Templates for the HTML output | |
| 199 SUMMARY_TEMPLATE = '''<html> | |
| 200 <head> | |
| 201 <style> | |
| 202 %s | |
| 203 </style> | |
| 204 </head> | |
| 205 <body> | |
| 206 <script type="application/javascript"> | |
| 207 %s | |
| 208 </script> | |
| 209 <script type="application/javascript"> | |
| 210 var files = [] | |
| 211 var code = {} | |
| 212 var summary = {} | |
| 213 %s | |
| 214 | |
| 215 render(files, code, summary); | |
| 216 </script> | |
| 217 </body> | |
| 218 </html> | |
| 219 ''' | |
| 220 | |
| 221 # Template for exporting the coverage information data for each file | |
| 222 FILE_TEMPLATE = ''' | |
| 223 // file %s | |
| 224 files.push("%s"); | |
| 225 code["%s"] = [%s]; | |
| 226 summary["%s"] = [%s];''' | |
| 227 | |
| 228 # TODO(sigmund): increase these thresholds to 95 and 70 | |
| 229 GOOD_THRESHOLD = 80 | |
| 230 OK_THRESHOLD = 50 | |
| 231 | |
| 232 def ReportSummary(outdir, result): | |
| 233 """ Analyzes the results and produces an ASCII and an HTML summary report. | |
| 234 Args: | |
| 235 outdir: directory where to generate the HTML report | |
| 236 result: a dictionary containing results of the coverage analysis. The | |
| 237 dictionary maps file names to a pair of 2 sets of numbers. The | |
| 238 first set contains executable covered lines, the second set | |
| 239 contains executable non-covered lines. Any missing line is | |
| 240 likely not translated into code (e.g. comments, an interface | |
| 241 body) or translated into declarations (e.g. a getter | |
| 242 declaration). | |
| 243 | |
| 244 """ | |
| 245 basepath = os.path.join(os.path.dirname(sys.argv[0]), "../") | |
| 246 html_summary_lines = [] | |
| 247 print_summary_lines = [] | |
| 248 for key in result.keys(): | |
| 249 filepath = os.path.relpath(os.path.join(basepath, key)) | |
| 250 if (os.path.exists(filepath) and | |
| 251 # exclude library directories under client/ | |
| 252 not "dom/generated" in filepath and | |
| 253 not "json/" in filepath): | |
| 254 linenum = 0 | |
| 255 realcode = 0 | |
| 256 total_covered = 0 | |
| 257 escaped_lines = [] | |
| 258 file_summary = [] | |
| 259 with codecs.open(filepath, 'r', 'utf-8') as f: | |
| 260 lines = f.read() | |
| 261 for line in lines.split('\n'): | |
| 262 linenum += 1 | |
| 263 stripline = line.strip() | |
| 264 if linenum in result[key][0]: | |
| 265 total_covered += 1 | |
| 266 file_summary.append("1") | |
| 267 realcode += 1 | |
| 268 elif linenum in result[key][1]: | |
| 269 file_summary.append("0") | |
| 270 realcode += 1 | |
| 271 else: | |
| 272 file_summary.append("2") | |
| 273 | |
| 274 escaped_lines.append( | |
| 275 '"' + line.replace('"','\\"').replace('<', '<" + "') + '"') | |
| 276 | |
| 277 percent = total_covered * 100 / realcode | |
| 278 | |
| 279 # append a pair, the first component is only used for sorting | |
| 280 print_summary_lines.append((filepath, "%s%3d%% (%4d of %4d) - %s%s" | |
| 281 % (GREEN_COLOR if (percent >= GOOD_THRESHOLD) else | |
| 282 YELLOW_COLOR if (percent >= OK_THRESHOLD) else | |
| 283 RED_COLOR, | |
| 284 percent, total_covered, realcode, | |
| 285 filepath, | |
| 286 NO_COLOR))) | |
| 287 | |
| 288 html_summary_lines.append(FILE_TEMPLATE % ( | |
| 289 filepath, filepath, | |
| 290 filepath, ",".join(escaped_lines), | |
| 291 filepath, ",".join(file_summary))) | |
| 292 | |
| 293 print_summary_lines.sort() | |
| 294 print "\n".join([s for (percent, s) in print_summary_lines]) | |
| 295 | |
| 296 outfile = os.path.join(outdir, 'coverage_summary.html') | |
| 297 resource_dir = os.path.abspath(os.path.dirname(sys.argv[0])) | |
| 298 | |
| 299 # Inject css and js code within the coverage page: | |
| 300 with open(os.path.join(resource_dir, 'coverage.css'), 'r') as f: | |
| 301 style = f.read() | |
| 302 with open(os.path.join(resource_dir, 'show_coverage.js'), 'r') as f: | |
| 303 jscode = f.read() | |
| 304 with codecs.open(outfile, 'w', 'utf-8') as f: | |
| 305 f.write(SUMMARY_TEMPLATE % (style, jscode, "\n".join(html_summary_lines))) | |
| 306 | |
| 307 print ("Detailed summary available at: %s" % outfile) | |
| 308 return 0 | |
| 309 | |
| 310 def ExecuteCommand(cmd, verbose): | |
| 311 """Execute a command in a subprocess. | |
| 312 """ | |
| 313 if verbose: print '\nExecuting: ' + ' '.join(cmd) | |
| 314 pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) | |
| 315 (output, err) = pipe.communicate() | |
| 316 if pipe.returncode != 0 and verbose: | |
| 317 print 'Execution failed: ' + output + '\n' + err | |
| 318 print output | |
| 319 print err | |
| 320 return pipe.returncode, output, err | |
| 321 | |
| 322 def main(): | |
| 323 global total_tests | |
| 324 parser = Flags() | |
| 325 (options, args) = parser.parse_args() | |
| 326 if not AreOptionsValid(options): | |
| 327 parser.print_help() | |
| 328 return 1 | |
| 329 | |
| 330 # Create dir for instrumented code | |
| 331 instrumented_dir = os.path.join(os.path.dirname(options.dir), | |
| 332 os.path.basename(options.dir) + "_instrumented") | |
| 333 | |
| 334 # Determine the set of tests to execute | |
| 335 prefix = options.tests_prefix | |
| 336 tests = [] | |
| 337 if os.path.exists(prefix) and os.path.isdir(prefix): | |
| 338 test_dir = prefix | |
| 339 test_file_prefix = None | |
| 340 else: | |
| 341 test_dir = os.path.dirname(prefix) | |
| 342 test_file_prefix = os.path.basename(prefix) | |
| 343 | |
| 344 for root, dirs, files in os.walk(test_dir): | |
| 345 for f in files: | |
| 346 if f.endswith('.app'): | |
| 347 path = os.path.join(root, f) | |
| 348 if test_file_prefix is None or path.startswith(prefix): | |
| 349 tests.append(path) | |
| 350 | |
| 351 # Run tests and collect results | |
| 352 result = dict() | |
| 353 total_tests = len(tests) | |
| 354 for test in tests: | |
| 355 testname = test.replace("/", "_") | |
| 356 script = testname + ".js" | |
| 357 PrintNewTest(test) | |
| 358 status = InstrumentScript(script, options.dir, instrumented_dir, | |
| 359 options.jscoverage, options.verbose) | |
| 360 if status: | |
| 361 print "skipped %s " % script | |
| 362 global last_line_length | |
| 363 last_line_length = 0 | |
| 364 else: | |
| 365 sourcemapfile = os.path.join( | |
| 366 os.path.join(options.sourcemapdir, test), test + ".js.map") | |
| 367 CollectCoverageForTest( | |
| 368 testname, options.controller, script, options.drt, | |
| 369 options.dir, instrumented_dir, sourcemapfile, result, options.verbose) | |
| 370 | |
| 371 print "" # end compact progress indication | |
| 372 ReportSummary(options.dir, result) | |
| 373 return 0 | |
| 374 | |
| 375 # special functions and variables to print progress compactly | |
| 376 current_test = 0 | |
| 377 current_test_name = '' | |
| 378 total_tests = 0 # to be filled in later after parsing options | |
| 379 last_line_length = 0 | |
| 380 | |
| 381 def PrintLine(s): | |
| 382 global last_line_length | |
| 383 if last_line_length > 0: | |
| 384 print "\r" + (" " * last_line_length) + "\r", | |
| 385 last_line_length = len(s) | |
| 386 print s, | |
| 387 sys.stdout.flush() | |
| 388 | |
| 389 def PrintNewTest(test): | |
| 390 global current_test | |
| 391 global current_test_name | |
| 392 global total_tests | |
| 393 global GREEN_COLOR | |
| 394 global NO_COLOR | |
| 395 current_test += 1 | |
| 396 current_test_name = test | |
| 397 PrintLine("%d of %d (%d%%): %s (%sstart%s)" % (current_test, total_tests, | |
| 398 (100 * current_test / total_tests), test, GREEN_COLOR, NO_COLOR)) | |
| 399 | |
| 400 def PrintNewStep(message): | |
| 401 global current_test | |
| 402 global current_test_name | |
| 403 global total_tests | |
| 404 global GREEN_COLOR | |
| 405 global NO_COLOR | |
| 406 PrintLine("%d of %d (%d%%): %s (%s%s%s)" % (current_test, total_tests, | |
| 407 (100 * current_test / total_tests), current_test_name, | |
| 408 GREEN_COLOR, message, NO_COLOR)) | |
| 409 | |
| 410 | |
| 411 def Flags(): | |
| 412 result = optparse.OptionParser() | |
| 413 result.add_option("-d", "--dir", | |
| 414 help="Directory where the compiled tests can be found.", | |
| 415 default=None) | |
| 416 result.add_option("--controller", | |
| 417 help="Path to the coverage controller js file.", | |
| 418 default=None) | |
| 419 result.add_option("-v", "--verbose", | |
| 420 help="Print a messages and progress", | |
| 421 default=False, | |
| 422 action="store_true") | |
| 423 result.add_option("-t", "--tests_prefix", | |
| 424 help="Prefix (typically a directory) where to crawl for test app files", | |
| 425 type="string", | |
| 426 action="store", | |
| 427 default=None) | |
| 428 result.add_option("--drt", | |
| 429 help="The location for DumpRenderTree in the local file system", | |
| 430 default=None) | |
| 431 result.add_option("--jscoverage", | |
| 432 help="The location for jscoverage in the local file system", | |
| 433 default=None) | |
| 434 result.add_option("--sourcemapdir", | |
| 435 help="The location for sourcemap files in the local file system", | |
| 436 default=None) | |
| 437 result.set_usage( | |
| 438 "coverage.py -d <cdart-output-dir> -t <test-dir> " | |
| 439 "--drt=<path-to-DumpRenderTree> " | |
| 440 "--jscoverage=<path-to-jscoverage> " | |
| 441 "--sourcemapdir=<path-to-dir>" | |
| 442 "[options]") | |
| 443 return result | |
| 444 | |
| 445 def AreOptionsValid(options): | |
| 446 return (options.dir and options.tests_prefix and options.drt | |
| 447 and options.jscoverage and options.sourcemapdir) | |
| 448 | |
| 449 if __name__ == '__main__': | |
| 450 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 9837113:
Move update.py to samples/swarm, remove a bunch of deprecated files for (Closed)
Base URL: http://dart.googlecode.com/svn/branches/bleeding_edge/dart/