OLD | NEW |
---|---|
1 # Copyright 2014 The Chromium Authors. All rights reserved. | 1 # Copyright 2014 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import contextlib | 5 import contextlib |
6 import ConfigParser | 6 import ConfigParser |
7 import glob | 7 import glob |
8 import imp | 8 import imp |
9 import inspect | 9 import inspect |
10 import logging | 10 import logging |
(...skipping 34 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
45 def __init__(self): | 45 def __init__(self): |
46 self._stream = StringIO() | 46 self._stream = StringIO() |
47 | 47 |
48 def reset(self): | 48 def reset(self): |
49 self._stream = StringIO() | 49 self._stream = StringIO() |
50 | 50 |
51 def __getattr__(self, key): | 51 def __getattr__(self, key): |
52 return getattr(self._stream, key) | 52 return getattr(self._stream, key) |
53 | 53 |
54 | 54 |
55 def get_python_root(path): | |
56 """Get the lowest directory with no __init__.py file. | |
57 | |
58 When ``path`` is pointing inside a Python package, this function returns the | |
59 directory directly containing this package. If ``path`` points outside of | |
60 a Python package, the it returns ``path``. | |
61 | |
62 Args: | |
63 path (str): arbitrary path | |
64 Returns: | |
65 root (str): ancestor directory, with no __init__.py file in it. | |
66 """ | |
67 if not os.path.exists(path): | |
68 raise ValueError('path must exist: %s') | |
69 | |
70 while path != os.path.dirname(path): | |
71 if not os.path.exists(os.path.join(path, '__init__.py')): | |
72 return path | |
73 path = os.path.dirname(path) | |
74 | |
75 # This is not supposed to happen, but in case somebody adds a __init__.py | |
76 # at the filesystem root ... | |
77 raise IOError("Unable to find a python root for %s" % path) | |
78 | |
79 | |
55 def get_package_path(package_name, path): | 80 def get_package_path(package_name, path): |
56 """Return path toward 'package_name'. | 81 """Return path toward 'package_name'. |
57 | 82 |
58 If path is None, search for a package in sys.path. | 83 If path is None, search for a package in sys.path. |
59 Otherwise, look for a direct subdirectory of path. | 84 Otherwise, look for a direct subdirectory of path. |
60 | 85 |
61 If no package is found, returns None. | 86 If no package is found, returns None. |
62 """ | 87 """ |
63 if path is None: | 88 if path is None: |
64 _, package_path, _ = imp.find_module(package_name) | 89 _, package_path, _ = imp.find_module(package_name) |
(...skipping 214 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
279 globs = ['%s%s' % (g, '*' if '*' not in g else '') for g in opts.test_glob] | 304 globs = ['%s%s' % (g, '*' if '*' not in g else '') for g in opts.test_glob] |
280 | 305 |
281 matcher = re.compile( | 306 matcher = re.compile( |
282 '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g) | 307 '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g) |
283 for g in globs if g[0] != '-')) | 308 for g in globs if g[0] != '-')) |
284 if matcher.pattern == '^$': | 309 if matcher.pattern == '^$': |
285 matcher = re.compile('^.*$') | 310 matcher = re.compile('^.*$') |
286 | 311 |
287 neg_matcher = re.compile( | 312 neg_matcher = re.compile( |
288 '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g[1:]) | 313 '^%s$' % '|'.join('(?:%s)' % glob.fnmatch.translate(g[1:]) |
289 for g in globs if g[0] == '-')) | 314 for g in globs if g[0] == '-')) |
iannucci
2014/11/12 20:26:24
I think neg_matcher got lost in the new world?
pgervais
2014/11/13 17:55:46
(discussed that offline) Yes, the neg_matcher was
| |
290 | 315 |
291 SENTINEL = object() | 316 SENTINEL = object() |
292 | 317 |
293 def generate_tests(): | 318 def generate_tests(): |
294 paths_seen = set() | 319 paths_seen = set() |
295 seen_tests = False | 320 seen_tests = False |
296 try: | 321 try: |
297 for gen in gens: | 322 for gen in gens: |
298 gen_cover_ctx = cover_ctx(include=util.get_cover_list(gen)) | 323 gen_cover_ctx = cover_ctx(include=util.get_cover_list(gen)) |
299 | 324 |
(...skipping 204 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
504 p.start() | 529 p.start() |
505 | 530 |
506 gen_loop_process(*test_gen_args) | 531 gen_loop_process(*test_gen_args) |
507 # Signal all run_loop_process that they can exit. | 532 # Signal all run_loop_process that they can exit. |
508 test_gen_finished.set() | 533 test_gen_finished.set() |
509 | 534 |
510 for p in procs: | 535 for p in procs: |
511 p.join() | 536 p.join() |
512 | 537 |
513 | 538 |
539 def parse_test_glob(test_glob): | |
540 """A test glob is composed of a path and a glob expression like: | |
iannucci
2014/11/12 20:26:24
do we still support the negative glob e.g. `-path/
pgervais
2014/11/13 17:55:46
Not currently (see above comment)
| |
541 '<path>:<glob>'. The path should point to a directory or a file inside | |
iannucci
2014/11/12 20:26:24
TODO: let's scan packages too so that <path> can a
pgervais
2014/11/13 17:55:46
TODO added.
| |
542 a Python package (it can be the root directory of that package). | |
543 The glob is a Python name used to filter tests. | |
544 | |
545 Example: | |
546 'my/nice/package/test1/:TestA*', the package root being 'my/nice/package': | |
547 this matches all tests whose name starts with 'TestA' inside all files | |
548 matching test1/*_test.py. | |
iannucci
2014/11/12 20:26:24
not strictly true, right? non-unittest tests would
pgervais
2014/11/13 17:55:46
True, the definition of a 'test name' is not very
| |
549 | |
550 Args: | |
551 test_glob (str): a test glob | |
552 Returns: | |
553 (path, test_filter): absolute path and test filter glob. | |
554 """ | |
555 parts = test_glob.split(':') | |
556 if len(parts) > 2: | |
557 raise ValueError('A test_glob should contain at most one colon (got %s)' | |
558 % test_glob) | |
559 if len(parts) == 2: | |
560 path, test_filter = parts | |
561 if '/' in test_filter: | |
iannucci
2014/11/12 20:26:24
Hm, I'm not sure this is strictly true... do we en
pgervais
2014/11/13 17:55:46
The test filter is only applied to test names, whi
| |
562 raise ValueError('A test filter cannot contain a slash (got %s)', | |
563 test_filter) | |
564 | |
565 if not test_filter: # empty string case | |
566 test_filter = '*' | |
567 else: | |
568 path, test_filter = parts[0], '*' | |
569 | |
570 path = os.path.abspath(path) | |
571 return path, test_filter | |
572 | |
573 | |
574 class PackageTestingContext(object): | |
iannucci
2014/11/12 20:26:24
may be worth having all this context stuff in it's
| |
575 def __init__(self, cwd, package_name, filters): | |
576 """Information to run a set of tests in a single package. | |
577 | |
578 See also parse_test_glob. | |
579 """ | |
580 self.cwd = cwd | |
581 self.package_name = package_name | |
582 # list of (path, filter) pairs. | |
583 # The path is where to look for tests for. Only tests whose name matches the | |
584 # glob are kept. | |
585 self.filters = filters | |
586 | |
587 @classmethod | |
588 def from_path(cls, path, filters='*'): | |
iannucci
2014/11/12 20:26:24
maybe make filters `('*',)` so that you don't need
pgervais
2014/11/13 17:55:47
This is one of the features of Python that I like:
| |
589 path = os.path.abspath(path) | |
590 cwd = get_python_root(path) | |
591 package_name = os.path.relpath(path, cwd).split(os.path.sep)[0] | |
592 # list of (path, filter) pairs. | |
593 # The path is where to look for tests for. Only tests whose name matches the | |
594 # glob are kept. | |
595 if isinstance(filters, basestring): | |
596 filters = [(path, filters)] | |
597 else: | |
598 filters = [(path, filt) for filt in filters] | |
599 | |
600 return cls(cwd, package_name, filters) | |
601 | |
602 @classmethod | |
603 def from_context_list(cls, contexts): | |
604 """Merge several PackageTestingContext pointing to the same package.""" | |
605 cwd = set(context.cwd for context in contexts) | |
606 assert len(cwd) == 1, \ | |
607 'from_context_list processes contexts with the same working '\ | |
608 'directory only.' | |
609 | |
610 package_name = set(context.package_name for context in contexts) | |
611 assert len(package_name) == 1, \ | |
612 'from_context_list processes contexts with the same package '\ | |
613 'name only.' | |
iannucci
2014/11/12 20:26:24
may be friendlier to have `def merge_contexts(cls,
pgervais
2014/11/13 17:55:46
If you consider only the RuntimeContext object, th
| |
614 | |
615 filters = [] | |
616 for context in contexts: | |
617 filters.extend(context.filters) | |
618 | |
619 return cls(cwd.pop(), package_name.pop(), filters) | |
620 | |
621 | |
622 class ProcessingContext(object): | |
623 def __init__(self, testing_contexts): | |
624 """Information to run a set of tasks in a given working directory. | |
625 | |
626 Args: | |
627 testing_contexts (list): list of PackageTestingContext instances | |
628 """ | |
629 self.cwd = testing_contexts[0].cwd | |
630 | |
631 # Merge testing_contexts by package | |
632 groups = {} | |
633 for context in testing_contexts: | |
634 if context.cwd != self.cwd: | |
635 raise ValueError('All package must have the same value for "cwd"') | |
636 groups.setdefault(context.package_name, []).append(context) | |
637 | |
638 self.testing_contexts = [PackageTestingContext.from_context_list(contexts) | |
639 for contexts in groups.itervalues()] | |
iannucci
2014/11/12 20:26:24
Yeah, I think the merge contexts function I mentio
| |
640 | |
641 | |
642 def get_runtime_contexts(test_globs): | |
643 """Compute the list of packages/filters to get tests from.""" | |
644 # Step 1: compute list of packages + subtree | |
645 testing_contexts = [] | |
646 for test_glob in test_globs: | |
647 path, test_filter = parse_test_glob(test_glob) | |
648 if os.path.exists(os.path.join(path, '__init__.py')): | |
649 testing_contexts.append( | |
650 PackageTestingContext.from_path(path, test_filter)) | |
651 else: | |
652 # Look for all packages in path. | |
653 subpaths = [] | |
654 black_list = get_config(path) | |
655 | |
656 for filename in filter(lambda x: x not in black_list, os.listdir(path)): | |
657 abs_filename = os.path.join(path, filename) | |
658 if (os.path.isdir(abs_filename) | |
659 and os.path.isfile(os.path.join(abs_filename, '__init__.py'))): | |
660 subpaths.append(abs_filename) | |
661 | |
662 testing_contexts.extend( | |
663 [PackageTestingContext.from_path(subpath, test_filter) | |
664 for subpath in subpaths]) | |
665 | |
666 # Step 2: group by working directory - one process per wd. | |
667 groups = {} | |
668 for context in testing_contexts: | |
669 groups.setdefault(context.cwd, []).append(context) | |
670 return [ProcessingContext(contexts) for contexts in groups.itervalues()] | |
671 | |
672 | |
514 def result_loop(cover_ctx, opts): | 673 def result_loop(cover_ctx, opts): |
515 """Run the specified operation in all paths in parallel. | 674 """Run the specified operation in all paths in parallel. |
516 | 675 |
517 Directories and packages to process are defined in opts.directory and | 676 Directories and packages to process are defined in opts.directory and |
518 opts.package. | 677 opts.package. |
519 | 678 |
520 The operation to perform (list/test/debug/train) is defined by opts.handler. | 679 The operation to perform (list/test/debug/train) is defined by opts.handler. |
521 """ | 680 """ |
522 | 681 |
682 runtime_contexts = get_runtime_context(opts.test_glob) | |
683 | |
523 def ensure_echo_on(): | 684 def ensure_echo_on(): |
524 """Restore echo on in the terminal. | 685 """Restore echo on in the terminal. |
525 | 686 |
526 This is useful when killing a pdb session with C-c. | 687 This is useful when killing a pdb session with C-c. |
527 """ | 688 """ |
528 try: | 689 try: |
529 import termios | 690 import termios |
530 except ImportError: | 691 except ImportError: |
531 termios = None | 692 termios = None |
532 if termios: | 693 if termios: |
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
603 | 764 |
604 if procs: | 765 if procs: |
605 error = opts.handler.result_stage_loop(opts, generate_objects(procs)) | 766 error = opts.handler.result_stage_loop(opts, generate_objects(procs)) |
606 except ResultStageAbort: | 767 except ResultStageAbort: |
607 pass | 768 pass |
608 | 769 |
609 if not kill_switch.is_set() and not result_queue.empty(): | 770 if not kill_switch.is_set() and not result_queue.empty(): |
610 error = True | 771 error = True |
611 | 772 |
612 return error, kill_switch.is_set() | 773 return error, kill_switch.is_set() |
OLD | NEW |