Change test running logic to run tests without any size annotation.

build/android/pylib/run_java_tests.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """Runs the Java tests. See more information on run_instrumentation_tests.py."""
 
 import fnmatch
 import logging
 import os
 import re
@@ skipping 511 matching lines @@
     options: Command line options.
     apks: list of APKs to use.
 
   Returns:
     A TestResults object holding the results of the Java tests.
 
   Raises:
     FatalTestException: when there's no attached the devices.
   """
   test_apk = apks[0]
+  # The default annotation for tests which do not have any sizes annotation.
+  default_size_annotation = 'SmallTest'
+
+  def testsMissingAnnotation(test_apk):

[frankf, 2012/12/05 23:54:17]

Unfortunately, we don't follow the google3 style guide for method names. ->
_GetTestsMissingAnnotation()

[shashi, 2012/12/06 00:27:26]

Done.

+    test_size_annotations = frozenset(['Smoke', 'SmallTest', 'MediumTest',
+                                       'LargeTest', 'EnormousTest', 'FlakyTest',
+                                       'DisabledTest', 'Manual', 'PerfTest'])
+    tests_missing_annotations = []
+    for test_method in test_apk.GetTestMethods():
+      annotations = frozenset(test_apk.GetTestAnnotations(test_method))
+      if (annotations.isdisjoint(test_size_annotations) and
+         not apk_info.ApkInfo.IsPythonDrivenTest(test_method)):
+        tests_missing_annotations.append(test_method)
+    return tests_missing_annotations
+
   if options.annotation:
     available_tests = test_apk.GetAnnotatedTests(options.annotation)
-    if len(options.annotation) == 1 and options.annotation[0] == 'SmallTest':
-      tests_without_annotation = [
-          m for m in
-          test_apk.GetTestMethods()
-          if not test_apk.GetTestAnnotations(m) and
-          not apk_info.ApkInfo.IsPythonDrivenTest(m)]
-      if tests_without_annotation:
-        tests_without_annotation.sort()
+    if (len(options.annotation) == 1 and
+        options.annotation[0] == default_size_annotation):

[frankf, 2012/12/05 23:54:17]

Why do we include un-annotated tests in SmallTest only if it's run by itself?
These tests are left out if no annotation/filter is specified (default
behaviour).

[shashi, 2012/12/06 00:27:26]

Currently build bots always specify size when running tests. So tests lacking
annotation are never run.
Ideally there should be no tests without any annotations. But there has been a
few. 
I saw that this logic has been in place to run un-annotated tests as small
tests, except that it does not work when a test has an annotation like PhoneOnly
but not size annotation.
I thought to improve this logic so that if someone accidently  disables a test
by removing an annotation we still run it.

On 2012/12/05 23:54:17, frankf wrote:

[frankf, 2012/12/06 01:01:55]

What I meant:

1. Unannotated tests are run:
./run_instrumentation.py -A SmallTest

2. Unannotated tests are not run:
./run_instrumentation.py -A SmallTest,LargeTest
./run_instrumentation.py

This is awkward since SmallTests is part of both invocations.
 
On 2012/12/06 00:27:26, shashi wrote:

[shashi, 2012/12/06 01:18:40]

Makes sense, we need not check if SmallTest is the only annotation.
On 2012/12/06 01:01:55, frankf wrote:

+      tests_missing_annotations =  testsMissingAnnotation(test_apk)
+      if tests_missing_annotations:
+        tests_missing_annotations.sort()

[frankf, 2012/12/05 23:54:17]

just have the method sort it for you.

[shashi, 2012/12/06 00:27:26]

Done.

         logging.warning('The following tests do not contain any annotation. '
-                        'Assuming "SmallTest":\n%s',
-                        '\n'.join(tests_without_annotation))
-        available_tests += tests_without_annotation
+                        'Assuming "%s":\n%s',
+                        default_size_annotation,
+                        '\n'.join(tests_missing_annotations))
+        available_tests += tests_missing_annotations
   else:
     available_tests = [m for m in test_apk.GetTestMethods()
                        if not apk_info.ApkInfo.IsPythonDrivenTest(m)]
   coverage = os.environ.get('EMMA_INSTRUMENT') == 'true'
 
   tests = []
   if options.test_filter:
     # |available_tests| are in adb instrument format: package.path.class#test.
     filter_without_hash = options.test_filter.replace('#', '.')
     tests = [t for t in available_tests
@@ skipping 17 matching lines @@
 
   logging.info('Will run: %s', str(tests))
 
   if len(attached_devices) > 1 and (coverage or options.wait_for_debugger):
     logging.warning('Coverage / debugger can not be sharded, '
                     'using first available device')
     attached_devices = attached_devices[:1]
   sharder = TestSharder(attached_devices, options, tests, apks)
   test_results = sharder.RunShardedTests()
   return test_results