PPAPI/NaCl: Make related tests run in 1 fixture

ppapi/tests/test_case.h

Index: ppapi/tests/test_case.h
diff --git a/ppapi/tests/test_case.h b/ppapi/tests/test_case.h
index c7699963b64f112e50a9e08f6e1cbb2cbdb3a051..47324b6f6c93667f5e4060436e084738c627b501 100644
--- a/ppapi/tests/test_case.h
+++ b/ppapi/tests/test_case.h
@@ -7,11 +7,13 @@
 
 #include <cmath>
 #include <limits>
+#include <map>
 #include <set>
 #include <string>
 
-#include "ppapi/c/pp_resource.h"
 #include "ppapi/c/dev/ppb_testing_dev.h"
+#include "ppapi/c/pp_resource.h"
+#include "ppapi/c/pp_time.h"
 #include "ppapi/cpp/dev/scrollbar_dev.h"
 #include "ppapi/cpp/message_loop.h"
 #include "ppapi/cpp/view.h"
@@ -43,10 +45,14 @@ class TestCase {
   virtual bool Init();
 
   // Override to implement the test case. It will be called after the plugin is
-  // first displayed, passing a string. If the string is empty, the
-  // should run all tests for this test case. Otherwise, it should run the test
-  // whose name matches test_filter exactly (if there is one). This should
-  // generally be implemented using the RUN_TEST* macros.
+  // first displayed, passing a string. If the string is empty, RunTests should
+  // run all tests for this test case. Otherwise, it must be a comma-delimited
+  // list of test names, possibly prefixed. E.g.:
+  //   "Foo_GoodTest,DISABLED_Foo_BadTest,Foo_OtherGoodTest"
+  // All listed tests which are not prefixed will be run.
+  //
+  // This should generally be implemented in a TestCase subclass using the
+  // RUN_TEST* macros.
   virtual void RunTests(const std::string& test_filter) = 0;
 
   static std::string MakeFailureMessage(const char* file, int line,
@@ -82,6 +88,13 @@ class TestCase {
 
   static void QuitMainMessageLoop(PP_Instance instance);
 
+  const std::map<std::string, bool>& remaining_tests() {
+    return remaining_tests_;
+  }
+  const std::set<std::string>& skipped_tests() {
+    return skipped_tests_;
+  }
+
  protected:
 #if !(defined __native_client__)
   // Overridden by each test to supply a ScriptableObject corresponding to the
@@ -101,9 +114,15 @@ class TestCase {
   // Makes sure the test is run over HTTP.
   bool EnsureRunningOverHTTP();
 
+  // Returns true if |filter| only contains a TestCase name, which normally
+  // means "run all tests". Some TestCases require special setup for individual
+  // tests, and can use this function to decide whether to ignore those tests.
+  bool ShouldRunAllTests(const std::string& filter);
+
   // Return true if the given test name matches the filter. This is true if
-  // (a) filter is empty or (b) test_name and filter match exactly.
-  bool MatchesFilter(const std::string& test_name, const std::string& filter);
+  // (a) filter is empty or (b) test_name matches a test name listed in filter
+  // exactly.
+  bool ShouldRunTest(const std::string& test_name, const std::string& filter);
 
   // Check for leaked resources and vars at the end of the test. If any exist,
   // return a string with some information about the error. Otherwise, return
@@ -113,6 +132,8 @@ class TestCase {
   // CheckResourcesAndVars will do nothing and return the same string.
   std::string CheckResourcesAndVars(std::string errors);
 
+  PP_TimeTicks NowInTimeTicks();
+
   // Run the given test method on a background thread and return the result.
   template <class T>
   std::string RunOnThread(std::string(T::*test_to_run)()) {
@@ -212,6 +233,24 @@ class TestCase {
   // Var ids that should be ignored when checking for leaks on shutdown.
   std::set<int64_t> ignored_leaked_vars_;
 
+  // The tests that were found in test_filter but have not yet been run. The
+  // bool indicates whether the test should be run (i.e., it will be false if
+  // the test name was prefixed in the test_filter string).
+  //
+  // This is initialized lazily the first time that ShouldRunTest is called by
+  // RunTests. When RunTests is finished, this should be empty. Any remaining
+  // tests are tests that were listed in the test_filter but didn't match
+  // any calls to ShouldRunTest, meaning it was probably a typo. TestingInstance
+  // should log this and consider it a failure.
+  std::map<std::string, bool> remaining_tests_;
+  // Flag indicating whether we have populated remaining_tests_ yet.
+  bool have_populated_remaining_tests_;
+
+  // If ShouldRunTest is called but the given test name doesn't match anything
+  // in the test_filter, the test name will be added here. This allows
+  // TestingInstance to detect when not all tests were listed.
+  std::set<std::string> skipped_tests_;
+
 #if !(defined __native_client__)
   // Holds the test object, if any was retrieved from CreateTestObject.
   pp::VarPrivate test_object_;
@@ -260,31 +299,42 @@ class TestCaseFactory {
 // RunTest function. This assumes the function name is TestFoo where Foo is the
 // test |name|.
 #define RUN_TEST(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_OPTIONAL); \
-    instance_->LogTest(#name, CheckResourcesAndVars(Test##name())); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest(#name, \
+                       CheckResourcesAndVars(Test##name()), \
+                       start_time); \
   }
 
 // Like RUN_TEST above but forces functions taking callbacks to complete
 // asynchronously on success or error.
 #define RUN_TEST_FORCEASYNC(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_REQUIRED); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
     instance_->LogTest(#name"ForceAsync", \
-                       CheckResourcesAndVars(Test##name())); \
+                       CheckResourcesAndVars(Test##name()), \
+                       start_time); \
   }
 
 #define RUN_TEST_BLOCKING(test_case, name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_BLOCKING); \
-    instance_->LogTest(#name"Blocking", \
-        CheckResourcesAndVars(RunOnThread(&test_case::Test##name))); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest( \
+        #name"Blocking", \
+        CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
+        start_time); \
   }
 
 #define RUN_TEST_BACKGROUND(test_case, name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
-    instance_->LogTest(#name"Background", \
-        CheckResourcesAndVars(RunOnThread(&test_case::Test##name))); \
+  if (ShouldRunTest(#name, test_filter)) { \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest( \
+        #name"Background", \
+        CheckResourcesAndVars(RunOnThread(&test_case::Test##name)), \
+        start_time); \
   }
 
 #define RUN_TEST_FORCEASYNC_AND_NOT(name, test_filter) \
@@ -303,7 +353,7 @@ class TestCaseFactory {
   } while (false)
 
 #define RUN_TEST_WITH_REFERENCE_CHECK(name, test_filter) \
-  if (MatchesFilter(#name, test_filter)) { \
+  if (ShouldRunTest(#name, test_filter)) { \
     set_callback_type(PP_OPTIONAL); \
     uint32_t objects = testing_interface_->GetLiveObjectsForInstance( \
         instance_->pp_instance()); \
@@ -313,7 +363,10 @@ class TestCaseFactory {
             instance_->pp_instance()) != objects) \
       error_message = MakeFailureMessage(__FILE__, __LINE__, \
           "reference leak check"); \
-    instance_->LogTest(#name, error_message); \
+    PP_TimeTicks start_time(NowInTimeTicks()); \
+    instance_->LogTest(#name, \
+                       error_message, \
+                       start_time); \
   }
 // TODO(dmichael): Add CheckResourcesAndVars above when Windows tests pass
 //                 cleanly. crbug.com/173503