ENH: added per test timeout support

Source/CTest/cmCTestBuildAndTestHandler.cxx

@@ -29,6 +29,7 @@ cmCTestBuildAndTestHandler::cmCTestBuildAndTestHandler()
   this->BuildTwoConfig         = false;
   this->BuildNoClean           = false;
   this->BuildNoCMake           = false;
+  this->Timeout = 0;
 }
 
 //----------------------------------------------------------------------
@@ -146,7 +147,6 @@ int cmCTestBuildAndTestHandler::RunCMakeAndTest(std::string* outstring)
   cmSystemTools::SetErrorCallback(CMakeMessageCallback, &cmakeOutString);
   cmSystemTools::SetStdoutCallback(CMakeStdoutCallback, &cmakeOutString);
   cmOStringStream out;
-  // What is this? double timeout = this->CTest->GetTimeOut();
 
   // if the generator and make program are not specified then it is an error
   if (!this->BuildGenerator.size() || !this->BuildMakeProgram.size())
@@ -217,7 +217,7 @@ int cmCTestBuildAndTestHandler::RunCMakeAndTest(std::string* outstring)
     *outstring =  out.str();
     }
 
-  // if not test was specified then we are done
+  // if no test was specified then we are done
   if (!this->TestCommand.size())
     {
     return 0;
@@ -361,8 +361,8 @@ int cmCTestBuildAndTestHandler::RunCMakeAndTest(std::string* outstring)
     out << this->TestCommandArgs[k] << " ";
     }
   out << "\n";
-  // What is this? this->TimeOut = timeout;
-  int runTestRes = this->CTest->RunTest(testCommand, &outs, &retval, 0);
+  int runTestRes = this->CTest->RunTest(testCommand, &outs, &retval, 0, 
+                                        this->Timeout);
   if(runTestRes != cmsysProcess_State_Exited || retval != 0)
     {
     out << "Failed to run test command: " << testCommand[0] << "\n";
@@ -432,6 +432,11 @@ int cmCTestBuildAndTestHandler::ProcessCommandLineArguments(
     idx++;
     this->ExecutableDirectory = allArgs[idx];
     }
+  if(currentArg.find("--test-timeout",0) == 0 && idx < allArgs.size() - 1)
+    {
+    idx++;
+    this->Timeout = atof(allArgs[idx].c_str());
+    }
   if(currentArg.find("--build-generator",0) == 0 && idx < allArgs.size() - 1)
     {
     idx++;

Source/CTest/cmCTestBuildAndTestHandler.h

@@ -75,6 +75,7 @@ protected:
   std::vector<std::string> TestCommandArgs;
   std::vector<std::string> BuildTargets;
   bool                     BuildNoCMake;
+  double                   Timeout;
 };
 
 #endif

Source/CTest/cmCTestTestHandler.cxx

@@ -559,60 +559,11 @@ int cmCTestTestHandler::ProcessHandler()
 }
 
 //----------------------------------------------------------------------
-void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
-                                          std::vector<cmStdString> &failed)
+void cmCTestTestHandler::ProcessOneTest(cmCTestTestProperties *it,
+                                        std::vector<cmStdString> &passed,
+                                        std::vector<cmStdString> &failed,
+                                        int cnt, int tmsize)
 {
-  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
-  this->TestList.clear();
-
-  this->GetListOfTests();
-  cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
-
-  this->StartTest = this->CTest->CurrentTime();
-  double elapsed_time_start = cmSystemTools::GetTime();
-
-  *this->LogFile << "Start testing: " << this->StartTest << std::endl
-    << "----------------------------------------------------------"
-    << std::endl;
-
-  // how many tests are in based on RegExp?
-  int inREcnt = 0;
-  cmCTestTestHandler::ListOfTests::iterator it;
-  for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
-    {
-    if (it->IsInBasedOnREOptions)
-      {
-      inREcnt ++;
-      }
-    }
-  // expand the test list based on the union flag
-  if (this->UseUnion)
-    {
-    this->ExpandTestsToRunInformation((int)tmsize);
-    }
-  else
-    {
-    this->ExpandTestsToRunInformation(inREcnt);
-    }
-
-  int cnt = 0;
-  inREcnt = 0;
-  std::string last_directory = "";
-  for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
-    {
-    cnt ++;
-    if (it->IsInBasedOnREOptions)
-      {
-      inREcnt++;
-      }
-
-    // if we are out of time then skip this test, we leave two minutes 
-    // to submit results
-    if (this->CTest->GetRemainingTimeAllowed() - 120 <= 0)
-      {
-      continue;
-      }
-
   const std::string& testname = it->Name;
   std::vector<std::string>& args = it->Args;
   cmCTestTestResult cres;
@@ -621,41 +572,9 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
   cres.ReturnValue = -1;
   cres.Status = cmCTestTestHandler::NOT_RUN;
   cres.TestCount = cnt;  
-
-    if (!(last_directory == it->Directory))
-      {
-      cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
-        "Changing directory into " << it->Directory.c_str() << "\n");
-      *this->LogFile << "Changing directory into: " << it->Directory.c_str()
-        << std::endl;
-      last_directory = it->Directory;
-      cmSystemTools::ChangeDirectory(it->Directory.c_str());
-      }
   cres.Name = testname;
   cres.Path = it->Directory.c_str();
   
-    if (this->UseUnion)
-      {
-      // if it is not in the list and not in the regexp then skip
-      if ((this->TestsToRun.size() &&
-           std::find(this->TestsToRun.begin(), this->TestsToRun.end(), cnt)
-           == this->TestsToRun.end()) && !it->IsInBasedOnREOptions)
-        {
-        continue;
-        }
-      }
-    else
-      {
-      // is this test in the list of tests to run? If not then skip it
-      if ((this->TestsToRun.size() &&
-           std::find(this->TestsToRun.begin(),
-             this->TestsToRun.end(), inREcnt)
-           == this->TestsToRun.end()) || !it->IsInBasedOnREOptions)
-        {
-        continue;
-        }
-      }
-
   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(3) << cnt << "/");
   cmCTestLog(this->CTest, HANDLER_OUTPUT, std::setw(3) << tmsize << " ");
   if ( this->MemCheck )
@@ -700,7 +619,7 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
       cres.FullCommandLine = actualCommand;
       this->TestResults.push_back( cres );
       failed.push_back(testname);
-        continue;
+      return;
       }
     }
   
@@ -727,7 +646,8 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
   
   
   cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT, std::endl
-      << (this->MemCheck?"MemCheck":"Test") << " command: " << testCommand
+             << (this->MemCheck?"MemCheck":"Test") 
+             << " command: " << testCommand
              << std::endl);
   *this->LogFile << cnt << "/" << tmsize
                  << " Test: " << testname.c_str() << std::endl;
@@ -751,7 +671,8 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
   
   if ( !this->CTest->GetShowOnly() )
     {
-      res = this->CTest->RunTest(arguments, &output, &retVal, this->LogFile);
+    res = this->CTest->RunTest(arguments, &output, &retVal, this->LogFile,
+                               it->Timeout);
     }
   
   clock_finish = cmSystemTools::GetTime();
@@ -908,19 +829,110 @@ void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
 
   if ( cres.Status == cmCTestTestHandler::COMPLETED )
     {
-      this->CleanTestOutput(output, static_cast<size_t>(
-          this->CustomMaximumPassedTestOutputSize));
+    this->CleanTestOutput(output, static_cast<size_t>
+                          (this->CustomMaximumPassedTestOutputSize));
     }
   else
     {
-      this->CleanTestOutput(output, static_cast<size_t>(
-          this->CustomMaximumFailedTestOutputSize));
+    this->CleanTestOutput(output, static_cast<size_t>
+                          (this->CustomMaximumFailedTestOutputSize));
     }
 
   cres.Output = output;
   cres.ReturnValue = retVal;
   cres.CompletionStatus = "Completed";
   this->TestResults.push_back( cres );
+}
+
+//----------------------------------------------------------------------
+void cmCTestTestHandler::ProcessDirectory(std::vector<cmStdString> &passed,
+                                          std::vector<cmStdString> &failed)
+{
+  std::string current_dir = cmSystemTools::GetCurrentWorkingDirectory();
+  this->TestList.clear();
+
+  this->GetListOfTests();
+  cmCTestTestHandler::ListOfTests::size_type tmsize = this->TestList.size();
+
+  this->StartTest = this->CTest->CurrentTime();
+  double elapsed_time_start = cmSystemTools::GetTime();
+
+  *this->LogFile << "Start testing: " << this->StartTest << std::endl
+    << "----------------------------------------------------------"
+    << std::endl;
+
+  // how many tests are in based on RegExp?
+  int inREcnt = 0;
+  cmCTestTestHandler::ListOfTests::iterator it;
+  for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
+    {
+    if (it->IsInBasedOnREOptions)
+      {
+      inREcnt ++;
+      }
+    }
+  // expand the test list based on the union flag
+  if (this->UseUnion)
+    {
+    this->ExpandTestsToRunInformation((int)tmsize);
+    }
+  else
+    {
+    this->ExpandTestsToRunInformation(inREcnt);
+    }
+
+  int cnt = 0;
+  inREcnt = 0;
+  std::string last_directory = "";
+  for ( it = this->TestList.begin(); it != this->TestList.end(); it ++ )
+    {
+    cnt ++;
+    if (it->IsInBasedOnREOptions)
+      {
+      inREcnt++;
+      }
+
+    // if we are out of time then skip this test, we leave two minutes 
+    // to submit results
+    if (this->CTest->GetRemainingTimeAllowed() - 120 <= 0)
+      {
+      continue;
+      }
+    
+    if (!(last_directory == it->Directory))
+      {
+      cmCTestLog(this->CTest, HANDLER_VERBOSE_OUTPUT,
+                 "Changing directory into " << it->Directory.c_str() << "\n");
+      *this->LogFile << "Changing directory into: " << it->Directory.c_str()
+                     << std::endl;
+      last_directory = it->Directory;
+      cmSystemTools::ChangeDirectory(it->Directory.c_str());
+      }
+
+    if (this->UseUnion)
+      {
+      // if it is not in the list and not in the regexp then skip
+      if ((this->TestsToRun.size() &&
+           std::find(this->TestsToRun.begin(), this->TestsToRun.end(), cnt)
+           == this->TestsToRun.end()) && !it->IsInBasedOnREOptions)
+        {
+        continue;
+        }
+      }
+    else
+      {
+      // is this test in the list of tests to run? If not then skip it
+      if ((this->TestsToRun.size() &&
+           std::find(this->TestsToRun.begin(),
+                     this->TestsToRun.end(), inREcnt)
+           == this->TestsToRun.end()) || !it->IsInBasedOnREOptions)
+        {
+        continue;
+        }
+      }
+    
+    // process this one test
+    this->ProcessOneTest(&(*it), passed, failed, cnt, tmsize);
     }
 
   this->EndTest = this->CTest->CurrentTime();
@@ -1647,6 +1659,10 @@ bool cmCTestTestHandler::SetTestsProperties(
             {
             rtit->WillFail = cmSystemTools::IsOn(val.c_str());
             }
+          if ( key == "TIMEOUT" )
+            {
+            rtit->Timeout = atof(val.c_str());
+            }
           if ( key == "FAIL_REGULAR_EXPRESSION" )
             {
             std::vector<std::string> lval;
@@ -1751,6 +1767,7 @@ bool cmCTestTestHandler::AddTest(const std::vector<std::string>& args)
   
   test.IsInBasedOnREOptions = true;
   test.WillFail = false;
+  test.Timeout = 0;
   if (this->UseIncludeRegExpFlag &&
     !this->IncludeTestsRegularExpression.find(testname.c_str()))
     {

Source/CTest/cmCTestTestHandler.h

@@ -85,6 +85,7 @@ public:
     std::map<cmStdString, cmStdString> Measurements;
     bool IsInBasedOnREOptions;
     bool WillFail;
+    double Timeout;
   };
 
   struct cmCTestTestResult
@@ -145,11 +146,18 @@ private:
   virtual void GenerateDartOutput(std::ostream& os);
 
   /**
-   * Run the test for a directory and any subdirectories
+   * Run the tests for a directory and any subdirectories
    */
   void ProcessDirectory(std::vector<cmStdString> &passed,
                         std::vector<cmStdString> &failed);
 
+  /**
+   *  Run one test
+   */
+  void ProcessOneTest(cmCTestTestProperties *props,
+                      std::vector<cmStdString> &passed,
+                      std::vector<cmStdString> &failed,
+                      int count, int tmsize);
 
   typedef std::vector<cmCTestTestProperties> ListOfTests;
   /**

Source/cmCTest.cxx

@@ -1074,7 +1074,7 @@ int cmCTest::RunMakeCommand(const char* command, std::string* output,
 //----------------------------------------------------------------------
 int cmCTest::RunTest(std::vector<const char*> argv,
                      std::string* output, int *retVal,
-                     std::ostream* log)
+                     std::ostream* log, double testTimeOut)
 {
   if(cmSystemTools::SameFile(argv[0], this->CTestSelf.c_str()) &&
      !this->ForceNewCTestProcess)
@@ -1087,6 +1087,14 @@ int cmCTest::RunTest(std::vector<const char*> argv,
       {
       if(argv[i])
         {
+        // if this test has a test command make sure we pass the timeout in
+        if (strcmp(argv[i],"--test-command") == 0 && testTimeOut)
+          {
+          args.push_back("--test-timeout");
+          cmOStringStream msg;
+          msg << testTimeOut;
+          args.push_back(msg.str());
+          }
         args.push_back(argv[i]);
         }
       }
@@ -1128,6 +1136,12 @@ int cmCTest::RunTest(std::vector<const char*> argv,
     {
     timeout = this->TimeOut;
     }
+  cmCTestLog(this, HANDLER_VERBOSE_OUTPUT, "-- timeout set to "
+             << testTimeOut << std::endl);
+  if (testTimeOut && testTimeOut < timeout)
+    {
+    timeout = testTimeOut;
+    }
   // always have at least 1 second if we got to here
   if (timeout <= 0)
     {

Source/cmCTest.h

@@ -232,7 +232,7 @@ public:
   //! Run command specialized for tests. Returns process status and retVal is
   // return value or exception.
   int RunTest(std::vector<const char*> args, std::string* output, int *retVal,
-    std::ostream* logfile);
+    std::ostream* logfile, double testTimeOut);
 
   /**
    * Execute handler and return its result. If the handler fails, it returns

Source/cmTest.cxx

@@ -117,6 +117,15 @@ void cmTest::DefineProperties(cmake *cm)
      "against the specified regular expressions and at least one of the"
      " regular expressions has to match, otherwise the test will fail.");
 
+  cm->DefineProperty
+    ("TIMEOUT", cmProperty::TEST, 
+     "How many seconds to allow for this test.",
+     "This property if set will limit a test to nto take more than "
+     "the specified number of seconds to run. If it exceeds that the "
+     "test process will be killed and ctest will move to the next test. "
+     "This setting takes precedence over DART_TESTING_TIMEOUT and "
+     "CTEST_TESTING_TIMOUT.");
+
   cm->DefineProperty
     ("WILL_FAIL", cmProperty::TEST, 
      "If set to true, this will invert the pass/fail flag of the test.",

Source/ctest.cxx

@@ -148,7 +148,7 @@ static const cmDocumentationEntry cmDocumentationOptions[] =
    "complete. Other options that affect this mode are --build-target "
    "--build-nocmake, --build-run-dir, "
    "--build-two-config, --build-exe-dir, --build-project,"
-   "--build-noclean, --build-options"},
+   "--build-noclean, --build-options, --test-timeout"},
   {"--build-target", "Specify a specific target to build.",
    "This option goes with the --build-and-test option, if left out the all "
    "target is built." },
@@ -168,6 +168,8 @@ static const cmDocumentationEntry cmDocumentationOptions[] =
 
   {"--test-command", "The test to run with the --build-and-test option.", ""
   },
+  {"--test-timeout", "The time limit in seconds for --test-command.", ""
+  },
   {"--tomorrow-tag", "Nightly or experimental starts with next day tag.",
    "This is useful if the build will not finish in one day." },
   {"--ctest-config", "The configuration file used to initialize CTest state "