Teuchos - Trilinos Tools Package Version of the Day
Loading...
Searching...
No Matches
Teuchos_UnitTestRepository.cpp
1// @HEADER
2// ***********************************************************************
3//
4// Teuchos: Common Tools Package
5// Copyright (2004) Sandia Corporation
6//
7// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8// license for use of this work by or on behalf of the U.S. Government.
9//
10// Redistribution and use in source and binary forms, with or without
11// modification, are permitted provided that the following conditions are
12// met:
13//
14// 1. Redistributions of source code must retain the above copyright
15// notice, this list of conditions and the following disclaimer.
16//
17// 2. Redistributions in binary form must reproduce the above copyright
18// notice, this list of conditions and the following disclaimer in the
19// documentation and/or other materials provided with the distribution.
20//
21// 3. Neither the name of the Corporation nor the names of the
22// contributors may be used to endorse or promote products derived from
23// this software without specific prior written permission.
24//
25// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36//
37// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38//
39// ***********************************************************************
40// @HEADER
41
42
47#include "Teuchos_Array.hpp"
48#include "Teuchos_Assert.hpp"
49#include "Teuchos_VerboseObject.hpp"
51#include "Teuchos_Assert.hpp"
52#include "Teuchos_Time.hpp"
53#include "Teuchos_StandardCatchMacros.hpp"
54
55
56namespace Teuchos {
57
58
59struct UnitTestData {
60
61 const Teuchos::UnitTestBase * unitTest;
62 std::string groupName;
63 std::string testName;
64 int insertionIndex;
65
66 UnitTestData(
67 Teuchos::UnitTestBase *unitTest_in,
68 const std::string groupName_in,
69 const std::string testName_in
70 )
71 : unitTest(unitTest_in), groupName(groupName_in), testName(testName_in),
72 insertionIndex(insersionIndexCounter_++)
73 {
74#ifdef TEUCHOS_DEBUG
75 TEUCHOS_ASSERT(unitTest_in);
76#endif
77 }
78
79private:
80 UnitTestData(); // Not defined!
81 static int insersionIndexCounter_;
82};
83
84
85int UnitTestData::insersionIndexCounter_ = 0;
86
87
88bool operator<(const UnitTestData &a, const UnitTestData &b)
89{
90 if (a.groupName < b.groupName) {
91 return true;
92 }
93 else if (a.groupName > b.groupName) {
94 return false;
95 }
96 return a.insertionIndex < b.insertionIndex;
97}
98
99
100
101std::string getUnitTestName(const std::string groupName,
102 const std::string testName)
103{
104 std::ostringstream oss;
105 oss << groupName<<"_"<<testName<<"_UnitTest";
106 return oss.str();
107}
108
109
110enum EShowTestDetails {
111 SHOW_TEST_DETAILS_ALL,
112 SHOW_TEST_DETAILS_TEST_NAMES,
113 SHOW_TEST_DETAILS_FINAL_RESULTS
114};
115
116
117bool strMatch( const std::string &fullMatchStr, const std::string &str )
118{
119
120 const std::string::size_type npos = std::string::npos;
121
122 const size_t strLen = str.length();
123 const size_t fullMatchStrLen = fullMatchStr.length();
124
125 if (fullMatchStrLen == 0) {
126 return true;
127 }
128
129 const bool beginGlob = fullMatchStr[0] == '*';
130 const bool endGlob = fullMatchStr[fullMatchStrLen-1] == '*';
131
132 const size_t matchStrLen =
133 fullMatchStrLen + (beginGlob ? -1 : 0) + (endGlob ? -1 : 0);
134
135 if (matchStrLen == 0) {
136 return true;
137 }
138
139 if (matchStrLen > strLen) {
140 return false;
141 }
142
143 if (beginGlob && endGlob) {
144 return str.find(fullMatchStr.substr(1, matchStrLen)) != npos;
145 }
146
147 if (endGlob) {
148 return fullMatchStr.substr(0, matchStrLen) == str.substr(0, matchStrLen);
149 }
150
151 if (beginGlob) {
152 return fullMatchStr.substr(1, matchStrLen) ==
153 str.substr(strLen-matchStrLen, matchStrLen);
154 }
155
156 return fullMatchStr == str;
157
158}
159
160
161} // namespace Teuchos
162
163
164
165
166namespace Teuchos {
167
168
169// Implementation class
170
171
172class UnitTestRepository::InstanceData {
173public:
174
175 typedef Teuchos::Array<UnitTestData> unitTests_t;
176
177 unitTests_t unitTests;
178 CommandLineProcessor clp;
179 EShowTestDetails showTestDetails;
180 bool globallyReduceUnitTestResult;
181 bool showSrcLocation;
182 bool showFailSrcLocation;
183 bool noOp;
184 std::string groupName;
185 std::string testName;
186 std::string notUnitTestName;
187 int testCounter;
188
189 InstanceData()
190 :clp(false),
191 showTestDetails(SHOW_TEST_DETAILS_TEST_NAMES),
192#if defined(HAVE_TEUCHOS_GLOBALLY_REDUCE_UNITTEST_RESULTS)
193 globallyReduceUnitTestResult(true),
194#else
195 globallyReduceUnitTestResult(false),
196#endif
197 showSrcLocation(false),
198 showFailSrcLocation(true),
199 noOp(false),
200 testCounter(0)
201 {}
202
203};
204
205
206// public
207
208
210{
211 return getData().clp;
212}
213
214
216 const bool globallyReduceUnitTestResult)
217{
218 getData().globallyReduceUnitTestResult = globallyReduceUnitTestResult;
219}
220
221
223{
224 return getData().globallyReduceUnitTestResult;
225}
226
227
229{
230
231 typedef InstanceData::unitTests_t unitTests_t;
232
233 using std::setprecision;
234
235 Time overallTimer("overallTimer", true);
236 Time timer("timer");
237
238 const int timerPrec = 3;
239
240 out << "\n***\n*** Unit test suite ...\n***\n\n";
241
242 InstanceData &data = getData();
243
244 const bool showAll = data.showTestDetails == SHOW_TEST_DETAILS_ALL;
245 const bool showTestNames = data.showTestDetails == SHOW_TEST_DETAILS_TEST_NAMES || showAll;
246
247 showTestFailureLocation(data.showFailSrcLocation);
248
249 bool success = true;
250 int testCounter = 0;
251 int numTestsRun = 0;
252 int numTestsFailed = 0;
253
254 Array<std::string> failedTests;
255
256 try {
257
258 out << "\nSorting tests by group name then by the order they were added ...";
259 timer.start(true);
260 std::sort( data.unitTests.begin(), data.unitTests.end() );
261 timer.stop();
262 out << " (time = "<<setprecision(timerPrec)<<timer.totalElapsedTime()<<")\n";
263
264 out << "\nRunning unit tests ...\n\n";
265 unitTests_t::iterator iter = data.unitTests.begin();
266 for ( ; iter != data.unitTests.end(); ++iter, ++testCounter ) {
267
268 const UnitTestData &utd = (*iter);
269
270 const std::string unitTestName = getUnitTestName(utd.groupName, utd.testName);
271
272 if (
273 (
274 strMatch(data.groupName, utd.groupName)
275 &&
276 strMatch(data.testName, utd.testName)
277 )
278 &&
279 (
280 data.notUnitTestName.length() == 0
281 ||
282 !strMatch(data.notUnitTestName, unitTestName)
283 )
284 )
285 {
286
287 ++numTestsRun;
288
289 std::ostringstream testHeaderOSS;
290 testHeaderOSS <<testCounter<<". "<<unitTestName<<" ... ";
291 const std::string testHeader = testHeaderOSS.str();
292
293 if (showAll)
294 out <<"\n";
295
296 if (showTestNames)
297 out <<testHeader<<std::flush;
298
299 {
300
302 RCP<FancyOStream> localOut;
303 if (showAll) {
304 out << "\n";
305 localOut = rcpFromRef(out);
306 }
307 else {
308 oss = rcp(new std::ostringstream);
309 localOut = fancyOStream(rcp_implicit_cast<std::ostream>(oss));
310 }
311
312 OSTab tab(out);
313
314 if (!data.noOp) {
315
316 timer.start(true);
317 const bool result = runUnitTestImpl(*utd.unitTest, *localOut);
318 timer.stop();
319
320 if (!result) {
321
322 failedTests.push_back(testHeader);
323
324 if (!showTestNames)
325 out <<testHeader<<"\n"<<std::flush;
326 else if (!showAll)
327 out <<"\n";
328
329 if (!is_null(oss))
330 out << oss->str();
331
332 out
333 <<"[FAILED] "
334 <<" "<<setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<< " sec)"
335 <<" "<<unitTestName<<"\n"
336 <<"Location: "<<utd.unitTest->unitTestFile()<<":"
337 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
338
339 if (!is_null(oss))
340 out << "\n";
341
342 success = false;
343
344 ++numTestsFailed;
345
346 }
347 else {
348
349 if (showTestNames)
350 out << "[Passed] "
351 << setprecision(timerPrec)<<"("<<timer.totalElapsedTime()<<" sec)\n";
352
353 if (showAll && data.showSrcLocation)
354 out
355 << "Location: "<<utd.unitTest->unitTestFile()<<":"
356 <<utd.unitTest->unitTestFileLineNumber()<<"\n";
357
358 }
359
360 }
361 else {
362
363 if (showTestNames)
364 out << "[Not Run]\n";
365
366 }
367
368 }
369
370 }
371
372 }
373
374 TEUCHOS_ASSERT_EQUALITY(testCounter, as<int>(data.unitTests.size()));
375
376 }
377 TEUCHOS_STANDARD_CATCH_STATEMENTS(true, out, success);
378
379 if (failedTests.size()) {
380 out << "\nThe following tests FAILED:\n";
381 for (Teuchos_Ordinal i = 0; i < failedTests.size(); ++i)
382 out << " " << failedTests[i] << "\n";
383 }
384
385 overallTimer.stop();
386 out << "\nTotal Time: " << setprecision(timerPrec)
387 << overallTimer.totalElapsedTime() << " sec\n";
388
389 out
390 << "\nSummary: total = " << testCounter
391 << ", run = " << numTestsRun;
392
393 if (!data.noOp) {
394 out
395 << ", passed = " << (numTestsRun-numTestsFailed)
396 << ", failed = " << numTestsFailed << "\n";
397 }
398 else {
399 out
400 << ", passed = ???"
401 << ", failed = ???\n";
402 }
403
404 return success;
405
406}
407
408
409int UnitTestRepository::runUnitTestsFromMain( int argc, char* argv[] )
410{
411
413
414 CommandLineProcessor &clp = getData().clp;
415 setUpCLP(outArg(clp));
417 clp.parse(argc,argv);
418 if ( parse_return != CommandLineProcessor::PARSE_SUCCESSFUL ) {
419 *out << "\nEnd Result: TEST FAILED" << std::endl;
420 return parse_return;
421 }
422
423 const bool success = runUnitTests(*out);
424
425 if (success)
426 *out << "\nEnd Result: TEST PASSED" << std::endl;
427 else
428 *out << "\nEnd Result: TEST FAILED" << std::endl;
429
430 clp.printFinalTimerSummary(out.ptr());
431
432 return (success ? 0 : 1);
433
434}
435
436
438 const std::string groupName, const std::string testName_in )
439{
440 InstanceData &data = getData();
441 std::string testName = testName_in;
442 data.unitTests.push_back(UnitTestData(unitTest, groupName, testName));
443}
444
445
447{
448 return (getData().showTestDetails == SHOW_TEST_DETAILS_ALL);
449}
450
451
452// private:
453
454
455UnitTestRepository::UnitTestRepository()
456{}
457
458
459void UnitTestRepository::setUpCLP(const Ptr<CommandLineProcessor>& clp)
460{
461
462 clp->addOutputSetupOptions(true);
463
464 const int numShowTestDetails = 3;
465 const EShowTestDetails showTestDetailsValues[numShowTestDetails] =
466 { SHOW_TEST_DETAILS_ALL,
467 SHOW_TEST_DETAILS_TEST_NAMES,
468 SHOW_TEST_DETAILS_FINAL_RESULTS
469 };
470 const char* showTestDetailsNames[numShowTestDetails] =
471 { "ALL",
472 "TEST_NAMES",
473 "FINAL_RESULTS"
474 };
475 clp->setOption(
476 "show-test-details", &getData().showTestDetails,
477 numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
478 "Level of detail to show in the tests"
479 );
480 clp->setOption(
481 "details", &getData().showTestDetails,
482 numShowTestDetails, showTestDetailsValues, showTestDetailsNames,
483 "Short for --show-test-details"
484 );
485
486 clp->setOption(
487 "show-src-location", "no-show-src-location", &getData().showSrcLocation,
488 "If true, then the location of the unit test source code is shown."
489 " Only meaningful if --show-test-details=ALL."
490 );
491
492 clp->setOption(
493 "show-fail-src-location", "no-show-fail-src-location", &getData().showFailSrcLocation,
494 "If true, then the location of every failed unit test check is printed."
495 );
496
497 clp->setOption(
498 "globally-reduce-test-result", "no-globally-reduce-test-result",
499 &getData().globallyReduceUnitTestResult,
500 "If true, individual unit test pass/fail is globally reduced across MPI processes."
501 );
502
503 clp->setOption(
504 "group-name", &getData().groupName,
505 "If specified, selects only tests that match the group name glob." );
506 clp->setOption(
507 "group", &getData().groupName,
508 "Short for --group-name." );
509
510 clp->setOption(
511 "test-name", &getData().testName,
512 "If specified, selects only tests that match the test name glob." );
513 clp->setOption(
514 "test", &getData().testName,
515 "Short for --test-name." );
516
517 clp->setOption(
518 "not-unit-test", &getData().notUnitTestName,
519 "If specified, full unit tests with glob matches will *not* be run." );
520
521 clp->setOption(
522 "no-op", "do-op", &getData().noOp,
523 "If --no-op, then only the names of the tests that would be run are run."
524 );
525
526}
527
528
529UnitTestRepository::InstanceData& UnitTestRepository::getData()
530{
531 static UnitTestRepository::InstanceData data;
532 return data;
533}
534
535
536bool UnitTestRepository::runUnitTestImpl(const UnitTestBase &unitTest,
537 FancyOStream &out)
538{
539 const bool result = unitTest.runUnitTest(out);
540 if (getData().globallyReduceUnitTestResult) {
541 const int globalSum = GlobalMPISession::sum(result ? 0 : 1);
542 if (globalSum == 0) {
543 return true;
544 }
545 else {
546 // Only print that there are failures on processes where the local
547 // unit test actally passed. On processes where the local unit test
548 // fails, users already know that test failed so there is no need to
549 // exlain it.
550 if (result) {
551 out << "NOTE: Global reduction shows failures on other processes!\n"
552 << "(rerun with --output-to-root-rank-only=-1 to see output\n"
553 << "from other processes to see what process failed!)\n";
554 }
555 else {
556 // The test failed on the root process so the user already knows it failed!
557 }
558 // Determine what processes have failing tests
559 const int numProcs = GlobalMPISession::getNProc();
560 Array<int> passFailFlags(numProcs);
561 GlobalMPISession::allGather( result ? 0 : 1, passFailFlags());
562 Array<int> procsThatFailed;
563 for ( int proc_k = 0; proc_k < numProcs; ++proc_k ) {
564 if (passFailFlags[proc_k] != 0) {
565 procsThatFailed.push_back(proc_k);
566 }
567 }
568 // Print what processes have the failing tests. If there is only one
569 // processes, don't print anything.
570 if (numProcs > 1) {
571 if (procsThatFailed.size() == numProcs) {
572 out << "NOTE: Unit test failed on all processes!\n";
573 // NOTE: when all the processes are failing it is useless to print
574 // out a list of all of the processes.
575 }
576 else {
577 out << "NOTE: Unit test failed on processes = " << procsThatFailed << "\n"
578 << "(rerun with --output-to-root-rank-only=<procID> to see output\n"
579 << "from individual processes where the unit test is failing!)\n";
580 }
581 }
582 return false;
583 }
584 }
585 return result;
586}
587
588
589} // namespace Teuchos
Templated array class derived from the STL std::vector.
Basic command line parser for input from (argc,argv[])
Teuchos header file which uses auto-configuration information to include necessary C++ headers.
Utilities to make writing tests easier.
Basic wall-clock timer class.
Unit testing support.
Unit testing support.
Replacement for std::vector that is compatible with the Teuchos Memory Management classes.
size_type size() const
void push_back(const value_type &x)
Class that helps parse command line input arguments from (argc,argv[]) and set options.
EParseCommandLineReturn
Return value for CommandLineProcessor::parse(). Note: These enums are all given non-negative values s...
EParseCommandLineReturn parse(int argc, char *argv[], std::ostream *errout=&std::cerr) const
Parse a command line.
void printFinalTimerSummary(const Ptr< std::ostream > &out=null)
Call to print timers so that they don't get printed in the destructor.
static int sum(int localVal)
Sum a set of integers across processes.
static int getNProc()
The number of processes in MPI_COMM_WORLD.
static void allGather(int localVal, const ArrayView< int > &allVals)
Global all-to-all of a set of integers across processes.
Smart reference counting pointer class for automatic garbage collection.
Ptr< T > ptr() const
Get a safer wrapper raw C++ pointer to the underlying object.
Wall-clock timer.
double totalElapsedTime(bool readCurrentTime=false) const
The total time in seconds accumulated by this timer.
void start(bool reset=false)
Start the timer, if the timer is enabled (see disable()).
double stop()
Stop the timer, if the timer is enabled (see disable()).
Unit test base class.
static bool verboseUnitTests()
Returns if unit tests are verbose or not.
static bool runUnitTests(FancyOStream &out)
Run the registered unit tests.
static bool getGloballyReduceTestResult()
Get if the unit tests should reduce across processes or not.
static CommandLineProcessor & getCLP()
Return the CLP to add options to.
static int runUnitTestsFromMain(int argc, char *argv[])
Run the unit tests from main() passing in (argc, argv).
static void addUnitTest(UnitTestBase *unitTest, const std::string groupName, const std::string testName)
Add an unit test (called indirectly through macros.
static void setGloballyReduceTestResult(const bool globallyReduceUnitTestResult)
Set if the unit tests should reduce pass/fail across processes.
static RCP< FancyOStream > getDefaultOStream()
Get the default output stream object.
std::ostream subclass that performs the magic of indenting data sent to an std::ostream object among ...
Tabbing class for helping to create formated, indented output for a basic_FancyOStream object.
#define TEUCHOS_ASSERT(assertion_test)
This macro is throws when an assert fails.
#define TEUCHOS_ASSERT_EQUALITY(val1, val2)
This macro is checks that to numbers are equal and if not then throws an exception with a good error ...
bool is_null(const std::shared_ptr< T > &p)
Returns true if p.get()==NULL.
#define TEUCHOS_STANDARD_CATCH_STATEMENTS(VERBOSE, ERR_STREAM, SUCCESS_FLAG)
Simple macro that catches and reports standard exceptions and other exceptions.
basic_FancyOStream< char > FancyOStream
bool showTestFailureLocation()
Return if TEUCHOS_PASS_FAIL(...) should print test failure location.
The Teuchos namespace contains all of the classes, structs and enums used by Teuchos,...
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Deprecated.