File indexing completed on 2024-05-19 12:37:22
0001 /*************************************************************************** 0002 * Copyright (C) 2013-2017 by Linuxstopmotion contributors; * 0003 * see the AUTHORS file for details. * 0004 * * 0005 * This program is free software; you can redistribute it and/or modify * 0006 * it under the terms of the GNU General Public License as published by * 0007 * the Free Software Foundation; either version 2 of the License, or * 0008 * (at your option) any later version. * 0009 * * 0010 * This program is distributed in the hope that it will be useful, * 0011 * but WITHOUT ANY WARRANTY; without even the implied warranty of * 0012 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * 0013 * GNU General Public License for more details. * 0014 * * 0015 * You should have received a copy of the GNU General Public License * 0016 * along with this program; if not, write to the * 0017 * Free Software Foundation, Inc., * 0018 * 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. * 0019 ***************************************************************************/ 0020 0021 #include "testundo.h" 0022 0023 #include "hash.h" 0024 #include "oomtestutil.h" 0025 #include "src/domain/undo/executor.h" 0026 #include "src/domain/undo/commandlogger.h" 0027 #include "src/domain/undo/filelogger.h" 0028 #include "src/domain/undo/random.h" 0029 #include "src/domain/animation/errorhandler.h" 0030 #include "src/foundation/stringwriter.h" 0031 0032 #include <stdlib.h> 0033 #include <string.h> 0034 #include <sstream> 0035 #include <stdio.h> 0036 #include <cerrno> 0037 #include <exception> 0038 #include <assert.h> 0039 #include <unistd.h> 0040 0041 #include <QtTest/QtTest> 0042 0043 0044 0045 class StringLoggerWrapper: public CommandLogger { 0046 CommandLogger* delegate; 0047 std::string* out; 0048 std::string pending; 0049 int committedUpTo; 0050 public: 0051 StringLoggerWrapper(std::string* output) : 0052 delegate(0), out(output), committedUpTo(0) { 0053 } 0054 /** 0055 * Create a logger that writes to a std::string and also passes writes 0056 * along to @a wrapped. 0057 * @param wrapped Ownership is not passed. 0058 * @param output The string to be logged to. Ownership is not passed. 0059 */ 0060 StringLoggerWrapper(std::string* output, CommandLogger* wrapped) : 0061 delegate(wrapped), out(output), committedUpTo(0) { 0062 } 0063 StringLoggerWrapper(const StringLoggerWrapper&); // unimplemented 0064 StringLoggerWrapper& operator=(const StringLoggerWrapper&); // unimplemented 0065 ~StringLoggerWrapper() { 0066 } 0067 void setOutputString(std::string* output) { 0068 out = output; 0069 } 0070 void output(std::string* to) { 0071 if (to) 0072 to->append(pending, 0, committedUpTo); 0073 pending.erase(0, committedUpTo); 0074 committedUpTo = 0; 0075 } 0076 void writePendingCommand(const char* command) { 0077 pending.resize(committedUpTo); 0078 pending.append(command); 0079 pending.append("!\n"); 0080 if (delegate) 0081 delegate->writePendingCommand(command); 0082 } 0083 void commit() { 0084 committedUpTo = pending.length(); 0085 if (delegate) 0086 delegate->commit(); 0087 output(out); 0088 } 0089 void writePendingUndo() { 0090 pending.resize(committedUpTo); 0091 pending.append("--undo---\n"); 0092 if (delegate) 0093 delegate->writePendingUndo(); 0094 } 0095 void writePendingRedo() { 0096 pending.resize(committedUpTo); 0097 pending.append("--redo---\n"); 0098 if (delegate) 0099 delegate->writePendingRedo(); 0100 } 0101 void setDelegate(CommandLogger* newLogger) { 0102 delegate = newLogger; 0103 } 0104 void flush() { 0105 output(out); 0106 if (delegate) 0107 delegate->flush(); 0108 } 0109 }; 0110 0111 ModelTestHelper::~ModelTestHelper() { 0112 } 0113 0114 // In order to test the executor and its ability to survive exceptions being 0115 // thrown, we chain together a load of executor steps, and compare the 0116 // result of running both. For example, we might run a load of commands 0117 // with a randomly-failing mallocker and compare the result against 0118 // running the commands from the log thus produced. 0119 class ExecutorStep { 0120 static int failures; 0121 long mallocCount; 0122 ExecutorStep* previous; 0123 std::string log[2]; 0124 StringLoggerWrapper stringLogger; 0125 RandomSource final; 0126 void setup(Executor& e, CommandLogger* logger, int whichLog) { 0127 cancelAnyMallocFailure(); 0128 log[whichLog].clear(); 0129 stringLogger.setOutputString(&log[whichLog]); 0130 stringLogger.setDelegate(logger); 0131 e.setCommandLogger(&stringLogger); 0132 } 0133 int activeLog; 0134 void finishRun(long mallocsAtStart, RandomSource& rng) { 0135 long end = mallocsSoFar(); 0136 mallocCount = end - mallocsAtStart; 0137 final = rng; 0138 } 0139 /** 0140 * Runs the step. Sets malloc count and log. 0141 */ 0142 void run(Executor& e, RandomSource& rng, int whichLog) { 0143 activeLog = whichLog; 0144 if (previous) 0145 previous->run(e, rng, whichLog); 0146 setup(e, logger(), whichLog); 0147 long start = mallocsSoFar(); 0148 try { 0149 doStep(e, rng); 0150 } catch(...) { 0151 finishRun(start, rng); 0152 throw; 0153 } 0154 finishRun(start, rng); 0155 } 0156 public: 0157 int getCurrentlyActiveLog() const { 0158 return activeLog; 0159 } 0160 ExecutorStep* getPrevious() const { 0161 return previous; 0162 } 0163 RandomSource finalRng() const { 0164 return final; 0165 } 0166 static int failureCount() { 0167 return failures; 0168 } 0169 ExecutorStep(ExecutorStep* following) 0170 : mallocCount(0), previous(following), stringLogger(&log[0]), 0171 activeLog(0) { 0172 } 0173 virtual ~ExecutorStep() { 0174 } 0175 virtual const char* name() const = 0; 0176 virtual void doStep(Executor& e, RandomSource& rng) = 0; 0177 virtual CommandLogger* logger() { 0178 return 0; 0179 } 0180 virtual void cleanup() { 0181 } 0182 virtual void appendCommandLog(std::string& out, int which) { 0183 stringLogger.output(&log[which]); 0184 out.append(log[which]); 0185 } 0186 void getLog(std::string& out, int which) { 0187 if (previous) 0188 previous->getLog(out, which); 0189 out.append(";\n"); 0190 out.append(name()); 0191 out.append(": "); 0192 appendCommandLog(out, which); 0193 } 0194 long getMallocCount() const { 0195 return mallocCount; 0196 } 0197 /** 0198 * Runs this series of steps and {@a other} and tests the results against 0199 * one another. {@a other} is run first. 0200 */ 0201 void runAndCheck(const char* name, ExecutorStep& other, Executor& executor, 0202 ModelTestHelper& helper, RandomSource rng, int testNum) { 0203 RandomSource r2 = rng; 0204 try { 0205 other.run(executor, rng, 0); 0206 } catch (std::exception& e) { 0207 other.cleanup(); 0208 std::string log; 0209 other.getLog(log, 0); 0210 std::ostringstream ss; 0211 ss << "Failed to run 'other' step in test '" << name 0212 << "' on iteration " << testNum 0213 << "\nSuccessful log:" << log; 0214 std::string s = ss.str(); 0215 QFAIL(s.c_str()); 0216 } 0217 other.cleanup(); 0218 Hash h = helper.hashModel(executor); 0219 std::string model; 0220 helper.dumpModel(model, executor); 0221 try { 0222 run(executor, r2, 1); 0223 } catch (std::exception& e) { 0224 cleanup(); 0225 std::string logS1; 0226 other.getLog(logS1, 0); 0227 std::string logS2; 0228 getLog(logS2, 1); 0229 std::ostringstream ss; 0230 ss << "Failed to run 'this' step in test '" << name 0231 << "' on iteration " << testNum 0232 << "\nOther log:" << logS1 0233 << "\nSuccessful portion of 'this' log:" << logS2; 0234 std::string s = ss.str(); 0235 QFAIL(s.c_str()); 0236 } 0237 cleanup(); 0238 Hash h2 = helper.hashModel(executor); 0239 if (h != h2) { 0240 ++failures; 0241 std::string logS1; 0242 other.getLog(logS1, 0); 0243 std::string logS2; 0244 getLog(logS2, 1); 0245 std::ostringstream ss; 0246 ss << "Failed test '" << name << "' on iteration " << testNum 0247 << "\nTesting:" << logS1 << "\nAgainst:" << logS2; 0248 std::string model2; 0249 helper.dumpModel(model2, executor); 0250 ss << "Resulting in:\n" << model << "And:\n" << model2; 0251 std::string s = ss.str(); 0252 QFAIL(s.c_str()); 0253 } 0254 } 0255 /** 0256 * Runs this series of steps and {@a other} and tests the results against 0257 * one another. {@a other} is run first. 0258 * Just like runAndCheck but pre-runs this.run. Useful if "this" contains a 0259 * FailingStep (whose delegate needs to be pre-run). 0260 */ 0261 void runAndCheckWithPreRun(const char* name, ExecutorStep& other, Executor& executor, 0262 ModelTestHelper& helper, RandomSource rng, int testNum) { 0263 RandomSource r1 = rng; 0264 try { 0265 run(executor, r1, 1); 0266 } catch (std::exception& e) { 0267 cleanup(); 0268 std::string logS1; 0269 other.getLog(logS1, 0); 0270 std::string logS2; 0271 getLog(logS2, 0); 0272 std::ostringstream ss; 0273 ss << "Failed to pre-run 'this' step in test '" << name 0274 << "' on iteration " << testNum 0275 << "\nOther log:" << logS1 0276 << "\nSuccessful portion of 'this' log:" << logS2; 0277 std::string s = ss.str(); 0278 QFAIL(s.c_str()); 0279 } 0280 cleanup(); 0281 runAndCheck(name, other, executor, helper, rng, testNum); 0282 } 0283 }; 0284 0285 int ExecutorStep::failures = 0; 0286 0287 FILE* fileOpen(const char* path, const char* mode) { 0288 FILE* fh = fopen(path, mode); 0289 if (!fh) { 0290 sleep(1); 0291 fh = fopen(path, mode); 0292 } 0293 if (!fh) { 0294 int err = errno; 0295 // for some reason errno can be 0 when fopen returns 0 0296 if (err == 0 || err == ENOMEM) { 0297 throw std::bad_alloc(); 0298 } 0299 StringWriter sw; 0300 sw.writeIdentifier("fopen failed for file"); 0301 sw.writeString(path); 0302 sw.writeIdentifier("with error code"); 0303 sw.writeInteger(err); 0304 sw.writeChar('('); 0305 sw.writeIdentifier(strerror(err)); 0306 sw.writeChar(')'); 0307 QWARN(sw.result()); 0308 } 0309 return fh; 0310 } 0311 0312 /** 0313 * Runs the delegate, failing one of its mallocs. 0314 * It works best as part of the "this" in a call to runAndCheck with 0315 * the delegate a non-failing part of the "other" argument. 0316 */ 0317 class FailingStep : public ExecutorStep { 0318 ExecutorStep* del; 0319 std::string nameString; 0320 bool fail; 0321 int totalFails; 0322 int noMallocsToFailCount; 0323 public: 0324 FailingStep(ExecutorStep* delegate) 0325 : ExecutorStep(delegate? delegate->getPrevious() : 0), 0326 del(delegate), nameString("failing "), fail(false), 0327 totalFails(0), noMallocsToFailCount(0) { 0328 nameString.append(del->name()); 0329 nameString.c_str(); 0330 } 0331 const char* name() const { 0332 return nameString.c_str(); 0333 } 0334 CommandLogger* logger() { 0335 return del->logger(); 0336 } 0337 bool failed() const { 0338 return fail; 0339 } 0340 int failedCount() const { 0341 return totalFails; 0342 } 0343 int noMallocsCount() const { 0344 return noMallocsToFailCount; 0345 } 0346 void doStep(Executor& e, RandomSource& rng) { 0347 fail = false; 0348 long mallocCount = del->getMallocCount(); 0349 if (mallocCount < 1) { 0350 del->doStep(e, rng); 0351 ++noMallocsToFailCount; 0352 return; 0353 } 0354 int muf = rng.getUniform(mallocCount - 1); 0355 setMallocsUntilFailure(muf); 0356 try { 0357 del->doStep(e, rng); 0358 } catch(...) { 0359 fail = true; 0360 ++totalFails; 0361 } 0362 cancelAnyMallocFailure(); 0363 } 0364 void cleanup() { 0365 del->cleanup(); 0366 } 0367 }; 0368 0369 class ExecutorInit : public ExecutorStep { 0370 ModelTestHelper& mth; 0371 public: 0372 ExecutorInit(ModelTestHelper& helper) : ExecutorStep(0), mth(helper) { 0373 } 0374 const char* name() const { 0375 return "initialize"; 0376 } 0377 void doStep(Executor& e, RandomSource&) { 0378 mth.resetModel(e); 0379 } 0380 }; 0381 0382 class ExecutorConstruct : public ExecutorStep { 0383 public: 0384 ExecutorConstruct(ExecutorStep* following) : ExecutorStep(following) { 0385 } 0386 const char* name() const { 0387 return "constructive commands"; 0388 } 0389 void doStep(Executor& e, RandomSource& rng) { 0390 e.executeRandomConstructiveCommands(rng); 0391 } 0392 }; 0393 0394 class ClearHistory : public ExecutorStep { 0395 public: 0396 ClearHistory(ExecutorStep* following) 0397 : ExecutorStep(following) { 0398 } 0399 const char* name() const { 0400 return "clear history"; 0401 } 0402 void doStep(Executor& e, RandomSource&) { 0403 e.clearHistory(); 0404 } 0405 }; 0406 0407 class ExecutorDo : public ExecutorStep { 0408 int min; 0409 int max; 0410 FileCommandLogger fLogger; 0411 const char* logFName; 0412 FILE* fh; 0413 public: 0414 ExecutorDo(ExecutorStep* following, const char* logFileName) 0415 : ExecutorStep(following), min(1), max(40), logFName(logFileName), 0416 fh(0) { 0417 } 0418 void setMinimumAndMaximumCommands(int minimum, int maximum) { 0419 min = minimum; 0420 max = maximum; 0421 } 0422 const char* name() const { 0423 return "random commands"; 0424 } 0425 void doStep(Executor& e, RandomSource& rng) { 0426 fh = fileOpen(logFName, "w"); 0427 assert(fh); 0428 fLogger.setLogFile(fh); 0429 int cc; 0430 e.executeRandomCommands(cc, rng, min, max); 0431 } 0432 void cleanup() { 0433 fLogger.getLogger()->flush(); 0434 fLogger.setLogFile(0); 0435 fh = 0; 0436 } 0437 CommandLogger* logger() { 0438 return fLogger.getLogger(); 0439 } 0440 }; 0441 0442 class ExecutorDoesAndRandomUndoesAndRedoes : public ExecutorStep { 0443 FileCommandLogger fLogger; 0444 const char* logFName; 0445 FILE* fh; 0446 public: 0447 ExecutorDoesAndRandomUndoesAndRedoes(ExecutorStep* following, 0448 const char* logFileName) 0449 : ExecutorStep(following), logFName(logFileName), fh(0) { 0450 } 0451 const char* name() const { 0452 return "random undoes and redoes"; 0453 } 0454 void doStep(Executor& e, RandomSource& rng) { 0455 fh = fileOpen(logFName, "w"); 0456 assert(fh); 0457 fLogger.setLogFile(fh); 0458 int commandCount = 0; 0459 e.executeRandomCommands(commandCount, rng, 1, 20); 0460 int maxUndoes = rng.getUniform(commandCount); 0461 for (int j = 0; j != maxUndoes; ++j) { 0462 e.undo(); 0463 } 0464 int redoCount = rng.getUniform(maxUndoes); 0465 for (int i = 0; i != redoCount; ++i) { 0466 e.redo(); 0467 } 0468 e.executeRandomCommands(commandCount, rng, 1, 3); 0469 } 0470 void cleanup() { 0471 fLogger.setLogFile(0); 0472 fh = 0; 0473 } 0474 CommandLogger* logger() { 0475 return fLogger.getLogger(); 0476 } 0477 }; 0478 0479 class ExecutorUndoAll : public ExecutorStep { 0480 public: 0481 ExecutorUndoAll(ExecutorStep* following) : ExecutorStep(following) { 0482 } 0483 const char* name() const { 0484 return "undo all"; 0485 } 0486 void doStep(Executor& e, RandomSource&) { 0487 while (e.canUndo()) { 0488 e.undo(); 0489 } 0490 } 0491 }; 0492 0493 class ExecutorRedoAll : public ExecutorStep { 0494 public: 0495 ExecutorRedoAll(ExecutorStep* following) : ExecutorStep(following) { 0496 } 0497 const char* name() const { 0498 return "redo all"; 0499 } 0500 void doStep(Executor& e, RandomSource&) { 0501 while (e.canRedo()) { 0502 e.redo(); 0503 } 0504 } 0505 }; 0506 0507 class ExecutorReplay : public ExecutorStep { 0508 const char* logFName; 0509 FILE* fh; 0510 enum { 0511 lineBufferSize = 1024 0512 }; 0513 char lineBuffer[lineBufferSize]; 0514 std::string replayed[2]; 0515 public: 0516 ExecutorReplay(ExecutorStep* following, const char* logFileName) 0517 : ExecutorStep(following), logFName(logFileName), fh(0) { 0518 } 0519 ~ExecutorReplay() { 0520 cleanup(); 0521 } 0522 const char* name() const { 0523 return "replay"; 0524 } 0525 void appendCommandLog(std::string& out, int which) { 0526 out.append(replayed[which]); 0527 } 0528 void doStep(Executor& e, RandomSource&) { 0529 cleanup(); 0530 int whichLog = getCurrentlyActiveLog(); 0531 replayed[whichLog].clear(); 0532 fh = fileOpen(logFName, "r"); 0533 assert(fh); 0534 while (fgets(lineBuffer, lineBufferSize, fh)) { 0535 e.executeFromLog(lineBuffer, *ErrorHandler::getThrower()); 0536 replayed[whichLog].append(lineBuffer); 0537 } 0538 } 0539 void cleanup() { 0540 if (fh) { 0541 fclose(fh); 0542 fh = 0; 0543 } 0544 } 0545 }; 0546 0547 // Use this if you want FailingStep to refer to a pair of steps. 0548 class TwoSteps : public ExecutorStep { 0549 ExecutorStep& s1; 0550 ExecutorStep& s2; 0551 std::string nameStr; 0552 public: 0553 // The steps that first and second are following are ignored here. 0554 TwoSteps(ExecutorStep& first, ExecutorStep& second, ExecutorStep* following) 0555 : ExecutorStep(following), s1(first), s2(second), nameStr(s1.name()) { 0556 nameStr.append(" then "); 0557 nameStr.append(s2.name()); 0558 nameStr.c_str(); 0559 } 0560 const char* name() const { 0561 return nameStr.c_str(); 0562 } 0563 void doStep(Executor& e, RandomSource& rng) { 0564 s1.doStep(e, rng); 0565 s2.doStep(e, rng); 0566 } 0567 }; 0568 0569 // Do then replay 0570 // Do then replay then undo 0571 // Do then replay then undo then redo 0572 // Do then replay then OOM[undo then redo] then undo 0573 // Do then replay then OOM[undo then redo] then redo 0574 // OOM[do] then replay 0575 // Also need: Do then undo some then redo some (checkpoint) then replay 0576 void testUndo(Executor& e, ModelTestHelper& helper) { 0577 FileCommandLogger fileLogger; 0578 static const char tmpDirTemplate[] = "/tmp/lsmXXXXXX"; 0579 char tmpDirName[sizeof(tmpDirTemplate)]; 0580 strncpy(tmpDirName, tmpDirTemplate, sizeof(tmpDirName)); 0581 mkdtemp(tmpDirName); 0582 std::string tmpFileName(tmpDirName); 0583 tmpFileName += "/command.log"; 0584 0585 // the tree of possible execution paths that we are going to check: 0586 // (1) Do = replay 0587 ExecutorInit init(helper); 0588 ExecutorConstruct construct(&init); 0589 ClearHistory clearHistory(&construct); 0590 ExecutorDo doStuff(&clearHistory, tmpFileName.c_str()); 0591 ExecutorReplay replay(&clearHistory, tmpFileName.c_str()); 0592 0593 // (2) Do, undo = replay, undo 0594 ExecutorUndoAll undoToConstruct(&doStuff); 0595 ExecutorUndoAll undoAfterReplay(&replay); 0596 0597 // (3) Do = replay, undo, redo 0598 ExecutorRedoAll redoAgain(&undoToConstruct); 0599 0600 // (4) Construct = Do, replay, OOM[undo then redo], undo 0601 TwoSteps undoThenRedo(undoToConstruct, redoAgain, &undoToConstruct); 0602 FailingStep failToUndoThenRedo(&undoThenRedo); 0603 ExecutorUndoAll undoAfterFail(&failToUndoThenRedo); 0604 0605 // (5) Do = replay, OOM[undo then redo], redo 0606 ExecutorRedoAll redoAfterFail(&failToUndoThenRedo); 0607 0608 // (6) OOM[Do] = replay 0609 FailingStep failToDo(&doStuff); 0610 0611 // (7) Do, undo some, redo some, do = replay 0612 ExecutorDoesAndRandomUndoesAndRedoes doesUndoesRedoes(&construct, 0613 tmpFileName.c_str()); 0614 0615 // (8) OOM[Do, undo some, redo sometestundo, do] = replay 0616 FailingStep failingDur(&doesUndoesRedoes); 0617 0618 const int commandCount = e.commandCount(); 0619 const int testCount = 4 * commandCount * commandCount; 0620 const int firstPhaseEnds = testCount / 2; 0621 const int secondPhaseEnds = (testCount * 3) / 4; 0622 bool oomLoaded = loadOomTestUtil(); 0623 QVERIFY2(oomLoaded, "Oom Test Util not loaded!"); 0624 std::string logString; 0625 RandomSource rng; 0626 StringLoggerWrapper stringLogger(&logString); 0627 // Now we will check pairs of steps in the tree against each other to check 0628 // that they produce identical results. 0629 for (int i = 0; i != testCount; ++i) { 0630 int minCommands = 1; 0631 int maxCommands = 40; 0632 if (i < secondPhaseEnds) { 0633 if (i < firstPhaseEnds) { 0634 maxCommands = 1; 0635 } else { 0636 minCommands = maxCommands = 2; 0637 } 0638 } 0639 doStuff.setMinimumAndMaximumCommands(minCommands, maxCommands); 0640 0641 // (1) 0642 replay.runAndCheck("replay", doStuff, e, helper, rng, i); 0643 // (2) 0644 undoAfterReplay.runAndCheck("undo after replay", 0645 undoToConstruct, e, helper, rng, i); 0646 // (3) 0647 redoAgain.runAndCheck("redo after replay and undo", 0648 doStuff, e, helper, rng, i); 0649 // (4) 0650 undoAfterFail.runAndCheck("undo after fail", 0651 construct, e, helper, rng, i); 0652 // (5) 0653 redoAfterFail.runAndCheck("redo after fail", 0654 doStuff, e, helper, rng, i); 0655 // (6) 0656 replay.runAndCheck("replays failing sequence correctly", 0657 failToDo, e, helper, rng, i); 0658 // (7) 0659 replay.runAndCheck("replays sequence of does, undoes and redoes correctly", 0660 doesUndoesRedoes, e, helper, rng, i); 0661 // (8) 0662 replay.runAndCheckWithPreRun( 0663 "replays failing sequence of does, undoes and redoes correctly", 0664 failingDur, e, helper, rng, i); 0665 0666 rng = redoAgain.finalRng(); 0667 } 0668 cancelAnyMallocFailure(); 0669 0670 QVERIFY2(0 == unlink(tmpFileName.c_str()), "Could not delete test command log file"); 0671 QVERIFY2(0 == rmdir(tmpDirName), "Could not delete test directory"); 0672 // The tests that rely on failing commands will pass if the fake malloc 0673 // somehow does not cause any command to fail (this will happen, for 0674 // example, if no allocations are actually made). If this happens too much 0675 // then these tests are not being useful, so we check here that at least 0676 // half of the tests run in each case did require recovering from a failure. 0677 QVERIFY2(testCount / 2 < failToUndoThenRedo.failedCount() 0678 || testCount < failToUndoThenRedo.noMallocsCount(), 0679 "failToUndoThenRedo didn't fail very often"); 0680 QVERIFY2(testCount / 2 < failToDo.failedCount(), 0681 "failToDo didn't fail very often"); 0682 }