File indexing completed on 2024-04-28 16:08:40

0001 /***************************************************************************
0002  *   Copyright (C) 2013-2017 by Linuxstopmotion contributors;              *
0003  *   see the AUTHORS file for details.                                     *
0004  *                                                                         *
0005  *   This program is free software; you can redistribute it and/or modify  *
0006  *   it under the terms of the GNU General Public License as published by  *
0007  *   the Free Software Foundation; either version 2 of the License, or     *
0008  *   (at your option) any later version.                                   *
0009  *                                                                         *
0010  *   This program is distributed in the hope that it will be useful,       *
0011  *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
0012  *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
0013  *   GNU General Public License for more details.                          *
0014  *                                                                         *
0015  *   You should have received a copy of the GNU General Public License     *
0016  *   along with this program; if not, write to the                         *
0017  *   Free Software Foundation, Inc.,                                       *
0018  *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
0019  ***************************************************************************/
0020 
0021 #include "testundo.h"
0022 
0023 #include "hash.h"
0024 #include "oomtestutil.h"
0025 #include "src/domain/undo/executor.h"
0026 #include "src/domain/undo/commandlogger.h"
0027 #include "src/domain/undo/filelogger.h"
0028 #include "src/domain/undo/random.h"
0029 #include "src/domain/animation/errorhandler.h"
0030 #include "src/foundation/stringwriter.h"
0031 
0032 #include <stdlib.h>
0033 #include <string.h>
0034 #include <sstream>
0035 #include <stdio.h>
0036 #include <cerrno>
0037 #include <exception>
0038 #include <assert.h>
0039 #include <unistd.h>
0040 
0041 #include <QtTest/QtTest>
0042 
0043 
0044 
0045 class StringLoggerWrapper: public CommandLogger {
0046     CommandLogger* delegate;
0047     std::string* out;
0048     std::string pending;
0049     int committedUpTo;
0050 public:
0051     StringLoggerWrapper(std::string* output) :
0052             delegate(0), out(output), committedUpTo(0) {
0053     }
0054     /**
0055      * Create a logger that writes to a std::string and also passes writes
0056      * along to @a wrapped.
0057      * @param wrapped Ownership is not passed.
0058      * @param output The string to be logged to. Ownership is not passed.
0059      */
0060     StringLoggerWrapper(std::string* output, CommandLogger* wrapped) :
0061             delegate(wrapped), out(output), committedUpTo(0) {
0062     }
0063     StringLoggerWrapper(const StringLoggerWrapper&); // unimplemented
0064     StringLoggerWrapper& operator=(const StringLoggerWrapper&); // unimplemented
0065     ~StringLoggerWrapper() {
0066     }
0067     void setOutputString(std::string* output) {
0068         out = output;
0069     }
0070     void output(std::string* to) {
0071         if (to)
0072             to->append(pending, 0, committedUpTo);
0073         pending.erase(0, committedUpTo);
0074         committedUpTo = 0;
0075     }
0076     void writePendingCommand(const char* command) {
0077         pending.resize(committedUpTo);
0078         pending.append(command);
0079         pending.append("!\n");
0080         if (delegate)
0081             delegate->writePendingCommand(command);
0082     }
0083     void commit() {
0084         committedUpTo = pending.length();
0085         if (delegate)
0086             delegate->commit();
0087         output(out);
0088     }
0089     void writePendingUndo() {
0090         pending.resize(committedUpTo);
0091         pending.append("--undo---\n");
0092         if (delegate)
0093             delegate->writePendingUndo();
0094     }
0095     void writePendingRedo() {
0096         pending.resize(committedUpTo);
0097         pending.append("--redo---\n");
0098         if (delegate)
0099             delegate->writePendingRedo();
0100     }
0101     void setDelegate(CommandLogger* newLogger) {
0102         delegate = newLogger;
0103     }
0104     void flush() {
0105         output(out);
0106         if (delegate)
0107             delegate->flush();
0108     }
0109 };
0110 
0111 ModelTestHelper::~ModelTestHelper() {
0112 }
0113 
0114 // In order to test the executor and its ability to survive exceptions being
0115 // thrown, we chain together a load of executor steps, and compare the
0116 // result of running both. For example, we might run a load of commands
0117 // with a randomly-failing mallocker and compare the result against
0118 // running the commands from the log thus produced.
0119 class ExecutorStep {
0120     static int failures;
0121     long mallocCount;
0122     ExecutorStep* previous;
0123     std::string log[2];
0124     StringLoggerWrapper stringLogger;
0125     RandomSource final;
0126     void setup(Executor& e, CommandLogger* logger, int whichLog) {
0127         cancelAnyMallocFailure();
0128         log[whichLog].clear();
0129         stringLogger.setOutputString(&log[whichLog]);
0130         stringLogger.setDelegate(logger);
0131         e.setCommandLogger(&stringLogger);
0132     }
0133     int activeLog;
0134     void finishRun(long mallocsAtStart, RandomSource& rng) {
0135         long end = mallocsSoFar();
0136         mallocCount = end - mallocsAtStart;
0137         final = rng;
0138     }
0139     /**
0140      * Runs the step. Sets malloc count and log.
0141      */
0142     void run(Executor& e, RandomSource& rng, int whichLog) {
0143         activeLog = whichLog;
0144         if (previous)
0145             previous->run(e, rng, whichLog);
0146         setup(e, logger(), whichLog);
0147         long start = mallocsSoFar();
0148         try {
0149             doStep(e, rng);
0150         } catch(...) {
0151             finishRun(start, rng);
0152             throw;
0153         }
0154         finishRun(start, rng);
0155     }
0156 public:
0157     int getCurrentlyActiveLog() const {
0158         return activeLog;
0159     }
0160     ExecutorStep* getPrevious() const {
0161         return previous;
0162     }
0163     RandomSource finalRng() const {
0164         return final;
0165     }
0166     static int failureCount() {
0167         return failures;
0168     }
0169     ExecutorStep(ExecutorStep* following)
0170         : mallocCount(0), previous(following), stringLogger(&log[0]),
0171           activeLog(0) {
0172     }
0173     virtual ~ExecutorStep() {
0174     }
0175     virtual const char* name() const = 0;
0176     virtual void doStep(Executor& e, RandomSource& rng) = 0;
0177     virtual CommandLogger* logger() {
0178         return 0;
0179     }
0180     virtual void cleanup() {
0181     }
0182     virtual void appendCommandLog(std::string& out, int which) {
0183         stringLogger.output(&log[which]);
0184         out.append(log[which]);
0185     }
0186     void getLog(std::string& out, int which) {
0187         if (previous)
0188             previous->getLog(out, which);
0189         out.append(";\n");
0190         out.append(name());
0191         out.append(": ");
0192         appendCommandLog(out, which);
0193     }
0194     long getMallocCount() const {
0195         return mallocCount;
0196     }
0197     /**
0198      * Runs this series of steps and {@a other} and tests the results against
0199      * one another. {@a other} is run first.
0200      */
0201     void runAndCheck(const char* name, ExecutorStep& other, Executor& executor,
0202             ModelTestHelper& helper, RandomSource rng, int testNum) {
0203         RandomSource r2 = rng;
0204         try {
0205             other.run(executor, rng, 0);
0206         } catch (std::exception& e) {
0207             other.cleanup();
0208             std::string log;
0209             other.getLog(log, 0);
0210             std::ostringstream ss;
0211             ss << "Failed to run 'other' step in test '" << name
0212                     << "' on iteration " << testNum
0213                     << "\nSuccessful log:" << log;
0214             std::string s = ss.str();
0215             QFAIL(s.c_str());
0216         }
0217         other.cleanup();
0218         Hash h = helper.hashModel(executor);
0219         std::string model;
0220         helper.dumpModel(model, executor);
0221         try {
0222             run(executor, r2, 1);
0223         } catch (std::exception& e) {
0224             cleanup();
0225             std::string logS1;
0226             other.getLog(logS1, 0);
0227             std::string logS2;
0228             getLog(logS2, 1);
0229             std::ostringstream ss;
0230             ss << "Failed to run 'this' step in test '" << name
0231                     << "' on iteration " << testNum
0232                     << "\nOther log:" << logS1
0233                     << "\nSuccessful portion of 'this' log:" << logS2;
0234             std::string s = ss.str();
0235             QFAIL(s.c_str());
0236         }
0237         cleanup();
0238         Hash h2 = helper.hashModel(executor);
0239         if (h != h2) {
0240             ++failures;
0241             std::string logS1;
0242             other.getLog(logS1, 0);
0243             std::string logS2;
0244             getLog(logS2, 1);
0245             std::ostringstream ss;
0246             ss << "Failed test '" << name << "' on iteration " << testNum
0247                     << "\nTesting:" << logS1 << "\nAgainst:" << logS2;
0248             std::string model2;
0249             helper.dumpModel(model2, executor);
0250             ss << "Resulting in:\n" << model << "And:\n" << model2;
0251             std::string s = ss.str();
0252             QFAIL(s.c_str());
0253         }
0254     }
0255     /**
0256      * Runs this series of steps and {@a other} and tests the results against
0257      * one another. {@a other} is run first.
0258      * Just like runAndCheck but pre-runs this.run. Useful if "this" contains a
0259      * FailingStep (whose delegate needs to be pre-run).
0260      */
0261     void runAndCheckWithPreRun(const char* name, ExecutorStep& other, Executor& executor,
0262             ModelTestHelper& helper, RandomSource rng, int testNum) {
0263         RandomSource r1 = rng;
0264         try {
0265             run(executor, r1, 1);
0266         } catch (std::exception& e) {
0267             cleanup();
0268             std::string logS1;
0269             other.getLog(logS1, 0);
0270             std::string logS2;
0271             getLog(logS2, 0);
0272             std::ostringstream ss;
0273             ss << "Failed to pre-run 'this' step in test '" << name
0274                     << "' on iteration " << testNum
0275                     << "\nOther log:" << logS1
0276                     << "\nSuccessful portion of 'this' log:" << logS2;
0277             std::string s = ss.str();
0278             QFAIL(s.c_str());
0279         }
0280         cleanup();
0281         runAndCheck(name, other, executor, helper, rng, testNum);
0282     }
0283 };
0284 
0285 int ExecutorStep::failures = 0;
0286 
0287 FILE* fileOpen(const char* path, const char* mode) {
0288     FILE* fh = fopen(path, mode);
0289     if (!fh) {
0290         sleep(1);
0291         fh = fopen(path, mode);
0292     }
0293     if (!fh) {
0294         int err = errno;
0295         // for some reason errno can be 0 when fopen returns 0
0296         if (err == 0 || err == ENOMEM) {
0297             throw std::bad_alloc();
0298         }
0299         StringWriter sw;
0300         sw.writeIdentifier("fopen failed for file");
0301         sw.writeString(path);
0302         sw.writeIdentifier("with error code");
0303         sw.writeInteger(err);
0304         sw.writeChar('(');
0305         sw.writeIdentifier(strerror(err));
0306         sw.writeChar(')');
0307         QWARN(sw.result());
0308     }
0309     return fh;
0310 }
0311 
0312 /**
0313  * Runs the delegate, failing one of its mallocs.
0314  * It works best as part of the "this" in a call to runAndCheck with
0315  * the delegate a non-failing part of the "other" argument.
0316  */
0317 class FailingStep : public ExecutorStep {
0318     ExecutorStep* del;
0319     std::string nameString;
0320     bool fail;
0321     int totalFails;
0322     int noMallocsToFailCount;
0323 public:
0324     FailingStep(ExecutorStep* delegate)
0325         : ExecutorStep(delegate? delegate->getPrevious() : 0),
0326           del(delegate), nameString("failing "), fail(false),
0327           totalFails(0), noMallocsToFailCount(0) {
0328         nameString.append(del->name());
0329         nameString.c_str();
0330     }
0331     const char* name() const {
0332         return nameString.c_str();
0333     }
0334     CommandLogger* logger() {
0335         return del->logger();
0336     }
0337     bool failed() const {
0338         return fail;
0339     }
0340     int failedCount() const {
0341         return totalFails;
0342     }
0343     int noMallocsCount() const {
0344         return noMallocsToFailCount;
0345     }
0346     void doStep(Executor& e, RandomSource& rng) {
0347         fail = false;
0348         long mallocCount = del->getMallocCount();
0349         if (mallocCount < 1) {
0350             del->doStep(e, rng);
0351             ++noMallocsToFailCount;
0352             return;
0353         }
0354         int muf = rng.getUniform(mallocCount - 1);
0355         setMallocsUntilFailure(muf);
0356         try {
0357             del->doStep(e, rng);
0358         } catch(...) {
0359             fail = true;
0360             ++totalFails;
0361         }
0362         cancelAnyMallocFailure();
0363     }
0364     void cleanup() {
0365         del->cleanup();
0366     }
0367 };
0368 
0369 class ExecutorInit : public ExecutorStep {
0370     ModelTestHelper& mth;
0371 public:
0372     ExecutorInit(ModelTestHelper& helper) : ExecutorStep(0), mth(helper) {
0373     }
0374     const char* name() const {
0375         return "initialize";
0376     }
0377     void doStep(Executor& e, RandomSource&) {
0378         mth.resetModel(e);
0379     }
0380 };
0381 
0382 class ExecutorConstruct : public ExecutorStep {
0383 public:
0384     ExecutorConstruct(ExecutorStep* following) : ExecutorStep(following) {
0385     }
0386     const char* name() const {
0387         return "constructive commands";
0388     }
0389     void doStep(Executor& e, RandomSource& rng) {
0390         e.executeRandomConstructiveCommands(rng);
0391     }
0392 };
0393 
0394 class ClearHistory : public ExecutorStep {
0395 public:
0396     ClearHistory(ExecutorStep* following)
0397         : ExecutorStep(following) {
0398     }
0399     const char* name() const {
0400         return "clear history";
0401     }
0402     void doStep(Executor& e, RandomSource&) {
0403         e.clearHistory();
0404     }
0405 };
0406 
0407 class ExecutorDo : public ExecutorStep {
0408     int min;
0409     int max;
0410     FileCommandLogger fLogger;
0411     const char* logFName;
0412     FILE* fh;
0413 public:
0414     ExecutorDo(ExecutorStep* following, const char* logFileName)
0415         : ExecutorStep(following), min(1), max(40), logFName(logFileName),
0416           fh(0) {
0417     }
0418     void setMinimumAndMaximumCommands(int minimum, int maximum) {
0419         min = minimum;
0420         max = maximum;
0421     }
0422     const char* name() const {
0423         return "random commands";
0424     }
0425     void doStep(Executor& e, RandomSource& rng) {
0426         fh = fileOpen(logFName, "w");
0427         assert(fh);
0428         fLogger.setLogFile(fh);
0429         int cc;
0430         e.executeRandomCommands(cc, rng, min, max);
0431     }
0432     void cleanup() {
0433         fLogger.getLogger()->flush();
0434         fLogger.setLogFile(0);
0435         fh = 0;
0436     }
0437     CommandLogger* logger() {
0438         return fLogger.getLogger();
0439     }
0440 };
0441 
0442 class ExecutorDoesAndRandomUndoesAndRedoes : public ExecutorStep {
0443     FileCommandLogger fLogger;
0444     const char* logFName;
0445     FILE* fh;
0446 public:
0447     ExecutorDoesAndRandomUndoesAndRedoes(ExecutorStep* following,
0448             const char* logFileName)
0449         : ExecutorStep(following), logFName(logFileName), fh(0) {
0450     }
0451     const char* name() const {
0452         return "random undoes and redoes";
0453     }
0454     void doStep(Executor& e, RandomSource& rng) {
0455         fh = fileOpen(logFName, "w");
0456         assert(fh);
0457         fLogger.setLogFile(fh);
0458         int commandCount = 0;
0459         e.executeRandomCommands(commandCount, rng, 1, 20);
0460         int maxUndoes = rng.getUniform(commandCount);
0461         for (int j = 0; j != maxUndoes; ++j) {
0462             e.undo();
0463         }
0464         int redoCount = rng.getUniform(maxUndoes);
0465         for (int i = 0; i != redoCount; ++i) {
0466             e.redo();
0467         }
0468         e.executeRandomCommands(commandCount, rng, 1, 3);
0469     }
0470     void cleanup() {
0471         fLogger.setLogFile(0);
0472         fh = 0;
0473     }
0474     CommandLogger* logger() {
0475         return fLogger.getLogger();
0476     }
0477 };
0478 
0479 class ExecutorUndoAll : public ExecutorStep {
0480 public:
0481     ExecutorUndoAll(ExecutorStep* following) : ExecutorStep(following) {
0482     }
0483     const char* name() const {
0484         return "undo all";
0485     }
0486     void doStep(Executor& e, RandomSource&) {
0487         while (e.canUndo()) {
0488             e.undo();
0489         }
0490     }
0491 };
0492 
0493 class ExecutorRedoAll : public ExecutorStep {
0494 public:
0495     ExecutorRedoAll(ExecutorStep* following) : ExecutorStep(following) {
0496     }
0497     const char* name() const {
0498         return "redo all";
0499     }
0500     void doStep(Executor& e, RandomSource&) {
0501         while (e.canRedo()) {
0502             e.redo();
0503         }
0504     }
0505 };
0506 
0507 class ExecutorReplay : public ExecutorStep {
0508     const char* logFName;
0509     FILE* fh;
0510     enum {
0511         lineBufferSize = 1024
0512     };
0513     char lineBuffer[lineBufferSize];
0514     std::string replayed[2];
0515 public:
0516     ExecutorReplay(ExecutorStep* following, const char* logFileName)
0517         : ExecutorStep(following),  logFName(logFileName), fh(0) {
0518     }
0519     ~ExecutorReplay() {
0520         cleanup();
0521     }
0522     const char* name() const {
0523         return "replay";
0524     }
0525     void appendCommandLog(std::string& out, int which) {
0526         out.append(replayed[which]);
0527     }
0528     void doStep(Executor& e, RandomSource&) {
0529         cleanup();
0530         int whichLog = getCurrentlyActiveLog();
0531         replayed[whichLog].clear();
0532         fh = fileOpen(logFName, "r");
0533         assert(fh);
0534         while (fgets(lineBuffer, lineBufferSize, fh)) {
0535             e.executeFromLog(lineBuffer, *ErrorHandler::getThrower());
0536             replayed[whichLog].append(lineBuffer);
0537         }
0538     }
0539     void cleanup() {
0540         if (fh) {
0541             fclose(fh);
0542             fh = 0;
0543         }
0544     }
0545 };
0546 
0547 // Use this if you want FailingStep to refer to a pair of steps.
0548 class TwoSteps : public ExecutorStep {
0549     ExecutorStep& s1;
0550     ExecutorStep& s2;
0551     std::string nameStr;
0552 public:
0553     // The steps that first and second are following are ignored here.
0554     TwoSteps(ExecutorStep& first, ExecutorStep& second, ExecutorStep* following)
0555         : ExecutorStep(following), s1(first), s2(second), nameStr(s1.name()) {
0556         nameStr.append(" then ");
0557         nameStr.append(s2.name());
0558         nameStr.c_str();
0559     }
0560     const char* name() const {
0561         return nameStr.c_str();
0562     }
0563     void doStep(Executor& e, RandomSource& rng) {
0564         s1.doStep(e, rng);
0565         s2.doStep(e, rng);
0566     }
0567 };
0568 
0569 // Do then replay
0570 // Do then replay then undo
0571 // Do then replay then undo then redo
0572 // Do then replay then OOM[undo then redo] then undo
0573 // Do then replay then OOM[undo then redo] then redo
0574 // OOM[do] then replay
0575 // Also need: Do then undo some then redo some (checkpoint) then replay
0576 void testUndo(Executor& e, ModelTestHelper& helper) {
0577     FileCommandLogger fileLogger;
0578     static const char tmpDirTemplate[] = "/tmp/lsmXXXXXX";
0579     char tmpDirName[sizeof(tmpDirTemplate)];
0580     strncpy(tmpDirName, tmpDirTemplate, sizeof(tmpDirName));
0581     mkdtemp(tmpDirName);
0582     std::string tmpFileName(tmpDirName);
0583     tmpFileName += "/command.log";
0584 
0585     // the tree of possible execution paths that we are going to check:
0586     // (1) Do = replay
0587     ExecutorInit init(helper);
0588     ExecutorConstruct construct(&init);
0589     ClearHistory clearHistory(&construct);
0590     ExecutorDo doStuff(&clearHistory, tmpFileName.c_str());
0591     ExecutorReplay replay(&clearHistory, tmpFileName.c_str());
0592 
0593     // (2) Do, undo = replay, undo
0594     ExecutorUndoAll undoToConstruct(&doStuff);
0595     ExecutorUndoAll undoAfterReplay(&replay);
0596 
0597     // (3) Do = replay, undo, redo
0598     ExecutorRedoAll redoAgain(&undoToConstruct);
0599 
0600     // (4) Construct = Do, replay, OOM[undo then redo], undo
0601     TwoSteps undoThenRedo(undoToConstruct, redoAgain, &undoToConstruct);
0602     FailingStep failToUndoThenRedo(&undoThenRedo);
0603     ExecutorUndoAll undoAfterFail(&failToUndoThenRedo);
0604 
0605     // (5) Do = replay, OOM[undo then redo], redo
0606     ExecutorRedoAll redoAfterFail(&failToUndoThenRedo);
0607 
0608     // (6) OOM[Do] = replay
0609     FailingStep failToDo(&doStuff);
0610 
0611     // (7) Do, undo some, redo some, do = replay
0612     ExecutorDoesAndRandomUndoesAndRedoes doesUndoesRedoes(&construct,
0613             tmpFileName.c_str());
0614 
0615     // (8) OOM[Do, undo some, redo sometestundo, do] = replay
0616     FailingStep failingDur(&doesUndoesRedoes);
0617 
0618     const int commandCount = e.commandCount();
0619     const int testCount = 4 * commandCount * commandCount;
0620     const int firstPhaseEnds = testCount / 2;
0621     const int secondPhaseEnds = (testCount * 3) / 4;
0622     bool oomLoaded = loadOomTestUtil();
0623     QVERIFY2(oomLoaded, "Oom Test Util not loaded!");
0624     std::string logString;
0625     RandomSource rng;
0626     StringLoggerWrapper stringLogger(&logString);
0627     // Now we will check pairs of steps in the tree against each other to check
0628     // that they produce identical results.
0629     for (int i = 0; i != testCount; ++i) {
0630         int minCommands = 1;
0631         int maxCommands = 40;
0632         if (i < secondPhaseEnds) {
0633             if (i < firstPhaseEnds) {
0634                 maxCommands = 1;
0635             } else {
0636                 minCommands = maxCommands = 2;
0637             }
0638         }
0639         doStuff.setMinimumAndMaximumCommands(minCommands, maxCommands);
0640 
0641         // (1)
0642         replay.runAndCheck("replay", doStuff, e, helper, rng, i);
0643         // (2)
0644         undoAfterReplay.runAndCheck("undo after replay",
0645                 undoToConstruct, e, helper, rng, i);
0646         // (3)
0647         redoAgain.runAndCheck("redo after replay and undo",
0648                 doStuff, e, helper, rng, i);
0649         // (4)
0650         undoAfterFail.runAndCheck("undo after fail",
0651                 construct, e, helper, rng, i);
0652         // (5)
0653         redoAfterFail.runAndCheck("redo after fail",
0654                 doStuff, e, helper, rng, i);
0655         // (6)
0656         replay.runAndCheck("replays failing sequence correctly",
0657                 failToDo, e, helper, rng, i);
0658         // (7)
0659         replay.runAndCheck("replays sequence of does, undoes and redoes correctly",
0660                 doesUndoesRedoes, e, helper, rng, i);
0661         // (8)
0662         replay.runAndCheckWithPreRun(
0663                 "replays failing sequence of does, undoes and redoes correctly",
0664                 failingDur, e, helper, rng, i);
0665 
0666         rng = redoAgain.finalRng();
0667     }
0668     cancelAnyMallocFailure();
0669 
0670     QVERIFY2(0 == unlink(tmpFileName.c_str()), "Could not delete test command log file");
0671     QVERIFY2(0 == rmdir(tmpDirName), "Could not delete test directory");
0672     // The tests that rely on failing commands will pass if the fake malloc
0673     // somehow does not cause any command to fail (this will happen, for
0674     // example, if no allocations are actually made). If this happens too much
0675     // then these tests are not being useful, so we check here that at least
0676     // half of the tests run in each case did require recovering from a failure.
0677     QVERIFY2(testCount / 2 < failToUndoThenRedo.failedCount()
0678             || testCount < failToUndoThenRedo.noMallocsCount(),
0679             "failToUndoThenRedo didn't fail very often");
0680     QVERIFY2(testCount / 2 < failToDo.failedCount(),
0681             "failToDo didn't fail very often");
0682 }