Warning, file /education/kstars/kstars/ekos/focus/focusalgorithms.cpp was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).

0001 /*
0002     SPDX-FileCopyrightText: 2019 Hy Murveit <hy-1@murveit.com>
0003 
0004     SPDX-License-Identifier: GPL-2.0-or-later
0005 */
0006 
0007 #include "focusalgorithms.h"
0008 #include "curvefit.h"
0009 
0010 #include "polynomialfit.h"
0011 #include <QVector>
0012 #include "kstars.h"
0013 #include "fitsviewer/fitsstardetector.h"
0014 #include "focusstars.h"
0015 #include <ekos_focus_debug.h>
0016 
0017 namespace Ekos
0018 {
0019 /**
0020  * @class LinearFocusAlgorithm
0021  * @short Autofocus algorithm that linearly samples HFR values.
0022  *
0023  * @author Hy Murveit
0024  * @version 1.0
0025  */
0026 class LinearFocusAlgorithm : public FocusAlgorithmInterface
0027 {
0028     public:
0029 
0030         // Constructor initializes a linear autofocus algorithm, starting at some initial position,
0031         // sampling HFR values, decreasing the position by some step size, until the algorithm believe's
0032         // it's seen a minimum. It then either:
0033         // a) tries to find that minimum in a 2nd pass. This is the LINEAR algorithm, or
0034         // b) moves to the minimum. This is the LINEAR 1 PASS algorithm.
0035         LinearFocusAlgorithm(const FocusParams &params);
0036 
0037         // After construction, this should be called to get the initial position desired by the
0038         // focus algorithm. It returns the initial position passed to the constructor if
0039         // it has no movement request.
0040         int initialPosition() override
0041         {
0042             return requestedPosition;
0043         }
0044 
0045         // Pass in the measurement for the last requested position. Returns the position for the next
0046         // requested measurement, or -1 if the algorithm's done or if there's an error.
0047         // If stars is not nullptr, then the relativeHFR scheme is used to modify the HFR value.
0048         int newMeasurement(int position, double value, const double starWeight, const QList<Edge*> *stars) override;
0049 
0050         FocusAlgorithmInterface *Copy() override;
0051 
0052         void getMeasurements(QVector<int> *pos, QVector<double> *val, QVector<double> *sds) const override
0053         {
0054             *pos = positions;
0055             *val = values;
0056             *sds = weights;
0057         }
0058 
0059         void getPass1Measurements(QVector<int> *pos, QVector<double> *val, QVector<double> *sds, QVector<bool> *out) const override
0060         {
0061             *pos = pass1Positions;
0062             *val = pass1Values;
0063             *sds = pass1Weights;
0064             *out = pass1Outliers;
0065         }
0066 
0067         double latestValue() const override
0068         {
0069             if (values.size() > 0)
0070                 return values.last();
0071             return -1;
0072         }
0073 
0074         bool isInFirstPass() const override
0075         {
0076             if (params.focusAlgorithm == Focus::FOCUS_LINEAR || params.focusAlgorithm == Focus::FOCUS_LINEAR1PASS)
0077                 return inFirstPass;
0078             else
0079                 return true;
0080         }
0081 
0082         QString getTextStatus(double R2 = 0) const override;
0083 
0084         int currentStep() const override
0085         {
0086             return numSteps;
0087         }
0088 
0089     private:
0090 
0091         // Called in newMeasurement. Sets up the next iteration.
0092         int completeIteration(int step, bool foundFit, double minPos, double minVal);
0093 
0094         // Does the bookkeeping for the final focus solution.
0095         int setupSolution(int position, double value, double sigma);
0096 
0097         // Perform the linear walk for FOCUS_WALK_FIXED_STEPS and FOCUS_WALK_CFZ_SHUFFLE
0098         int linearWalk(int position, double value, const double starWeight);
0099 
0100         // Calculate the curve fitting goal based on the algorithm parameters and progress
0101         CurveFitting::FittingGoal getGoal(int numSteps);
0102 
0103         // Get the minimum number of datapoints before attempting the first curve fit
0104         int getCurveMinPoints();
0105 
0106         // Analyze data for donuts are remove
0107         void removeDonuts();
0108 
0109         // Calc the next step size for Linear1Pass for FOCUS_WALK_FIXED_STEPS and FOCUS_WALK_CFZ_SHUFFLE
0110         int getNextStepSize();
0111 
0112         // Called when we've found a solution, e.g. the HFR value is within tolerance of the desired value.
0113         // It it returns true, then it's decided tht we should try one more sample for a possible improvement.
0114         // If it returns false, then "play it safe" and use the current position as the solution.
0115         bool setupPendingSolution(int position);
0116 
0117         // Determines the desired focus position for the first sample.
0118         void computeInitialPosition();
0119 
0120         // Get the first step focus position
0121         int getFirstPosition();
0122 
0123         // Sets the internal state for re-finding the minimum, and returns the requested next
0124         // position to sample.
0125         // Called by Linear. L1P calls finishFirstPass instead
0126         int setupSecondPass(int position, double value, double margin = 2.0);
0127 
0128         // Called by L1P to finish off the first pass, setting state variables.
0129         int finishFirstPass(int position, double value);
0130 
0131         // Peirce's criterion
0132         // Returns the squared threshold error deviation for outlier identification
0133         // using Peirce's criterion based on Gould's methodology.
0134         // Arguments:
0135         // N = total number of observations
0136         // n = max number of outliers to be removed
0137         // m = number of model unknowns
0138         // Returns: squared error threshold (x2)
0139         double peirce_criterion(double N, double n, double m);
0140 
0141         // Used in the 2nd pass. Focus is getting worse. Requires several consecutive samples getting worse.
0142         bool gettingWorse();
0143 
0144         // If one of the last 2 samples are as good or better than the 2nd best so far, return true.
0145         bool bestSamplesHeuristic();
0146 
0147         // Adds to the debug log a line summarizing the result of running this algorithm.
0148         void debugLog();
0149 
0150         // Use the detailed star measurements to adjust the HFR value.
0151         // See comment above method definition.
0152         double relativeHFR(double origHFR, const QList<Edge*> *stars);
0153 
0154         // Calculate the standard deviation (=sigma) of the star HFRs
0155         double calculateStarSigma(const bool useWeights, const QList<Edge*> *stars);
0156 
0157         // Used to time the focus algorithm.
0158         QElapsedTimer stopWatch;
0159 
0160         // A vector containing the HFR values sampled by this algorithm so far.
0161         QVector<double> values;
0162         QVector<double> pass1Values;
0163         // A vector containing star weights
0164         QVector<double> weights;
0165         QVector<double> pass1Weights;
0166         // A vector containing the focus positions corresponding to the HFR values stored above.
0167         QVector<int> positions;
0168         QVector<int> pass1Positions;
0169         // A vector containing whether or not the datapoint has been identified as an outlier.
0170         QVector<bool> pass1Outliers;
0171 
0172         // Focus position requested by this algorithm the previous step.
0173         int requestedPosition;
0174         // The position where the first pass is begun. Usually requestedPosition unless there's a restart.
0175         int passStartPosition;
0176         // Number of iterations processed so far. Used to see it doesn't exceed params.maxIterations.
0177         int numSteps;
0178         // The best value in the first pass. The 2nd pass attempts to get within
0179         // tolerance of this value.
0180         double firstPassBestValue;
0181         // The position of the minimum found in the first pass.
0182         int firstPassBestPosition;
0183         // The sampling interval--the recommended number of focuser steps moved inward each iteration
0184         // of the first pass.
0185         int stepSize;
0186         // The minimum focus position to use. Computed from the focuser limits and maxTravel.
0187         int minPositionLimit;
0188         // The maximum focus position to use. Computed from the focuser limits and maxTravel.
0189         int maxPositionLimit;
0190         // Counter for the number of times a v-curve minimum above the current position was found,
0191         // which implies the initial focus sweep has passed the minimum and should be terminated.
0192         int numPolySolutionsFound;
0193         // Counter for the number of times a v-curve minimum above the passStartPosition was found,
0194         // which implies the sweep should restart at a higher position.
0195         int numRestartSolutionsFound;
0196         // The index (into values and positions) when the most recent 2nd pass started.
0197         int secondPassStartIndex;
0198         // True if performing the first focus sweep.
0199         bool inFirstPass;
0200         // True if the 2nd pass has found a solution, and it's now optimizing the solution.
0201         bool solutionPending;
0202         // When we're near done, but the HFR just got worse, we may retry the current position
0203         // in case the HFR value was noisy.
0204         int retryNumber = 0;
0205 
0206         // Keep the solution-pending position/value for the status messages.
0207         double solutionPendingValue = 0;
0208         int solutionPendingPosition = 0;
0209 
0210         // RelativeHFR variables
0211         bool relativeHFREnabled = false;
0212         QVector<FocusStars> starLists;
0213         double bestRelativeHFR = 0;
0214         int bestRelativeHFRIndex = 0;
0215 };
0216 
0217 // Copies the object. Used in testing to examine alternate possible inputs given
0218 // the current state.
0219 FocusAlgorithmInterface *LinearFocusAlgorithm::Copy()
0220 {
0221     LinearFocusAlgorithm *alg = new LinearFocusAlgorithm(params);
0222     *alg = *this;
0223     return dynamic_cast<FocusAlgorithmInterface*>(alg);
0224 }
0225 
0226 FocusAlgorithmInterface *MakeLinearFocuser(const FocusAlgorithmInterface::FocusParams &params)
0227 {
0228     return new LinearFocusAlgorithm(params);
0229 }
0230 
0231 LinearFocusAlgorithm::LinearFocusAlgorithm(const FocusParams &focusParams)
0232     : FocusAlgorithmInterface(focusParams)
0233 {
0234     stopWatch.start();
0235     // These variables don't get reset if we restart the algorithm.
0236     numSteps = 0;
0237     maxPositionLimit = std::min(params.maxPositionAllowed, params.startPosition + params.maxTravel);
0238     minPositionLimit = std::max(params.minPositionAllowed, params.startPosition - params.maxTravel);
0239     computeInitialPosition();
0240 }
0241 
0242 QString LinearFocusAlgorithm::getTextStatus(double R2) const
0243 {
0244     if (params.focusAlgorithm == Focus::FOCUS_LINEAR)
0245     {
0246         if (done)
0247         {
0248             if (focusSolution > 0)
0249                 return QString("Solution: %1").arg(focusSolution);
0250             else
0251                 return "Failed";
0252         }
0253         if (solutionPending)
0254             return QString("Pending: %1,  %2").arg(solutionPendingPosition).arg(solutionPendingValue, 0, 'f', 2);
0255         if (inFirstPass)
0256             return "1st Pass";
0257         else
0258             return QString("2nd Pass. 1st: %1,  %2")
0259                    .arg(firstPassBestPosition).arg(firstPassBestValue, 0, 'f', 2);
0260     }
0261     else // Linear 1 Pass
0262     {
0263         QString text = "L1P";
0264         if (params.filterName != "")
0265             text.append(QString(" [%1]").arg(params.filterName));
0266         if (params.curveFit == CurveFitting::FOCUS_QUADRATIC)
0267             text.append(": Quadratic");
0268         else if (params.curveFit == CurveFitting::FOCUS_HYPERBOLA)
0269         {
0270             text.append(": Hyperbola");
0271             text.append(params.useWeights ? " (W)" : " (U)");
0272         }
0273         else if (params.curveFit == CurveFitting::FOCUS_PARABOLA)
0274         {
0275             text.append(": Parabola");
0276             text.append(params.useWeights ? " (W)" : " (U)");
0277         }
0278 
0279         if (inFirstPass)
0280             return text;
0281         else if (!done)
0282             return text.append(". Moving to Solution");
0283         else
0284         {
0285             if (focusSolution > 0)
0286             {
0287                 if (params.curveFit == CurveFitting::FOCUS_QUADRATIC)
0288                     return text.append(" Solution: %1").arg(focusSolution);
0289                 else
0290                     // Add R2 to 2 decimal places. Round down to be conservative
0291                     return text.append(" Solution: %1, R²=%2").arg(focusSolution).arg(trunc(R2 * 100.0) / 100.0, 0, 'f', 2);
0292             }
0293             else
0294                 return text.append(" Failed");
0295         }
0296     }
0297 }
0298 
0299 void LinearFocusAlgorithm::computeInitialPosition()
0300 {
0301     // These variables get reset if the algorithm is restarted.
0302     stepSize = params.initialStepSize;
0303     inFirstPass = true;
0304     solutionPending = false;
0305     retryNumber = 0;
0306     firstPassBestValue = -1;
0307     firstPassBestPosition = 0;
0308     numPolySolutionsFound = 0;
0309     numRestartSolutionsFound = 0;
0310     secondPassStartIndex = -1;
0311 
0312     qCDebug(KSTARS_EKOS_FOCUS)
0313             << QString("Linear: Travel %1 initStep %2 pos %3 min %4 max %5 maxIters %6 tolerance %7 minlimit %8 maxlimit %9 init#steps %10 algo %11 backlash %12 curvefit %13 useweights %14")
0314             .arg(params.maxTravel).arg(params.initialStepSize).arg(params.startPosition).arg(params.minPositionAllowed)
0315             .arg(params.maxPositionAllowed).arg(params.maxIterations).arg(params.focusTolerance).arg(minPositionLimit)
0316             .arg(maxPositionLimit).arg(params.initialOutwardSteps).arg(params.focusAlgorithm).arg(params.backlash)
0317             .arg(params.curveFit).arg(params.useWeights);
0318 
0319     requestedPosition = std::min(maxPositionLimit, getFirstPosition());
0320     passStartPosition = requestedPosition;
0321     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: initialPosition %1 sized %2")
0322                                .arg(requestedPosition).arg(params.initialStepSize);
0323 }
0324 
0325 // Calculate the first position relative to the start position
0326 // Generally this is just the number of outward points * step size.
0327 // For LINEAR1PASS most walks have a constant step size so again it is the number of outward points * step size.
0328 // For LINEAR1PASS and FOCUS_WALK_CFZ_SHUFFLE the middle 3 or 4 points are separated by half step size
0329 int LinearFocusAlgorithm::getFirstPosition()
0330 {
0331     if (params.focusAlgorithm != Focus::FOCUS_LINEAR1PASS)
0332         return static_cast<int>(params.startPosition + params.initialOutwardSteps * params.initialStepSize);
0333 
0334     int firstPosition;
0335     double outSteps, numFullStepsOut, numHalfStepsOut;
0336 
0337     switch (params.focusWalk)
0338     {
0339         case Focus::FOCUS_WALK_CLASSIC:
0340             firstPosition = static_cast<int>(params.startPosition + params.initialOutwardSteps * params.initialStepSize);
0341             break;
0342 
0343         case Focus::FOCUS_WALK_FIXED_STEPS:
0344             outSteps = (params.numSteps - 1) / 2.0f;
0345             firstPosition = params.startPosition + (outSteps * params.initialStepSize);
0346             break;
0347 
0348         case Focus::FOCUS_WALK_CFZ_SHUFFLE:
0349             if (params.numSteps % 2)
0350             {
0351                 // Odd number of steps
0352                 numHalfStepsOut = 1.0f;
0353                 numFullStepsOut = ((params.numSteps - 1) / 2.0f) - numHalfStepsOut;
0354             }
0355             else
0356             {
0357                 // Even number of steps
0358                 numHalfStepsOut = 1.5f;
0359                 numFullStepsOut = (params.numSteps / 2.0f) - 2.0f;
0360             }
0361             firstPosition = params.startPosition + (numFullStepsOut * params.initialStepSize) + (numHalfStepsOut *
0362                             (params.initialStepSize / 2.0f));
0363 
0364             break;
0365 
0366         default:
0367             firstPosition = static_cast<int>(params.startPosition + params.initialOutwardSteps * params.initialStepSize);
0368             break;
0369     }
0370 
0371     return firstPosition;
0372 }
0373 
0374 // The Problem:
0375 // The HFR values passed in to these focus algorithms are simply the mean or median HFR values
0376 // for all stars detected (with some other constraints). However, due to randomness in the
0377 // star detection scheme, two sets of star measurements may use different sets of actual stars to
0378 // compute their mean HFRs. This adds noise to determining best focus (e.g. including a
0379 // wider star in one set but not the other will tend to increase the HFR for that star set).
0380 // relativeHFR tries to correct for this by comparing two sets of stars only using their common stars.
0381 //
0382 // The Solution:
0383 // We maintain a reference set of stars, along with the HFR computed for those reference stars (bestRelativeHFR).
0384 // We find the common set of stars between an input star set and those reference stars. We compute
0385 // HFRs for the input stars in that common set, and for the reference common set as well.
0386 // The ratio of those common-set HFRs multiply bestRelativeHFR to generate the relative HFR for this
0387 // input star set--that is, it's the HFR "relative to the reference set". E.g. if the common-set HFR
0388 // for the input stars is twice worse than that from the reference common stars, then the relative HFR
0389 // would be 2 * bestRelativeHFR.
0390 // The reference set is the best HFR set of stars seen so far, but only from the 1st pass.
0391 // Thus, in the 2nd pass, the reference stars would be the best HFR set from the 1st pass.
0392 //
0393 // In unusual cases, e.g. when the common set can't be computed, we just return the input origHFR.
0394 double LinearFocusAlgorithm::relativeHFR(double origHFR, const QList<Edge*> *stars)
0395 {
0396     constexpr double minHFR = 0.3;
0397     if (origHFR < minHFR) return origHFR;
0398 
0399     const int currentIndex = values.size();
0400     const bool isFirstSample = (currentIndex == 0);
0401     double relativeHFR = origHFR;
0402 
0403     if (isFirstSample && stars != nullptr)
0404         relativeHFREnabled = true;
0405 
0406     if (relativeHFREnabled && stars == nullptr)
0407     {
0408         // Error--we have been getting relativeHFR stars, and now we're not.
0409         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: Error: Inconsistent relativeHFR, disabling.");
0410         relativeHFREnabled = false;
0411     }
0412 
0413     if (!relativeHFREnabled)
0414         return origHFR;
0415 
0416     if (starLists.size() != positions.size())
0417     {
0418         // Error--inconsistent data structures!
0419         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: Error: Inconsistent relativeHFR data structures, disabling.");
0420         relativeHFREnabled = false;
0421         return origHFR;
0422     }
0423 
0424     // Copy the input stars.
0425     // FocusStars enables us to measure HFR relatively consistently,
0426     // by using the same stars when comparing two measurements.
0427     constexpr int starDistanceThreshold = 5;  // Max distance (pixels) for two positions to be considered the same star.
0428     FocusStars fs(*stars, starDistanceThreshold);
0429     starLists.push_back(fs);
0430     auto &currentStars = starLists.back();
0431 
0432     if (isFirstSample)
0433     {
0434         relativeHFR = currentStars.getHFR();
0435         if (relativeHFR <= 0)
0436         {
0437             // Fall back to the simpler scheme.
0438             relativeHFREnabled = false;
0439             return origHFR;
0440         }
0441         // 1st measurement. By definition this is the best HFR.
0442         bestRelativeHFR = relativeHFR;
0443         bestRelativeHFRIndex = currentIndex;
0444     }
0445     else
0446     {
0447         // HFR computed relative to the best measured so far.
0448         auto &bestStars = starLists[bestRelativeHFRIndex];
0449         double hfr = currentStars.relativeHFR(bestStars, bestRelativeHFR);
0450         if (hfr > 0)
0451         {
0452             relativeHFR = hfr;
0453         }
0454         else
0455         {
0456             relativeHFR = currentStars.getHFR();
0457             if (relativeHFR <= 0)
0458                 return origHFR;
0459         }
0460 
0461         // In the 1st pass we compute the current HFR relative to the best HFR measured yet.
0462         // In the 2nd pass we compute the current HFR relative to the best HFR in the 1st pass.
0463         if (inFirstPass && relativeHFR <= bestRelativeHFR)
0464         {
0465             bestRelativeHFR = relativeHFR;
0466             bestRelativeHFRIndex = currentIndex;
0467         }
0468     }
0469 
0470     qCDebug(KSTARS_EKOS_FOCUS) << QString("RelativeHFR: orig %1 computed %2 relative %3")
0471                                .arg(origHFR).arg(currentStars.getHFR()).arg(relativeHFR);
0472 
0473     return relativeHFR;
0474 }
0475 
0476 int LinearFocusAlgorithm::newMeasurement(int position, double value, const double starWeight, const QList<Edge*> *stars)
0477 {
0478     if (params.focusAlgorithm == Focus::FOCUS_LINEAR1PASS && (params.focusWalk == Focus::FOCUS_WALK_FIXED_STEPS
0479             || params.focusWalk == Focus::FOCUS_WALK_CFZ_SHUFFLE))
0480         return linearWalk(position, value, starWeight);
0481 
0482     double minPos = -1.0, minVal = 0;
0483     bool foundFit = false;
0484 
0485     double origValue = value;
0486     // For QUADRATIC continue to use the relativeHFR functionality
0487     // For HYPERBOLA and PARABOLA the stars used for the HDR calculation and the sigma calculation
0488     // should be the same. For now, we will use the full set of stars and therefore not adjust the HFR
0489     if (params.curveFit == CurveFitting::FOCUS_QUADRATIC)
0490         value = relativeHFR(value, stars);
0491 
0492     // For LINEAR 1 PASS don't bother with a full 2nd pass just jump to the solution
0493     // Do the step out and back in to deal with backlash
0494     if (params.focusAlgorithm == Focus::FOCUS_LINEAR1PASS && !inFirstPass)
0495     {
0496         return setupSolution(position, value, starWeight);
0497     }
0498 
0499     int thisStepSize = stepSize;
0500     ++numSteps;
0501     if (inFirstPass)
0502         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: step %1, newMeasurement(%2, %3 -> %4, %5)")
0503                                    .arg(numSteps).arg(position).arg(origValue).arg(value)
0504                                    .arg(stars == nullptr ? 0 : stars->size());
0505     else
0506         qCDebug(KSTARS_EKOS_FOCUS)
0507                 << QString("Linear: step %1, newMeasurement(%2, %3 -> %4, %5) 1stBest %6 %7").arg(numSteps)
0508                 .arg(position).arg(origValue).arg(value).arg(stars == nullptr ? 0 : stars->size())
0509                 .arg(firstPassBestPosition).arg(firstPassBestValue, 0, 'f', 3);
0510 
0511     const int LINEAR_POSITION_TOLERANCE = params.initialStepSize;
0512     if (abs(position - requestedPosition) > LINEAR_POSITION_TOLERANCE)
0513     {
0514         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: error didn't get the requested position");
0515         return requestedPosition;
0516     }
0517     // Have we already found a solution?
0518     if (focusSolution != -1)
0519     {
0520         doneString = i18n("Called newMeasurement after a solution was found.");
0521         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: error %1").arg(doneString);
0522         debugLog();
0523         return -1;
0524     }
0525 
0526     // Store the sample values.
0527     positions.push_back(position);
0528     values.push_back(value);
0529     weights.push_back(starWeight);
0530     if (inFirstPass)
0531     {
0532         pass1Positions.push_back(position);
0533         pass1Values.push_back(value);
0534         pass1Weights.push_back(starWeight);
0535         pass1Outliers.push_back(false);
0536     }
0537 
0538     // If we're in the 2nd pass and either
0539     // - the current value is within tolerance, or
0540     // - we're optimizing because we've previously found a within-tolerance solution,
0541     // then setupPendingSolution decides whether to continue optimizing or to complete.
0542     if (solutionPending ||
0543             (!inFirstPass && (value < firstPassBestValue * (1.0 + params.focusTolerance))))
0544     {
0545         if (setupPendingSolution(position))
0546             // We can continue to look a little further.
0547             return completeIteration(retryNumber > 0 ? 0 : stepSize, false, -1.0f, -1.0f);
0548         else
0549             // Finish now
0550             return setupSolution(position, value, starWeight);
0551     }
0552 
0553     if (inFirstPass)
0554     {
0555         constexpr int kNumPolySolutionsRequired = 2;
0556         constexpr int kNumRestartSolutionsRequired = 3;
0557         //constexpr double kDecentValue = 3.0;
0558 
0559         if (values.size() >= getCurveMinPoints())
0560         {
0561             if (params.curveFit == CurveFitting::FOCUS_QUADRATIC)
0562             {
0563                 PolynomialFit fit(2, positions, values);
0564                 foundFit = fit.findMinimum(position, 0, params.maxPositionAllowed, &minPos, &minVal);
0565                 if (!foundFit)
0566                 {
0567                     // I've found that the first sample can be odd--perhaps due to backlash.
0568                     // Try again skipping the first sample, if we have sufficient points.
0569                     if (values.size() > getCurveMinPoints())
0570                     {
0571                         PolynomialFit fit2(2, positions.mid(1), values.mid(1));
0572                         foundFit = fit2.findMinimum(position, 0, params.maxPositionAllowed, &minPos, &minVal);
0573                         minPos = minPos + 1;
0574                     }
0575                 }
0576             }
0577             else // Hyperbola or Parabola so use the LM solver
0578             {
0579                 auto goal = getGoal(numSteps);
0580                 params.curveFitting->fitCurve(goal, positions, values, weights, pass1Outliers, params.curveFit, params.useWeights,
0581                                               params.optimisationDirection);
0582 
0583                 foundFit = params.curveFitting->findMinMax(position, 0, params.maxPositionAllowed, &minPos, &minVal,
0584                            static_cast<CurveFitting::CurveFit>(params.curveFit),
0585                            params.optimisationDirection);
0586             }
0587 
0588             if (foundFit)
0589             {
0590                 const int distanceToMin = static_cast<int>(position - minPos);
0591                 qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: fit(%1): %2 = %3 @ %4 distToMin %5")
0592                                            .arg(positions.size()).arg(minPos).arg(minVal).arg(position).arg(distanceToMin);
0593                 if (distanceToMin >= 0)
0594                 {
0595                     // The minimum is further inward.
0596                     numPolySolutionsFound = 0;
0597                     numRestartSolutionsFound = 0;
0598                     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: Solutions reset %1 = %2").arg(minPos).arg(minVal);
0599 
0600                     // The next code block is an optimisation where the user starts focus off a long way from prime focus.
0601                     // The idea is that by detecting this situation we can warp quickly to the solution.
0602                     // However, when the solver is solving on a small number of points (e.g. 6 or 7) its possible that the
0603                     // best fit curve produces a solution much further in than what turns out to be the actual focus point.
0604                     // This code therefore results in inappropriate warping. So this code is now disabled.
0605                     /*if (value / minVal > kDecentValue)
0606                     {
0607                         // Only skip samples if the value is a long way from the minimum.
0608                         const int stepsToMin = distanceToMin / stepSize;
0609                         // Temporarily increase the step size if the minimum is very far inward.
0610                         if (stepsToMin >= 8)
0611                             thisStepSize = stepSize * 4;
0612                         else if (stepsToMin >= 4)
0613                             thisStepSize = stepSize * 2;
0614                     }*/
0615                 }
0616                 else if (!bestSamplesHeuristic())
0617                 {
0618                     // We have potentially passed the bottom of the curve,
0619                     // but it's possible it is further back than the start of our sweep.
0620                     if (minPos > passStartPosition)
0621                     {
0622                         numRestartSolutionsFound++;
0623                         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: RESTART Solution #%1 %2 = %3 @ %4")
0624                                                    .arg(numRestartSolutionsFound).arg(minPos).arg(minVal).arg(position);
0625                     }
0626                     else
0627                     {
0628                         numPolySolutionsFound++;
0629                         numRestartSolutionsFound = 0;
0630                         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: Solution #%1: %2 = %3 @ %4")
0631                                                    .arg(numPolySolutionsFound).arg(minPos).arg(minVal).arg(position);
0632                     }
0633                 }
0634 
0635                 // The LINEAR algo goes >2 steps past the minimum. The LINEAR 1 PASS algo uses the InitialOutwardSteps parameter
0636                 // in order to have some user control over the number of steps past the minimum when fitting the curve.
0637                 // With LINEAR 1 PASS setupSecondPass with a margin of 0.0 as no need to move the focuser further
0638                 // This will result in a step out, past the solution of either "backlash" id set, or 5 * step size, followed
0639                 // by a step in to the solution. By stepping out and in, backlash will be neutralised.
0640                 int howFarToGo;
0641                 if (params.focusAlgorithm == Focus::FOCUS_LINEAR)
0642                     howFarToGo = kNumPolySolutionsRequired;
0643                 else
0644                     howFarToGo = std::max(1, static_cast<int> (params.initialOutwardSteps) - 1);
0645 
0646                 if (numPolySolutionsFound >= howFarToGo)
0647                 {
0648                     // We found a minimum. Setup the 2nd pass. We could use either the polynomial min or the
0649                     // min measured star as the target HFR. I've seen using the polynomial minimum to be
0650                     // sometimes too conservative, sometimes too low. For now using the min sample.
0651                     double minMeasurement = *std::min_element(values.begin(), values.end());
0652                     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: 1stPass solution @ %1: pos %2 val %3, min measurement %4")
0653                                                .arg(position).arg(minPos).arg(minVal).arg(minMeasurement);
0654 
0655                     // For Linear setup the second pass with a margin of 2
0656                     if (params.focusAlgorithm == Focus::FOCUS_LINEAR)
0657                         return setupSecondPass(static_cast<int>(minPos), minMeasurement, 2.0);
0658                     else
0659                         // For Linear 1 Pass do a round on the double minPos parameter before casting to an int
0660                         // as the cast will truncate and potentially change the position down by 1
0661                         // The code to display v-curve rounds so this keeps things consistent.
0662                         // Could make this change for Linear as well but this will then need testfocus to change
0663                         return finishFirstPass(static_cast<int>(round(minPos)), minMeasurement);
0664                 }
0665                 else if (numRestartSolutionsFound >= kNumRestartSolutionsRequired)
0666                 {
0667                     // We need to restart--that is the error is rising and thus our initial position
0668                     // was already past the minimum. Restart the algorithm with a greater initial position.
0669                     // If the min position from the polynomial solution is not too far from the current start position
0670                     // use that, but don't let it go too far away.
0671                     const double highestStartPosition = params.startPosition + params.initialOutwardSteps * params.initialStepSize;
0672                     params.startPosition = std::min(minPos, highestStartPosition);
0673                     computeInitialPosition();
0674                     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: Restart. Current pos %1, min pos %2, min val %3, re-starting at %4")
0675                                                .arg(position).arg(minPos).arg(minVal).arg(requestedPosition);
0676                     return requestedPosition;
0677                 }
0678 
0679             }
0680             else
0681             {
0682                 // Minimum failed indicating the 2nd-order polynomial is an inverted U--it has a maximum,
0683                 // but no minimum.  This is, of course, not a sensible solution for the focuser, but can
0684                 // happen with noisy data and perhaps small step sizes. We still might be able to take advantage,
0685                 // and notice whether the polynomial is increasing or decreasing locally. For now, do nothing.
0686                 qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: ******** No poly min: Poly must be inverted");
0687             }
0688         }
0689         else
0690         {
0691             // Don't have enough samples to reliably fit a polynomial.
0692             // Simply step the focus in one more time and iterate.
0693         }
0694     }
0695     else if (gettingWorse())
0696     {
0697         // Doesn't look like we'll find something close to the min. Retry the 2nd pass.
0698         // NOTE: this branch will only be executed for the 2 pass version of Linear
0699         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: getting worse, re-running 2nd pass");
0700         return setupSecondPass(firstPassBestPosition, firstPassBestValue);
0701     }
0702 
0703     return completeIteration(thisStepSize, foundFit, minPos, minVal);
0704 }
0705 
0706 // Get the curve fitting goal, based on the "walk".
0707 // Start with STANDARD but at the end of the pass use BEST
0708 CurveFitting::FittingGoal LinearFocusAlgorithm::getGoal(int numSteps)
0709 {
0710     if (params.focusWalk == Focus::FOCUS_WALK_CLASSIC)
0711     {
0712         if (numSteps > 2.0 * params.initialOutwardSteps)
0713             return CurveFitting::FittingGoal::BEST;
0714         else
0715             return CurveFitting::FittingGoal::STANDARD;
0716     }
0717     else // FOCUS_WALK_FIXED_STEPS or FOCUS_WALK_CFZ_SHUFFLE
0718     {
0719         if (numSteps >= params.numSteps)
0720             return CurveFitting::FittingGoal::BEST;
0721         else
0722             return CurveFitting::FittingGoal::STANDARD;
0723     }
0724 }
0725 
0726 // Get the minimum number of points to start fitting a curve
0727 // The default is 5. Try to get past the minimum so the curve shape is hyperbolic/parabolic
0728 int LinearFocusAlgorithm::getCurveMinPoints()
0729 {
0730     if (params.focusAlgorithm != Focus::FOCUS_LINEAR1PASS)
0731         return 5;
0732 
0733     if (params.focusWalk == Focus::FOCUS_WALK_CLASSIC)
0734         return params.initialOutwardSteps + 2;
0735 
0736     // For donut busting don't try drawing an intermediate curve... as we need to scrub data first
0737     if (params.donutBuster)
0738         return params.numSteps;
0739 
0740     return params.numSteps / 2 + 2;
0741 }
0742 
0743 // Process next step for LINEAR1PASS for walks: FOCUS_WALK_FIXED_STEPS and FOCUS_WALK_CFZ_SHUFFLE
0744 int LinearFocusAlgorithm::linearWalk(int position, double value, const double starWeight)
0745 {
0746     if (!inFirstPass)
0747     {
0748         return setupSolution(position, value, starWeight);
0749     }
0750 
0751     bool foundFit = false;
0752     double minPos = -1.0, minVal = 0;
0753     ++numSteps;
0754     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear1Pass: step %1, linearWalk %2, %3")
0755                                .arg(numSteps).arg(position).arg(value);
0756 
0757     // If we are within stepsize ticks of where we should be then fine, otherwise try again
0758     if (abs(position - requestedPosition) > params.initialStepSize)
0759     {
0760         qCDebug(KSTARS_EKOS_FOCUS) << QString("linearWalk error: position %1, requested position %2").arg(position)
0761                                    .arg(requestedPosition);
0762         return requestedPosition;
0763     }
0764 
0765     // Store the sample values.
0766     positions.push_back(position);
0767     values.push_back(value);
0768     weights.push_back(starWeight);
0769     pass1Positions.push_back(position);
0770     pass1Values.push_back(value);
0771     pass1Weights.push_back(starWeight);
0772     pass1Outliers.push_back(false);
0773 
0774     if (values.size() < getCurveMinPoints())
0775     {
0776         // Don't have enough samples to reliably fit a curve.
0777         // Simply step the focus in one more time and iterate.
0778     }
0779     else
0780     {
0781         if (params.curveFit == CurveFitting::FOCUS_QUADRATIC)
0782             qCDebug(KSTARS_EKOS_FOCUS) << QString("linearWalk called incorrectly for FOCUS_QUADRATIC");
0783         else
0784         {
0785             // Hyperbola or Parabola so use the LM solver
0786             auto goal = getGoal(numSteps);
0787             // Check for Donuts
0788             if (params.donutBuster)
0789                 removeDonuts();
0790             params.curveFitting->fitCurve(goal, positions, values, weights, pass1Outliers, params.curveFit, params.useWeights,
0791                                           params.optimisationDirection);
0792 
0793             foundFit = params.curveFitting->findMinMax(position, 0, params.maxPositionAllowed, &minPos, &minVal,
0794                        static_cast<CurveFitting::CurveFit>(params.curveFit), params.optimisationDirection);
0795 
0796             if (numSteps >= params.numSteps)
0797             {
0798                 // linearWalk has now completed either successfully (foundFit=true) or not
0799                 if (foundFit)
0800                     return finishFirstPass(static_cast<int>(round(minPos)), minVal);
0801                 else
0802                 {
0803                     done = true;
0804                     doneString = i18n("Failed to fit curve to data.");
0805                     qCDebug(KSTARS_EKOS_FOCUS) << QString("LinearWalk: %1").arg(doneString);
0806                     debugLog();
0807                     return -1;
0808                 }
0809             }
0810         }
0811     }
0812 
0813     int nextStepSize = getNextStepSize();
0814     return completeIteration(nextStepSize, foundFit, minPos, minVal);
0815 }
0816 
0817 // Check for donuts
0818 // Assumption is that we are starting near to focus so that we should have a reasonable v-curve
0819 // near the central datapoints but the wings may contain donuts=poor quality data.
0820 // 1. Make sure we have enough data to work with
0821 // 2. Find the central minimum datapoint
0822 // 3. Move outwards on each side of the minimum checking the next datapoint. If value drops this could be bad data
0823 // Only remove the wings - bad data near the centre won't be filtered out.
0824 void LinearFocusAlgorithm::removeDonuts()
0825 {
0826     // 1. We need at least 7 datapoints to do any analysis
0827     if (values.size() <= 7)
0828         return;
0829 
0830     QVector<bool> queryOutliers = pass1Outliers;
0831 
0832     // 2. Firstly find the central minimum datapoint
0833     int centre = values.size() / 2;
0834     int minElement = centre;
0835     double minValue = values[centre];
0836     for (int i = 1; i < 4; i++)
0837     {
0838         if (values[centre + i] < minValue)
0839         {
0840             minValue = values[centre + i];
0841             minElement = centre + i;
0842         }
0843         if (values[centre - i] < minValue)
0844         {
0845             minValue = values[centre - i];
0846             minElement = centre - i;
0847         }
0848     }
0849 
0850     // 3a. Move outward from the central minimum checking for increasing values
0851     double threshold = values[minElement];
0852     double nextValue, deltaValue;
0853     double tolerance = 0.5; //
0854     for (int i = minElement; i > 0; i--)
0855     {
0856         nextValue = values[i - 1];
0857         if (nextValue < threshold)
0858             // Smaller value indicates bad data
0859             queryOutliers[i - 1] = true;
0860         else
0861         {
0862             deltaValue = (nextValue - values[i]) * tolerance;
0863             threshold = nextValue + deltaValue;
0864         }
0865     }
0866 
0867     // Starting at the furthest outward point, move inward to minimum point, marking bad data (donuts) as outliers
0868     for (int i = 0; i < minElement - 3; i++)
0869     {
0870         if (queryOutliers[i])
0871             pass1Outliers[i] = true;
0872         else
0873             // Stop the process at the first good datapoint - don't discard all bad data
0874             break;
0875     }
0876 
0877     // 3b. Move inward from the central minimum checking for increasing values
0878     threshold = values[minElement];
0879     for (int i = minElement; i < values.size() - 1; i++)
0880     {
0881         nextValue = values[i + 1];
0882         if (nextValue < threshold)
0883             // Smaller value indicates bad data
0884             queryOutliers[i + 1] = true;
0885         else
0886         {
0887             deltaValue = (nextValue - values[i]) * tolerance;
0888             threshold = nextValue + deltaValue;
0889         }
0890     }
0891 
0892     // Starting at the furthest outward point, move inward to minimum point, marking bad data (donuts) as outliers
0893     for (int i = values.size() - 1; i > minElement + 3; i--)
0894     {
0895         if (queryOutliers[i])
0896             pass1Outliers[i] = true;
0897         else
0898             // Stop the process at the first good datapoint - don't discard all bad data
0899             break;
0900     }
0901 
0902     QString str;
0903     for (int i = pass1Outliers.size() - 1; i >= 0; i--)
0904         str.append((pass1Outliers[i]) ? 'Y' : 'N');
0905     qCDebug(KSTARS_EKOS_FOCUS) << QString("Donut analysis: %1").arg(str);
0906 
0907 }
0908 
0909 // Function to calculate the next step size for LINEAR1PASS for walks: FOCUS_WALK_FIXED_STEPS and FOCUS_WALK_CFZ_SHUFFLE
0910 int LinearFocusAlgorithm::getNextStepSize()
0911 {
0912     int nextStepSize, lower, upper;
0913 
0914     switch (params.focusWalk)
0915     {
0916         case Focus::FOCUS_WALK_CLASSIC:
0917             nextStepSize = stepSize;
0918             break;
0919         case Focus::FOCUS_WALK_FIXED_STEPS:
0920             nextStepSize = stepSize;
0921             break;
0922         case Focus::FOCUS_WALK_CFZ_SHUFFLE:
0923             if (params.numSteps % 2)
0924             {
0925                 // Odd number of steps
0926                 lower = (params.numSteps - 3) / 2;
0927                 upper = params.numSteps - lower;
0928             }
0929             else
0930             {
0931                 // Evem number of steps
0932                 lower = (params.numSteps - 4) / 2;
0933                 upper = (params.numSteps - lower);
0934             }
0935 
0936             if (numSteps <= lower)
0937                 nextStepSize = stepSize;
0938             else if (numSteps >= upper)
0939                 nextStepSize = stepSize;
0940             else
0941                 nextStepSize = stepSize / 2;
0942             break;
0943         default:
0944             nextStepSize = stepSize;
0945             break;
0946     }
0947 
0948     return nextStepSize;
0949 }
0950 
0951 int LinearFocusAlgorithm::setupSolution(int position, double value, double weight)
0952 {
0953     focusSolution = position;
0954     focusValue = value;
0955     done = true;
0956     doneString = i18n("Solution found.");
0957     if (params.focusAlgorithm == Focus::FOCUS_LINEAR)
0958         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: solution @ %1 = %2 (best %3)")
0959                                    .arg(position).arg(value).arg(firstPassBestValue);
0960     else // Linear 1 Pass
0961     {
0962         double delta = fabs(value - firstPassBestValue);
0963         QString str("Linear: solution @ ");
0964         str.append(QString("%1 = %2 (expected %3) delta=%4").arg(position).arg(value).arg(firstPassBestValue).arg(delta));
0965 
0966         if (params.useWeights && weight > 0.0)
0967         {
0968             // TODO fix this for weights
0969             double numSigmas = delta * weight;
0970             str.append(QString(" or %1 sigma %2 than expected")
0971                        .arg(numSigmas).arg(value <= firstPassBestValue ? "better" : "worse"));
0972             if (value <= firstPassBestValue || numSigmas < 1)
0973                 str.append(QString(". GREAT result"));
0974             else if (numSigmas < 3)
0975                 str.append(QString(". OK result"));
0976             else
0977                 str.append(QString(". POOR result. If this happens repeatedly it may be a sign of poor backlash compensation."));
0978         }
0979 
0980         qCInfo(KSTARS_EKOS_FOCUS) << str;
0981     }
0982     debugLog();
0983     return -1;
0984 }
0985 
0986 int LinearFocusAlgorithm::completeIteration(int step, bool foundFit, double minPos, double minVal)
0987 {
0988     if (params.focusAlgorithm == Focus::FOCUS_LINEAR && numSteps == params.maxIterations - 2)
0989     {
0990         // If we're close to exceeding the iteration limit, retry this pass near the old minimum position.
0991         // For Linear 1 Pass we are still in the first pass so no point retrying the 2nd pass. Just carry on and
0992         // fail in 2 more datapoints if necessary.
0993         const int minIndex = static_cast<int>(std::min_element(values.begin(), values.end()) - values.begin());
0994         return setupSecondPass(positions[minIndex], values[minIndex], 0.5);
0995     }
0996     else if (numSteps > params.maxIterations)
0997     {
0998         // Fail. Exceeded our alloted number of iterations.
0999         done = true;
1000         doneString = i18n("Too many steps.");
1001         qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: error %1").arg(doneString);
1002         debugLog();
1003         return -1;
1004     }
1005 
1006     // Setup the next sample.
1007     requestedPosition = requestedPosition - step;
1008 
1009     // Make sure the next sample is within bounds.
1010     if (requestedPosition < minPositionLimit)
1011     {
1012         // The position is too low. Pick the min value and go to (or retry) a 2nd iteration.
1013         if (params.focusAlgorithm == Focus::FOCUS_LINEAR)
1014         {
1015             const int minIndex = static_cast<int>(std::min_element(values.begin(), values.end()) - values.begin());
1016             qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: reached end without Vmin. Restarting %1 pos %2 value %3")
1017                                        .arg(minIndex).arg(positions[minIndex]).arg(values[minIndex]);
1018             return setupSecondPass(positions[minIndex], values[minIndex]);
1019         }
1020         else // Linear 1 Pass
1021         {
1022             if (foundFit && minPos >= minPositionLimit)
1023                 // Although we ran out of room, we have got a viable, although not perfect, solution
1024                 return finishFirstPass(static_cast<int>(round(minPos)), minVal);
1025             else
1026             {
1027                 // We can't travel far enough to find a solution so fail as focuser parameters require user attention
1028                 done = true;
1029                 doneString = i18n("Solution lies outside max travel.");
1030                 qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: error %1").arg(doneString);
1031                 debugLog();
1032                 return -1;
1033             }
1034         }
1035     }
1036     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: requesting position %1").arg(requestedPosition);
1037     return requestedPosition;
1038 }
1039 
1040 // This is called in the 2nd pass, when we've found a sample point that's within tolerance
1041 // of the 1st pass' best value. Since it's within tolerance, the 2nd pass might finish, using
1042 // the current value as the final solution. However, this method checks to see if we might go
1043 // a bit further. Once solutionPending is true, it's committed to finishing soon.
1044 // It goes further if:
1045 // - the current HFR value is an improvement on the previous 2nd-pass value,
1046 // - the number of steps taken so far is not close the max number of allowable steps.
1047 // If instead the HFR got worse, we retry the current position a few times to see if it might
1048 // get better before giving up.
1049 bool LinearFocusAlgorithm::setupPendingSolution(int position)
1050 {
1051     const int length = values.size();
1052     const int secondPassIndex = length - secondPassStartIndex;
1053     const double thisValue = values[length - 1];
1054 
1055     // ReferenceValue is the value used in the notGettingWorse computation.
1056     // Basically, it's the previous value, or the value before retries.
1057     double referenceValue = (secondPassIndex <= 1) ? 1e6 : values[length - 2];
1058     if (retryNumber > 0 && length - (2 + retryNumber) >= 0)
1059         referenceValue = values[length - (2 + retryNumber)];
1060 
1061     // NotGettingWorse: not worse than the previous (non-repeat) 2nd pass sample, or the 1st 2nd pass sample.
1062     const bool notGettingWorse = (secondPassIndex <= 1) || (thisValue <= referenceValue);
1063 
1064     // CouldGoFurther: Not near a boundary in position or number of steps.
1065     const bool couldGoFather = (numSteps < params.maxIterations - 2) && (position - stepSize > minPositionLimit);
1066 
1067     // Allow passing the 1st pass' minimum HFR position, but take smaller steps and don't retry as much.
1068     int maxNumRetries = 3;
1069     if (position - stepSize / 2 < firstPassBestPosition)
1070     {
1071         stepSize = std::max(2, params.initialStepSize / 4);
1072         maxNumRetries = 1;
1073     }
1074 
1075     if (notGettingWorse && couldGoFather)
1076     {
1077         qCDebug(KSTARS_EKOS_FOCUS)
1078                 << QString("Linear: %1: Position(%2) & HFR(%3) -- Pass1: %4 %5, solution pending, searching further")
1079                 .arg(length).arg(position).arg(thisValue, 0, 'f', 3).arg(firstPassBestPosition).arg(firstPassBestValue, 0, 'f', 3);
1080         solutionPending = true;
1081         solutionPendingPosition = position;
1082         solutionPendingValue = thisValue;
1083         retryNumber = 0;
1084         return true;
1085     }
1086     else if (solutionPending && couldGoFather && retryNumber < maxNumRetries &&
1087              (secondPassIndex > 1) && (thisValue >= referenceValue))
1088     {
1089         qCDebug(KSTARS_EKOS_FOCUS)
1090                 << QString("Linear: %1: Position(%2) & HFR(%3) -- Pass1: %4 %5, solution pending, got worse, retrying")
1091                 .arg(length).arg(position).arg(thisValue, 0, 'f', 3).arg(firstPassBestPosition).arg(firstPassBestValue, 0, 'f', 3);
1092         // Try this poisition again.
1093         retryNumber++;
1094         return true;
1095     }
1096     qCDebug(KSTARS_EKOS_FOCUS)
1097             << QString("Linear: %1: Position(%2) & HFR(%3) -- Pass1: %4 %5, finishing, can't go further")
1098             .arg(length).arg(position).arg(thisValue, 0, 'f', 3).arg(firstPassBestPosition).arg(firstPassBestValue, 0, 'f', 3);
1099     retryNumber = 0;
1100     return false;
1101 }
1102 
1103 void LinearFocusAlgorithm::debugLog()
1104 {
1105     QString str("Linear: points=[");
1106     for (int i = 0; i < positions.size(); ++i)
1107     {
1108         str.append(QString("(%1, %2, %3)").arg(positions[i]).arg(values[i]).arg(weights[i]));
1109         if (i < positions.size() - 1)
1110             str.append(", ");
1111     }
1112     str.append(QString("];iterations=%1").arg(numSteps));
1113     str.append(QString(";duration=%1").arg(stopWatch.elapsed() / 1000));
1114     str.append(QString(";solution=%1").arg(focusSolution));
1115     str.append(QString(";value=%1").arg(focusValue));
1116     str.append(QString(";filter='%1'").arg(params.filterName));
1117     str.append(QString(";temperature=%1").arg(params.temperature));
1118     str.append(QString(";focusalgorithm=%1").arg(params.focusAlgorithm));
1119     str.append(QString(";backlash=%1").arg(params.backlash));
1120     str.append(QString(";curvefit=%1").arg(params.curveFit));
1121     str.append(QString(";useweights=%1").arg(params.useWeights));
1122 
1123     qCDebug(KSTARS_EKOS_FOCUS) << str;
1124 }
1125 
1126 // Called by Linear to set state for the second pass
1127 int LinearFocusAlgorithm::setupSecondPass(int position, double value, double margin)
1128 {
1129     firstPassBestPosition = position;
1130     firstPassBestValue = value;
1131     inFirstPass = false;
1132     solutionPending = false;
1133     secondPassStartIndex = values.size();
1134 
1135     // Arbitrarily go back "margin" steps above the best position.
1136     // Could be a problem if backlash were worse than that many steps.
1137     requestedPosition = std::min(static_cast<int>(firstPassBestPosition + stepSize * margin), maxPositionLimit);
1138     stepSize = params.initialStepSize / 2;
1139     qCDebug(KSTARS_EKOS_FOCUS) << QString("Linear: 2ndPass starting at %1 step %2").arg(requestedPosition).arg(stepSize);
1140     return requestedPosition;
1141 }
1142 
1143 // Called by L1P to finish off the first pass by setting state variables
1144 int LinearFocusAlgorithm::finishFirstPass(int position, double value)
1145 {
1146     firstPassBestPosition = position;
1147     firstPassBestValue = value;
1148     inFirstPass = false;
1149     requestedPosition = position;
1150 
1151     if (params.refineCurveFit)
1152     {
1153         // We've completed pass1 so, if required, reprocess all the datapoints excluding outliers
1154         // First, we need the distance of each datapoint from the curve
1155         std::vector<std::pair<int, double>> curveDeltas;
1156 
1157         params.curveFitting->calculateCurveDeltas(params.curveFit, curveDeltas);
1158 
1159         // Check for outliers if there are enough datapoints
1160         if (curveDeltas.size() > 6)
1161         {
1162             // Build a vector of the square of the curve deltas
1163             std::vector<double> curveDeltasX2Vec;
1164             for (int i = 0; i < curveDeltas.size(); i++)
1165                 curveDeltasX2Vec.push_back(pow(curveDeltas[i].second, 2.0));
1166 
1167             auto curveDeltasX2Mean = Mathematics::RobustStatistics::ComputeLocation(Mathematics::RobustStatistics::LOCATION_MEAN,
1168                                      curveDeltasX2Vec);
1169 
1170             // Order the curveDeltas, highest first, then check against the limit
1171             // Remove points over the limit, but don't remove too many points to compromise the curve
1172             double maxOutliers;
1173             if (curveDeltas.size() < 7)
1174                 maxOutliers = 1;
1175             else if (curveDeltas.size() < 11)
1176                 maxOutliers = 2;
1177             else
1178                 maxOutliers = 3;
1179 
1180             double modelUnknowns = params.curveFit == CurveFitting::FOCUS_PARABOLA ? 3.0 : 4.0;
1181 
1182             // Use Peirce's Criterion to get the outlier threshold
1183             // Note this operates on the square of the curve deltas
1184             double pc = peirce_criterion(static_cast<double> (curveDeltas.size()), maxOutliers, modelUnknowns);
1185             double pc_threshold = sqrt(pc * curveDeltasX2Mean);
1186 
1187             // Sort the curve deltas, largest first
1188             std::sort(curveDeltas.begin(), curveDeltas.end(),
1189                       [](const std::pair<int, double> &a, const std::pair<int, double> &b)
1190             {
1191                 return ( a.second > b.second );
1192             });
1193 
1194             // Save off pass1 variables in case they need to be reset later
1195             auto bestPass1Outliers = pass1Outliers;
1196 
1197             // Work out how many outliers to exclude
1198             int outliers = 0;
1199             for (int i = 0; i < curveDeltas.size(); i++)
1200             {
1201                 if(curveDeltas[i].second <= pc_threshold)
1202                     break;
1203                 else
1204                 {
1205                     pass1Outliers[curveDeltas[i].first] = true;
1206                     outliers++;
1207                 }
1208                 if (outliers >= maxOutliers)
1209                     break;
1210             }
1211 
1212             // If there are any outliers then try to improve the curve fit by removing these
1213             // datapoints and reruning the curve fitting process. Note that this may not improve
1214             // the fit so we need to be prepared to reinstate the original solution.
1215             QVector<double> currentCoefficients;
1216             if (outliers > 0 && params.curveFitting->getCurveParams(params.curveFit, currentCoefficients))
1217             {
1218                 double currentR2 = params.curveFitting->calculateR2(static_cast<CurveFitting::CurveFit>(params.curveFit));
1219 
1220                 // Try another curve fit on the data without outliers
1221                 params.curveFitting->fitCurve(CurveFitting::FittingGoal::BEST, pass1Positions, pass1Values, pass1Weights,
1222                                               pass1Outliers, params.curveFit, params.useWeights, params.optimisationDirection);
1223                 double minPos, minVal;
1224                 bool foundFit = params.curveFitting->findMinMax(position, 0, params.maxPositionAllowed, &minPos, &minVal,
1225                                 static_cast<CurveFitting::CurveFit>(params.curveFit),
1226                                 params.optimisationDirection);
1227                 if (foundFit)
1228                 {
1229                     double newR2 = params.curveFitting->calculateR2(static_cast<CurveFitting::CurveFit>(params.curveFit));
1230                     if (newR2 > currentR2)
1231                     {
1232                         // We have a better solution so we'll use it
1233                         requestedPosition = minPos;
1234                         qCDebug(KSTARS_EKOS_FOCUS) << QString("Refine Curve Fit improved focus from %1 to %2. R2 improved from %3 to %4")
1235                                                    .arg(position).arg(minPos).arg(currentR2).arg(newR2);
1236                     }
1237                     else
1238                     {
1239                         // New solution is not better than previous one so go with the previous one
1240                         qCDebug(KSTARS_EKOS_FOCUS) <<
1241                                                    QString("Refine Curve Fit could not improve on the original solution");
1242                         pass1Outliers = bestPass1Outliers;
1243                         params.curveFitting->setCurveParams(params.curveFit, currentCoefficients);
1244                     }
1245                 }
1246                 else
1247                 {
1248                     qCDebug(KSTARS_EKOS_FOCUS) <<
1249                                                QString("Refine Curve Fit failed to fit curve whilst refining curve fit. Running with original solution");
1250                     pass1Outliers = bestPass1Outliers;
1251                     params.curveFitting->setCurveParams(params.curveFit, currentCoefficients);
1252                 }
1253             }
1254         }
1255     }
1256     return requestedPosition;
1257 }
1258 
1259 // Peirce's criterion based on Gould's methodology for outlier identification
1260 // See https://en.wikipedia.org/wiki/Peirce%27s_criterion for details incl Peirce's original paper
1261 // and Gould's paper both referenced in the notes. The wikipedia entry also includes some code fragments
1262 // that form the basis of this function.
1263 //
1264 // Arguments:
1265 // N = total number of observations
1266 // n = number of outliers to be removed
1267 // m = number of model unknowns
1268 // Returns: squared error threshold (x2)
1269 double LinearFocusAlgorithm::peirce_criterion(double N, double n, double m)
1270 {
1271     double x2 = 0.0;
1272     if (N > 1)
1273     {
1274         // Calculate Q (Nth root of Gould's equation B):
1275         double Q = (pow(n, (n / N)) * pow((N - n), ((N - n) / N))) / N;
1276 
1277         // Initialize R values
1278         double r_new = 1.0;
1279         double r_old = 0.0;
1280 
1281         // Start iteration to converge on R:
1282         while (abs(r_new - r_old) > (N * 2.0e-16))
1283         {
1284             // Calculate Lamda
1285             // (1 / (N - n) th root of Gould's equation B
1286             double ldiv = pow(r_new, n);
1287             if (ldiv == 0)
1288                 ldiv = 1.0e-6;
1289 
1290             double Lamda = pow((pow(Q, N) / (ldiv)), (1.0 / (N - n)));
1291 
1292             // Calculate x -squared(Gould's equation C)
1293             x2 = 1.0 + (N - m - n) / n * (1.0 - pow(Lamda, 2.0));
1294 
1295             if (x2 < 0.0)
1296             {
1297                 // If x2 goes negative, return 0
1298                 x2 = 0.0;
1299                 r_old = r_new;
1300             }
1301             else
1302             {
1303                 // Use x-squared to update R(Gould 's equation D)
1304                 r_old = r_new;
1305                 r_new = exp((x2 - 1) / 2.0) * gsl_sf_erfc(sqrt(x2) / sqrt(2.0));
1306             }
1307         }
1308     }
1309 
1310     return x2;
1311 }
1312 
1313 // Return true if one of the 2 recent samples is among the best 2 samples so far.
1314 bool LinearFocusAlgorithm::bestSamplesHeuristic()
1315 {
1316     const int length = values.size();
1317     if (length < 5) return true;
1318     QVector<double> tempValues = values;
1319     std::nth_element(tempValues.begin(), tempValues.begin() + 2, tempValues.end());
1320     double secondBest = tempValues[1];
1321     if ((values[length - 1] <= secondBest) || (values[length - 2] <= secondBest))
1322         return true;
1323     return false;
1324 }
1325 
1326 // Return true if there are "streak" consecutive values which are successively worse.
1327 bool LinearFocusAlgorithm::gettingWorse()
1328 {
1329     // Must have this many consecutive values getting worse.
1330     constexpr int streak = 3;
1331     const int length = values.size();
1332     if (secondPassStartIndex < 0)
1333         return false;
1334     if (length < streak + 1)
1335         return false;
1336     // This insures that all the values we're checking are in the latest 2nd pass.
1337     if (length - secondPassStartIndex < streak + 1)
1338         return false;
1339     for (int i = length - 1; i >= length - streak; --i)
1340         if (values[i] <= values[i - 1])
1341             return false;
1342     return true;
1343 }
1344 
1345 }
1346