File indexing completed on 2024-04-28 15:09:03

 
 #include "opsastrometryindexfiles.h"
 
 #include "align.h"
 #include "kstars.h"
 #include "ksutils.h"
 #include "Options.h"
 #include "ksnotification.h"
 
 #include <KConfigDialog>
 #include <KMessageBox>
 
 namespace Ekos
 {
 OpsAstrometryIndexFiles::OpsAstrometryIndexFiles(Align *parent) : QDialog(KStars::Instance())
 {
     setupUi(this);
 
     downloadSpeed       = 100;
     actualdownloadSpeed = downloadSpeed;
     alignModule         = parent;
     manager             = new QNetworkAccessManager();
 
     indexURL->setText("http://broiler.astrometry.net/~dstn/");
 
     //Get a pointer to the KConfigDialog
     // m_ConfigDialog = KConfigDialog::exists( "alignsettings" );
     connect(openIndexFileDirectory, SIGNAL(clicked()), this, SLOT(slotOpenIndexFileDirectory()));
     connect(indexURL, &QLineEdit::textChanged, [&]()
     {
         indexURL->text();
     });
 
 
     astrometryIndex[2.8]  = "00";
     astrometryIndex[4.0]  = "01";
     astrometryIndex[5.6]  = "02";
     astrometryIndex[8]    = "03";
     astrometryIndex[11]   = "04";
     astrometryIndex[16]   = "05";
     astrometryIndex[22]   = "06";
     astrometryIndex[30]   = "07";
     astrometryIndex[42]   = "08";
     astrometryIndex[60]   = "09";
     astrometryIndex[85]   = "10";
     astrometryIndex[120]  = "11";
     astrometryIndex[170]  = "12";
     astrometryIndex[240]  = "13";
     astrometryIndex[340]  = "14";
     astrometryIndex[480]  = "15";
     astrometryIndex[680]  = "16";
     astrometryIndex[1000] = "17";
     astrometryIndex[1400] = "18";
     astrometryIndex[2000] = "19";
 
     QList<QCheckBox *> checkboxes = findChildren<QCheckBox *>();
 
     connect(indexLocations, static_cast<void (QComboBox::*)(int)>(&QComboBox::currentIndexChanged), this,
             &OpsAstrometryIndexFiles::slotUpdate);
 
     for (auto &checkBox : checkboxes)
     {
         connect(checkBox, &QCheckBox::clicked, this, &OpsAstrometryIndexFiles::downloadOrDeleteIndexFiles);
     }
 
     QList<QProgressBar *> progressBars = findChildren<QProgressBar *>();
     QList<QLabel *> qLabels = findChildren<QLabel *>();
     QList<QPushButton *> qButtons = findChildren<QPushButton *>();
 
     for (auto &bar : progressBars)
     {
         if(bar->objectName().contains("progress"))
         {
             bar->setVisible(false);
             bar->setTextVisible(false);
         }
     }
 
     for (auto &button : qButtons)
     {
         if(button->objectName().contains("cancel"))
         {
             button->setVisible(false);
         }
     }
 
     for (QLabel * label : qLabels)
     {
         if(label->text().contains("info") || label->text().contains("perc"))
         {
             label->setVisible(false);
         }
     }
 
     connect(addIndexFileDirectory, &QAbstractButton::clicked, this, [this]()
     {
         QString dir = QFileDialog::getExistingDirectory(this, "Load Index File Directory",
                       QDir::homePath(),
                       QFileDialog::ShowDirsOnly
                       | QFileDialog::DontResolveSymlinks);
         if (dir.isEmpty())
             return;
         addDirectoryToList(dir);
     });
     connect(removeIndexFileDirectory, &QAbstractButton::clicked, this, [this]()
     {
         if(indexLocations->currentIndex() != 0)
             removeDirectoryFromList(indexLocations->currentText());
     });
 
 
 }
 
 void OpsAstrometryIndexFiles::showEvent(QShowEvent *)
 {
     updateIndexDirectoryList();
 
 }
 
 void OpsAstrometryIndexFiles::updateIndexDirectoryList()
 {
     // This is needed because they might have directories stored in the config file.
     // So we can't just use the options folder list.
     QStringList astrometryDataDirs = KSUtils::getAstrometryDataDirs();
 
     indexLocations->clear();
     if(astrometryDataDirs.count() > 1)
         indexLocations->addItem("All Sources");
     indexLocations->addItems(astrometryDataDirs);
     slotUpdate();
 }
 
 void OpsAstrometryIndexFiles::addDirectoryToList(QString directory)
 {
     QDir dir(directory);
     if(!dir.exists())
         return;
     QString directoryPath = dir.absolutePath();
 
     QStringList indexFileDirs = Options::astrometryIndexFolderList();
     if(indexFileDirs.contains(directoryPath))
         return;
     indexFileDirs.append(directoryPath);
     Options::setAstrometryIndexFolderList(indexFileDirs);
     updateIndexDirectoryList();
 }
 
 void OpsAstrometryIndexFiles::removeDirectoryFromList(QString directory)
 {
     QStringList indexFileDirs = Options::astrometryIndexFolderList();
     if(indexFileDirs.contains(directory))
     {
         indexFileDirs.removeOne(directory);
         Options::setAstrometryIndexFolderList(indexFileDirs);
         updateIndexDirectoryList();
     }
 }
 
 void OpsAstrometryIndexFiles::slotUpdate()
 {
     QList<QCheckBox *> checkboxes = findChildren<QCheckBox *>();
 
     for (auto &checkBox : checkboxes)
     {
         checkBox->setChecked(false);
     }
 
     if(indexLocations->count() == 0)
         return;
 
     double fov_w, fov_h, fov_pixscale;
 
     // Values in arcmins. Scale in arcsec per pixel
     alignModule->getFOVScale(fov_w, fov_h, fov_pixscale);
 
     double fov_check = qMax(fov_w, fov_h);
 
     FOVOut->setText(QString("%1' x %2'").arg(QString::number(fov_w, 'f', 2), QString::number(fov_h, 'f', 2)));
 
     QStringList nameFilter("*.fits");
 
     QStringList astrometryDataDirs = Options::astrometryIndexFolderList();
 
     bool allDirsSelected = (indexLocations->currentIndex() == 0 && astrometryDataDirs.count() > 1);
     bool folderIsWriteable;
 
     QStringList astrometryDataDirsToIndex;
 
     if(allDirsSelected)
     {
         folderDetails->setText(i18n("Downloads Disabled, this is not a directory, it is a list of all index files."));
         folderIsWriteable = false;
         astrometryDataDirsToIndex = astrometryDataDirs;
         openIndexFileDirectory->setEnabled(false);
     }
     else
     {
         QString folderPath = indexLocations->currentText();
         folderIsWriteable = QFileInfo(folderPath).isWritable();
         if(folderIsWriteable)
             folderDetails->setText(i18n("Downloads Enabled, the directory exists and is writeable."));
         else
             folderDetails->setText(i18n("Downloads Disabled, directory permissions issue."));
         if(!QFileInfo::exists(folderPath))
             folderDetails->setText(i18n("Downloads Disabled, directory does not exist."));
         astrometryDataDirsToIndex << folderPath;
         openIndexFileDirectory->setEnabled(true);
     }
     folderDetails->setCursorPosition(0);
 
     //This loop checks all the folders that are supposed to be checked for the files
     //It checks the box if it finds them
     for(auto &astrometryDataDir : astrometryDataDirsToIndex)
     {
         QDir directory(astrometryDataDir);
         QStringList indexList = directory.entryList(nameFilter);
 
         for (auto &indexName : indexList)
         {
             if (fileCountMatches(directory, indexName))
             {
                 indexName                = indexName.replace('-', '_').left(10);
                 QCheckBox *indexCheckBox = findChild<QCheckBox *>(indexName);
                 if (indexCheckBox)
                     indexCheckBox->setChecked(true);
             }
         }
     }
 
     for (auto &checkBox : checkboxes)
     {
         checkBox->setEnabled(folderIsWriteable);
         checkBox->setIcon(QIcon(":/icons/astrometry-optional.svg"));
         checkBox->setToolTip(i18n("Optional"));
         checkBox->setStyleSheet("");
     }
 
     float last_skymarksize = 2;
 
     for (auto &skymarksize : astrometryIndex.keys())
     {
         QString indexName1        = "index_41" + astrometryIndex.value(skymarksize);
         QString indexName2        = "index_42" + astrometryIndex.value(skymarksize);
         QString indexName3        = "index_52" + astrometryIndex.value(skymarksize);
         QCheckBox *indexCheckBox1 = findChild<QCheckBox *>(indexName1);
         QCheckBox *indexCheckBox2 = findChild<QCheckBox *>(indexName2);
         QCheckBox *indexCheckBox3 = findChild<QCheckBox *>(indexName3);
         if ((skymarksize >= 0.40 * fov_check && skymarksize <= 0.9 * fov_check) ||
                 (fov_check > last_skymarksize && fov_check < skymarksize))
         {
             if (indexCheckBox1)
             {
                 indexCheckBox1->setIcon(QIcon(":/icons/astrometry-required.svg"));
                 indexCheckBox1->setToolTip(i18n("Required"));
             }
             if (indexCheckBox2)
             {
                 indexCheckBox2->setIcon(QIcon(":/icons/astrometry-required.svg"));
                 indexCheckBox2->setToolTip(i18n("Required"));
             }
             if (indexCheckBox3)
             {
                 indexCheckBox3->setIcon(QIcon(":/icons/astrometry-required.svg"));
                 indexCheckBox3->setToolTip(i18n("Required"));
             }
         }
         else if (skymarksize >= 0.10 * fov_check && skymarksize <= fov_check)
         {
             if (indexCheckBox1)
             {
                 indexCheckBox1->setIcon(QIcon(":/icons/astrometry-recommended.svg"));
                 indexCheckBox1->setToolTip(i18n("Recommended"));
             }
             if (indexCheckBox2)
             {
                 indexCheckBox2->setIcon(QIcon(":/icons/astrometry-recommended.svg"));
                 indexCheckBox2->setToolTip(i18n("Recommended"));
             }
             if (indexCheckBox3)
             {
                 indexCheckBox3->setIcon(QIcon(":/icons/astrometry-recommended.svg"));
                 indexCheckBox3->setToolTip(i18n("Recommended"));
             }
         }
 
         last_skymarksize = skymarksize;
     }
 
     //This loop goes over all the directories and adds a stylesheet to change the look of the checkbox text
     //if the File is installed in any directory.  Note that this indicator is then used below in the
     //Index File download function to check if they really want to do install a file that is installed.
     for(QString astrometryDataDir : astrometryDataDirs)
     {
         QDir directory(astrometryDataDir);
         QStringList indexList = directory.entryList(nameFilter);
 
         for (auto &indexName : indexList)
         {
             if (fileCountMatches(directory, indexName))
             {
                 indexName                = indexName.replace('-', '_').left(10);
                 QCheckBox *indexCheckBox = findChild<QCheckBox *>(indexName);
                 if (indexCheckBox)
                     indexCheckBox->setStyleSheet("QCheckBox{font-weight: bold; color:green}");
             }
         }
     }
 }
 
 int OpsAstrometryIndexFiles::indexFileCount(QString indexName)
 {
     int count = 0;
     if(indexName.contains("4207") || indexName.contains("4206") || indexName.contains("4205"))
         count = 12;
     else if(indexName.contains("4204") || indexName.contains("4203") || indexName.contains("4202")
             || indexName.contains("4201") || indexName.contains("4200") || indexName.contains("5206")
             || indexName.contains("5205") || indexName.contains("5204") || indexName.contains("5203")
             || indexName.contains("5202") || indexName.contains("5201") || indexName.contains("5200"))
         count = 48;
     else
         count = 1;
     return count;
 }
 
 bool OpsAstrometryIndexFiles::fileCountMatches(QDir directory, QString indexName)
 {
     QString indexNameMatch = indexName.left(10) + "*.fits";
     QStringList list = directory.entryList(QStringList(indexNameMatch));
     return list.count() == indexFileCount(indexName);
 }
 
 void OpsAstrometryIndexFiles::slotOpenIndexFileDirectory()
 {
     if(indexLocations->count() == 0)
         return;
     QUrl path = QUrl::fromLocalFile(indexLocations->currentText());
     QDesktopServices::openUrl(path);
 }
 
 bool OpsAstrometryIndexFiles::astrometryIndicesAreAvailable()
 {
     QUrl indexUrl = QUrl(this->indexURL->text());
     QNetworkReply *response = manager->get(QNetworkRequest(QUrl(indexUrl.url(QUrl::RemovePath))));
     QTimer timeout(this);
     timeout.setInterval(5000);
     timeout.setSingleShot(true);
     timeout.start();
     while (!response->isFinished())
     {
         if (!timeout.isActive())
         {
             response->deleteLater();
             return false;
         }
         qApp->processEvents();
     }
 
     timeout.stop();
     bool wasSuccessful = (response->error() == QNetworkReply::NoError);
     response->deleteLater();
 
     return wasSuccessful;
 }
 
 void OpsAstrometryIndexFiles::downloadIndexFile(const QString &URL, const QString &fileN, QCheckBox *checkBox,
         int currentIndex, int maxIndex, double fileSize)
 {
     QElapsedTimer downloadTime;
     downloadTime.start();
 
     QString indexString = QString::number(currentIndex);
     if (currentIndex < 10)
         indexString = '0' + indexString;
 
     QString indexSeriesName             = checkBox->text().remove('&');
     QProgressBar *indexDownloadProgress = findChild<QProgressBar *>(indexSeriesName.replace('-', '_').left(10) + "_progress");
     QLabel *indexDownloadInfo           = findChild<QLabel *>(indexSeriesName.replace('-', '_').left(10) + "_info");
     QPushButton *indexDownloadCancel    = findChild<QPushButton *>(indexSeriesName.replace('-', '_').left(10) + "_cancel");
     QLabel *indexDownloadPerc    = findChild<QLabel *>(indexSeriesName.replace('-', '_').left(10) + "_perc");
 
     setDownloadInfoVisible(indexSeriesName, checkBox, true);
 
     if(indexDownloadInfo)
     {
         if (indexDownloadProgress && maxIndex > 0)
             indexDownloadProgress->setValue(currentIndex * 100 / maxIndex);
         indexDownloadInfo->setText("(" + QString::number(currentIndex) + '/' + QString::number(maxIndex + 1) + ") ");
     }
 
     QString indexURL = URL;
 
     indexURL.replace('*', indexString);
 
     QNetworkReply *response = manager->get(QNetworkRequest(QUrl(indexURL)));
 
     //Shut it down after too much time elapses.
     //If the filesize is less  than 4 MB, it sets the timeout for 1 minute or 60000 ms.
     //If it's larger, it assumes a bad download rate of 1 Mbps (100 bytes/ms)
     //and the calculation estimates the time in milliseconds it would take to download.
     int timeout = 60000;
     if(fileSize > 4000000)
         timeout = fileSize / downloadSpeed;
     //qDebug()<<"Filesize: "<< fileSize << ", timeout: " << timeout;
 
     QMetaObject::Connection *cancelConnection = new QMetaObject::Connection();
     QMetaObject::Connection *replyConnection = new QMetaObject::Connection();
     QMetaObject::Connection *percentConnection = new QMetaObject::Connection();
 
     if(indexDownloadPerc)
     {
         *percentConnection = connect(response, &QNetworkReply::downloadProgress,
                                      [ = ](qint64 bytesReceived, qint64 bytesTotal)
         {
             if (indexDownloadProgress)
             {
                 indexDownloadProgress->setValue(bytesReceived);
                 indexDownloadProgress->setMaximum(bytesTotal);
             }
             indexDownloadPerc->setText(QString::number(bytesReceived * 100 / bytesTotal) + '%');
         });
 
     }
 
     timeoutTimer.disconnect();
     connect(&timeoutTimer, &QTimer::timeout, this, [&]()
     {
         KSNotification::error(
             i18n("Download Timed out.  Either the network is not fast enough, the file is not accessible, or you are not connected."));
         disconnectDownload(cancelConnection, replyConnection, percentConnection);
         if(response)
         {
             response->abort();
             response->deleteLater();
         }
         setDownloadInfoVisible(indexSeriesName, checkBox, false);
     });
     timeoutTimer.start(timeout);
 
     *cancelConnection = connect(indexDownloadCancel, &QPushButton::clicked,
                                 [ = ]()
     {
         qDebug() << Q_FUNC_INFO << "Download Cancelled.";
         timeoutTimer.stop();
         disconnectDownload(cancelConnection, replyConnection, percentConnection);
         if(response)
         {
             response->abort();
             response->deleteLater();
         }
         setDownloadInfoVisible(indexSeriesName, checkBox, false);
     });
 
     *replyConnection = connect(response, &QNetworkReply::finished, this,
                                [ = ]()
     {
         timeoutTimer.stop();
         if(response)
         {
             disconnectDownload(cancelConnection, replyConnection, percentConnection);
             setDownloadInfoVisible(indexSeriesName, checkBox, false);
             response->deleteLater();
             if (response->error() != QNetworkReply::NoError)
                 return;
 
             QByteArray responseData = response->readAll();
             QString indexFileN      = fileN;
 
             indexFileN.replace('*', indexString);
 
             QFile file(indexFileN);
             if (QFileInfo(QFileInfo(file).path()).isWritable())
             {
                 if (!file.open(QIODevice::WriteOnly))
                 {
                     KSNotification::error(i18n("File Write Error"));
                     slotUpdate();
                     return;
                 }
                 else
                 {
                     file.write(responseData.data(), responseData.size());
                     file.close();
                     int downloadedFileSize = QFileInfo(file).size();
                     int dtime = downloadTime.elapsed();
                     actualdownloadSpeed = (actualdownloadSpeed + (downloadedFileSize / dtime)) / 2;
                     qDebug() << Q_FUNC_INFO << "Filesize: " << downloadedFileSize << ", time: " << dtime << ", inst speed: " <<
                              downloadedFileSize / dtime <<
                              ", averaged speed: " << actualdownloadSpeed;
 
                 }
             }
             else
             {
                 KSNotification::error(i18n("Astrometry Folder Permissions Error"));
             }
 
             if (currentIndex == maxIndex)
             {
                 slotUpdate();
             }
             else
                 downloadIndexFile(URL, fileN, checkBox, currentIndex + 1, maxIndex, fileSize);
         }
     });
 }
 
 void OpsAstrometryIndexFiles::setDownloadInfoVisible(QString indexSeriesName, QCheckBox *checkBox, bool set)
 {
     Q_UNUSED(checkBox);
 
     QProgressBar *indexDownloadProgress = findChild<QProgressBar *>(indexSeriesName.replace('-', '_').left(10) + "_progress");
     QLabel *indexDownloadInfo           = findChild<QLabel *>(indexSeriesName.replace('-', '_').left(10) + "_info");
     QPushButton *indexDownloadCancel    = findChild<QPushButton *>(indexSeriesName.replace('-', '_').left(10) + "_cancel");
     QLabel *indexDownloadPerc          = findChild<QLabel *>(indexSeriesName.replace('-', '_').left(10) + "_perc");
     if (indexDownloadProgress)
         indexDownloadProgress->setVisible(set);
     if (indexDownloadInfo)
         indexDownloadInfo->setVisible(set);
     if (indexDownloadCancel)
         indexDownloadCancel->setVisible(set);
     if (indexDownloadPerc)
         indexDownloadPerc->setVisible(set);
 }
 void OpsAstrometryIndexFiles::disconnectDownload(QMetaObject::Connection *cancelConnection,
         QMetaObject::Connection *replyConnection, QMetaObject::Connection *percentConnection)
 {
     if(cancelConnection)
         disconnect(*cancelConnection);
     if(replyConnection)
         disconnect(*replyConnection);
     if(percentConnection)
         disconnect(*percentConnection);
 }
 
 void OpsAstrometryIndexFiles::downloadOrDeleteIndexFiles(bool checked)
 {
     QCheckBox *checkBox = qobject_cast<QCheckBox *>(QObject::sender());
 
     if (indexLocations->count() == 0)
         return;
 
     QString astrometryDataDir = indexLocations->currentText();
     if(!QFileInfo::exists(astrometryDataDir))
     {
         KSNotification::sorry(
             i18n("The selected Index File directory does not exist.  Please either create it or choose another."));
     }
 
     if (checkBox)
     {
         QString indexSeriesName = checkBox->text().remove('&');
         QString filePath        = astrometryDataDir + '/' + indexSeriesName;
         QString fileNumString   = indexSeriesName.mid(8, 2);
 
         if (checked)
         {
             if(!checkBox->styleSheet().isEmpty()) //This means that the checkbox has a stylesheet so the index file was installed someplace.
             {
                 if (KMessageBox::Cancel == KMessageBox::warningContinueCancel(
                             nullptr, i18n("The file %1 already exists in another directory.  Are you sure you want to download it to this directory as well?",
                                           indexSeriesName),
                             i18n("Install File(s)"), KStandardGuiItem::cont(),
                             KStandardGuiItem::cancel(), "install_index_files_warning"))
                 {
                     slotUpdate();
                     return;
                 }
             }
             checkBox->setChecked(!checked);
             if (astrometryIndicesAreAvailable())
             {
                 QString BASE_URL;
                 QString URL;
 
                 if (this->indexURL->text().endsWith("/"))
                 {
                     BASE_URL = this->indexURL->text();
                 }
                 else
                 {
                     BASE_URL = this->indexURL->text() + "/";
                 }
 
                 if (indexSeriesName.startsWith(QLatin1String("index-41")))
                     URL = BASE_URL + "4100/" + indexSeriesName;
                 else if (indexSeriesName.startsWith(QLatin1String("index-42")))
                     URL = BASE_URL + "4200/" + indexSeriesName;
                 else if (indexSeriesName.startsWith(QLatin1String("index-52")))
                     URL = "https://portal.nersc.gov/project/cosmo/temp/dstn/index-5200/LITE/" + indexSeriesName;
 
                 int maxIndex = indexFileCount(indexSeriesName) - 1;
 
                 double fileSize = 1E11 * qPow(astrometryIndex.key(fileNumString),
                                               -1.909); //This estimates the file size based on skymark size obtained from the index number.
                 if(maxIndex != 0)
                     fileSize /= maxIndex; //FileSize is divided between multiple files for some index series.
                 downloadIndexFile(URL, filePath, checkBox, 0, maxIndex, fileSize);
             }
             else
             {
                 KSNotification::sorry(i18n("Could not contact Astrometry Index Server."));
             }
         }
         else
         {
             if (KMessageBox::Continue == KMessageBox::warningContinueCancel(
                         nullptr, i18n("Are you sure you want to delete these index files? %1", indexSeriesName),
                         i18n("Delete File(s)"), KStandardGuiItem::cont(),
                         KStandardGuiItem::cancel(), "delete_index_files_warning"))
             {
                 if (QFileInfo(astrometryDataDir).isWritable())
                 {
                     QStringList nameFilter("*.fits");
                     QDir directory(astrometryDataDir);
                     QStringList indexList = directory.entryList(nameFilter);
                     for (auto &fileName : indexList)
                     {
                         if (fileName.contains(indexSeriesName.left(10)))
                         {
                             if (!directory.remove(fileName))
                             {
                                 KSNotification::error(i18n("File Delete Error"));
                                 slotUpdate();
                                 return;
                             }
                             slotUpdate();
                         }
                     }
                 }
                 else
                 {
                     KSNotification::error(i18n("Astrometry Folder Permissions Error"));
                     slotUpdate();
                 }
             }
         }
     }
 }
 }