File indexing completed on 2024-12-29 04:11:45

0001 /***************************************************************************
0002  *                                                                         *
0003  *   Copyright : (C) 2003 The University of Toronto                        *
0004  *   email     : netterfield@astro.utoronto.ca                             *
0005  *                                                                         *
0006  *   This program is free software; you can redistribute it and/or modify  *
0007  *   it under the terms of the GNU General Public License as published by  *
0008  *   the Free Software Foundation; either version 2 of the License, or     *
0009  *   (at your option) any later version.                                   *
0010  *                                                                         *
0011  ***************************************************************************/
0012 
0013 #include "asciisource.h"
0014 #include "asciidatainterfaces.h"
0015 
0016 #include "curve.h"
0017 #include "colorsequence.h"
0018 #include "objectstore.h"
0019 
0020 #include "math_kst.h"
0021 
0022 #include "kst_atof.h"
0023 #include "measuretime.h"
0024 #include "debug.h"
0025 
0026 #include <QFile>
0027 #include <QFileInfo>
0028 #include <QMessageBox>
0029 #include <QThread>
0030 #include <QtConcurrentRun>
0031 #include <QFutureSynchronizer>
0032 #include <QLabel>
0033 #include <QApplication>
0034 #include <QVBoxLayout>
0035 #include <QProgressBar>
0036 
0037 
0038 #include <ctype.h>
0039 #include <stdlib.h>
0040 
0041 
0042 using namespace Kst;
0043 
0044 const int BIG_READ=100000;
0045 
0046 //-------------------------------------------------------------------------------------------
0047 struct ms : QThread
0048 {
0049   static void sleep(int t) { QThread::msleep(t); }
0050 };
0051 
0052 
0053 //-------------------------------------------------------------------------------------------
0054 static const QString asciiTypeString = "ASCII file";
0055 
0056 
0057 //-------------------------------------------------------------------------------------------
0058 const QString AsciiSource::asciiTypeKey()
0059 {
0060   return asciiTypeString;
0061 }
0062 
0063 
0064 //-------------------------------------------------------------------------------------------
0065 AsciiSource::AsciiSource(Kst::ObjectStore *store, QSettings *cfg, const QString& filename, const QString& type, const QDomElement& e) :
0066   Kst::DataSource(store, cfg, filename, type),
0067   _reader(_config),
0068   _fileBuffer(),
0069   _busy(false),
0070   _read_count_max(-1),
0071   _read_count(0),
0072   _showFieldProgress(false),
0073   is(new DataInterfaceAsciiString(*this)),
0074   iv(new DataInterfaceAsciiVector(*this)),
0075   _updatesDisabled(true)
0076 {
0077   setInterface(is);
0078   setInterface(iv);
0079 
0080   reset();
0081 
0082   _source = asciiTypeString;
0083   if (!type.isEmpty() && type != asciiTypeString) {
0084     return;
0085   }
0086 
0087   _config.readGroup(*cfg, filename);
0088   if (!e.isNull()) {
0089     _config.load(e);
0090   }
0091 
0092   // TODO only works for local files
0093   setUpdateType((UpdateCheckType)_config._updateType.value());
0094 
0095   _valid = true;
0096   registerChange();
0097   internalDataSourceUpdate();
0098   _progressTimer.restart();
0099 }
0100 
0101 
0102 //-------------------------------------------------------------------------------------------
0103 AsciiSource::~AsciiSource()
0104 {
0105 }
0106 
0107 
0108 //-------------------------------------------------------------------------------------------
0109 void AsciiSource::reset()
0110 {
0111   // forget about cached data
0112   _fileBuffer.clear();
0113   _reader.clear();
0114   _haveWarned = false;
0115 
0116   //_valid = false;
0117   _fileSize = 0;
0118   _lastFileSize = 0;
0119   _haveHeader = false;
0120   _fieldListComplete = false;
0121 
0122   _fieldList.clear();
0123   _fieldLookup.clear();
0124   _scalarList.clear();
0125   _strings.clear();
0126 
0127   Object::reset();
0128 
0129   _strings = fileMetas();
0130 
0131   prepareRead(0);
0132 }
0133 
0134 //-------------------------------------------------------------------------------------------
0135 bool AsciiSource::initRowIndex()
0136 {
0137   _reader.clear();
0138   _fileSize = 0;
0139 
0140   if (_config._dataLine > 0) {
0141     QFile file(_filename);
0142     if (!AsciiFileBuffer::openFile(file)) {
0143       return false;
0144     }
0145     qint64 header_row = 0;
0146     qint64 left = _config._dataLine;
0147     while (left > 0) {
0148       QByteArray line = file.readLine();
0149       if (line.isEmpty() || file.atEnd()) {
0150         return false;
0151       }
0152       --left;
0153       if (header_row != _config._fieldsLine && header_row != _config._unitsLine) {
0154         _strings[QString("Header %1").arg(header_row, 2, 10, QChar('0'))] = QString::fromAscii(line).trimmed();
0155       }
0156       header_row++;
0157     }
0158     _reader.setRow0Begin(file.pos());
0159   }
0160 
0161   return true;
0162 }
0163 
0164 //-------------------------------------------------------------------------------------------
0165 void AsciiSource::updateLists() {
0166   _fieldList = fieldListFor(_filename, _config);
0167   QStringList units;
0168   if (_config._readUnits) {
0169     units += unitListFor(_filename, _config);
0170     for (int index = 0; index < _fieldList.size(); ++index) {
0171       if (index >= units.size()) {
0172         break; // Missing units => the user's fault, but at least don't crash
0173       }
0174       _fieldUnits[_fieldList[index]] = units[index];
0175     }
0176   }
0177   _fieldListComplete = _fieldList.count() > 1;
0178 
0179   _fieldLookup.clear();
0180   for (int i = 0; i < _fieldList.size(); i++)
0181       _fieldLookup[_fieldList[i]] = i;
0182 
0183   // Re-update the scalar list
0184   _scalarList = scalarListFor(_filename, _config);
0185 
0186 }
0187 
0188 //-------------------------------------------------------------------------------------------
0189 Kst::Object::UpdateType AsciiSource::internalDataSourceUpdate()
0190 {
0191   return internalDataSourceUpdate(true);
0192 }
0193 
0194 
0195 //-------------------------------------------------------------------------------------------
0196 Kst::Object::UpdateType AsciiSource::internalDataSourceUpdate(bool read_completely)
0197 {
0198   //MeasureTime t("AsciiSource::internalDataSourceUpdate: " + _filename);
0199   if (_busy)
0200     return NoChange;
0201 
0202   // forget about cached data
0203   _fileBuffer.clear();
0204 
0205   if (!_haveHeader) {
0206     _haveHeader = initRowIndex();
0207     if (!_haveHeader) {
0208       return NoChange;
0209     }
0210   }
0211   updateLists();
0212 
0213   QFile file(_filename);
0214   if (!AsciiFileBuffer::openFile(file)) {
0215     // Qt: If the device is closed, the size returned will not reflect the actual size of the device.
0216     return NoChange;
0217   }
0218 
0219   if (_updatesDisabled) {
0220     _fileSize = 0;
0221   } else {
0222     _fileSize = file.size();
0223   }
0224 
0225   bool force_update = true;
0226   if (_fileSize == file.size()) {
0227     force_update = false;
0228   }
0229 
0230   _fileCreationTime_t = QFileInfo(file).created().toTime_t();
0231 
0232   int col_count = _fieldList.size() - 1; // minus INDEX
0233 
0234   bool new_data = false;
0235   // emit progress message if there are more than 100 MB to parse
0236   if (_fileSize - _lastFileSize > 100 * 1024 * 1024 && read_completely) {
0237     _showFieldProgress = true;
0238     emitProgress(1, tr("Parsing '%1' ...").arg(_filename));
0239     QApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
0240     QFuture<bool> future = QtConcurrent::run(&_reader, &AsciiDataReader::findAllDataRows, read_completely, &file, _fileSize, col_count);
0241     _busy = true;
0242     while (_busy) {
0243       if (future.isFinished()) {
0244         try {
0245           new_data = future;
0246         } catch ( const std::exception&) {
0247           // TODO out of memory?
0248         }
0249         _busy = false;
0250         emitProgress(50, tr("Finished parsing '%1'").arg(_filename));
0251       } else {
0252         ms::sleep(500);
0253         emitProgress(1 + 99.0 * _reader.progressValue() / 100.0, tr("Parsing '%1': %2 rows").arg(_filename).arg(QString::number(_reader.progressRows())));
0254         QApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
0255       }
0256     }
0257   } else {
0258     _showFieldProgress = false;
0259     new_data = _reader.findAllDataRows(read_completely, &file, _fileSize, col_count);
0260   }
0261 
0262   _lastFileSize = _fileSize;
0263 
0264   return (!new_data && !force_update ? NoChange : Updated);
0265 }
0266 
0267 
0268 //-------------------------------------------------------------------------------------------
0269 int AsciiSource::columnOfField(const QString& field) const
0270 {
0271   if (_fieldLookup.contains(field)) {
0272     return _fieldLookup[field];
0273   }
0274 
0275   if (_fieldListComplete) {
0276     return -1;
0277   }
0278 
0279   bool ok = false;
0280   int col = field.toInt(&ok);
0281   if (ok) {
0282     return col;
0283   }
0284 
0285   return -1;
0286 }
0287 
0288 
0289 //-------------------------------------------------------------------------------------------
0290 bool AsciiSource::useSlidingWindow(qint64 bytesToRead)  const
0291 {
0292   return _config._limitFileBuffer && _config._limitFileBufferSize < bytesToRead;
0293 }
0294 
0295 
0296 //-------------------------------------------------------------------------------------------
0297 int AsciiSource::readField(double *v, const QString& field, int s, int n)
0298 {
0299   _actualField = field;
0300   if (n>BIG_READ) {
0301     updateFieldMessage(tr("Reading field: "));
0302   }
0303 
0304   // TODO multi threading problem: could trigger a dead-lock
0305   // FIXME: Debug::trace() here is a memory leak, which is serious for large quickly updating files.
0306   //Debug::trace(QString("AsciiSource::readField() %1  s=%2  n=%3").arg(field.leftJustified(15)).arg(QString("%1").arg(s, 10)).arg(n));
0307 
0308   int read = tryReadField(v, field, s, n);
0309 
0310   if (isTime(field)) {
0311     if (_config._indexInterpretation == AsciiSourceConfig::FixedRate ) {
0312       double rate = _config._dataRate.value();
0313       if (rate>0) {
0314         rate = 1.0/rate;
0315       } else {
0316         rate = 1.0;
0317       }
0318 
0319       for (int i=0; i<read; i++) {
0320         v[i] *= rate;
0321       }
0322     }
0323 
0324     double dT = 0.0;
0325     if (_config._offsetDateTime.value()) {
0326       dT = (double)_config._dateTimeOffset.value().toTime_t();
0327     } else if (_config._offsetRelative.value()) {
0328       dT = _config._relativeOffset.value();
0329     } else if (_config._offsetFileDate.value()) {
0330       dT = _fileCreationTime_t;
0331     }
0332 
0333     for (int i=0; i<read; i++) {
0334       v[i] += dT;
0335     }
0336 
0337   }
0338 
0339   QString msg("%1.\nTry without threads or use a different file buffer limit when using threads for reading.");
0340   if (read == abs(n)) { // when reading only 1 sample, n could be -1 instead of 1 to signal 1 sample, not 1 frame.
0341     return read;
0342   } else if (read > 0) {
0343     if (!_haveWarned)
0344       QMessageBox::warning(0, "Error while reading ASCII file", msg.arg("The file was read only partially"));
0345     _haveWarned = true;
0346     return read;
0347   } else if (read == 0) {
0348     if (!_haveWarned) {
0349       // TODO Why is nothing read? Only log once because of the danger of a dead-lock
0350       Debug::warning("AsciiSource: 0 bytes read from file");
0351     }
0352     _haveWarned = true;
0353   } else if (read == -3) {
0354     if (!_haveWarned)
0355       QMessageBox::warning(0, "Error while reading ASCII file", "The file could not be opened for reading");
0356     _haveWarned = true;
0357   }
0358 
0359   emitProgress(100, QString());
0360   return 0;
0361 }
0362 
0363 
0364 //-------------------------------------------------------------------------------------------
0365 bool AsciiSource::useThreads() const
0366 {
0367   // only use threads for files > 1 MB
0368   return _config._useThreads && _fileSize > 1 * 1024 * 1024;
0369 }
0370 
0371 //-------------------------------------------------------------------------------------------
0372 void AsciiSource::prepareRead(int count)
0373 {
0374   _read_count_max = count;
0375   _read_count = 0;
0376   _progress = 0;
0377   _progressSteps = 0;
0378 }
0379 
0380 //-------------------------------------------------------------------------------------------
0381 void AsciiSource::readingDone()
0382 {
0383   // clear
0384   emit progress(100, "");
0385 }
0386 
0387 //-------------------------------------------------------------------------------------------
0388 int AsciiSource::tryReadField(double *v, const QString& field, int s, int n)
0389 {
0390   if (n < 0) {
0391     n = 1; /* n < 0 means read one sample, not frame - irrelevent here */
0392   }
0393 
0394   if (field == "INDEX") {
0395     for (int i = 0; i < n; i++) {
0396       v[i] = double(s + i);
0397     }
0398     if (n>BIG_READ) {
0399       updateFieldMessage(tr("INDEX created"));
0400     }
0401     return n;
0402   }
0403 
0404   int col = columnOfField(field);
0405   if (col == -1) {
0406     _read_count_max = -1;
0407     return -2;
0408   }
0409 
0410   // check if the already in buffer
0411   const qint64 begin = _reader.beginOfRow(s);
0412   const qint64 bytesToRead = _reader.beginOfRow(s + n) - begin;
0413   if ((begin != _fileBuffer.begin()) || (bytesToRead != _fileBuffer.bytesRead())) {
0414     QFile* file = new QFile(_filename);
0415     if (!AsciiFileBuffer::openFile(*file)) {
0416       delete file;
0417       _read_count_max = -1;
0418       return -3;
0419     }
0420 
0421     // prepare file buffer
0422 
0423     _fileBuffer.setFile(file);
0424 
0425     int numThreads;
0426     if (!useThreads()) {
0427       numThreads = 1;
0428     } else {
0429       numThreads = QThread::idealThreadCount();
0430       numThreads = (numThreads > 0) ? numThreads : 1;
0431     }
0432 
0433     if (useSlidingWindow(bytesToRead)) {
0434       if (useThreads()) {
0435         _fileBuffer.useSlidingWindowWithChunks(_reader.rowIndex(), begin, bytesToRead, _config._limitFileBufferSize, numThreads);
0436       } else {
0437         _fileBuffer.useSlidingWindow(_reader.rowIndex(), begin, bytesToRead, _config._limitFileBufferSize);
0438       }
0439     } else {
0440       _fileBuffer.useOneWindowWithChunks(_reader.rowIndex(), begin, bytesToRead, numThreads);
0441     }
0442 
0443     if (_fileBuffer.bytesRead() == 0) {
0444       _fileBuffer.clear();
0445       _read_count_max = -1;
0446       return 0;
0447     }
0448 
0449     _reader.detectLineEndingType(*file);
0450   }
0451 
0452   // now start reading
0453   LexicalCast::NaNMode nanMode;
0454   switch (_config._nanValue.value()) {
0455   case 0: nanMode = LexicalCast::NullValue; break;
0456   case 1: nanMode = LexicalCast::NaNValue; break;
0457 #ifndef KST_NO_THREAD_LOCAL
0458   case 2: nanMode = LexicalCast::PreviousValue; break;
0459 #endif
0460   default:nanMode = LexicalCast::NullValue; break;
0461   }
0462   LexicalCast::AutoReset useDot(_config._useDot, nanMode);
0463 
0464 
0465   if (field == _config._indexVector && _config._indexInterpretation == AsciiSourceConfig::FormattedTime) {
0466     LexicalCast::instance().setTimeFormat(_config._timeAsciiFormatString);
0467   }
0468 
0469   QVector<QVector<AsciiFileData> >& slidingWindow = _fileBuffer.fileData();
0470   int sampleRead = 0;
0471 
0472   _progressSteps = 0;
0473   for (int i = 0; i < slidingWindow.size(); i++) {
0474       _progressSteps += slidingWindow[i].size() * 2;
0475   }
0476   if (_read_count_max == -1) {
0477     _progress = 0;
0478   } else {
0479     _progressSteps *= _read_count_max;
0480   }
0481 
0482   for (int i = 0; i < slidingWindow.size(); i++) {
0483 
0484     int read;
0485     if (useThreads())
0486       read = parseWindowMultithreaded(slidingWindow[i], col, v, s, field);
0487     else
0488       read = parseWindowSinglethreaded(slidingWindow[i], col, v, s, field, sampleRead);
0489 
0490     // something went wrong abort reading
0491     if (read == 0) {
0492       break;
0493     }
0494 
0495     sampleRead += read;
0496   }
0497 
0498   if (useSlidingWindow(bytesToRead)) {
0499     // only buffering the complete file makes sense
0500     _fileBuffer.clear();
0501   }
0502 
0503   if (n>BIG_READ) {
0504     updateFieldMessage(tr("Finished reading: "));
0505   }
0506 
0507   _read_count++;
0508   if (_read_count_max == _read_count)
0509     _read_count_max = -1;
0510 
0511   return sampleRead;
0512 }
0513 
0514 
0515 //-------------------------------------------------------------------------------------------
0516 int AsciiSource::parseWindowSinglethreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field, int sRead)
0517 {
0518   int read = 0;
0519   for (int i = 0; i < window.size(); i++) {
0520     Q_ASSERT(sRead + start ==  window[i].rowBegin());
0521     if (!window[i].read() || window[i].bytesRead() == 0)
0522       return 0;
0523     _progress++;
0524     read += _reader.readFieldFromChunk(window[i], col, v, start, field);
0525     _progress += window.size();
0526   }
0527   return read;
0528 }
0529 
0530 
0531 //-------------------------------------------------------------------------------------------
0532 int AsciiSource::parseWindowMultithreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field)
0533 {
0534   updateFieldProgress(tr("reading ..."));
0535   for (int i = 0; i < window.size(); i++) {
0536     if (!window[i].read()) {
0537       return 0;
0538     }
0539     _progress++;
0540     updateFieldProgress(tr("reading ..."));
0541   }
0542 
0543   updateFieldProgress(tr("parsing ..."));
0544   QFutureSynchronizer<int> readFutures;
0545   foreach (const AsciiFileData& chunk, window) {
0546     QFuture<int> future = QtConcurrent::run(&_reader, &AsciiDataReader::readFieldFromChunk, chunk, col, v, start, field);
0547     readFutures.addFuture(future);
0548   }
0549   readFutures.waitForFinished();
0550   _progress += window.size();
0551   updateFieldProgress(tr("parsing ..."));
0552   int sampleRead = 0;
0553   foreach (const QFuture<int> future, readFutures.futures()) {
0554     sampleRead += future.result();
0555   }
0556   return sampleRead;
0557 }
0558 
0559 //-------------------------------------------------------------------------------------------
0560 void AsciiSource::emitProgress(int percent, const QString& message)
0561 {
0562   if (_progressTimer.elapsed() < 500) {
0563     // don't flood the gui with progress messages
0564     return;
0565   }
0566   emit progress(percent, message);
0567   _progressTimer.restart();
0568   QApplication::processEvents(QEventLoop::ExcludeUserInputEvents);
0569 }
0570 
0571 //-------------------------------------------------------------------------------------------
0572 void AsciiSource::updateFieldMessage(const QString& message)
0573 {
0574    // hide progress bar
0575    emitProgress(100, message + _actualField);
0576 }
0577 
0578 //-------------------------------------------------------------------------------------------
0579 void AsciiSource::updateFieldProgress(const QString& message)
0580 {
0581   if (_read_count_max == 0) {
0582     //emitProgress(-1, ""); // indicate "busy"  FIXME: commented out because it doesn't go away.
0583   } else {
0584     if (_progressSteps != 0 && _read_count_max != -1) {
0585       emitProgress(50 + 50 * _progress / _progressSteps, _actualField + ": " + message);
0586     }
0587   }
0588 }
0589 
0590 //-------------------------------------------------------------------------------------------
0591 QString AsciiSource::fileType() const
0592 {
0593   return asciiTypeString;
0594 }
0595 
0596 //-------------------------------------------------------------------------------------------
0597 void AsciiSource::setUpdateType(UpdateCheckType updateType)
0598 {
0599     if (_config._updateType != updateType) {
0600         //Q_ASSERT(AsciiSourceConfig().readGroup(*_cfg, _filename) == _config);
0601         _config._updateType = updateType;
0602         _config.saveGroup(*_cfg, _filename);
0603     }
0604     DataSource::setUpdateType(updateType);
0605 }
0606 
0607 
0608 //-------------------------------------------------------------------------------------------
0609 bool AsciiSource::isEmpty() const
0610 {
0611   return _reader.numberOfFrames() < 1;
0612 }
0613 
0614 
0615 //-------------------------------------------------------------------------------------------
0616 QStringList AsciiSource::scalarListFor(const QString& filename, AsciiSourceConfig)
0617 {
0618   QFile file(filename);
0619   if (!AsciiFileBuffer::openFile(file)) {
0620     return QStringList();
0621   }
0622   return QStringList() << "FRAMES";
0623 }
0624 
0625 
0626 //-------------------------------------------------------------------------------------------
0627 QStringList AsciiSource::stringListFor(const QString& filename, AsciiSourceConfig)
0628 {
0629   QFile file(filename);
0630   if (!AsciiFileBuffer::openFile(file)) {
0631     return QStringList();
0632   }
0633   return QStringList() << "FILE";
0634 }
0635 
0636 
0637 //-------------------------------------------------------------------------------------------
0638 int AsciiSource::splitHeaderLine(const QByteArray& line, const AsciiSourceConfig& cfg, QStringList* stringList)
0639 {
0640   QStringList dummy;
0641   QStringList& parts(stringList ? *stringList : dummy);
0642   parts.clear();
0643   const QRegExp regexColumnDelimiter(QString("[%1]").arg(QRegExp::escape(cfg._columnDelimiter.value())));
0644 
0645   if (cfg._columnType == AsciiSourceConfig::Custom && !cfg._columnDelimiter.value().isEmpty()) {
0646     parts += QString(line).trimmed().split(regexColumnDelimiter, QString::SkipEmptyParts);
0647   } else if (cfg._columnType == AsciiSourceConfig::Fixed) {
0648     int cnt = line.length() / cfg._columnWidth;
0649     for (int i = 0; i < cnt; ++i) {
0650       QString sub = line.mid(i * cfg._columnWidth).left(cfg._columnWidth);
0651       parts += sub.trimmed();
0652     }
0653   } else {
0654     if (!stringList) {
0655       //MeasureTime t("AsciiDataReader::countColumns()");
0656       int columns = AsciiDataReader::splitColumns(line, AsciiCharacterTraits::IsWhiteSpace());
0657 
0658       // The following assert crashes (sometimes?) when kst is pointed at an
0659       // executable.  So... rather than crashing, lets just bail.
0660       if (columns != QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts).size()) {
0661         return 0;
0662       }
0663       Q_ASSERT(columns == QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts).size());
0664       return columns;
0665     } else {
0666       //MeasureTime t("AsciiDataReader::countColumns(parts)");
0667       AsciiDataReader::splitColumns(line, AsciiCharacterTraits::IsWhiteSpace(), &parts);
0668       Q_ASSERT(parts == QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts));
0669     }
0670   }
0671   return parts.count();
0672 }
0673 
0674 
0675 //-------------------------------------------------------------------------------------------
0676 QStringList AsciiSource::fieldListFor(const QString& filename, AsciiSourceConfig cfg)
0677 {
0678   QFile file(filename);
0679   if (!AsciiFileBuffer::openFile(file)) {
0680     return QStringList();
0681   }
0682 
0683   QStringList fields;
0684   fields += "INDEX";
0685 
0686   if (cfg._readFields) {
0687     int fieldsLine = cfg._fieldsLine;
0688     int currentLine = 0; // Explicit line counter, to make the code easier to understand
0689     while (currentLine < cfg._dataLine) {
0690       const QByteArray line = file.readLine();
0691       int r = line.size();
0692       if (currentLine == fieldsLine && r >= 0) {
0693         QStringList parts;
0694         AsciiSource::splitHeaderLine(line, cfg, &parts);
0695         fields += parts;
0696         break;
0697       }
0698       currentLine++;
0699     }
0700     QStringList trimmed;
0701     foreach(const QString& str, fields) {
0702       trimmed << str.trimmed();
0703     }
0704     return trimmed;
0705   }
0706 
0707 
0708   QRegExp regex;
0709   if (cfg._columnType == AsciiSourceConfig::Custom && !cfg._columnDelimiter.value().isEmpty()) {
0710     regex.setPattern(QString("^[%1]*[%2].*").arg(QRegExp::escape(cfg._columnDelimiter.value())).arg(cfg._delimiters));
0711   } else {
0712     regex.setPattern(QString("^\\s*[%1].*").arg(cfg._delimiters));
0713   }
0714 
0715   bool done = false;
0716   int skip = cfg._dataLine;
0717   //FIXME This is a hack which should eventually be fixed by specifying
0718   // the starting frame of the data when calling KstDataSource::fieldListForSource
0719   // and KstDataSource::fieldList.  If the skip value is not specified, then
0720   // we scan a few lines and take the maximum number of fields that we find.
0721   int maxcnt;
0722   if (skip > 0) {
0723     maxcnt = -1;
0724   } else {
0725     maxcnt = 0;
0726   }
0727   int cnt;
0728   int nextscan = 0;
0729   int curscan = 0;
0730   while (!file.atEnd() && !done && (nextscan < 200)) {
0731     QByteArray line = file.readLine();
0732     int r = line.size();
0733     if (skip > 0) { //keep skipping until desired line
0734       --skip;
0735       if (r < 0) {
0736         return fields;
0737       }
0738       continue;
0739     }
0740     if (maxcnt >= 0) { //original skip value == 0, so scan some lines
0741       if (curscan >= nextscan) {
0742         if (r > 1 && !regex.exactMatch(line)) {
0743           cnt = splitHeaderLine(line, cfg);
0744           if (cnt > maxcnt) {
0745             maxcnt = cnt;
0746           }
0747         } else if (r < 0) {
0748           return fields;
0749         }
0750         nextscan += nextscan + 1;
0751       }
0752       curscan++;
0753       continue;
0754     }
0755     if (r > 1 && !regex.exactMatch(line)) { //at desired line, find count
0756       maxcnt = splitHeaderLine(line, cfg);
0757       done = true;
0758     } else if (r < 0) {
0759       return fields;
0760     }
0761   }
0762 
0763   for (int i = 1; i <= maxcnt; ++i) {
0764     fields += tr("Column %1").arg(i).trimmed();
0765   }
0766 
0767   return fields;
0768 }
0769 
0770 
0771 //-------------------------------------------------------------------------------------------
0772 QStringList AsciiSource::unitListFor(const QString& filename, AsciiSourceConfig cfg)
0773 {
0774   QFile file(filename);
0775   if (!AsciiFileBuffer::openFile(file)) {
0776     return QStringList();
0777   }
0778 
0779   QStringList units;
0780   units += ""; // To go with INDEX
0781 
0782   int unitsLine = cfg._unitsLine;
0783   int currentLine = 0;
0784   while (currentLine < cfg._dataLine) {
0785     const QByteArray line = file.readLine();
0786     int r = line.size();
0787     if (currentLine == unitsLine && r >= 0) {
0788       QStringList parts;
0789       AsciiSource::splitHeaderLine(line, cfg, &parts);
0790       units += parts;
0791       break;
0792     }
0793     currentLine++;
0794   }
0795   QStringList trimmed;
0796   foreach(const QString& str, units) {
0797     trimmed << str.trimmed();
0798   }
0799   return trimmed;
0800 }
0801 
0802 
0803 //-------------------------------------------------------------------------------------------
0804 void AsciiSource::save(QXmlStreamWriter &s)
0805 {
0806   Kst::DataSource::save(s);
0807   _config.save(s);
0808 }
0809 
0810 
0811 //-------------------------------------------------------------------------------------------
0812 void AsciiSource::parseProperties(QXmlStreamAttributes &properties)
0813 {
0814   _config.parseProperties(properties);
0815   reset();
0816   internalDataSourceUpdate();
0817 }
0818 
0819 
0820 //-------------------------------------------------------------------------------------------
0821 bool AsciiSource::supportsTimeConversions() const
0822 {
0823   return false; //fieldList().contains(_config._indexVector) && _config._indexInterpretation != AsciiSourceConfig::Unknown && _config._indexInterpretation != AsciiSourceConfig::INDEX;
0824 }
0825 
0826 
0827 //-------------------------------------------------------------------------------------------
0828 int AsciiSource::sampleForTime(double ms, bool *ok)
0829 {
0830   switch (_config._indexInterpretation) {
0831   case AsciiSourceConfig::Seconds:
0832     // FIXME: make sure "seconds" exists in _indexVector
0833     if (ok) {
0834       *ok = true;
0835     }
0836     return 0;
0837   case AsciiSourceConfig::CTime:
0838     // FIXME: make sure "seconds" exists in _indexVector (different than above?)
0839     if (ok) {
0840       *ok = true;
0841     }
0842     return 0;
0843   default:
0844     return Kst::DataSource::sampleForTime(ms, ok);
0845   }
0846 }
0847 
0848 
0849 //-------------------------------------------------------------------------------------------
0850 const QString& AsciiSource::typeString() const
0851 {
0852   return asciiTypeString;
0853 }
0854 
0855 
0856 //-------------------------------------------------------------------------------------------
0857 int AsciiSource::sampleForTime(const QDateTime& time, bool *ok)
0858 {
0859   switch (_config._indexInterpretation) {
0860   case AsciiSourceConfig::Seconds:
0861     // FIXME: make sure "time" exists in _indexVector
0862     if (ok) {
0863       *ok = true;
0864     }
0865     return time.toTime_t();
0866   case AsciiSourceConfig::CTime:
0867     // FIXME: make sure "time" exists in _indexVector (different than above?)
0868     if (ok) {
0869       *ok = true;
0870     }
0871     return time.toTime_t();
0872   default:
0873     return Kst::DataSource::sampleForTime(time, ok);
0874   }
0875 }
0876 
0877 //-------------------------------------------------------------------------------------------
0878 bool AsciiSource::isTime(const QString &field) const
0879 {
0880   return (_config._indexInterpretation.value() != AsciiSourceConfig::NoInterpretation) &&
0881       (_config._indexInterpretation.value() != AsciiSourceConfig::Unknown) &&
0882       (field == _config._indexVector);
0883 }
0884 
0885 //-------------------------------------------------------------------------------------------
0886 QString AsciiSource::timeFormat() const
0887 {
0888   if (_config._indexInterpretation.value() != AsciiSourceConfig::FormattedTime) {
0889     return QString("");
0890   }
0891   else {
0892     return _config._timeAsciiFormatString;
0893   }
0894 }
0895 
0896 //-------------------------------------------------------------------------------------------
0897 Kst::ObjectList<Kst::Object> AsciiSource::autoCurves(ObjectStore& objectStore)
0898 {
0899   // here we could do more sophisticated stuff when generating a list of curves
0900   return ObjectList<Kst::Object>();
0901 }
0902 
0903 
0904 
0905 // vim: ts=2 sw=2 et