File indexing completed on 2024-12-29 04:11:45
0001 /*************************************************************************** 0002 * * 0003 * Copyright : (C) 2003 The University of Toronto * 0004 * email : netterfield@astro.utoronto.ca * 0005 * * 0006 * This program is free software; you can redistribute it and/or modify * 0007 * it under the terms of the GNU General Public License as published by * 0008 * the Free Software Foundation; either version 2 of the License, or * 0009 * (at your option) any later version. * 0010 * * 0011 ***************************************************************************/ 0012 0013 #include "asciisource.h" 0014 #include "asciidatainterfaces.h" 0015 0016 #include "curve.h" 0017 #include "colorsequence.h" 0018 #include "objectstore.h" 0019 0020 #include "math_kst.h" 0021 0022 #include "kst_atof.h" 0023 #include "measuretime.h" 0024 #include "debug.h" 0025 0026 #include <QFile> 0027 #include <QFileInfo> 0028 #include <QMessageBox> 0029 #include <QThread> 0030 #include <QtConcurrentRun> 0031 #include <QFutureSynchronizer> 0032 #include <QLabel> 0033 #include <QApplication> 0034 #include <QVBoxLayout> 0035 #include <QProgressBar> 0036 0037 0038 #include <ctype.h> 0039 #include <stdlib.h> 0040 0041 0042 using namespace Kst; 0043 0044 const int BIG_READ=100000; 0045 0046 //------------------------------------------------------------------------------------------- 0047 struct ms : QThread 0048 { 0049 static void sleep(int t) { QThread::msleep(t); } 0050 }; 0051 0052 0053 //------------------------------------------------------------------------------------------- 0054 static const QString asciiTypeString = "ASCII file"; 0055 0056 0057 //------------------------------------------------------------------------------------------- 0058 const QString AsciiSource::asciiTypeKey() 0059 { 0060 return asciiTypeString; 0061 } 0062 0063 0064 //------------------------------------------------------------------------------------------- 0065 AsciiSource::AsciiSource(Kst::ObjectStore *store, QSettings *cfg, const QString& filename, const QString& type, const QDomElement& e) : 0066 Kst::DataSource(store, cfg, filename, type), 0067 _reader(_config), 0068 _fileBuffer(), 0069 _busy(false), 0070 _read_count_max(-1), 0071 _read_count(0), 0072 _showFieldProgress(false), 0073 is(new DataInterfaceAsciiString(*this)), 0074 iv(new DataInterfaceAsciiVector(*this)), 0075 _updatesDisabled(true) 0076 { 0077 setInterface(is); 0078 setInterface(iv); 0079 0080 reset(); 0081 0082 _source = asciiTypeString; 0083 if (!type.isEmpty() && type != asciiTypeString) { 0084 return; 0085 } 0086 0087 _config.readGroup(*cfg, filename); 0088 if (!e.isNull()) { 0089 _config.load(e); 0090 } 0091 0092 // TODO only works for local files 0093 setUpdateType((UpdateCheckType)_config._updateType.value()); 0094 0095 _valid = true; 0096 registerChange(); 0097 internalDataSourceUpdate(); 0098 _progressTimer.restart(); 0099 } 0100 0101 0102 //------------------------------------------------------------------------------------------- 0103 AsciiSource::~AsciiSource() 0104 { 0105 } 0106 0107 0108 //------------------------------------------------------------------------------------------- 0109 void AsciiSource::reset() 0110 { 0111 // forget about cached data 0112 _fileBuffer.clear(); 0113 _reader.clear(); 0114 _haveWarned = false; 0115 0116 //_valid = false; 0117 _fileSize = 0; 0118 _lastFileSize = 0; 0119 _haveHeader = false; 0120 _fieldListComplete = false; 0121 0122 _fieldList.clear(); 0123 _fieldLookup.clear(); 0124 _scalarList.clear(); 0125 _strings.clear(); 0126 0127 Object::reset(); 0128 0129 _strings = fileMetas(); 0130 0131 prepareRead(0); 0132 } 0133 0134 //------------------------------------------------------------------------------------------- 0135 bool AsciiSource::initRowIndex() 0136 { 0137 _reader.clear(); 0138 _fileSize = 0; 0139 0140 if (_config._dataLine > 0) { 0141 QFile file(_filename); 0142 if (!AsciiFileBuffer::openFile(file)) { 0143 return false; 0144 } 0145 qint64 header_row = 0; 0146 qint64 left = _config._dataLine; 0147 while (left > 0) { 0148 QByteArray line = file.readLine(); 0149 if (line.isEmpty() || file.atEnd()) { 0150 return false; 0151 } 0152 --left; 0153 if (header_row != _config._fieldsLine && header_row != _config._unitsLine) { 0154 _strings[QString("Header %1").arg(header_row, 2, 10, QChar('0'))] = QString::fromAscii(line).trimmed(); 0155 } 0156 header_row++; 0157 } 0158 _reader.setRow0Begin(file.pos()); 0159 } 0160 0161 return true; 0162 } 0163 0164 //------------------------------------------------------------------------------------------- 0165 void AsciiSource::updateLists() { 0166 _fieldList = fieldListFor(_filename, _config); 0167 QStringList units; 0168 if (_config._readUnits) { 0169 units += unitListFor(_filename, _config); 0170 for (int index = 0; index < _fieldList.size(); ++index) { 0171 if (index >= units.size()) { 0172 break; // Missing units => the user's fault, but at least don't crash 0173 } 0174 _fieldUnits[_fieldList[index]] = units[index]; 0175 } 0176 } 0177 _fieldListComplete = _fieldList.count() > 1; 0178 0179 _fieldLookup.clear(); 0180 for (int i = 0; i < _fieldList.size(); i++) 0181 _fieldLookup[_fieldList[i]] = i; 0182 0183 // Re-update the scalar list 0184 _scalarList = scalarListFor(_filename, _config); 0185 0186 } 0187 0188 //------------------------------------------------------------------------------------------- 0189 Kst::Object::UpdateType AsciiSource::internalDataSourceUpdate() 0190 { 0191 return internalDataSourceUpdate(true); 0192 } 0193 0194 0195 //------------------------------------------------------------------------------------------- 0196 Kst::Object::UpdateType AsciiSource::internalDataSourceUpdate(bool read_completely) 0197 { 0198 //MeasureTime t("AsciiSource::internalDataSourceUpdate: " + _filename); 0199 if (_busy) 0200 return NoChange; 0201 0202 // forget about cached data 0203 _fileBuffer.clear(); 0204 0205 if (!_haveHeader) { 0206 _haveHeader = initRowIndex(); 0207 if (!_haveHeader) { 0208 return NoChange; 0209 } 0210 } 0211 updateLists(); 0212 0213 QFile file(_filename); 0214 if (!AsciiFileBuffer::openFile(file)) { 0215 // Qt: If the device is closed, the size returned will not reflect the actual size of the device. 0216 return NoChange; 0217 } 0218 0219 if (_updatesDisabled) { 0220 _fileSize = 0; 0221 } else { 0222 _fileSize = file.size(); 0223 } 0224 0225 bool force_update = true; 0226 if (_fileSize == file.size()) { 0227 force_update = false; 0228 } 0229 0230 _fileCreationTime_t = QFileInfo(file).created().toTime_t(); 0231 0232 int col_count = _fieldList.size() - 1; // minus INDEX 0233 0234 bool new_data = false; 0235 // emit progress message if there are more than 100 MB to parse 0236 if (_fileSize - _lastFileSize > 100 * 1024 * 1024 && read_completely) { 0237 _showFieldProgress = true; 0238 emitProgress(1, tr("Parsing '%1' ...").arg(_filename)); 0239 QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); 0240 QFuture<bool> future = QtConcurrent::run(&_reader, &AsciiDataReader::findAllDataRows, read_completely, &file, _fileSize, col_count); 0241 _busy = true; 0242 while (_busy) { 0243 if (future.isFinished()) { 0244 try { 0245 new_data = future; 0246 } catch ( const std::exception&) { 0247 // TODO out of memory? 0248 } 0249 _busy = false; 0250 emitProgress(50, tr("Finished parsing '%1'").arg(_filename)); 0251 } else { 0252 ms::sleep(500); 0253 emitProgress(1 + 99.0 * _reader.progressValue() / 100.0, tr("Parsing '%1': %2 rows").arg(_filename).arg(QString::number(_reader.progressRows()))); 0254 QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); 0255 } 0256 } 0257 } else { 0258 _showFieldProgress = false; 0259 new_data = _reader.findAllDataRows(read_completely, &file, _fileSize, col_count); 0260 } 0261 0262 _lastFileSize = _fileSize; 0263 0264 return (!new_data && !force_update ? NoChange : Updated); 0265 } 0266 0267 0268 //------------------------------------------------------------------------------------------- 0269 int AsciiSource::columnOfField(const QString& field) const 0270 { 0271 if (_fieldLookup.contains(field)) { 0272 return _fieldLookup[field]; 0273 } 0274 0275 if (_fieldListComplete) { 0276 return -1; 0277 } 0278 0279 bool ok = false; 0280 int col = field.toInt(&ok); 0281 if (ok) { 0282 return col; 0283 } 0284 0285 return -1; 0286 } 0287 0288 0289 //------------------------------------------------------------------------------------------- 0290 bool AsciiSource::useSlidingWindow(qint64 bytesToRead) const 0291 { 0292 return _config._limitFileBuffer && _config._limitFileBufferSize < bytesToRead; 0293 } 0294 0295 0296 //------------------------------------------------------------------------------------------- 0297 int AsciiSource::readField(double *v, const QString& field, int s, int n) 0298 { 0299 _actualField = field; 0300 if (n>BIG_READ) { 0301 updateFieldMessage(tr("Reading field: ")); 0302 } 0303 0304 // TODO multi threading problem: could trigger a dead-lock 0305 // FIXME: Debug::trace() here is a memory leak, which is serious for large quickly updating files. 0306 //Debug::trace(QString("AsciiSource::readField() %1 s=%2 n=%3").arg(field.leftJustified(15)).arg(QString("%1").arg(s, 10)).arg(n)); 0307 0308 int read = tryReadField(v, field, s, n); 0309 0310 if (isTime(field)) { 0311 if (_config._indexInterpretation == AsciiSourceConfig::FixedRate ) { 0312 double rate = _config._dataRate.value(); 0313 if (rate>0) { 0314 rate = 1.0/rate; 0315 } else { 0316 rate = 1.0; 0317 } 0318 0319 for (int i=0; i<read; i++) { 0320 v[i] *= rate; 0321 } 0322 } 0323 0324 double dT = 0.0; 0325 if (_config._offsetDateTime.value()) { 0326 dT = (double)_config._dateTimeOffset.value().toTime_t(); 0327 } else if (_config._offsetRelative.value()) { 0328 dT = _config._relativeOffset.value(); 0329 } else if (_config._offsetFileDate.value()) { 0330 dT = _fileCreationTime_t; 0331 } 0332 0333 for (int i=0; i<read; i++) { 0334 v[i] += dT; 0335 } 0336 0337 } 0338 0339 QString msg("%1.\nTry without threads or use a different file buffer limit when using threads for reading."); 0340 if (read == abs(n)) { // when reading only 1 sample, n could be -1 instead of 1 to signal 1 sample, not 1 frame. 0341 return read; 0342 } else if (read > 0) { 0343 if (!_haveWarned) 0344 QMessageBox::warning(0, "Error while reading ASCII file", msg.arg("The file was read only partially")); 0345 _haveWarned = true; 0346 return read; 0347 } else if (read == 0) { 0348 if (!_haveWarned) { 0349 // TODO Why is nothing read? Only log once because of the danger of a dead-lock 0350 Debug::warning("AsciiSource: 0 bytes read from file"); 0351 } 0352 _haveWarned = true; 0353 } else if (read == -3) { 0354 if (!_haveWarned) 0355 QMessageBox::warning(0, "Error while reading ASCII file", "The file could not be opened for reading"); 0356 _haveWarned = true; 0357 } 0358 0359 emitProgress(100, QString()); 0360 return 0; 0361 } 0362 0363 0364 //------------------------------------------------------------------------------------------- 0365 bool AsciiSource::useThreads() const 0366 { 0367 // only use threads for files > 1 MB 0368 return _config._useThreads && _fileSize > 1 * 1024 * 1024; 0369 } 0370 0371 //------------------------------------------------------------------------------------------- 0372 void AsciiSource::prepareRead(int count) 0373 { 0374 _read_count_max = count; 0375 _read_count = 0; 0376 _progress = 0; 0377 _progressSteps = 0; 0378 } 0379 0380 //------------------------------------------------------------------------------------------- 0381 void AsciiSource::readingDone() 0382 { 0383 // clear 0384 emit progress(100, ""); 0385 } 0386 0387 //------------------------------------------------------------------------------------------- 0388 int AsciiSource::tryReadField(double *v, const QString& field, int s, int n) 0389 { 0390 if (n < 0) { 0391 n = 1; /* n < 0 means read one sample, not frame - irrelevent here */ 0392 } 0393 0394 if (field == "INDEX") { 0395 for (int i = 0; i < n; i++) { 0396 v[i] = double(s + i); 0397 } 0398 if (n>BIG_READ) { 0399 updateFieldMessage(tr("INDEX created")); 0400 } 0401 return n; 0402 } 0403 0404 int col = columnOfField(field); 0405 if (col == -1) { 0406 _read_count_max = -1; 0407 return -2; 0408 } 0409 0410 // check if the already in buffer 0411 const qint64 begin = _reader.beginOfRow(s); 0412 const qint64 bytesToRead = _reader.beginOfRow(s + n) - begin; 0413 if ((begin != _fileBuffer.begin()) || (bytesToRead != _fileBuffer.bytesRead())) { 0414 QFile* file = new QFile(_filename); 0415 if (!AsciiFileBuffer::openFile(*file)) { 0416 delete file; 0417 _read_count_max = -1; 0418 return -3; 0419 } 0420 0421 // prepare file buffer 0422 0423 _fileBuffer.setFile(file); 0424 0425 int numThreads; 0426 if (!useThreads()) { 0427 numThreads = 1; 0428 } else { 0429 numThreads = QThread::idealThreadCount(); 0430 numThreads = (numThreads > 0) ? numThreads : 1; 0431 } 0432 0433 if (useSlidingWindow(bytesToRead)) { 0434 if (useThreads()) { 0435 _fileBuffer.useSlidingWindowWithChunks(_reader.rowIndex(), begin, bytesToRead, _config._limitFileBufferSize, numThreads); 0436 } else { 0437 _fileBuffer.useSlidingWindow(_reader.rowIndex(), begin, bytesToRead, _config._limitFileBufferSize); 0438 } 0439 } else { 0440 _fileBuffer.useOneWindowWithChunks(_reader.rowIndex(), begin, bytesToRead, numThreads); 0441 } 0442 0443 if (_fileBuffer.bytesRead() == 0) { 0444 _fileBuffer.clear(); 0445 _read_count_max = -1; 0446 return 0; 0447 } 0448 0449 _reader.detectLineEndingType(*file); 0450 } 0451 0452 // now start reading 0453 LexicalCast::NaNMode nanMode; 0454 switch (_config._nanValue.value()) { 0455 case 0: nanMode = LexicalCast::NullValue; break; 0456 case 1: nanMode = LexicalCast::NaNValue; break; 0457 #ifndef KST_NO_THREAD_LOCAL 0458 case 2: nanMode = LexicalCast::PreviousValue; break; 0459 #endif 0460 default:nanMode = LexicalCast::NullValue; break; 0461 } 0462 LexicalCast::AutoReset useDot(_config._useDot, nanMode); 0463 0464 0465 if (field == _config._indexVector && _config._indexInterpretation == AsciiSourceConfig::FormattedTime) { 0466 LexicalCast::instance().setTimeFormat(_config._timeAsciiFormatString); 0467 } 0468 0469 QVector<QVector<AsciiFileData> >& slidingWindow = _fileBuffer.fileData(); 0470 int sampleRead = 0; 0471 0472 _progressSteps = 0; 0473 for (int i = 0; i < slidingWindow.size(); i++) { 0474 _progressSteps += slidingWindow[i].size() * 2; 0475 } 0476 if (_read_count_max == -1) { 0477 _progress = 0; 0478 } else { 0479 _progressSteps *= _read_count_max; 0480 } 0481 0482 for (int i = 0; i < slidingWindow.size(); i++) { 0483 0484 int read; 0485 if (useThreads()) 0486 read = parseWindowMultithreaded(slidingWindow[i], col, v, s, field); 0487 else 0488 read = parseWindowSinglethreaded(slidingWindow[i], col, v, s, field, sampleRead); 0489 0490 // something went wrong abort reading 0491 if (read == 0) { 0492 break; 0493 } 0494 0495 sampleRead += read; 0496 } 0497 0498 if (useSlidingWindow(bytesToRead)) { 0499 // only buffering the complete file makes sense 0500 _fileBuffer.clear(); 0501 } 0502 0503 if (n>BIG_READ) { 0504 updateFieldMessage(tr("Finished reading: ")); 0505 } 0506 0507 _read_count++; 0508 if (_read_count_max == _read_count) 0509 _read_count_max = -1; 0510 0511 return sampleRead; 0512 } 0513 0514 0515 //------------------------------------------------------------------------------------------- 0516 int AsciiSource::parseWindowSinglethreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field, int sRead) 0517 { 0518 int read = 0; 0519 for (int i = 0; i < window.size(); i++) { 0520 Q_ASSERT(sRead + start == window[i].rowBegin()); 0521 if (!window[i].read() || window[i].bytesRead() == 0) 0522 return 0; 0523 _progress++; 0524 read += _reader.readFieldFromChunk(window[i], col, v, start, field); 0525 _progress += window.size(); 0526 } 0527 return read; 0528 } 0529 0530 0531 //------------------------------------------------------------------------------------------- 0532 int AsciiSource::parseWindowMultithreaded(QVector<AsciiFileData>& window, int col, double* v, int start, const QString& field) 0533 { 0534 updateFieldProgress(tr("reading ...")); 0535 for (int i = 0; i < window.size(); i++) { 0536 if (!window[i].read()) { 0537 return 0; 0538 } 0539 _progress++; 0540 updateFieldProgress(tr("reading ...")); 0541 } 0542 0543 updateFieldProgress(tr("parsing ...")); 0544 QFutureSynchronizer<int> readFutures; 0545 foreach (const AsciiFileData& chunk, window) { 0546 QFuture<int> future = QtConcurrent::run(&_reader, &AsciiDataReader::readFieldFromChunk, chunk, col, v, start, field); 0547 readFutures.addFuture(future); 0548 } 0549 readFutures.waitForFinished(); 0550 _progress += window.size(); 0551 updateFieldProgress(tr("parsing ...")); 0552 int sampleRead = 0; 0553 foreach (const QFuture<int> future, readFutures.futures()) { 0554 sampleRead += future.result(); 0555 } 0556 return sampleRead; 0557 } 0558 0559 //------------------------------------------------------------------------------------------- 0560 void AsciiSource::emitProgress(int percent, const QString& message) 0561 { 0562 if (_progressTimer.elapsed() < 500) { 0563 // don't flood the gui with progress messages 0564 return; 0565 } 0566 emit progress(percent, message); 0567 _progressTimer.restart(); 0568 QApplication::processEvents(QEventLoop::ExcludeUserInputEvents); 0569 } 0570 0571 //------------------------------------------------------------------------------------------- 0572 void AsciiSource::updateFieldMessage(const QString& message) 0573 { 0574 // hide progress bar 0575 emitProgress(100, message + _actualField); 0576 } 0577 0578 //------------------------------------------------------------------------------------------- 0579 void AsciiSource::updateFieldProgress(const QString& message) 0580 { 0581 if (_read_count_max == 0) { 0582 //emitProgress(-1, ""); // indicate "busy" FIXME: commented out because it doesn't go away. 0583 } else { 0584 if (_progressSteps != 0 && _read_count_max != -1) { 0585 emitProgress(50 + 50 * _progress / _progressSteps, _actualField + ": " + message); 0586 } 0587 } 0588 } 0589 0590 //------------------------------------------------------------------------------------------- 0591 QString AsciiSource::fileType() const 0592 { 0593 return asciiTypeString; 0594 } 0595 0596 //------------------------------------------------------------------------------------------- 0597 void AsciiSource::setUpdateType(UpdateCheckType updateType) 0598 { 0599 if (_config._updateType != updateType) { 0600 //Q_ASSERT(AsciiSourceConfig().readGroup(*_cfg, _filename) == _config); 0601 _config._updateType = updateType; 0602 _config.saveGroup(*_cfg, _filename); 0603 } 0604 DataSource::setUpdateType(updateType); 0605 } 0606 0607 0608 //------------------------------------------------------------------------------------------- 0609 bool AsciiSource::isEmpty() const 0610 { 0611 return _reader.numberOfFrames() < 1; 0612 } 0613 0614 0615 //------------------------------------------------------------------------------------------- 0616 QStringList AsciiSource::scalarListFor(const QString& filename, AsciiSourceConfig) 0617 { 0618 QFile file(filename); 0619 if (!AsciiFileBuffer::openFile(file)) { 0620 return QStringList(); 0621 } 0622 return QStringList() << "FRAMES"; 0623 } 0624 0625 0626 //------------------------------------------------------------------------------------------- 0627 QStringList AsciiSource::stringListFor(const QString& filename, AsciiSourceConfig) 0628 { 0629 QFile file(filename); 0630 if (!AsciiFileBuffer::openFile(file)) { 0631 return QStringList(); 0632 } 0633 return QStringList() << "FILE"; 0634 } 0635 0636 0637 //------------------------------------------------------------------------------------------- 0638 int AsciiSource::splitHeaderLine(const QByteArray& line, const AsciiSourceConfig& cfg, QStringList* stringList) 0639 { 0640 QStringList dummy; 0641 QStringList& parts(stringList ? *stringList : dummy); 0642 parts.clear(); 0643 const QRegExp regexColumnDelimiter(QString("[%1]").arg(QRegExp::escape(cfg._columnDelimiter.value()))); 0644 0645 if (cfg._columnType == AsciiSourceConfig::Custom && !cfg._columnDelimiter.value().isEmpty()) { 0646 parts += QString(line).trimmed().split(regexColumnDelimiter, QString::SkipEmptyParts); 0647 } else if (cfg._columnType == AsciiSourceConfig::Fixed) { 0648 int cnt = line.length() / cfg._columnWidth; 0649 for (int i = 0; i < cnt; ++i) { 0650 QString sub = line.mid(i * cfg._columnWidth).left(cfg._columnWidth); 0651 parts += sub.trimmed(); 0652 } 0653 } else { 0654 if (!stringList) { 0655 //MeasureTime t("AsciiDataReader::countColumns()"); 0656 int columns = AsciiDataReader::splitColumns(line, AsciiCharacterTraits::IsWhiteSpace()); 0657 0658 // The following assert crashes (sometimes?) when kst is pointed at an 0659 // executable. So... rather than crashing, lets just bail. 0660 if (columns != QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts).size()) { 0661 return 0; 0662 } 0663 Q_ASSERT(columns == QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts).size()); 0664 return columns; 0665 } else { 0666 //MeasureTime t("AsciiDataReader::countColumns(parts)"); 0667 AsciiDataReader::splitColumns(line, AsciiCharacterTraits::IsWhiteSpace(), &parts); 0668 Q_ASSERT(parts == QString(line).trimmed().split(QRegExp("\\s"), QString::SkipEmptyParts)); 0669 } 0670 } 0671 return parts.count(); 0672 } 0673 0674 0675 //------------------------------------------------------------------------------------------- 0676 QStringList AsciiSource::fieldListFor(const QString& filename, AsciiSourceConfig cfg) 0677 { 0678 QFile file(filename); 0679 if (!AsciiFileBuffer::openFile(file)) { 0680 return QStringList(); 0681 } 0682 0683 QStringList fields; 0684 fields += "INDEX"; 0685 0686 if (cfg._readFields) { 0687 int fieldsLine = cfg._fieldsLine; 0688 int currentLine = 0; // Explicit line counter, to make the code easier to understand 0689 while (currentLine < cfg._dataLine) { 0690 const QByteArray line = file.readLine(); 0691 int r = line.size(); 0692 if (currentLine == fieldsLine && r >= 0) { 0693 QStringList parts; 0694 AsciiSource::splitHeaderLine(line, cfg, &parts); 0695 fields += parts; 0696 break; 0697 } 0698 currentLine++; 0699 } 0700 QStringList trimmed; 0701 foreach(const QString& str, fields) { 0702 trimmed << str.trimmed(); 0703 } 0704 return trimmed; 0705 } 0706 0707 0708 QRegExp regex; 0709 if (cfg._columnType == AsciiSourceConfig::Custom && !cfg._columnDelimiter.value().isEmpty()) { 0710 regex.setPattern(QString("^[%1]*[%2].*").arg(QRegExp::escape(cfg._columnDelimiter.value())).arg(cfg._delimiters)); 0711 } else { 0712 regex.setPattern(QString("^\\s*[%1].*").arg(cfg._delimiters)); 0713 } 0714 0715 bool done = false; 0716 int skip = cfg._dataLine; 0717 //FIXME This is a hack which should eventually be fixed by specifying 0718 // the starting frame of the data when calling KstDataSource::fieldListForSource 0719 // and KstDataSource::fieldList. If the skip value is not specified, then 0720 // we scan a few lines and take the maximum number of fields that we find. 0721 int maxcnt; 0722 if (skip > 0) { 0723 maxcnt = -1; 0724 } else { 0725 maxcnt = 0; 0726 } 0727 int cnt; 0728 int nextscan = 0; 0729 int curscan = 0; 0730 while (!file.atEnd() && !done && (nextscan < 200)) { 0731 QByteArray line = file.readLine(); 0732 int r = line.size(); 0733 if (skip > 0) { //keep skipping until desired line 0734 --skip; 0735 if (r < 0) { 0736 return fields; 0737 } 0738 continue; 0739 } 0740 if (maxcnt >= 0) { //original skip value == 0, so scan some lines 0741 if (curscan >= nextscan) { 0742 if (r > 1 && !regex.exactMatch(line)) { 0743 cnt = splitHeaderLine(line, cfg); 0744 if (cnt > maxcnt) { 0745 maxcnt = cnt; 0746 } 0747 } else if (r < 0) { 0748 return fields; 0749 } 0750 nextscan += nextscan + 1; 0751 } 0752 curscan++; 0753 continue; 0754 } 0755 if (r > 1 && !regex.exactMatch(line)) { //at desired line, find count 0756 maxcnt = splitHeaderLine(line, cfg); 0757 done = true; 0758 } else if (r < 0) { 0759 return fields; 0760 } 0761 } 0762 0763 for (int i = 1; i <= maxcnt; ++i) { 0764 fields += tr("Column %1").arg(i).trimmed(); 0765 } 0766 0767 return fields; 0768 } 0769 0770 0771 //------------------------------------------------------------------------------------------- 0772 QStringList AsciiSource::unitListFor(const QString& filename, AsciiSourceConfig cfg) 0773 { 0774 QFile file(filename); 0775 if (!AsciiFileBuffer::openFile(file)) { 0776 return QStringList(); 0777 } 0778 0779 QStringList units; 0780 units += ""; // To go with INDEX 0781 0782 int unitsLine = cfg._unitsLine; 0783 int currentLine = 0; 0784 while (currentLine < cfg._dataLine) { 0785 const QByteArray line = file.readLine(); 0786 int r = line.size(); 0787 if (currentLine == unitsLine && r >= 0) { 0788 QStringList parts; 0789 AsciiSource::splitHeaderLine(line, cfg, &parts); 0790 units += parts; 0791 break; 0792 } 0793 currentLine++; 0794 } 0795 QStringList trimmed; 0796 foreach(const QString& str, units) { 0797 trimmed << str.trimmed(); 0798 } 0799 return trimmed; 0800 } 0801 0802 0803 //------------------------------------------------------------------------------------------- 0804 void AsciiSource::save(QXmlStreamWriter &s) 0805 { 0806 Kst::DataSource::save(s); 0807 _config.save(s); 0808 } 0809 0810 0811 //------------------------------------------------------------------------------------------- 0812 void AsciiSource::parseProperties(QXmlStreamAttributes &properties) 0813 { 0814 _config.parseProperties(properties); 0815 reset(); 0816 internalDataSourceUpdate(); 0817 } 0818 0819 0820 //------------------------------------------------------------------------------------------- 0821 bool AsciiSource::supportsTimeConversions() const 0822 { 0823 return false; //fieldList().contains(_config._indexVector) && _config._indexInterpretation != AsciiSourceConfig::Unknown && _config._indexInterpretation != AsciiSourceConfig::INDEX; 0824 } 0825 0826 0827 //------------------------------------------------------------------------------------------- 0828 int AsciiSource::sampleForTime(double ms, bool *ok) 0829 { 0830 switch (_config._indexInterpretation) { 0831 case AsciiSourceConfig::Seconds: 0832 // FIXME: make sure "seconds" exists in _indexVector 0833 if (ok) { 0834 *ok = true; 0835 } 0836 return 0; 0837 case AsciiSourceConfig::CTime: 0838 // FIXME: make sure "seconds" exists in _indexVector (different than above?) 0839 if (ok) { 0840 *ok = true; 0841 } 0842 return 0; 0843 default: 0844 return Kst::DataSource::sampleForTime(ms, ok); 0845 } 0846 } 0847 0848 0849 //------------------------------------------------------------------------------------------- 0850 const QString& AsciiSource::typeString() const 0851 { 0852 return asciiTypeString; 0853 } 0854 0855 0856 //------------------------------------------------------------------------------------------- 0857 int AsciiSource::sampleForTime(const QDateTime& time, bool *ok) 0858 { 0859 switch (_config._indexInterpretation) { 0860 case AsciiSourceConfig::Seconds: 0861 // FIXME: make sure "time" exists in _indexVector 0862 if (ok) { 0863 *ok = true; 0864 } 0865 return time.toTime_t(); 0866 case AsciiSourceConfig::CTime: 0867 // FIXME: make sure "time" exists in _indexVector (different than above?) 0868 if (ok) { 0869 *ok = true; 0870 } 0871 return time.toTime_t(); 0872 default: 0873 return Kst::DataSource::sampleForTime(time, ok); 0874 } 0875 } 0876 0877 //------------------------------------------------------------------------------------------- 0878 bool AsciiSource::isTime(const QString &field) const 0879 { 0880 return (_config._indexInterpretation.value() != AsciiSourceConfig::NoInterpretation) && 0881 (_config._indexInterpretation.value() != AsciiSourceConfig::Unknown) && 0882 (field == _config._indexVector); 0883 } 0884 0885 //------------------------------------------------------------------------------------------- 0886 QString AsciiSource::timeFormat() const 0887 { 0888 if (_config._indexInterpretation.value() != AsciiSourceConfig::FormattedTime) { 0889 return QString(""); 0890 } 0891 else { 0892 return _config._timeAsciiFormatString; 0893 } 0894 } 0895 0896 //------------------------------------------------------------------------------------------- 0897 Kst::ObjectList<Kst::Object> AsciiSource::autoCurves(ObjectStore& objectStore) 0898 { 0899 // here we could do more sophisticated stuff when generating a list of curves 0900 return ObjectList<Kst::Object>(); 0901 } 0902 0903 0904 0905 // vim: ts=2 sw=2 et