File indexing completed on 2024-04-21 15:07:18

 /**
  * parsetrigrams.cpp
  *
  * Parse a corpus of data and generate trigrams
  *
  * SPDX-FileCopyrightText: 2013 Martin Sandsmark <martin.sandsmark@kde.org>
  *
  * SPDX-License-Identifier: LGPL-2.1-or-later
  */
 
 #include "guesslanguage.h"
 
 #include <QDebug>
 #include <QFile>
 #include <QHash>
 #include <QString>
 
 int main(int argc, char *argv[])
 {
     if (argc < 3) {
         qWarning() << argv[0] << "corpus.txt outfile.trigram";
         return -1;
     }
 
     QFile file(QString::fromLocal8Bit(argv[1]));
     if (!file.open(QIODevice::ReadOnly | QFile::Text)) {
         qWarning() << "Unable to open corpus:" << argv[1];
         return -1;
     }
     QTextStream stream(&file);
     // Not needed with Qt6, UTF-8 is the default
 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
     stream.setCodec("UTF-8");
 #endif
 
     QFile outFile(QString::fromLocal8Bit(argv[2]));
     if (!outFile.open(QIODevice::WriteOnly)) {
         qWarning() << "Unable to open output file" << argv[2];
         return -1;
     }
 
     QHash<QString, int> model;
     qDebug() << "Reading in" << file.size() << "bytes";
     QString trigram = stream.read(3);
     QString contents = stream.readAll();
     qDebug() << "finished reading!";
     qDebug() << "Building model...";
     for (int i = 0; i < contents.size(); i++) {
         if (!contents[i].isPrint()) {
             continue;
         }
         model[trigram]++;
         trigram[0] = trigram[1];
         trigram[1] = trigram[2];
         trigram[2] = contents[i];
     }
     qDebug() << "model built!";
 
     qDebug() << "Sorting...";
     QMultiMap<int, QString> orderedTrigrams;
 
     for (auto it = model.cbegin(); it != model.cend(); ++it) {
         const QString data = it.key();
         Q_ASSERT(data.size() >= 3);
         bool hasTwoSpaces = ((data.size() > 1 && data[0].isSpace() && data[1].isSpace()) //
                              || (data.size() > 2 && data[1].isSpace() && data[2].isSpace()));
 
         if (!hasTwoSpaces) {
             orderedTrigrams.insert(it.value(), data);
         }
     }
 
     qDebug() << "Sorted!";
 
     qDebug() << "Weeding out...";
 
     auto i = orderedTrigrams.begin();
     while (orderedTrigrams.size() > Sonnet::MAXGRAMS) {
         i = orderedTrigrams.erase(i);
     }
     qDebug() << "Weeded!";
 
     qDebug() << "Storing...";
     i = orderedTrigrams.end();
     int count = 0;
     QTextStream outStream(&outFile);
     // Not needed with Qt6, UTF-8 is the default
 #if QT_VERSION < QT_VERSION_CHECK(6, 0, 0)
     outStream.setCodec("UTF-8");
 #endif
 
     while (i != orderedTrigrams.begin()) {
         --i;
         outStream << *i << "\t\t\t" << count++ << '\n';
     }
 }