File indexing completed on 2024-09-08 13:28:58

 // clang-format off
 /*
  * KDiff3 - Text Diff And Merge Tool
  *
  * SPDX-FileCopyrightText: 2002-2011 Joachim Eibl, joachim.eibl at gmx.de
  * SPDX-FileCopyrightText: 2018-2020 Michael Reeves reeves.87@gmail.com
  * SPDX-License-Identifier: GPL-2.0-or-later
  */
 // clang-format on
 
 /* Features of class SourceData:
 - Read a file (from the given URL) or accept data via a string.
 - Allocate and free buffers as necessary.
 - Run a preprocessor, when specified.
 - Run the line-matching preprocessor, when specified.
 - Run other preprocessing steps: Uppercase, ignore comments,
                                  remove carriage return, ignore numbers.
 
 Order of operation:
  1. If data was given via a string then save it to a temp file. (see setData())
  2. If the specified file is nonlocal (URL) copy it to a temp file. (TODO revisit this)
  3. If a preprocessor was specified, run the input file through it.
  4. Read the output of the preprocessor.
  5. If Uppercase was specified: Turn the read data to uppercase.
  6. Write the result to a temp file.
  7. If a line-matching preprocessor was specified, run the temp file through it.
  8. Read the output of the line-matching preprocessor.
  9. If ignore numbers was specified, strip the LMPP-output of all numbers.
 10. If ignore comments was specified, strip the LMPP-output of comments.
 
 Optimizations: Skip unneeded steps.
 */
 #include "SourceData.h"
 
 #include "CommentParser.h"
 #include "compat.h"
 #include "diff.h"
 #include "EncodedDataStream.h"
 #include "LineRef.h"
 #include "Logging.h"
 #include "Utils.h"
 
 #include <algorithm>         // for min
 #include <memory>
 #include <optional>
 #include <vector>            // for vector
 
 #include <QtGlobal>
 
 #include <QByteArray>
 #include <QProcess>
 #include <QScopedPointer>
 #include <QString>
 #include <QTemporaryFile>
 #include <QTextCodec>
 #include <QVector>
 
 extern std::unique_ptr<Options> gOptions;
 
 void SourceData::reset()
 {
     mFromClipBoard = false;
     mEncoding = u8"UTF-8";
     m_fileAccess = FileAccess();
     m_normalData.reset();
     m_lmppData.reset();
     if(!m_tempInputFileName.isEmpty())
     {
         m_tempFile.remove();
         m_tempInputFileName = "";
     }
 
     mErrors.clear();
 }
 
 void SourceData::setFilename(const QString& filename)
 {
     if(filename.isEmpty())
     {
         reset();
     }
     else
     {
         setFileAccess(FileAccess(filename));
     }
 }
 
 bool SourceData::isEmpty() const
 {
     return getFilename().isEmpty();
 }
 
 bool SourceData::hasData() const
 {
     return m_normalData.m_pBuf != nullptr;
 }
 
 bool SourceData::isValid() const
 {
     return isEmpty() || hasData();
 }
 
 QString SourceData::getFilename() const
 {
     return m_fileAccess.absoluteFilePath();
 }
 
 QString SourceData::getAliasName() const
 {
     return m_aliasName.isEmpty() ? m_fileAccess.prettyAbsPath() : m_aliasName;
 }
 
 void SourceData::setAliasName(const QString& name)
 {
     m_aliasName = name;
 }
 
 void SourceData::setFileAccess(const FileAccess& fileAccess)
 {
     mFromClipBoard = false;
 
     m_fileAccess = fileAccess;
     m_aliasName = QString();
     if(!m_tempInputFileName.isEmpty())
     {
         m_tempFile.remove();
         m_tempInputFileName = "";
     }
 
     mErrors.clear();
 }
 
 void SourceData::setEncoding(const char* encoding)
 {
     mEncoding = encoding;
 }
 
 void SourceData::setData(const QString& data)
 {
     mErrors.clear();
     // Create a temp file for preprocessing:
     if(m_tempInputFileName.isEmpty())
     {
         FileAccess::createTempFile(m_tempFile);
         m_tempInputFileName = m_tempFile.fileName();
     }
     m_fileAccess = FileAccess(m_tempInputFileName);
     QByteArray ba = QTextCodec::codecForName("UTF-8")->fromUnicode(data);
     bool bSuccess = m_fileAccess.writeFile(ba.constData(), ba.length());
     if(!bSuccess)
     {
         mErrors.append(i18n("Writing clipboard data to temp file failed."));
         return;
     }
     else
     {
         m_aliasName = i18n("From Clipboard");
         mFromClipBoard = true;
     }
 }
 
 const std::shared_ptr<LineDataVector>& SourceData::getLineDataForDiff() const
 {
     if(m_lmppData.m_pBuf == nullptr)
     {
         return m_normalData.m_v;
     }
     else
     {
         return m_lmppData.m_v;
     }
 }
 
 const std::shared_ptr<LineDataVector>& SourceData::getLineDataForDisplay() const
 {
     return m_normalData.m_v;
 }
 
 LineType SourceData::getSizeLines() const
 {
     return SafeInt<LineType>(m_normalData.lineCount());
 }
 
 qint64 SourceData::getSizeBytes() const
 {
     return m_normalData.byteCount();
 }
 
 const char* SourceData::getBuf() const
 {
     return m_normalData.m_pBuf.get();
 }
 
 const QString& SourceData::getText() const
 {
     return *m_normalData.m_unicodeBuf;
 }
 
 bool SourceData::isText() const
 {
     return m_normalData.isText() || m_normalData.isEmpty();
 }
 
 bool SourceData::isIncompleteConversion() const
 {
     return m_normalData.m_bIncompleteConversion;
 }
 
 bool SourceData::isFromBuffer() const
 {
     return mFromClipBoard;
 }
 
 bool SourceData::isBinaryEqualWith(const QSharedPointer<SourceData>& other) const
 {
     return m_fileAccess.exists() && other->m_fileAccess.exists() &&
            getSizeBytes() == other->getSizeBytes() &&
            (getSizeBytes() == 0 || memcmp(getBuf(), other->getBuf(), getSizeBytes()) == 0);
 }
 
 /*
     Warning: Do not call this function without re-running the comparison or
     otherwise resetting the DiffTextWindows as these store a pointer to the file
     data stored here.
 */
 void SourceData::FileData::reset()
 {
     m_pBuf.reset();
     m_v->clear();
     mDataSize = 0;
     mLineCount = 0;
     m_bIsText = false;
     m_bIncompleteConversion = false;
     m_eLineEndStyle = eLineEndStyleUndefined;
 }
 
 bool SourceData::FileData::readFile(FileAccess& file)
 {
     reset();
     if(file.fileName().isEmpty())
     {
         return true;
     }
 
     if(!file.isNormal())
         return true;
 
     mDataSize = file.sizeForReading();
     /*
         If the extra bytes are removed an unknown heap currption issue is triggered in the
         diff code. I don't have time to track this down to its true root cause.
     */
     m_pBuf = std::make_unique<char[]>(mDataSize + 100); // Alloc 100 byte extra: Safety hack, not nice but does no harm.
                                                         // Some extra bytes at the end of the buffer are needed by
                                                         // the diff algorithm. See also GnuDiff::diff_2_files().
     bool bSuccess = file.readFile(m_pBuf.get(), mDataSize);
     if(!bSuccess)
     {
         m_pBuf = nullptr;
         mDataSize = 0;
     }
     else
     {
         //null terminate buffer
         m_pBuf[mDataSize + 1] = 0;
         m_pBuf[mDataSize + 2] = 0;
         m_pBuf[mDataSize + 3] = 0;
         m_pBuf[mDataSize + 4] = 0;
     }
     return bSuccess;
 }
 
 bool SourceData::FileData::readFile(const QString& filename)
 {
     reset();
     if(filename.isEmpty())
     {
         return true;
     }
 
     FileAccess fa(filename);
     return readFile(fa);
 }
 
 bool SourceData::saveNormalDataAs(const QString& fileName)
 {
     return m_normalData.writeFile(fileName);
 }
 
 bool SourceData::FileData::writeFile(const QString& filename)
 {
     if(filename.isEmpty())
     {
         return true;
     }
 
     FileAccess fa(filename);
     bool bSuccess = fa.writeFile(m_pBuf.get(), mDataSize);
     return bSuccess;
 }
 
 //Deprecated
 void SourceData::FileData::copyBufFrom(const FileData& src) //TODO: Remove me.
 {
     reset();
     mDataSize = src.mDataSize;
     m_pBuf = std::make_unique<char[]>(mDataSize + 100);
     assert(src.m_pBuf != nullptr);
     memcpy(m_pBuf.get(), src.m_pBuf.get(), mDataSize);
 }
 
 std::optional<const char*> SourceData::detectEncoding(const QString& fileName)
 {
     QFile f(fileName);
     if(f.open(QIODevice::ReadOnly))
     {
         char buf[400];
 
         qint64 size = f.read(buf, sizeof(buf));
         FileOffset skipBytes = 0;
         return detectEncoding(buf, size, skipBytes);
     }
     return {};
 }
 
 void SourceData::readAndPreprocess(const char* encoding, bool bAutoDetect)
 {
     QTemporaryFile fileIn1, fileOut1;
     QString fileNameIn1;
     QString fileNameOut1;
     QString fileNameIn2;
     QString fileNameOut2;
 
     mEncoding = encoding;
 
     // Detect the input for the preprocessing operations
     if(!mFromClipBoard)
     {
         //Routine result of directory compare finding a file that isn't in all locations.
         if(!m_fileAccess.isValid()) return;
 
         assert(!m_fileAccess.exists() || !m_fileAccess.isDir());
         if(!m_fileAccess.isNormal())
         {
             mErrors.append(i18n("%1 is not a normal file.", m_fileAccess.prettyAbsPath()));
             return;
         }
 
         if(m_fileAccess.isLocal())
         {
             fileNameIn1 = m_fileAccess.absoluteFilePath();
         }
         else // File is not local: create a temporary local copy:
         {
             if(m_tempInputFileName.isEmpty())
             {
                 m_fileAccess.createLocalCopy();
                 m_tempInputFileName = m_fileAccess.getTempName();
             }
 
             fileNameIn1 = m_tempInputFileName;
         }
         if(bAutoDetect)
         {
             mEncoding = detectEncoding(fileNameIn1).value_or(encoding);
         }
     }
     else // The input was set via setData(), probably from clipboard.
     {
         /*
             Used to happen during early startup this is now a bug.
         */
         assert(!m_tempInputFileName.isEmpty());
 
         fileNameIn1 = m_tempInputFileName;
         mEncoding = "UTF-8";
     }
     const char* pEncoding1 = getEncoding();
     const char* pEncoding2 = getEncoding();
     const QString overSizedFile = i18nc("Error message. %1 = filepath", "File %1 too large to process. Skipping.", fileNameIn1);
 
     m_normalData.reset();
     m_lmppData.reset();
 
     FileAccess faIn(fileNameIn1);
     qint64 fileInSize = faIn.size();
 
     if(faIn.exists() && !faIn.isBrokenLink())
     {
         try
         {
             // Run the first preprocessor
             if(gOptions->m_PreProcessorCmd.isEmpty())
             {
                 // No preprocessing: Read the file directly:
                 if(!m_normalData.readFile(faIn))
                 {
                     mErrors.append(faIn.getStatusText());
                     return;
                 }
             }
             else
             {
                 unsigned char b;
                 //Don't fail the preprocessor command if the file can't be read.
                 if(!faIn.readFile(&b, 1))
                 {
                     mErrors.append(faIn.getStatusText());
                     mErrors.append(i18n("    Temp file is: %1", fileNameIn1));
                     return;
                 }
 
                 QTemporaryFile tmpInPPFile;
                 QString fileNameInPP = fileNameIn1;
 
                 if(pEncoding1 != gOptions->mEncodingPP)
                 {
                     // Before running the preprocessor convert to the format that the preprocessor expects.
                     FileAccess::createTempFile(tmpInPPFile);
                     fileNameInPP = tmpInPPFile.fileName();
                     pEncoding1 = gOptions->mEncodingPP;
                     convertFileEncoding(fileNameIn1, encoding, fileNameInPP, pEncoding1);
                 }
 
                 QString ppCmd = gOptions->m_PreProcessorCmd;
                 FileAccess::createTempFile(fileOut1);
                 fileNameOut1 = fileOut1.fileName();
 
                 QProcess ppProcess;
                 ppProcess.setStandardInputFile(fileNameInPP);
                 ppProcess.setStandardOutputFile(fileNameOut1);
                 QString program;
                 QStringList args;
                 QString errorReason = Utils::getArguments(ppCmd, program, args);
                 if(errorReason.isEmpty())
                 {
                     ppProcess.start(program, args);
                     ppProcess.waitForFinished(-1);
                 }
                 else
                     errorReason = "\n(" + errorReason + ')';
 
                 bool bSuccess = errorReason.isEmpty() && m_normalData.readFile(fileNameOut1);
                 if(fileInSize > 0 && (!bSuccess || m_normalData.byteCount() == 0))
                 {
                     mErrors.append(
                         i18n("Preprocessing possibly failed. Check this command:\n\n  %1"
                              "\n\nThe preprocessing command will be disabled now.",
                              ppCmd) +
                         errorReason);
                     gOptions->m_PreProcessorCmd = "";
 
                     pEncoding1 = getEncoding();
                 }
             }
         }
         catch(const std::bad_alloc&)
         {
             m_normalData.reset();
             mErrors.append(overSizedFile);
             return;
         }
 
         if(!m_normalData.preprocess(pEncoding1, false))
         {
             mErrors.append(overSizedFile);
             return;
         }
         //exit early for non text data further processing assumes a text file as input
         if(!m_normalData.isText())
             return;
 
         // LineMatching Preprocessor
         if(!gOptions->m_LineMatchingPreProcessorCmd.isEmpty())
         {
             QTemporaryFile tempOut2, fileInPP;
             fileNameIn2 = fileNameOut1.isEmpty() ? fileNameIn1 : fileNameOut1;
             QString fileNameInPP = fileNameIn2;
             pEncoding2 = pEncoding1;
             if(pEncoding2 != gOptions->mEncodingPP)
             {
                 // Before running the preprocessor convert to the format that the preprocessor expects.
                 FileAccess::createTempFile(fileInPP);
                 fileNameInPP = fileInPP.fileName();
                 pEncoding2 = gOptions->mEncodingPP;
                 convertFileEncoding(fileNameIn2, pEncoding1, fileNameInPP, pEncoding2);
             }
 
             QString ppCmd = gOptions->m_LineMatchingPreProcessorCmd;
             FileAccess::createTempFile(tempOut2);
             fileNameOut2 = tempOut2.fileName();
             QProcess ppProcess;
             ppProcess.setStandardInputFile(fileNameInPP);
             ppProcess.setStandardOutputFile(fileNameOut2);
             QString program;
             QStringList args;
             QString errorReason = Utils::getArguments(ppCmd, program, args);
             if(errorReason.isEmpty())
             {
                 ppProcess.start(program, args);
                 ppProcess.waitForFinished(-1);
             }
             else
                 errorReason = "\n(" + errorReason + ')';
 
             bool bSuccess = errorReason.isEmpty() && m_lmppData.readFile(fileNameOut2);
             if(FileAccess(fileNameIn2).size() > 0 && (!bSuccess || m_lmppData.byteCount() == 0))
             {
                 mErrors.append(
                     i18n("The line-matching-preprocessing possibly failed. Check this command:\n\n  %1"
                          "\n\nThe line-matching-preprocessing command will be disabled now.", ppCmd) +
                     errorReason);
                 gOptions->m_LineMatchingPreProcessorCmd = "";
                 if(!m_lmppData.readFile(fileNameIn2))
                 {
                     mErrors.append(i18nc("Read error message. %1 = filepath", "Failed to read file: %1", fileNameIn2));
                     return;
                 }
             }
         }
         else if(gOptions->ignoreComments() || gOptions->m_bIgnoreCase)
         {
             // We need a copy of the normal data.
             m_lmppData.copyBufFrom(m_normalData);
         }
     }
     else
     {
         //exit early for nonexistent files
         return;
     }
 
     if(!m_lmppData.preprocess(pEncoding2, true))
     {
         mErrors.append(overSizedFile);
         return;
     }
 
     assert(m_lmppData.isText());
     //TODO: Needed?
     if(m_lmppData.lineCount() < m_normalData.lineCount())
     {
         // Preprocessing command may result in smaller data buffer so adjust size
         for(qint64 i = m_lmppData.lineCount(); i < m_normalData.lineCount(); ++i)
         { // Set all empty lines to point to the end of the buffer.
             m_lmppData.m_v->push_back(LineData(m_lmppData.m_unicodeBuf, m_lmppData.m_unicodeBuf->length()));
         }
 
         m_lmppData.mLineCount = m_normalData.lineCount();
     }
 
     // Ignore comments
     if(gOptions->ignoreComments() && hasData())
     {
         qint64 vSize = std::min(m_normalData.lineCount(), m_lmppData.lineCount());
 
         for(qint64 i = 0; i < vSize; ++i)
         {
             //TODO: Phase this out. We should not be messing with these flags outside the parser.
             (*m_normalData.m_v)[i].setPureComment((*m_lmppData.m_v)[i].isPureComment());
             (*m_normalData.m_v)[i].setSkipable((*m_lmppData.m_v)[i].isSkipable());
         }
     }
 }
 
 /** Prepare the linedata vector for every input line.*/
 bool SourceData::FileData::preprocess(const QByteArray& encoding, bool removeComments)
 {
     if(m_pBuf == nullptr)
         return true;
 
     QString line;
     QChar curChar, prevChar = '\0';
     LineType lines = 0;
     QtSizeType lastOffset = 0;
     FileOffset skipBytes = 0;
     QScopedPointer<CommentParser> parser(new DefaultCommentParser());
 
     // detect line end style
     QVector<e_LineEndStyle> vOrigDataLineEndStyle;
     m_eLineEndStyle = eLineEndStyleUndefined;
 
     QByteArray pCodec = detectEncoding(m_pBuf.get(), mDataSize, skipBytes).value_or(encoding);
     if(pCodec != encoding)
         skipBytes = 0;
 
     if(mDataSize - skipBytes > limits<qint32>::max())
     {
         reset();
         return false;
     }
 
     try
     {
         const QByteArray ba = QByteArray::fromRawData(m_pBuf.get() + skipBytes, (QtSizeType)(mDataSize - skipBytes));
         EncodedDataStream ds(ba);
 
         mHasBOM = skipBytes != 0;
         ds.setEncoding(encoding);
         ds.setGenerateByteOrderMark(skipBytes != 0);
 
         m_bIncompleteConversion = false;
         m_unicodeBuf->clear();
 
         assert(m_unicodeBuf->length() == 0);
 
         mHasEOLTermination = false;
         bool skipNextRead = false;
         while(!ds.atEnd())
         {
             line.clear();
             if(lines >= limits<LineType>::max() - 5)
             {
                 reset();
                 return false;
             }
 
             if(!skipNextRead)
             {
                 prevChar = curChar;
                 ds.readChar(curChar);
             }
             else
                 skipNextRead = false;
 
             QtSizeType firstNonwhite = 0;
             bool foundNonWhite = false;
 
             while(curChar != '\n' && curChar != '\r')
             {
                 if(curChar.isNull() || curChar.isNonCharacter())
                 {
                     m_v->clear();
                     return true;
                 }
 
                 if(curChar == QChar::ReplacementCharacter)
                     m_bIncompleteConversion = true;
 
                 line.append(curChar);
                 if(!curChar.isSpace() && !foundNonWhite)
                 {
                     firstNonwhite = line.length();
                     foundNonWhite = true;
                 }
 
                 if(ds.atEnd())
                     break;
 
                 prevChar = curChar;
                 ds.readChar(curChar);
             }
 
             switch(curChar.unicode())
             {
                 case '\n':
                     vOrigDataLineEndStyle.push_back(eLineEndStyleUnix);
                     break;
                 case '\r':
                     if((FileOffset)lastOffset < mDataSize)
                     {
                         prevChar = curChar;
                         ds.readChar(curChar);
 
                         if(curChar == '\n')
                         {
                             vOrigDataLineEndStyle.push_back(eLineEndStyleDos);
                             break;
                         }
                         //work around for lack of seek API in QDataStream
                         skipNextRead = true;
                     }
 
                     //old mac style ending.
                     vOrigDataLineEndStyle.push_back(eLineEndStyleOldMac);
                     break;
             }
             parser->processLine(line);
             if(removeComments)
                 parser->removeComment(line);
             //Qt6 intrudes 64bit sizes
             if(line.size() >= limits<LineType>::max())
             {
                 reset();
                 return false;
             }
 
             ++lines;
             m_v->push_back(LineData(m_unicodeBuf, lastOffset, line.length(), firstNonwhite, parser->isSkipable(), parser->isPureComment()));
             //The last line may not have an EOL mark. In that case don't add one to our buffer.
             m_unicodeBuf->append(line);
             if(curChar == '\n' || curChar == '\r' || prevChar == '\r')
             {
                 //kdiff3 internally uses only unix style endings for simplicity.
                 m_unicodeBuf->append('\n');
             }
 
             assert(m_unicodeBuf->length() != lastOffset);
             lastOffset = m_unicodeBuf->length();
         }
 
         /*
             Process trailing new line as if there were a blank non-terminated line after it.
             But do nothing to the data buffer since this is a phantom line needed for internal purposes.
         */
         if(curChar == '\n' || curChar == '\r')
         {
             mHasEOLTermination = true;
             ++lines;
 
             parser->processLine("");
             m_v->push_back(LineData(m_unicodeBuf, lastOffset, 0, 0, parser->isSkipable(), parser->isPureComment()));
         }
 
         m_v->push_back(LineData(m_unicodeBuf, lastOffset));
 
         m_bIsText = true;
 
         if(!vOrigDataLineEndStyle.isEmpty())
             m_eLineEndStyle = vOrigDataLineEndStyle[0];
 
         mLineCount = lines;
         return true;
     }
     catch(const std::bad_alloc&)
     {
         reset();
         return false;
     }
 }
 
 // Convert the input file from input encoding to output encoding and write it to the output file.
 bool SourceData::convertFileEncoding(const QString& fileNameIn, const QByteArray& pCodecIn,
                                      const QString& fileNameOut, const QByteArray& pCodecOut)
 {
     QFile in(fileNameIn);
     if(!in.open(QIODevice::ReadOnly))
         return false;
     EncodedDataStream inStream(&in);
     inStream.setEncoding(pCodecIn);
 
     QFile out(fileNameOut);
     if(!out.open(QIODevice::WriteOnly))
         return false;
     EncodedDataStream outStream(&out);
     outStream.setEncoding(pCodecOut);
 
     QString data;
     while(!inStream.atEnd())
     {
         QChar c;
         inStream.readChar(c);
         data += c;
     }
     outStream << data;
 
     return true;
 }
 
 std::optional<const char*> SourceData::getEncodingFromTag(const QByteArray& s, const QByteArray& encodingTag)
 {
     QtSizeType encodingPos = s.indexOf(encodingTag);
     if(encodingPos >= 0)
     {
         QtSizeType apostrophPos = s.indexOf('"', encodingPos + encodingTag.length());
         QtSizeType apostroph2Pos = s.indexOf('\'', encodingPos + encodingTag.length());
         char apostroph = '"';
         if(apostroph2Pos >= 0 && (apostrophPos < 0 || apostroph2Pos < apostrophPos))
         {
             apostroph = '\'';
             apostrophPos = apostroph2Pos;
         }
 
         QtSizeType encodingEnd = s.indexOf(apostroph, apostrophPos + 1);
         if(encodingEnd >= 0) // e.g.: <meta charset="utf-8"> or <?xml version="1.0" encoding="ISO-8859-1"?>
         {
             QByteArray encoding = s.mid(apostrophPos + 1, encodingEnd - (apostrophPos + 1));
             if(QTextCodec::codecForName(encoding))
                 return encoding;
         }
         else // e.g.: <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         {
             QByteArray encoding = s.mid(encodingPos + encodingTag.length(), apostrophPos - (encodingPos + encodingTag.length()));
             if(QTextCodec::codecForName(encoding))
                 return encoding;
         }
     }
     return {};
 }
 
 std::optional<const char*> SourceData::detectEncoding(const char* buf, qint64 size, FileOffset& skipBytes)
 {
     if(size >= 2)
     {
         if(buf[0] == '\xFF' && buf[1] == '\xFE')
         {
             skipBytes = 2;
             return "UTF-16LE";
         }
 
         if(buf[0] == '\xFE' && buf[1] == '\xFF')
         {
             skipBytes = 2;
             return "UTF-16BE";
         }
     }
     if(size >= 3)
     {
         if(buf[0] == '\xEF' && buf[1] == '\xBB' && buf[2] == '\xBF')
         {
             skipBytes = 3;
             return "UTF-8-BOM";
         }
     }
     skipBytes = 0;
     QByteArray s;
     /*
         We don't need the whole file here just the header.
     */
     if(size <= 5000)
         s = QByteArray(buf, (QtSizeType)size);
     else
         s = QByteArray(buf, 5000);
 
     QtSizeType xmlHeaderPos = s.indexOf("<?xml");
     if(xmlHeaderPos >= 0)
     {
         QtSizeType xmlHeaderEnd = s.indexOf("?>", xmlHeaderPos);
         if(xmlHeaderEnd >= 0)
         {
             std::optional<const char*> encoding = getEncodingFromTag(s.mid(xmlHeaderPos, xmlHeaderEnd - xmlHeaderPos), "encoding=");
             if(encoding.has_value())
                 return encoding;
         }
     }
     else // HTML
     {
         QtSizeType metaHeaderPos = s.indexOf("<meta");
         while(metaHeaderPos >= 0)
         {
             QtSizeType metaHeaderEnd = s.indexOf(">", metaHeaderPos);
             if(metaHeaderEnd >= 0)
             {
                 std::optional<const char*> encoding = getEncodingFromTag(s.mid(metaHeaderPos, metaHeaderEnd - metaHeaderPos), "charset=");
                 if(encoding.has_value())
                     return encoding;
 
                 metaHeaderPos = s.indexOf("<meta", metaHeaderEnd);
             }
             else
                 break;
         }
     }
     //Attempt to detect non-bom UTF8. This is a very common encoding.
     return detectUTF8(s);
 }
 
 std::optional<const char*> SourceData::detectUTF8(const QByteArray& data)
 {
     QTextCodec* utf8 = QTextCodec::codecForName("UTF-8");
 
     QTextCodec::ConverterState state;
     utf8->toUnicode(data.constData(), SafeInt<qint32>(data.size()), &state);
 
     if(state.invalidChars == 0)
         for (qint32 i = 0; i < data.size()-state.remainingChars; i++)
             if ((unsigned)data.at(i) > 127)
                 return "UTF-8";
 
     return {};
 }