File indexing completed on 2024-05-05 16:10:18

 /*
     This file is part of the KDE libraries
 
     Copyright (C) 1997 Martin Jones (mjones@kde.org)
               (C) 1997 Torben Weis (weis@kde.org)
               (C) 1998 Waldo Bastian (bastian@kde.org)
               (C) 1999 Lars Knoll (knoll@kde.org)
               (C) 1999 Antti Koivisto (koivisto@kde.org)
               (C) 2001-2003 Dirk Mueller (mueller@kde.org)
               (C) 2004-2008 Apple Computer, Inc.
               (C) 2006-2008 Germain Garand (germain@ebooksfrance.org)
 
     This library is free software; you can redistribute it and/or
     modify it under the terms of the GNU Library General Public
     License as published by the Free Software Foundation; either
     version 2 of the License, or (at your option) any later version.
 
     This library is distributed in the hope that it will be useful,
     but WITHOUT ANY WARRANTY; without even the implied warranty of
     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
     Library General Public License for more details.
 
     You should have received a copy of the GNU Library General Public License
     along with this library; see the file COPYING.LIB.  If not, write to
     the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
     Boston, MA 02110-1301, USA.
 */
 //----------------------------------------------------------------------------
 //
 // KDE HTML Widget - Tokenizers
 
 // #define TOKEN_DEBUG 1
 // #define TOKEN_DEBUG 2
 
 #include "htmltokenizer.h"
 #include "html_documentimpl.h"
 #include "htmlparser.h"
 #include "dtd.h"
 
 #include <misc/loader.h>
 
 #include <khtmlview.h>
 #include <khtml_part.h>
 #include <xml/dom_docimpl.h>
 #include <ecma/kjs_proxy.h>
 #include <kcharsets.h>
 #include <ctype.h>
 #include <assert.h>
 #include <QVariant>
 #include "khtml_debug.h"
 #include <stdlib.h>
 
 #include "kentities_p.h"
 #include "htmlprospectivetokenizer.h"
 
 #define PROSPECTIVE_TOKENIZER_ENABLED 1
 
 using namespace khtml;
 
 static const QChar commentStart [] = { '<', '!', '-', '-', QChar::Null };
 static const char doctypeStart [] = "<!doctype";
 static const char publicStart [] = "public";
 static const char systemStart [] = "system";
 
 static const char scriptEnd [] = "</script";
 static const char xmpEnd [] = "</xmp";
 static const char styleEnd [] =  "</style";
 static const char textareaEnd [] = "</textarea";
 static const char titleEnd [] = "</title";
 
 #ifndef NDEBUG
 static const int sTokenizerChunkSize = 2048;
 static const int sTokenizerFastYieldDelay = 220;
 static const int sTokenizerYieldDelay = 650;
 #else
 static const int sTokenizerChunkSize = 4096;
 static const int sTokenizerFastYieldDelay = 180;
 static const int sTokenizerYieldDelay = 450;
 #endif
 
 #define KHTML_ALLOC_QCHAR_VEC( N ) (QChar*) malloc( sizeof(QChar)*( N ) )
 #define KHTML_REALLOC_QCHAR_VEC(P, N ) (QChar*) realloc(P, sizeof(QChar)*( N ))
 #define KHTML_DELETE_QCHAR_VEC( P ) free((char*)( P ))
 
 // Full support for MS Windows extensions to Latin-1.
 // Technically these extensions should only be activated for pages
 // marked "windows-1252" or "cp1252", but
 // in the standard Microsoft way, these extensions infect hundreds of thousands
 // of web pages.  Note that people with non-latin-1 Microsoft extensions
 // are SOL.
 //
 // See: http://www.microsoft.com/globaldev/reference/WinCP.asp
 //      http://www.bbsinc.com/iso8859.html
 //      http://www.obviously.com/
 //
 // There may be better equivalents
 #if 0
 #define fixUpChar(x)
 #else
 #define fixUpChar(x) \
     switch ((x).unicode()) \
     { \
     case 0x80: (x) = 0x20ac; break; \
     case 0x82: (x) = 0x201a;    break; \
     case 0x83: (x) = 0x0192; break; \
     case 0x84: (x) = 0x201e;    break; \
     case 0x85: (x) = 0x2026; break; \
     case 0x86: (x) = 0x2020; break; \
     case 0x87: (x) = 0x2021; break; \
     case 0x88: (x) = 0x02C6; break; \
     case 0x89: (x) = 0x2030; break; \
     case 0x8A: (x) = 0x0160; break; \
     case 0x8b: (x) = 0x2039;    break; \
     case 0x8C: (x) = 0x0152; break; \
     case 0x8E: (x) = 0x017D; break; \
     case 0x91: (x) = 0x2018;   break; \
     case 0x92: (x) = 0x2019;   break; \
     case 0x93: (x) = 0x201C;    break; \
     case 0x94: (x) = 0X201D;    break; \
     case 0x95: (x) = 0x2022;    break; \
     case 0x96: (x) = 0x2013;    break; \
     case 0x97: (x) = 0x2014;    break; \
     case 0x98: (x) = 0x02DC;    break; \
     case 0x99: (x) = 0x2122; break; \
     case 0x9A: (x) = 0x0161; break; \
     case 0x9b: (x) = 0x203A;    break; \
     case 0x9C: (x) = 0x0153; break; \
     case 0x9E: (x) = 0x017E; break; \
     case 0x9F: (x) = 0x0178; break; \
     default: break; \
     }
 #endif
 // ----------------------------------------------------------------------------
 
 HTMLTokenizer::HTMLTokenizer(DOM::DocumentImpl *_doc, KHTMLView *_view)
 {
     view = _view;
     buffer = nullptr;
     rawContent = nullptr;
     rawContentSize = rawContentMaxSize = rawContentResync = rawContentSinceLastEntity = 0;
     charsets = KCharsets::charsets();
     parser = new KHTMLParser(_view, _doc);
     m_executingScript = 0;
     m_externalScriptsTimerId = 0;
     m_tokenizerYieldDelay = sTokenizerFastYieldDelay;
     m_yieldTimer = 0;
     m_prospectiveTokenizer = nullptr;
     onHold = false;
     m_documentTokenizer = true;
     m_hasScriptsWaitingForStylesheets = false;
 
     reset();
 }
 
 HTMLTokenizer::HTMLTokenizer(DOM::DocumentImpl *_doc, DOM::DocumentFragmentImpl *i)
 {
     view = nullptr;
     buffer = nullptr;
     rawContent = nullptr;
     rawContentSize = rawContentMaxSize = rawContentResync = rawContentSinceLastEntity = 0;
     charsets = KCharsets::charsets();
     parser = new KHTMLParser(i, _doc);
     m_executingScript = 0;
     m_externalScriptsTimerId = 0;
     m_tokenizerYieldDelay = sTokenizerFastYieldDelay;
     m_yieldTimer = 0;
     m_prospectiveTokenizer = nullptr;
     onHold = false;
     m_documentTokenizer = false;
     m_hasScriptsWaitingForStylesheets = false;
 
     reset();
 }
 
 void HTMLTokenizer::setNormalYieldDelay()
 {
     m_tokenizerYieldDelay = sTokenizerYieldDelay;
 }
 
 void HTMLTokenizer::reset()
 {
     assert(m_executingScript == 0);
     Q_ASSERT(onHold == false);
     m_abort = false;
 
     while (!cachedScript.isEmpty()) {
         cachedScript.dequeue()->deref(this);
     }
 
     if (buffer) {
         KHTML_DELETE_QCHAR_VEC(buffer);
     }
     buffer = dest = nullptr;
     size = 0;
 
     if (rawContent) {
         KHTML_DELETE_QCHAR_VEC(rawContent);
     }
     rawContent = nullptr;
     rawContentSize = rawContentMaxSize = rawContentResync = 0;
 
     if (m_yieldTimer > 0) {
         killTimer(m_yieldTimer);
         m_yieldTimer = 0;
     }
 
     if (m_externalScriptsTimerId > 0) {
         killTimer(m_externalScriptsTimerId);
         m_externalScriptsTimerId = 0;
     }
     currToken.reset();
     doctypeToken.reset();
     javascript = false;
 }
 
 void HTMLTokenizer::begin()
 {
     m_executingScript = 0;
     onHold = false;
     reset();
     size = 254;
     buffer = KHTML_ALLOC_QCHAR_VEC(255);
     dest = buffer;
     tag = NoTag;
     pending = NonePending;
     discard = NoneDiscard;
     pre = false;
     prePos = 0;
     plaintext = false;
     xmp = false;
     processingInstruction = false;
     script = false;
     escaped = false;
     style = false;
     skipLF = false;
     select = false;
     comment = false;
     doctype = false;
     doctypeComment = NoDoctypeComment;
     doctypeAllowComment = false;
     server = false;
     textarea = false;
     title = false;
     startTag = false;
     tquote = NoQuote;
     searchCount = 0;
     doctypeSearchCount = 0;
     doctypeSecondarySearchCount = 0;
     Entity = NoEntity;
     noMoreData = false;
     brokenComments = false;
     brokenServer = false;
     lineno = 0;
     scriptStartLineno = 0;
     tagStartLineno = 0;
 }
 
 void HTMLTokenizer::processListing(TokenizerString list)
 {
     bool old_pre = pre;
 
     // This function adds the listing 'list' as
     // preformatted text-tokens to the token-collection
     // thereby converting TABs.
     if (!style) {
         pre = true;
     }
     prePos = 0;
 
     while (!list.isEmpty()) {
         checkBuffer(3 * TAB_SIZE);
 
         if (skipLF && (list->unicode() != '\n')) {
             skipLF = false;
         }
 
         if (skipLF) {
             skipLF = false;
             ++list;
         } else if ((list->unicode() == '\n') || (list->unicode() == '\r')) {
             if (discard == LFDiscard) {
                 // Ignore this LF
                 discard = NoneDiscard; // We have discarded 1 LF
             } else {
                 // Process this LF
                 if (pending) {
                     addPending();
                 }
 
                 // we used to do it not at all and we want to have
                 // it fixed for textarea. So here we are
                 if (textarea) {
                     prePos++;
                     *dest++ = *list;
                 } else {
                     pending = LFPending;
                 }
             }
             /* Check for MS-DOS CRLF sequence */
             if (list->unicode() == '\r') {
                 skipLF = true;
             }
             ++list;
         } else if ((list->unicode() == ' ') || (list->unicode() == '\t')) {
             if (pending) {
                 addPending();
             }
             if (*list == ' ') {
                 pending = SpacePending;
             } else {
                 pending = TabPending;
             }
 
             ++list;
         } else {
             discard = NoneDiscard;
             if (pending) {
                 addPending();
             }
 
             prePos++;
             *dest++ = *list;
             ++list;
         }
 
     }
 
     if ((pending == SpacePending) || (pending == TabPending)) {
         addPending();
     } else {
         pending = NonePending;
     }
 
     prePos = 0;
     pre = old_pre;
 }
 
 void HTMLTokenizer::parseRawContent(TokenizerString &src)
 {
     // The 'raw content' mode is a very lax tokenizing mode
     // that will absorb anything but the exact closing tag
     // that made us enter this mode, *except* if it inside a comment.
     //
     // Any other tag or comment will be passed verbatim to the parser as part
     // of the content. It is used for script, style, and a few others.
     //
     assert(textarea || title || !Entity);
     assert(!tag);
     assert(xmp + textarea + title + style + script == 1);
     if (script) {
         scriptStartLineno = lineno + src.lineCount();
     }
 
     if (comment) {
         parseComment(src);
     }
 
     while (!src.isEmpty()) {
         checkRawContentBuffer();
         unsigned char ch = src->toLatin1();
         if (!rawContentResync && !brokenComments && !xmp && ch == '-' &&
                 rawContentSize >= 3 && ((!textarea && !title) || rawContentSinceLastEntity >= 3) && !src.escaped() &&
                 QString::fromRawData(rawContent + rawContentSize - 3, 3) == "<!-") {
             comment = true;
             rawContent[ rawContentSize++ ] = ch;
             ++src;
             parseComment(src);
             continue;
         }
         if (rawContentResync && !tquote && (ch == '>')) {
             ++src;
             rawContentSize = rawContentResync - 1;
             rawContentResync = 0;
             rawContent[ rawContentSize ] = rawContent[ rawContentSize + 1 ] = 0;
             if (script) {
                 scriptHandler();
             } else {
                 processListing(TokenizerString(rawContent, rawContentSize));
                 processToken();
                 if (style)         {
                     currToken.tid = ID_STYLE + ID_CLOSE_TAG;
                 } else if (textarea) {
                     currToken.tid = ID_TEXTAREA + ID_CLOSE_TAG;
                 } else if (title) {
                     currToken.tid = ID_TITLE + ID_CLOSE_TAG;
                 } else if (xmp)  {
                     currToken.tid = ID_XMP + ID_CLOSE_TAG;
                 }
                 processToken();
                 script = style = textarea = title = xmp = false;
                 tquote = NoQuote;
                 rawContentSize = rawContentResync = 0;
             }
             return;
         }
         // possible end of tagname, lets check.
         if (!rawContentResync && !escaped && !src.escaped() && (ch == '>' || ch == '/' || ch <= ' ') && ch &&
                 rawContentSize >= searchStopperLen && ((!textarea && !title) || rawContentSinceLastEntity >= searchStopperLen) &&
                 QString::compare(QString::fromRawData(rawContent + rawContentSize - searchStopperLen, searchStopperLen),
                                  QLatin1String(searchStopper), Qt::CaseInsensitive) == 0) {
 
             // the purpose of rawContentResync is to look for an end tag that could possibly be of the form:
             // </endtag   junk="more junk>\"><>"     >
             // IOW, once the '</endtag' sequence has been found, the rest of the tag must still be validated,
             // so this micro-tokenizer switches to rawContentResync state until '>' is finally found.
             rawContentResync = rawContentSize - searchStopperLen + 1;
             tquote = NoQuote;
             continue;
         }
         if (rawContentResync && !escaped) {
             if (ch == '\"') {
                 tquote = (tquote == NoQuote) ? DoubleQuote : ((tquote == SingleQuote) ? SingleQuote : NoQuote);
             } else if (ch == '\'') {
                 tquote = (tquote == NoQuote) ? SingleQuote : (tquote == DoubleQuote) ? DoubleQuote : NoQuote;
             } else if (tquote != NoQuote && (ch == '\r' || ch == '\n')) {
                 tquote = NoQuote;
             }
         }
         escaped = (!escaped && ch == '\\');
         if (!rawContentResync && (textarea || title) && !src.escaped() && ch == '&') {
             QChar *rawContentDest = rawContent + rawContentSize;
             ++src;
             parseEntity(src, rawContentDest, true);
             rawContentSize = rawContentDest - rawContent;
         } else {
             rawContent[ rawContentSize++ ] = *src;
             ++src;
             ++rawContentSinceLastEntity;
         }
     }
 }
 
 void HTMLTokenizer::scriptHandler()
 {
     QString currentScriptSrc = scriptSrc;
     scriptSrc.clear();
 
     processListing(TokenizerString(rawContent, rawContentSize));
     QString exScript(buffer, dest - buffer);
 
     processToken();
     currToken.tid = ID_SCRIPT + ID_CLOSE_TAG;
     processToken();
 
     // Scripts following a frameset element should not be executed or even loaded in the case of extern scripts.
     bool followingFrameset = (parser->doc()->body() && parser->doc()->body()->id() == ID_FRAMESET);
     bool effectiveScript = !parser->skipMode() && !followingFrameset;
     bool deferredScript = false;
 
     if (effectiveScript) {
         CachedScript *cs = nullptr;
 
         // forget what we just got, load from src url instead
         if (!currentScriptSrc.isEmpty() && javascript) {
             const QString completeScriptUrl = parser->doc()->completeURL(currentScriptSrc);
             cs = parser->doc()->docLoader()->requestScript(completeScriptUrl, scriptSrcCharset);
         }
 
         if (cs) {
             cachedScript.enqueue(cs);
             pendingQueue.push(src);
             int scriptCount = cachedScript.count();
             setSrc(TokenizerString());
             rawContentSize = rawContentResync = 0;
             cs->ref(this);
             if (cachedScript.count() == scriptCount) {
                 deferredScript = true;
             }
         } else if (currentScriptSrc.isNull()/*no src attribute*/ && view && javascript) {
             pendingQueue.push(src);
             setSrc(TokenizerString());
             rawContentSize = rawContentResync = 0;
             scriptExecution(exScript, QString(), tagStartLineno /*scriptStartLineno*/);
         } else {
             // script was filtered or disallowed
             effectiveScript = false;
         }
     }
 
     script = false;
     rawContentSize = rawContentResync = 0;
 
     if (!effectiveScript) {
         return;
     }
 
     if (!m_executingScript && cachedScript.isEmpty()) {
         src.append(pendingQueue.pop());
     } else if (cachedScript.isEmpty()) {
         write(pendingQueue.pop(), false);
     } else if (!deferredScript && pendingQueue.count() > 1) {
         TokenizerString t = pendingQueue.pop();
         pendingQueue.top().prepend(t);
     }
 #if PROSPECTIVE_TOKENIZER_ENABLED
     if (!cachedScript.isEmpty() && !m_executingScript) {
         if (!m_prospectiveTokenizer) {
             m_prospectiveTokenizer = new ProspectiveTokenizer(parser->docPtr());
         }
         if (!m_prospectiveTokenizer->inProgress() && !pendingQueue.isEmpty()) {
             m_prospectiveTokenizer->begin();
             m_prospectiveTokenizer->write(pendingQueue.top());
         }
     }
 #endif
 
 }
 
 void HTMLTokenizer::scriptExecution(const QString &str, const QString &scriptURL,
                                     int baseLine)
 {
     bool oldscript = script;
     m_executingScript++;
     script = false;
     QString url;
     if (scriptURL.isNull() && view) {
         url = static_cast<DocumentImpl *>(view->part()->document().handle())->URL().url();
     } else {
         url = scriptURL;
     }
 
     if (view) {
         view->part()->executeScript(url, baseLine, Node(), str);
     }
     m_executingScript--;
     script = oldscript;
 }
 
 void HTMLTokenizer::parseComment(TokenizerString &src)
 {
     checkRawContentBuffer(src.length());
     while (src.length()) {
         rawContent[ rawContentSize++ ] = *src;
 
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
         qDebug("comment is now: *%s*", src.toString().left(16).toLatin1().constData());
 #endif
 
         if (src->unicode() == '>') {
             bool handleBrokenComments =  brokenComments && !(script || style);
             bool scriptEnd = false;
             if (rawContentSize > 2 && rawContent[rawContentSize - 3] == '-' &&
                     rawContent[rawContentSize - 2] == '-') {
                 scriptEnd = true;
             }
 
             if (handleBrokenComments || scriptEnd) {
                 ++src;
                 if (!(title || script || xmp || textarea || style)) {
                     checkRawContentBuffer();
                     rawContent[ rawContentSize ] = 0;
                     rawContent[ rawContentSize + 1 ] = 0;
                     currToken.tid = ID_COMMENT;
                     int size = scriptEnd ? rawContentSize - 3 : rawContentSize - 1;
                     processListing(TokenizerString(rawContent, size));
                     processToken();
                     currToken.tid = ID_COMMENT + ID_CLOSE_TAG;
                     processToken();
                     rawContentSize = 0;
                 }
                 comment = false;
                 return; // Finished parsing comment
             }
         }
         ++src;
     }
 }
 
 void HTMLTokenizer::parseDoctypeComment(TokenizerString &src)
 {
     while (!src.isEmpty()) {
         QChar c = *src;
         switch (doctypeComment) {
         case DoctypeCommentHalfBegin: {
             if (c != '-') {
                 // Ooops, it's not comment
                 doctypeComment = DoctypeCommentBogus;
                 return;
             } else {
                 // Doctype comment begins
                 doctypeComment = DoctypeComment;
                 ++src;
             }
             break;
         }
         case DoctypeComment: {
             if (c == '-') {
                 // Perhaps this is end of comment
                 doctypeComment = DoctypeCommentHalfEnd;
                 ++src;
             } else {
                 // Keep scanning for '--'
                 ++src;
             }
             break;
         }
         case DoctypeCommentHalfEnd: {
             if (c == '-') {
                 // Doctype comment ends
                 doctypeComment = DoctypeCommentEnd;
                 return;
             } else {
                 // It's not '--'
                 ++src;
                 doctypeComment = DoctypeComment;
             }
             break;
         }
         default: {
             assert(!"Undefined doctype comment state");
             break;
         }
         }
     }
 }
 
 void HTMLTokenizer::parseDoctype(TokenizerString &src)
 {
     while (!src.isEmpty() && doctype) {
         QChar c;
         bool isWhitespace = false;
         int dontAdvance = 0;
         if (doctypeComment == DoctypeCommentEnd) {
             doctypeComment = NoDoctypeComment;
             isWhitespace = true;
         } else if (doctypeComment == DoctypeCommentBogus) {
             doctypeComment = NoDoctypeComment;
             c = '-';
             dontAdvance++;
         } else {
             c = *src;
             if (doctypeAllowComment) {
                 if (!doctypeComment && c == '-') {
                     doctypeComment = DoctypeCommentHalfBegin;
                     ++src;
                 }
                 if (doctypeComment) {
                     parseDoctypeComment(src);
                     continue;
                 }
                 isWhitespace = c == '\r' || c == '\n' || c == '\t' || c == ' ';
             }
         }
 
         switch (doctypeToken.state) {
         case DoctypeBegin: {
             doctypeToken.state = DoctypeBeforeName;
             if (isWhitespace) {
                 // nothing
             }
             break;
         }
         case DoctypeBeforeName: {
             if (c == '>') {
                 // Malformed. Just exit.
                 doctype = false;
             } else if (isWhitespace) {
                 // nothing
             } else {
                 dontAdvance++;
                 doctypeToken.state = DoctypeName;
             }
             break;
         }
         case DoctypeName: {
             if (c == '>') {
                 // Valid doctype. Emit it.
                 doctype = false;
                 processDoctypeToken();
             } else if (isWhitespace) {
                 doctypeSearchCount = 0; // Used now to scan for PUBLIC
                 doctypeSecondarySearchCount = 0; // Used now to scan for SYSTEM
                 doctypeToken.state = DoctypeAfterName;
             } else {
                 doctypeToken.name.append(c);
             }
             break;
         }
         case DoctypeAfterName: {
             if (c == '>') {
                 // Valid doctype. Emit it.
                 doctype = false;
                 processDoctypeToken();
             } else if (c == '[') {
                 if (doctypeSearchCount > 0 || doctypeSecondarySearchCount > 0) { // is there any public/system indicator before?
                     doctypeSearchCount = doctypeSecondarySearchCount = 0;
                     doctypeToken.state = DoctypeBogus;
                 }
                 // Found internal subset
                 doctypeToken.state = DoctypeInternalSubset;
                 doctypeAllowComment = false;
             } else if (!isWhitespace) {
                 if (c.toLower() == publicStart[doctypeSearchCount]) {
                     doctypeSearchCount++;
                     if (doctypeSearchCount == 6)
                         // Found 'PUBLIC' sequence
                     {
                         doctypeToken.state = DoctypeBeforePublicID;
                     }
                 } else if (doctypeSearchCount > 0) {
                     doctypeSearchCount = 0;
                     doctypeToken.state = DoctypeBogus;
                 } else if (c.toLower() == systemStart[doctypeSecondarySearchCount]) {
                     doctypeSecondarySearchCount++;
                     if (doctypeSecondarySearchCount == 6)
                         // Found 'SYSTEM' sequence
                     {
                         doctypeToken.state = DoctypeBeforeSystemID;
                     }
                 } else {
                     doctypeSecondarySearchCount = 0;
                     doctypeToken.state = DoctypeBogus;
                 }
             } else {
                 // Whitespace keeps us in the after name state
             }
             break;
         }
         case DoctypeBeforePublicID: {
             if (c == '\"' || c == '\'') {
                 tquote = c == '\"' ? DoubleQuote : SingleQuote;
                 doctypeToken.state = DoctypePublicID;
                 doctypeAllowComment = false;
             } else if (c == '>') {
                 // Considered bogus. Don't process the doctype.
                 doctype = false;
             } else if (isWhitespace) {
                 // nothing
             } else {
                 doctypeToken.state = DoctypeBogus;
             }
             break;
         }
         case DoctypePublicID: {
             if ((c == '\"' && tquote == DoubleQuote) || (c == '\'' && tquote == SingleQuote)) {
                 doctypeToken.state = DoctypeAfterPublicID;
                 doctypeAllowComment = true;
             } else if (c == '>') {
                 // Considered bogus. Don't process the doctype.
                 doctype = false;
             } else {
                 doctypeToken.publicID.append(c);
             }
             break;
         }
         case DoctypeAfterPublicID: {
             if (c == '\"' || c == '\'') {
                 tquote = c == '\"' ? DoubleQuote : SingleQuote;
                 doctypeToken.state = DoctypeSystemID;
             } else if (c == '>') {
                 // Valid doctype. Emit it now.
                 doctype = false;
                 processDoctypeToken();
             } else if (isWhitespace) {
                 // nothing
             } else if (c == '[') {
                 // Found internal subset
                 doctypeToken.state = DoctypeInternalSubset;
                 doctypeAllowComment = false;
             } else {
                 doctypeToken.state = DoctypeBogus;
             }
             break;
         }
         case DoctypeBeforeSystemID: {
             if (c == '\"' || c == '\'') {
                 tquote = c == '\"' ? DoubleQuote : SingleQuote;
                 doctypeToken.state = DoctypeSystemID;
                 doctypeAllowComment = false;
             } else if (c == '>') {
                 // Considered bogus. Don't process the doctype.
                 doctype = false;
             } else if (isWhitespace) {
                 // nothing
             } else {
                 doctypeToken.state = DoctypeBogus;
             }
             break;
         }
         case DoctypeSystemID: {
             if ((c == '\"' && tquote == DoubleQuote) || (c == '\'' && tquote == SingleQuote)) {
                 doctypeToken.state = DoctypeAfterSystemID;
                 doctypeAllowComment = true;
             } else if (c == '>') {
                 // Considered bogus. Don't process the doctype.
                 doctype = false;
             } else {
                 doctypeToken.systemID.append(c);
             }
             break;
         }
         case DoctypeAfterSystemID: {
             if (c == '>') {
                 // Valid doctype. Emit it now.
                 doctype = false;
                 processDoctypeToken();
             } else if (isWhitespace) {
                 // nothing
             } else if (c == '[') {
                 // Found internal subset
                 doctypeToken.state = DoctypeInternalSubset;
                 doctypeAllowComment = false;
             } else {
                 doctypeToken.state = DoctypeBogus;
             }
             break;
         }
         case DoctypeInternalSubset: {
             if (c == ']') {
                 // Done
                 doctypeToken.state = DoctypeAfterInternalSubset;
                 doctypeAllowComment = true;
             } else {
                 doctypeToken.internalSubset.append(c);
             }
             break;
         }
         case DoctypeAfterInternalSubset: {
             if (c == '>') {
                 // Valid doctype. Emit it now.
                 doctype = false;
                 processDoctypeToken();
             } else if (isWhitespace) {
                 // nothing
             } else {
                 doctypeToken.state = DoctypeBogus;
             }
             break;
         }
         case DoctypeBogus: {
             if (c == '>') {
                 // Done with the bogus doctype.
                 doctype = false;
             } else {
                 // Just keep scanning for '>'
             }
             break;
         }
         default:
             break;
         }
         if (!dontAdvance) {
             ++src;
         } else if (dontAdvance == 1) {
             continue;
         } else { // double dontAdvance++, do workaround
             doctypeComment = DoctypeCommentBogus;
         }
     }
 }
 
 void HTMLTokenizer::parseServer(TokenizerString &src)
 {
     checkRawContentBuffer(src.length());
     while (!src.isEmpty()) {
         rawContent[ rawContentSize++ ] = *src;
         if (src->unicode() == '>' &&
                 rawContentSize > 1 && rawContent[rawContentSize - 2] == '%') {
             ++src;
             server = false;
             rawContentSize = 0;
             return; // Finished parsing server include
         }
         ++src;
     }
 }
 
 void HTMLTokenizer::parseProcessingInstruction(TokenizerString &src)
 {
     char oldchar = 0;
     while (!src.isEmpty()) {
         unsigned char chbegin = src->toLatin1();
         if (chbegin == '\'') {
             tquote = tquote == SingleQuote ? NoQuote : SingleQuote;
         } else if (chbegin == '\"') {
             tquote = tquote == DoubleQuote ? NoQuote : DoubleQuote;
         }
         // Look for '?>'
         // some crappy sites omit the "?" before it, so
         // we look for an unquoted '>' instead. (IE compatible)
         else if (chbegin == '>' && (!tquote || oldchar == '?')) {
             // We got a '?>' sequence
             processingInstruction = false;
             ++src;
             discard = LFDiscard;
             return; // Finished parsing comment!
         }
         ++src;
         oldchar = chbegin;
     }
 }
 
 void HTMLTokenizer::parseText(TokenizerString &src)
 {
     while (!src.isEmpty()) {
         // do we need to enlarge the buffer?
         checkBuffer();
 
         // ascii is okay because we only do ascii comparisons
         unsigned char chbegin = src->toLatin1();
 
         if (skipLF && (chbegin != '\n')) {
             skipLF = false;
         }
 
         if (skipLF) {
             skipLF = false;
             ++src;
         } else if ((chbegin == '\n') || (chbegin == '\r')) {
             if (chbegin == '\r') {
                 skipLF = true;
             }
 
             *dest++ = '\n';
             ++src;
         } else {
             *dest++ = *src;
             ++src;
         }
     }
 }
 
 void HTMLTokenizer::parseEntity(TokenizerString &src, QChar *&dest, bool start)
 {
     if (start) {
         cBufferPos = 0;
         entityLen = 0;
         Entity = SearchEntity;
     }
 
     while (!src.isEmpty()) {
         ushort cc = src->unicode();
         switch (Entity) {
         case NoEntity:
             return;
 
             break;
         case SearchEntity:
             if (cc == '#') {
                 cBuffer[cBufferPos++] = cc;
                 ++src;
                 Entity = NumericSearch;
             } else {
                 Entity = EntityName;
             }
 
             break;
 
         case NumericSearch:
             if (cc == 'x' || cc == 'X') {
                 cBuffer[cBufferPos++] = cc;
                 ++src;
                 Entity = Hexadecimal;
             } else if (cc >= '0' && cc <= '9') {
                 Entity = Decimal;
             } else {
                 Entity = SearchSemicolon;
             }
 
             break;
 
         case Hexadecimal: {
             int uc = EntityChar.unicode();
             int ll = qMin<uint>(src.length(), 8);
             while (ll--) {
                 QChar csrc(src->toLower());
                 cc = csrc.cell();
 
                 if (csrc.row() || !((cc >= '0' && cc <= '9') || (cc >= 'a' && cc <= 'f'))) {
                     break;
                 }
                 uc = uc * 16 + (cc - (cc < 'a' ? '0' : 'a' - 10));
                 cBuffer[cBufferPos++] = cc;
                 ++src;
             }
             EntityChar = QChar(uc);
             Entity = SearchSemicolon;
             break;
         }
         case Decimal: {
             int uc = EntityChar.unicode();
             int ll = qMin(src.length(), 9 - cBufferPos);
             while (ll--) {
                 cc = src->cell();
 
                 if (src->row() || !(cc >= '0' && cc <= '9')) {
                     Entity = SearchSemicolon;
                     break;
                 }
 
                 uc = uc * 10 + (cc - '0');
                 cBuffer[cBufferPos++] = cc;
                 ++src;
             }
             EntityChar = QChar(uc);
             if (cBufferPos == 9) {
                 Entity = SearchSemicolon;
             }
             break;
         }
         case EntityName: {
             int ll = qMin(src.length(), 9 - cBufferPos);
             while (ll--) {
                 QChar csrc = *src;
                 cc = csrc.cell();
 
                 if (csrc.row() || !((cc >= 'a' && cc <= 'z') ||
                                     (cc >= '0' && cc <= '9') || (cc >= 'A' && cc <= 'Z'))) {
                     Entity = SearchSemicolon;
                     break;
                 }
 
                 cBuffer[cBufferPos++] = cc;
                 ++src;
 
                 // be IE compatible and interpret even unterminated entities
                 // outside tags. like "foo &nbspstuff bla".
                 if (tag == NoTag) {
                     int code;
                     const bool found = kde_findEntity(cBuffer, cBufferPos, &code);
                     if (found && code < 256) {
                         EntityChar = code;
                         entityLen = cBufferPos;
                     }
                 }
             }
             if (cBufferPos == 9) {
                 Entity = SearchSemicolon;
             }
             if (Entity == SearchSemicolon) {
                 if (cBufferPos > 1) {
                     int code;
                     const bool found = kde_findEntity(cBuffer, cBufferPos, &code);
                     // IE only accepts unterminated entities < 256,
                     // Gecko accepts them all, but only outside tags
                     if (found && (tag == NoTag || code < 256 || *src == ';')) {
                         EntityChar = code;
                         entityLen = cBufferPos;
                     }
                 }
             }
             break;
         }
         case SearchSemicolon:
 #ifdef TOKEN_DEBUG
             qCDebug(KHTML_LOG) << "ENTITY " << EntityChar.unicode();
 #endif
             fixUpChar(EntityChar);
 
             if (*src == ';') {
                 ++src;
             }
 
             if (!EntityChar.isNull()) {
                 checkBuffer();
                 if (entityLen > 0 && entityLen < cBufferPos) {
                     int rem = cBufferPos - entityLen;
                     src.prepend(TokenizerString(QString::fromLatin1(cBuffer + entityLen, rem)));
                 }
                 src.push(EntityChar);
                 rawContentSinceLastEntity = -1;
             } else {
 #ifdef TOKEN_DEBUG
                 qCDebug(KHTML_LOG) << "unknown entity!";
 #endif
                 checkBuffer(11);
                 // ignore the sequence, add it to the buffer as plaintext
                 *dest++ = '&';
                 for (unsigned int i = 0; i < cBufferPos; i++) {
                     dest[i] = cBuffer[i];
                 }
                 dest += cBufferPos;
                 rawContentSinceLastEntity += cBufferPos + 1;
                 if (pre) {
                     prePos += cBufferPos + 1;
                 }
             }
 
             Entity = NoEntity;
             EntityChar = QChar::Null;
             return;
         };
     }
 }
 
 void HTMLTokenizer::parseTag(TokenizerString &src)
 {
     assert(!Entity);
     checkRawContentBuffer(src.length());
 
     while (!src.isEmpty()) {
         checkBuffer();
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
         uint l = 0;
         while (l < src.length() && (src.toString()[l]).toLatin1() != '>') {
             l++;
         }
         qDebug("src is now: *%s*, tquote: %d", src.toString().left(l).toLatin1().constData(), tquote);
 #endif
         switch (tag) {
         case NoTag:
             return;
         case TagName: {
 #if defined(TOKEN_DEBUG) &&  TOKEN_DEBUG > 1
             qDebug("TagName");
 #endif
             if (searchCount > 0) {
                 if (*src == commentStart[searchCount]) {
                     searchCount++;
                     if (searchCount == 2) {
                         doctypeSearchCount++;    // A '!' is also part of doctype, so we are moving through that still as well
                     } else {
                         doctypeSearchCount = 0;
                     }
 
                     if (searchCount == 4) {
 #ifdef TOKEN_DEBUG
                         qCDebug(KHTML_LOG) << "Found comment";
 #endif
                         // Found '<!--' sequence
                         ++src;
                         dest = buffer; // ignore the previous part of this tag
                         tag = NoTag;
 
                         comment = true;
                         parseComment(src);
                         return; // Finished parsing tag!
                     }
                     // cuts of high part, is okay
                     cBuffer[cBufferPos++] = src->cell();
                     ++src;
                     break;
                 } else {
                     searchCount = 0;    // Stop looking for '<!--' sequence
                 }
             }
 
             if (doctypeSearchCount > 0) {
                 if ((*src).toLower() == doctypeStart[doctypeSearchCount]) {
                     doctypeSearchCount++;
                     cBuffer[cBufferPos++] = src->cell();
                     ++src;
                     if (doctypeSearchCount == 9) {
                         // Found '<!DOCTYPE' sequence
                         tag = NoTag;
                         doctypeAllowComment = true;
                         doctypeComment = NoDoctypeComment;
                         doctypeToken.reset();
                         doctype = true;
 
                         parseDoctype(src);
                         return;
                     }
                     break;
                 } else {
                     doctypeSearchCount = 0;    // Stop looking for '<!DOCTYPE' sequence
                 }
             }
 
             bool finish = false;
             unsigned int ll = qMin(src.length(), CBUFLEN - cBufferPos);
             while (ll--) {
                 ushort curchar = src->unicode();
                 if (curchar <= ' ' || curchar == '>') {
                     finish = true;
                     break;
                 }
                 // this is a nasty performance trick. will work for the A-Z
                 // characters, but not for others. if it contains one,
                 // we fail anyway
                 char cc = curchar;
                 cBuffer[cBufferPos++] = cc | 0x20;
                 ++src;
             }
 
             // Disadvantage: we add the possible rest of the tag
             // as attribute names. ### judge if this causes problems
             if (finish || CBUFLEN == cBufferPos) {
                 bool beginTag;
                 char *ptr = cBuffer;
                 unsigned int len = cBufferPos;
                 cBuffer[cBufferPos] = '\0';
                 if ((cBufferPos > 0) && (*ptr == '/')) {
                     // End Tag
                     beginTag = false;
                     ptr++;
                     len--;
                 } else
                     // Start Tag
                 {
                     beginTag = true;
                 }
                 // Accept empty xml tags like <br/>
                 if (len > 1 && ptr[len - 1] == '/') {
                     ptr[--len] = '\0';
                     // if it is like <br/> and not like <input/ value=foo>, take it as flat
                     if (*src == '>') {
                         currToken.flat = true;
                     }
                 }
 
                 uint tagID = 0;
                 if (!tagID) {
                     DOMString tagName(ptr);
                     if (Element::khtmlValidQualifiedName(tagName)) {
                         safeLocalName = LocalName::fromString(tagName, IDS_NormalizeLower);
                         tagID = safeLocalName.id();
                     }
 #ifdef TOKEN_DEBUG
                     QByteArray tmp(ptr, len + 1);
                     qCDebug(KHTML_LOG) << "Unknown tag: \"" << tmp.data() << "\"";
 #endif
                 }
                 if (tagID) {
 #ifdef TOKEN_DEBUG
                     QByteArray tmp(ptr, len + 1);
                     qCDebug(KHTML_LOG) << "found tag id=" << tagID << ": " << tmp.data();
 #endif
                     currToken.tid = beginTag ? tagID : tagID + ID_CLOSE_TAG;
                 }
                 dest = buffer;
                 tag = SearchAttribute;
                 cBufferPos = 0;
             }
             break;
         }
         case SearchAttribute: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("SearchAttribute");
 #endif
             bool atespace = false;
             ushort curchar;
             while (!src.isEmpty()) {
                 curchar = src->unicode();
                 if (curchar > ' ') {
                     if (curchar == '<' || curchar == '>') {
                         tag = SearchEnd;
                     } else if (atespace && (curchar == '\'' || curchar == '"')) {
                         tag = SearchValue;
                         *dest++ = 0;
                         attrName = DOMString("");
                     } else {
                         tag = AttributeName;
                     }
 
                     cBufferPos = 0;
                     break;
                 }
                 atespace = true;
                 ++src;
             }
             break;
         }
         case AttributeName: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("AttributeName");
 #endif
             ushort curchar;
             int ll = qMin(src.length(), CBUFLEN - cBufferPos);
 
             while (ll--) {
                 curchar = src->unicode();
                 if (curchar <= '>') {
                     if (curchar <= ' ' || curchar == '=' || curchar == '>') {
                         unsigned int a;
                         cBuffer[cBufferPos] = '\0';
                         a = LocalName::fromString(DOMString(cBuffer), IDS_NormalizeLower).id(); // ### still deep copy?
                         if (a > ATTR_LAST_ATTR) {
                             a = 0;
                         }
 
                         if (!a) {
                             // did we just get /> or e.g checked/>
                             if (curchar == '>' && cBufferPos >= 1 && cBuffer[cBufferPos - 1] == '/') {
                                 currToken.flat = true;
                                 cBuffer[cBufferPos - 1] = '\0';
                                 if (cBufferPos > 1) {
                                     a = LocalName::fromString(DOMString(cBuffer), IDS_NormalizeLower).id();
                                 }
                                 if (a > ATTR_LAST_ATTR) {
                                     a = 0;
                                 }
                                 cBuffer[cBufferPos - 1] = '/';
                             }
                             if (!a) {
                                 attrName = DOMString(cBuffer, cBufferPos);
                             }
                         }
 
                         dest = buffer;
                         *dest++ = a;
 #ifdef TOKEN_DEBUG
                         if (!a || (cBufferPos && *cBuffer == '!')) {
                             qCDebug(KHTML_LOG) << "Unknown attribute: *" << QByteArray(cBuffer, cBufferPos + 1).data() << "*";
                         } else {
                             qCDebug(KHTML_LOG) << "Known attribute: " << QByteArray(cBuffer, cBufferPos + 1).data();
                         }
 #endif
 
                         tag = SearchEqual;
                         break;
                     }
                 }
                 cBuffer[cBufferPos++] =
                     (curchar >= 'A' && curchar <= 'Z') ? curchar | 0x20 : curchar;
                 ++src;
             }
             if (cBufferPos == CBUFLEN) {
                 cBuffer[cBufferPos] = '\0';
                 attrName = DOMString(cBuffer, cBufferPos);
                 dest = buffer;
                 *dest++ = 0;
                 tag = SearchEqual;
             }
             break;
         }
         case SearchEqual: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("SearchEqual");
 #endif
             ushort curchar;
             bool atespace = false;
             while (!src.isEmpty()) {
                 curchar = src->unicode();
                 if (curchar > ' ') {
                     if (curchar == '=') {
 #ifdef TOKEN_DEBUG
                         qCDebug(KHTML_LOG) << "found equal";
 #endif
                         tag = SearchValue;
                         ++src;
                     } else if (atespace && (curchar == '\'' || curchar == '"')) {
                         tag = SearchValue;
                         *dest++ = 0;
                         attrName = DOMString("");
                     } else {
                         DOMString v("");
                         currToken.addAttribute(parser->docPtr(), buffer, attrName, v);
                         dest = buffer;
                         tag = SearchAttribute;
                     }
                     break;
                 }
                 atespace = true;
                 ++src;
             }
             break;
         }
         case SearchValue: {
             ushort curchar;
             while (!src.isEmpty()) {
                 curchar = src->unicode();
                 if (curchar > ' ') {
                     if ((curchar == '\'' || curchar == '\"')) {
                         tquote = curchar == '\"' ? DoubleQuote : SingleQuote;
                         tag = QuotedValue;
                         ++src;
                     } else {
                         tag = Value;
                     }
 
                     break;
                 }
                 ++src;
             }
             break;
         }
         case QuotedValue: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("QuotedValue");
 #endif
             ushort curchar;
             while (!src.isEmpty()) {
                 checkBuffer();
 
                 curchar = src->unicode();
                 if (curchar <= '\'' && !src.escaped()) {
                     // ### attributes like '&{blaa....};' are supposed to be treated as jscript.
                     if (curchar == '&') {
                         ++src;
                         parseEntity(src, dest, true);
                         break;
                     } else if ((tquote == SingleQuote && curchar == '\'') ||
                                (tquote == DoubleQuote && curchar == '\"')) {
                         // some <input type=hidden> rely on trailing spaces. argh
                         while (dest > buffer + 1 && (*(dest - 1) == '\n' || *(dest - 1) == '\r')) {
                             dest--;    // remove trailing newlines
                         }
                         DOMString v(buffer + 1, dest - buffer - 1);
                         currToken.addAttribute(parser->docPtr(), buffer, attrName, v);
 
                         dest = buffer;
                         tag = SearchAttribute;
                         tquote = NoQuote;
                         ++src;
                         break;
                     }
                 }
                 *dest++ = *src;
                 ++src;
             }
             break;
         }
         case Value: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("Value");
 #endif
             ushort curchar;
             while (!src.isEmpty()) {
                 checkBuffer();
                 curchar = src->unicode();
                 if (curchar <= '>' && !src.escaped()) {
                     // parse Entities
                     if (curchar == '&') {
                         ++src;
                         parseEntity(src, dest, true);
                         break;
                     }
                     // no quotes. Every space means end of value
                     // '/' does not delimit in IE!
                     // HTML5: must not contain any literal space characters, any U+0022 QUOTATION MARK (") characters,
                     // U+0027 APOSTROPHE (') characters, U+003D EQUALS SIGN (=) characters, U+003C LESS-THAN SIGN (<) characters,
                     // U+003E GREATER-THAN SIGN (>) characters, or U+0060 GRAVE ACCENT (`) characters, and must not be the empty string.
                     // Real life: images.google.com uses URLs including form arguments (foo=bar)
                     // in unquoted parameters --- with an html5 <!doctype html> DTD.
                     // Real life takes priority, so we accept at least =
                     if (curchar <= ' ' || curchar == '>' || curchar == '\'' || curchar == '"' || curchar == '<' || /*curchar == '=' ||*/ curchar == '`') {
                         DOMString v(buffer + 1, dest - buffer - 1);
                         currToken.addAttribute(parser->docPtr(), buffer, attrName, v);
                         dest = buffer;
                         tag = SearchAttribute;
                         break;
                     }
                 }
 
                 *dest++ = *src;
                 ++src;
             }
             break;
         }
         case SearchEnd: {
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 1
             qDebug("SearchEnd");
 #endif
             while (!src.isEmpty()) {
                 if (*src == '<' || *src == '>') {
                     break;
                 }
 
                 if (*src == '/') {
                     currToken.flat = true;
                 }
 
                 ++src;
             }
             if (src.isEmpty() && *src != '<' && *src != '>') {
                 break;
             }
 
             searchCount = 0; // Stop looking for '<!--' sequence
             tag = NoTag;
             tquote = NoQuote;
             if (*src == '>') {
                 ++src;
             }
 
             if (!currToken.tid) { //stop if tag is unknown
                 return;
             }
 
             uint tagID = currToken.tid;
 #if defined(TOKEN_DEBUG) && TOKEN_DEBUG > 0
             qCDebug(KHTML_LOG) << "appending Tag: " << tagID;
 #endif
             // When parsing HTML flat tags like <div /> should
             // be ignored, the only exception is SCRIPT, and
             // tags with forbidden end-tags
             if (tagID < ID_CLOSE_TAG && tagID != ID_SCRIPT &&
                     DOM::endTagRequirement(tagID) != DOM::FORBIDDEN &&
                     parser->doc()->htmlMode() != DocumentImpl::XHtml) {
                 currToken.flat = false;
             }
 
             bool beginTag = !currToken.flat && (tagID < ID_CLOSE_TAG);
             HTMLScriptElementImpl *prevScriptElem = nullptr;
 
             if (tagID >= ID_CLOSE_TAG) {
                 tagID -= ID_CLOSE_TAG;
             } else if (tagID == ID_SCRIPT) {
                 prevScriptElem = parser->currentScriptElement();
                 DOMStringImpl *a = nullptr;
                 scriptSrc.clear(); scriptSrcCharset.clear();
                 if (currToken.attrs &&  /* potentially have a ATTR_SRC ? */
                         view &&  /* are we a regular tokenizer or just for innerHTML ? */
                         parser->doc()->view()->part()->jScriptEnabled() /* jscript allowed at all? */
                    ) {
                     if ((a = currToken.attrs->getValue(ATTR_SRC))) {
                         scriptSrc = DOMString(a).trimSpaces().string();
                     }
                     if ((a = currToken.attrs->getValue(ATTR_CHARSET))) {
                         scriptSrcCharset = DOMString(a).string().trimmed();
                     }
                     if (scriptSrcCharset.isEmpty() && view) {
                         scriptSrcCharset = parser->doc()->view()->part()->encoding();
                     }
                 }
                 javascript = true;
             }
 
             processToken();
 
             if (javascript) {
                 HTMLScriptElementImpl *sc = parser->currentScriptElement();
                 javascript = (sc && sc != prevScriptElem) ? sc->isValidScript() : false;
             }
 
             if (parser->selectMode() && beginTag) {
                 discard = AllDiscard;
             }
 
             switch (tagID) {
             case ID_LISTING:
             case ID_PRE:
                 pre = beginTag;
                 if (beginTag) {
                     discard = LFDiscard;
                 }
                 prePos = 0;
                 break;
             case ID_BR:
                 prePos = 0;
                 break;
             case ID_SCRIPT:
                 if (beginTag) {
                     searchStopper = scriptEnd;
                     searchStopperLen = 8;
                     script = true;
                     parseRawContent(src);
                 } else if (tagID < ID_CLOSE_TAG) { // Handle <script src="foo"/>
                     script = true;
                     scriptHandler();
                 }
                 break;
             case ID_STYLE:
                 if (beginTag) {
                     searchStopper = styleEnd;
                     searchStopperLen = 7;
                     style = true;
                     parseRawContent(src);
                 }
                 break;
             case ID_TEXTAREA:
                 if (beginTag) {
                     searchStopper = textareaEnd;
                     searchStopperLen = 10;
                     textarea = true;
                     discard = NoneDiscard;
                     rawContentSinceLastEntity = 0;
                     parseRawContent(src);
                 }
                 break;
             case ID_TITLE:
                 if (beginTag) {
                     searchStopper = titleEnd;
                     searchStopperLen = 7;
                     title = true;
                     rawContentSinceLastEntity = 0;
                     parseRawContent(src);
                 }
                 break;
             case ID_XMP:
                 if (beginTag) {
                     searchStopper = xmpEnd;
                     searchStopperLen = 5;
                     xmp = true;
                     parseRawContent(src);
                 }
                 break;
             case ID_SELECT:
                 select = beginTag;
                 break;
             case ID_PLAINTEXT:
                 plaintext = beginTag;
                 break;
             }
             return; // Finished parsing tag!
         }
         } // end switch
     }
     return;
 }
 
 void HTMLTokenizer::addPending()
 {
     if (select && !(comment || script)) {
         *dest++ = ' ';
     } else {
         switch (pending) {
         case LFPending:  *dest++ = QLatin1Char('\n'); prePos = 0; break;
         case SpacePending: *dest++ = QLatin1Char(' '); ++prePos; break;
         case TabPending: {
             // Don't expand tabs inside <textarea> or script
             int p = TAB_SIZE - (prePos % TAB_SIZE);
             if (textarea || script) {
                 *dest++ = QLatin1Char('\t');
             } else {
                 for (int x = 0; x < p; x++) {
                     *dest++ = QLatin1Char(' ');
                 }
             }
             prePos += p;
             break;
         }
         case NonePending:
             assert(0);
         }
     }
 
     pending = NonePending;
 }
 
 inline bool HTMLTokenizer::continueProcessing(int &processedCount)
 {
     // We don't want to be checking elapsed time with every character, so we only check after we've
     // processed a certain number of characters. We also do not do suspension if we're
     // parsing something like innerHTML.
     if (!m_executingScript && processedCount > sTokenizerChunkSize && cachedScript.isEmpty()) {
         processedCount = 0;
         if (m_time.elapsed() > m_tokenizerYieldDelay && m_documentTokenizer) {
             m_yieldTimer = startTimer(0);
             m_tokenizerYieldDelay = sTokenizerFastYieldDelay;
             return false;
         }
     }
     processedCount++;
     return true;
 }
 
 #include "khtmlpart_p.h"
 void HTMLTokenizer::write(const TokenizerString &str, bool appendData)
 {
 #ifdef TOKEN_DEBUG
     qCDebug(KHTML_LOG) << this << " Tokenizer::write(\"" << str.toString() << "\"," << appendData << ")";
 #endif
     if (!buffer) {
         return;
     }
 
     if ((m_executingScript && appendData) || cachedScript.count()) {
         // don't parse; we will do this later
         if (pendingQueue.isEmpty()) {
             pendingQueue.push(str);
         } else if (appendData) {
             pendingQueue.bottom().append(str);
         } else {
             pendingQueue.top().append(str);
         }
 #if PROSPECTIVE_TOKENIZER_ENABLED
         if (m_prospectiveTokenizer && m_prospectiveTokenizer->inProgress() && appendData) {
             m_prospectiveTokenizer->write(str);
         }
 #endif
         return;
     }
 
 #if PROSPECTIVE_TOKENIZER_ENABLED
     if (m_prospectiveTokenizer && m_prospectiveTokenizer->inProgress() && appendData) {
         m_prospectiveTokenizer->end();
     }
 #endif
 
     if (onHold) {
         src.append(str);
         return;
     }
 
     if (!src.isEmpty()) {
         src.append(str);
     } else {
         setSrc(str);
     }
 
     // Once a timer is set, it has control of when the tokenizer continues.
     if (m_yieldTimer > 0) {
         return;
     }
 
     int processedCount = 0;
     m_time.start();
 
     while (!src.isEmpty()) {
         if (m_abort || !continueProcessing(processedCount)) {
             break;
         }
         // do we need to enlarge the buffer?
         checkBuffer();
 
         ushort cc = src->unicode();
 
         if (skipLF && (cc != '\n')) {
             skipLF = false;
         }
 
         if (skipLF) {
             skipLF = false;
             ++src;
         } else if (Entity) {
             parseEntity(src, dest);
         } else if (plaintext) {
             parseText(src);
         } else if (script) {
             parseRawContent(src);
         } else if (style) {
             parseRawContent(src);
         } else if (xmp) {
             parseRawContent(src);
         } else if (textarea) {
             parseRawContent(src);
         } else if (title) {
             parseRawContent(src);
         } else if (comment) {
             parseComment(src);
         } else if (doctypeComment && doctypeComment != DoctypeCommentEnd && doctypeComment != DoctypeCommentBogus) {
             parseDoctypeComment(src);
         } else if (doctype) {
             parseDoctype(src);
         } else if (server) {
             parseServer(src);
         } else if (processingInstruction) {
             parseProcessingInstruction(src);
         } else if (tag) {
             parseTag(src);
         } else if (startTag) {
             startTag = false;
 
             switch (cc) {
             case '/':
                 break;
             case '!': {
                 // <!-- comment --> or <!DOCTYPE ...>
                 searchCount = 1; // Look for '<!--' sequence to start comment...
                 doctypeSearchCount = 1; // ... or for '<!DOCTYPE' sequence to start doctype
                 break;
             }
             case '?': {
                 // xml processing instruction
                 processingInstruction = true;
                 tquote = NoQuote;
                 parseProcessingInstruction(src);
                 continue;
             }
             case '%':
                 if (!brokenServer) {
                     // <% server stuff, handle as comment %>
                     server = true;
                     tquote = NoQuote;
                     parseServer(src);
                     continue;
                 }
             // else fall through
             default: {
                 if (((cc >= 'a') && (cc <= 'z')) || ((cc >= 'A') && (cc <= 'Z'))) {
                     // Start of a Start-Tag
                 } else {
                     // Invalid tag
                     // Add as is
                     if (pending) {
                         addPending();
                     }
                     *dest = '<';
                     dest++;
                     continue;
                 }
             }
             }; // end case
 
             // According to SGML any LF immediately after a starttag, or
             // immediately before an endtag should be ignored.
             // ### Gecko and MSIE though only ignores LF immediately after
             // starttags and only for PRE elements -- asj (28/06-2005)
             if (pending) {
                 if (!select) {
                     addPending();
                 } else {
                     pending = NonePending;
                 }
             }
 
             // Cancel unused discards
             discard = NoneDiscard;
             // if (!endTag) discard = LFDiscard;
 
             processToken();
 
             cBufferPos = 0;
             tag = TagName;
             parseTag(src);
         } else if (cc == '&' && !src.escaped()) {
             ++src;
             if (pending) {
                 addPending();
             }
             discard = NoneDiscard;
             parseEntity(src, dest, true);
         } else if (cc == '<' && !src.escaped()) {
             tagStartLineno = lineno + src.lineCount();
             ++src;
             discard = NoneDiscard;
             startTag = true;
         } else if ((cc == '\n') || (cc == '\r')) {
             if (discard == SpaceDiscard) {
                 discard = NoneDiscard;
             }
 
             if (discard == LFDiscard) {
                 // Ignore one LF
                 discard = NoneDiscard;
             } else if (discard == AllDiscard) {
                 // Ignore
             } else {
                 if (select && !script) {
                     pending = LFPending;
                 } else {
                     if (pending) {
                         addPending();
                     }
                     pending = LFPending;
                 }
             }
 
             /* Check for MS-DOS CRLF sequence */
             if (cc == '\r') {
                 skipLF = true;
             }
             ++src;
         } else if ((cc == ' ') || (cc == '\t')) {
             if (discard == LFDiscard) {
                 discard = NoneDiscard;
             }
 
             if (discard == SpaceDiscard) {
                 // Ignore one space
                 discard = NoneDiscard;
             } else if (discard == AllDiscard) {
                 // Ignore
             } else {
                 if (select && !script) {
                     if (!pending) {
                         pending = SpacePending;
                     }
                 } else {
                     if (pending) {
                         addPending();
                     }
                     if (cc == ' ') {
                         pending = SpacePending;
                     } else {
                         pending = TabPending;
                     }
                 }
             }
 
             ++src;
         } else {
             if (pending) {
                 addPending();
             }
 
             discard = NoneDiscard;
             if (pre) {
                 prePos++;
             }
             *dest = *src;
             fixUpChar(*dest);
             ++dest;
             ++src;
         }
     }
 
     if (noMoreData && cachedScript.isEmpty() && !m_executingScript && m_yieldTimer <= 0) {
         end();    // this actually causes us to be deleted
     }
 }
 
 void HTMLTokenizer::timerEvent(QTimerEvent *e)
 {
     if (e->timerId() == m_yieldTimer) {
         killTimer(m_yieldTimer);
         m_yieldTimer = 0;
         write(TokenizerString(), true);
     } else if (e->timerId() == m_externalScriptsTimerId) {
         if (view && view->hasLayoutPending()) {
             // all stylesheets are loaded but the style modifications
             // they triggered have yet to be applied, BBIAB
             return;
         }
         killTimer(m_externalScriptsTimerId);
         m_externalScriptsTimerId = 0;
         notifyFinished(nullptr);
     }
 }
 
 void HTMLTokenizer::end()
 {
     if (buffer) {
         // parseTag is using the buffer for different matters
         if (!tag) {
             processToken();
         }
 
         if (buffer) {
             KHTML_DELETE_QCHAR_VEC(buffer);
         }
 
         if (rawContent) {
             KHTML_DELETE_QCHAR_VEC(rawContent);
         }
 
         rawContent = nullptr;
         rawContentSize = rawContentMaxSize = rawContentResync = 0;
         buffer = nullptr;
     }
     emit finishedParsing();
 }
 
 void HTMLTokenizer::finish()
 {
     // The purpose of this iteration is to recover from 'raw content' tokenizing mode.
     // In this mode, any error such as the lack of a closing tag (for the considered element) or of a closing comment,
     // would result in the entire document being absorbed in one node.
     // When it happens, we simply put back in the input buffer what this mode's output has accumulated so far,
     // and retokenize after either disabling the 'raw content' mode (by setting the corresponding members to false)
     // or after setting a few flags disabling some lax parsing 'features' (brokenComments/brokenServer).
     while ((title || comment || server) && rawContent && rawContentSize) {
         // we've found an unmatched comment start
         if (comment) {
             brokenComments = true;
         } else if (server) {
             brokenServer = true;
         }
 
         checkRawContentBuffer();
         rawContent[ rawContentSize ] = 0;
         rawContent[ rawContentSize + 1 ] = 0;
         int pos;
         QString food;
         if (title || style || script || textarea) {
             rawContentSinceLastEntity = 0;
             food.setUnicode(rawContent, rawContentSize);
         } else if (server) {
             food = "<";
             food += QString(rawContent, rawContentSize);
         } else {
             pos = QString::fromRawData(rawContent, rawContentSize).indexOf('>');
             food.setUnicode(rawContent + pos + 1, rawContentSize - pos - 1); // deep copy
         }
         KHTML_DELETE_QCHAR_VEC(rawContent);
         rawContent = nullptr;
         rawContentSize = rawContentMaxSize = rawContentResync = 0;
 
         comment = server = title = false;
         if (!food.isEmpty()) {
             write(food, true);
         }
     }
     // this indicates we will not receive any more data... but if we are waiting on
     // an external script to load, we can't finish parsing until that is done
     noMoreData = true;
     if (cachedScript.isEmpty() && !m_executingScript && !onHold && m_yieldTimer <= 0) {
         end();    // this actually causes us to be deleted
     }
 }
 
 void HTMLTokenizer::processToken()
 {
     KJSProxy *jsProxy = view ? view->part()->jScript() : nullptr;
     if (jsProxy) {
         jsProxy->setEventHandlerLineno(tagStartLineno);
     }
     if (dest > buffer) {
 #if 0
         if (currToken.tid) {
             qDebug("unexpected token id: %d, str: *%s*", currToken.tid, QString::fromRawData(buffer, dest - buffer).toLatin1().constData());
             assert(0);
         }
 
 #endif
         currToken.text = new DOMStringImpl(buffer, dest - buffer);
         currToken.text->ref();
         if (currToken.tid != ID_COMMENT) {
             currToken.tid = ID_TEXT;
         }
     } else if (!currToken.tid) {
         currToken.reset();
         if (jsProxy) {
             jsProxy->setEventHandlerLineno(lineno + src.lineCount());
         }
         return;
     }
 
     dest = buffer;
 
 #ifdef TOKEN_DEBUG
     QString text;
     bool closing = (currToken.tid > ID_CLOSE_TAG);
     int rid = currToken.tid - (closing ? ID_CLOSE_TAG : 0);
     if (currToken.text) {
         text = QString::fromRawData(currToken.text->s, currToken.text->l);
     }
     qCDebug(KHTML_LOG) << "Token -->" << LocalName::fromId(localNamePart(rid)).toString()
              << "id =" << currToken.tid << "closing =" << closing;
     if (currToken.flat) {
         qCDebug(KHTML_LOG) << "Token is FLAT!";
     }
     if (!text.isNull()) {
         qCDebug(KHTML_LOG) << "text: \"" << text << "\"";
     }
     unsigned long l = currToken.attrs ? currToken.attrs->length() : 0;
 
     if (l) {
         qCDebug(KHTML_LOG) << "Attributes: " << l;
         for (unsigned long i = 0; i < l; ++i) {
             NodeImpl::Id tid = currToken.attrs->idAt(i);
             DOMString value = currToken.attrs->valueAt(i);
             qCDebug(KHTML_LOG) << "    " << tid << " " << LocalName::fromId(localNamePart(tid)).toString()
                      << "=\"" << value.string() << "\"";
         }
     }
 #endif
 
     // In some cases, parseToken() can cause javascript code to be executed
     // (for example, when setting an attribute that causes an event handler
     // to be created). So we need to protect against re-entrancy into the parser
     m_executingScript++;
 
     // pass the token over to the parser, the parser DOES NOT delete the token
     parser->parseToken(&currToken);
 
     m_executingScript--;
 
     if (currToken.flat && currToken.tid != ID_TEXT && !parser->noSpaces()) {
         discard = NoneDiscard;
     }
 
     currToken.reset();
     if (jsProxy) {
         jsProxy->setEventHandlerLineno(0);
     }
 }
 
 void HTMLTokenizer::processDoctypeToken()
 {
     // qCDebug(KHTML_LOG) << "Process DoctypeToken (name: " << doctypeToken.name << ", publicID: " << doctypeToken.publicID << ", systemID: " << doctypeToken.systemID;
     doctypeToken.publicID = doctypeToken.publicID.simplified();
     doctypeToken.systemID = doctypeToken.systemID.simplified();
     parser->parseDoctypeToken(&doctypeToken);
 }
 
 HTMLTokenizer::~HTMLTokenizer()
 {
     reset();
     delete m_prospectiveTokenizer;
     delete parser;
 }
 
 void HTMLTokenizer::enlargeBuffer(int len)
 {
     int newsize = qMax(size * 2, size + len);
     int oldoffs = (dest - buffer);
 
     buffer = KHTML_REALLOC_QCHAR_VEC(buffer, newsize);
     dest = buffer + oldoffs;
     size = newsize;
 }
 
 void HTMLTokenizer::enlargeRawContentBuffer(int len)
 {
     int newsize = qMax(rawContentMaxSize * 2, rawContentMaxSize + len);
     rawContent = KHTML_REALLOC_QCHAR_VEC(rawContent, newsize);
     rawContentMaxSize = newsize;
 }
 
 void HTMLTokenizer::notifyFinished(CachedObject *finishedObj)
 {
     Q_UNUSED(finishedObj);
     assert(!cachedScript.isEmpty());
     // Make external scripts wait for external stylesheets.
     // FIXME: This needs to be done for inline scripts too.
     m_hasScriptsWaitingForStylesheets = !parser->doc()->haveStylesheetsLoaded();
     if (m_hasScriptsWaitingForStylesheets) {
         // qCDebug(KHTML_LOG) << "Delaying script execution until stylesheets have loaded.";
         return;
     }
     // qCDebug(KHTML_LOG) << (finishedObj ? "Processing an external script"  : "Continuing processing of delayed external scripts");
 
     bool done = false;
     m_scriptTime.start();
     while (!done && cachedScript.head()->isLoaded()) {
         if (!continueProcessingScripts()) {
             break;
         }
 
         CachedScript *cs = cachedScript.dequeue();
         DOMString scriptSource = cs->script();
 #ifdef TOKEN_DEBUG
         qCDebug(KHTML_LOG) << "External script is:" << endl << scriptSource.string();
 #endif
         setSrc(TokenizerString());
 
         // make sure we forget about the script before we execute the new one
         // infinite recursion might happen otherwise
         QString cachedScriptUrl(cs->url().string());
         cs->deref(this);
 
         scriptExecution(scriptSource.string(), cachedScriptUrl);
 
         done = cachedScript.isEmpty();
         if (done) {
             assert(!m_hasScriptsWaitingForStylesheets);
         } else if (m_hasScriptsWaitingForStylesheets) {
             // flag has changed during the script execution,
             // so we need to wait for stylesheets again.
             done = true;
         }
         // 'script' is true when we are called synchronously from
         // scriptHandler(). In that case scriptHandler() will take care
         // of the pending queue.
         if (!script) {
             while (pendingQueue.count() > 1) {
                 TokenizerString t = pendingQueue.pop();
                 pendingQueue.top().prepend(t);
             }
             if (done) {
                 write(pendingQueue.pop(), false);
             }
             // we might be deleted at this point, do not
             // access any members.
         }
     }
 }
 
 bool HTMLTokenizer::continueProcessingScripts()
 {
     if (m_externalScriptsTimerId) {
         return false;
     }
     if (m_scriptTime.elapsed() > m_tokenizerYieldDelay && m_documentTokenizer) {
         if ((m_externalScriptsTimerId = startTimer(0))) {
             return false;
         }
     }
     return true;
 }
 
 void HTMLTokenizer::executeScriptsWaitingForStylesheets()
 {
     assert(parser->doc()->haveStylesheetsLoaded());
     if (m_hasScriptsWaitingForStylesheets) {
         notifyFinished(nullptr);
     }
 }
 
 bool HTMLTokenizer::isWaitingForScripts() const
 {
     return cachedScript.count();
 }
 
 bool HTMLTokenizer::isExecutingScript() const
 {
     return (m_executingScript > 0);
 }
 
 void HTMLTokenizer::setSrc(const TokenizerString &source)
 {
     lineno += src.lineCount();
     src = source;
     src.resetLineCount();
 }
 
 void HTMLTokenizer::setOnHold(bool _onHold)
 {
     if (onHold == _onHold) {
         return;
     }
     onHold = _onHold;
 }