File indexing completed on 2025-02-09 04:28:37

 /*
   This file is part of the KTextTemplate library
 
   SPDX-FileCopyrightText: 2009, 2010, 2011 Stephen Kelly <steveire@gmail.com>
 
   SPDX-License-Identifier: LGPL-2.1-or-later
 
 */
 
 #include "lexer_p.h"
 
 using namespace KTextTemplate;
 
 using TextProcessingState = State<TextProcessingMachine::Type>;
 using TextProcessingTransition = TextProcessingMachine::Transition;
 
 using ChurningState = LexerObject<TextProcessingState, NullTest, MarksClearer>;
 using FinalizeTokenState = LexerObject<TextProcessingState, NullTest, TokenFinalizer>;
 using EofHandler = LexerObject<TextProcessingTransition, NullTest, TokenFinalizer>;
 using EofHandlerWithTrimming = LexerObject<TextProcessingTransition, NullTest, TokenFinalizerWithTrimming>;
 
 using MaybeTemplateSyntaxHandler = CharacterTransition<'{'>;
 
 using TagStartHandler = CharacterTransition<'%', MarkStartSyntax>;
 using CommentStartHandler = CharacterTransition<'#', MarkStartSyntax>;
 using TagEndHandler = CharacterTransition<'%'>;
 using CommentEndHandler = CharacterTransition<'#'>;
 using BeginValueHandler = CharacterTransition<'{', MarkStartSyntax>;
 using MaybeEndValueHandler = CharacterTransition<'}'>;
 using NewlineHandler = CharacterTransition<'\n', MarkNewline>;
 using EndTemplateSyntaxHandler = CharacterTransition<'}', MarkEndSyntax>;
 using NotEndTemplateSyntaxHandler = NegateCharacterTransition<'}'>;
 
 using NotBeginTemplateSyntaxHandler = LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, OrTest<CharacterTest<'#'>, CharacterTest<'%'>>>>>;
 
 using NotBeginTemplateSyntaxOrNewlineHandler =
     LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, OrTest<CharacterTest<'#'>, OrTest<CharacterTest<'%'>, CharacterTest<'\n'>>>>>>;
 
 using NotTagCommentOrNewlineHandler =
     LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'#'>, OrTest<CharacterTest<'%'>, CharacterTest<'\n'>>>>>;
 
 using NonWhitespaceLineTextHandler = LexerObject<TextProcessingTransition, Negate<OrTest<IsSpace, CharacterTest<'{'>>>>;
 
 using WhitespaceNonNewlineHandler = LexerObject<TextProcessingTransition, AndTest<Negate<CharacterTest<'\n'>>, IsSpace>>;
 
 using FinalizingLineTextHandler = LexerObject<TextProcessingTransition, Negate<OrTest<CharacterTest<'{'>, IsSpace>>, TokenFinalizer>;
 
 using SyntaxBoundaryNewlineHandler = CharacterTransition<'\n', TokenFinalizerWithTrimmingAndNewline>;
 using SyntaxBoundaryHandler = CharacterTransition<'{', FinalizeAndMarkStartSyntax>;
 
 template<typename Transition>
 void addTransition(TextProcessingState *source, Lexer *lexer, TextProcessingState *target)
 {
     auto tr = new Transition(lexer, source);
     tr->setTargetState(target);
 }
 
 TextProcessingMachine *createMachine(Lexer *lexer, Lexer::TrimType type)
 {
     auto machine = new TextProcessingMachine;
 
     auto notFinished = new TextProcessingState(machine);
     auto finished = new TextProcessingState(machine);
     machine->setInitialState(notFinished);
 
     auto processingText = new ChurningState(lexer, notFinished);
     auto processingPostNewline = new TextProcessingState(notFinished);
     auto processingBeginTemplateSyntax = new TextProcessingState(notFinished);
     auto processingTag = new TextProcessingState(notFinished);
     auto processingComment = new TextProcessingState(notFinished);
     auto processingValue = new TextProcessingState(notFinished);
     auto maybeProcessingValue = new TextProcessingState(notFinished);
     auto processingEndTag = new TextProcessingState(notFinished);
     auto processingEndComment = new TextProcessingState(notFinished);
     auto processingEndValue = new TextProcessingState(notFinished);
     TextProcessingState *processingPostTemplateSyntax;
 
     if (type == Lexer::SmartTrim)
         processingPostTemplateSyntax = new TextProcessingState(notFinished);
     else
         processingPostTemplateSyntax = new FinalizeTokenState(lexer, notFinished);
     auto processingPostTemplateSyntaxWhitespace = new TextProcessingState(notFinished);
 
     if (type == Lexer::SmartTrim)
         notFinished->setInitialState(processingPostNewline);
     else
         notFinished->setInitialState(processingText);
 
     if (type == Lexer::SmartTrim) {
         addTransition<NewlineHandler>(processingText, lexer, processingPostNewline);
 
         addTransition<NewlineHandler>(processingPostNewline, lexer, processingPostNewline);
         addTransition<MaybeTemplateSyntaxHandler>(processingPostNewline, lexer, processingBeginTemplateSyntax);
         addTransition<NonWhitespaceLineTextHandler>(processingPostNewline, lexer, processingText);
     }
     addTransition<MaybeTemplateSyntaxHandler>(processingText, lexer, processingBeginTemplateSyntax);
 
     addTransition<TagStartHandler>(processingBeginTemplateSyntax, lexer, processingTag);
     addTransition<CommentStartHandler>(processingBeginTemplateSyntax, lexer, processingComment);
     addTransition<BeginValueHandler>(processingBeginTemplateSyntax, lexer, maybeProcessingValue);
 
     if (type == Lexer::SmartTrim) {
         addTransition<NotBeginTemplateSyntaxOrNewlineHandler>(processingBeginTemplateSyntax, lexer, processingText);
         addTransition<NewlineHandler>(processingBeginTemplateSyntax, lexer, processingPostNewline);
     } else {
         addTransition<NotBeginTemplateSyntaxHandler>(processingBeginTemplateSyntax, lexer, processingText);
     }
 
     addTransition<NewlineHandler>(processingTag, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
     addTransition<TagEndHandler>(processingTag, lexer, processingEndTag);
 
     addTransition<NewlineHandler>(processingComment, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
     addTransition<CommentEndHandler>(processingComment, lexer, processingEndComment);
 
     addTransition<TagStartHandler>(maybeProcessingValue, lexer, processingTag);
     addTransition<CommentStartHandler>(maybeProcessingValue, lexer, processingComment);
     addTransition<NotTagCommentOrNewlineHandler>(maybeProcessingValue, lexer, processingValue);
     addTransition<NewlineHandler>(maybeProcessingValue, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
 
     addTransition<NewlineHandler>(processingValue, lexer, type == Lexer::SmartTrim ? processingPostNewline : processingText);
     addTransition<MaybeEndValueHandler>(processingValue, lexer, processingEndValue);
 
     addTransition<NewlineHandler>(processingEndTag, lexer, processingPostNewline);
     addTransition<NotEndTemplateSyntaxHandler>(processingEndTag, lexer, processingTag);
     addTransition<EndTemplateSyntaxHandler>(processingEndTag, lexer, processingPostTemplateSyntax);
 
     addTransition<NewlineHandler>(processingEndComment, lexer, processingPostNewline);
     addTransition<NotEndTemplateSyntaxHandler>(processingEndComment, lexer, processingComment);
     addTransition<EndTemplateSyntaxHandler>(processingEndComment, lexer, processingPostTemplateSyntax);
 
     addTransition<NewlineHandler>(processingEndValue, lexer, processingPostNewline);
     addTransition<NotEndTemplateSyntaxHandler>(processingEndValue, lexer, processingValue);
     addTransition<EndTemplateSyntaxHandler>(processingEndValue, lexer, processingPostTemplateSyntax);
 
     if (type != Lexer::SmartTrim) {
         processingPostTemplateSyntax->setUnconditionalTransition(processingText);
     } else {
         addTransition<SyntaxBoundaryNewlineHandler>(processingPostTemplateSyntax, lexer, processingPostNewline);
         addTransition<WhitespaceNonNewlineHandler>(processingPostTemplateSyntax, lexer, processingPostTemplateSyntaxWhitespace);
         addTransition<FinalizingLineTextHandler>(processingPostTemplateSyntax, lexer, processingText);
         addTransition<SyntaxBoundaryHandler>(processingPostTemplateSyntax, lexer, processingBeginTemplateSyntax);
 
         // NOTE: We only have to transition to this if there was whitespace
         // before the opening tag. Maybe store that in an external state property?
         // Actually, this may be a bug if we try to finalize with trimming and
         // there is no leading whitespace.
         addTransition<SyntaxBoundaryNewlineHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingPostNewline);
         addTransition<FinalizingLineTextHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingText);
         addTransition<SyntaxBoundaryHandler>(processingPostTemplateSyntaxWhitespace, lexer, processingBeginTemplateSyntax);
     }
 
     {
         auto handler = new EofHandler(lexer, notFinished);
         handler->setTargetState(finished);
         notFinished->setEndTransition(handler);
     }
 
     if (type == Lexer::SmartTrim) {
         {
             auto handler = new EofHandlerWithTrimming(lexer, processingPostTemplateSyntaxWhitespace);
             handler->setTargetState(finished);
             processingPostTemplateSyntaxWhitespace->setEndTransition(handler);
         }
         {
             auto handler = new EofHandlerWithTrimming(lexer, processingPostTemplateSyntax);
             handler->setTargetState(finished);
             processingPostTemplateSyntax->setEndTransition(handler);
         }
     }
     return machine;
 }
 
 Lexer::Lexer(const QString &templateString)
     : m_templateString(templateString)
 {
 }
 
 Lexer::~Lexer() = default;
 
 void Lexer::clearMarkers()
 {
     m_startSyntaxPosition = -1;
     m_endSyntaxPosition = -1;
     m_newlinePosition = -1;
 }
 
 void Lexer::reset()
 {
     m_tokenList.clear();
     m_lineCount = 0;
     m_upto = 0;
     m_processedUpto = 0;
     clearMarkers();
 }
 
 QList<Token> Lexer::tokenize(TrimType type)
 {
     auto machine = createMachine(this, type);
 
     machine->start();
 
     auto it = m_templateString.constBegin();
     const auto end = m_templateString.constEnd();
 
     reset();
     for (; it != end; ++it, ++m_upto)
         machine->processCharacter(it);
 
     machine->finished();
 
     machine->stop();
 
     delete machine;
 
     return m_tokenList;
 }
 
 void Lexer::markStartSyntax()
 {
     m_startSyntaxPosition = m_upto;
 }
 
 void Lexer::markEndSyntax()
 {
     m_endSyntaxPosition = m_upto + 1;
 }
 
 void Lexer::markNewline()
 {
     m_newlinePosition = m_upto;
     ++m_lineCount;
 }
 
 void Lexer::finalizeToken()
 {
     auto nextPosition = m_upto;
     const auto validSyntax = m_endSyntaxPosition > m_startSyntaxPosition && (m_startSyntaxPosition >= m_processedUpto);
 
     if (validSyntax) {
         Q_ASSERT(m_startSyntaxPosition >= 0);
         nextPosition = m_startSyntaxPosition - 1;
     }
     finalizeToken(nextPosition, validSyntax);
 }
 
 void Lexer::finalizeTokenWithTrimmedWhitespace()
 {
     auto nextPosition = m_upto;
     // We know this to be true because the state machine has already guaranteed
     // it. This method is only called from transition and state actions which
     // occur after valid syntax.
     // TODO Investigate performance and other implications of changing the state
     // machine to assure similar in finalizeToken()
     Q_ASSERT(m_endSyntaxPosition > m_startSyntaxPosition);
 
     Q_ASSERT(m_startSyntaxPosition >= 0);
     if (m_newlinePosition >= 0 && m_newlinePosition >= m_processedUpto)
         nextPosition = qMin(m_startSyntaxPosition - 1, m_newlinePosition);
     else
         nextPosition = m_startSyntaxPosition - 1;
     finalizeToken(nextPosition, true);
 }
 
 void Lexer::finalizeToken(int nextPosition, bool processSyntax)
 {
     {
         Token token;
         token.content = m_templateString.mid(m_processedUpto, nextPosition - m_processedUpto);
         token.tokenType = TextToken;
         token.linenumber = m_lineCount;
         m_tokenList.append(token);
     }
 
     m_processedUpto = nextPosition;
 
     if (!processSyntax)
         return;
 
     m_processedUpto = m_endSyntaxPosition;
 
     const auto differentiator = m_templateString.at(m_startSyntaxPosition);
     if (differentiator == QLatin1Char('#'))
         return;
 
     Token syntaxToken;
     syntaxToken.content = m_templateString.mid(m_startSyntaxPosition + 1, m_endSyntaxPosition - m_startSyntaxPosition - 3).trimmed();
     syntaxToken.linenumber = m_lineCount;
 
     if (differentiator == QLatin1Char('{')) {
         syntaxToken.tokenType = VariableToken;
     } else {
         Q_ASSERT(differentiator == QLatin1Char('%'));
         syntaxToken.tokenType = BlockToken;
     }
     m_tokenList.append(syntaxToken);
 }