File indexing completed on 2024-03-24 16:05:55

0001 /*
0002     SPDX-FileCopyrightText: 2007 Andreas Pakulat <apaku@gmx.de>
0003     SPDX-FileCopyrightText: 2007 Piyush verma <piyush.verma@gmail.com>
0004     SPDX-FileCopyrightText: 2010-2013 Sven Brauch <svenbrauch@googlemail.com>
0005 
0006     SPDX-License-Identifier: GPL-2.0-or-later
0007 */
0008 
0009 #include "pythonparsejob.h"
0010 
0011 #include "pythondebug.h"
0012 #include "pythonhighlighting.h"
0013 #include "pythoneditorintegrator.h"
0014 #include "dumpchain.h"
0015 #include "parsesession.h"
0016 #include "pythonlanguagesupport.h"
0017 #include "declarationbuilder.h"
0018 #include "usebuilder.h"
0019 #include "kshell.h"
0020 #include "duchain/helpers.h"
0021 #include "pep8kcm/kcm_pep8.h"
0022 
0023 #include <language/duchain/duchainlock.h>
0024 #include <language/duchain/duchain.h>
0025 #include <language/duchain/topducontext.h>
0026 #include <language/duchain/dumpdotgraph.h>
0027 #include <serialization/indexedstring.h>
0028 #include <language/duchain/duchainutils.h>
0029 #include <language/backgroundparser/urlparselock.h>
0030 #include <language/backgroundparser/backgroundparser.h>
0031 #include <language/highlighting/codehighlighting.h>
0032 #include <language/interfaces/iastcontainer.h>
0033 #include <language/checks/controlflowgraph.h>
0034 #include <language/checks/dataaccessrepository.h>
0035 #include <interfaces/icore.h>
0036 #include <interfaces/iprojectcontroller.h>
0037 #include <interfaces/ilanguagecontroller.h>
0038 #include <interfaces/idocumentcontroller.h>
0039 #include <util/foregroundlock.h>
0040 #include <project/projectmodel.h>
0041 #include <util/path.h>
0042 
0043 #include <ktexteditor/document.h>
0044 
0045 #include <QReadLocker>
0046 #include <QThread>
0047 #include <QCoreApplication>
0048 #include <QProcess>
0049 #include <QTemporaryFile>
0050 #include <QDebug>
0051 #include <KConfigGroup>
0052 
0053 #include <custom-definesandincludes/idefinesandincludesmanager.h>
0054 
0055 using namespace KDevelop;
0056 
0057 namespace Python
0058 {
0059 
0060 ParseJob::ParseJob(const IndexedString &url, ILanguageSupport* languageSupport)
0061         : KDevelop::ParseJob(url, languageSupport)
0062         , m_ast(nullptr)
0063         , m_duContext(nullptr)
0064 {
0065     IDefinesAndIncludesManager* iface = IDefinesAndIncludesManager::manager();
0066     auto project = ICore::self()->projectController()->findProjectForUrl(url.toUrl());
0067     if ( project ) {
0068         foreach (Path path, iface->includes(project->projectItem(), IDefinesAndIncludesManager::UserDefined)) {
0069             m_cachedCustomIncludes.append(path.toUrl());
0070         }
0071         QMutexLocker lock(&Helper::cacheMutex);
0072         Helper::cachedCustomIncludes[project] = m_cachedCustomIncludes;
0073     }
0074 }
0075 
0076 ParseJob::~ParseJob()
0077 {
0078 }
0079 
0080 CodeAst *ParseJob::ast() const
0081 {
0082     Q_ASSERT( isFinished() && m_ast );
0083     return m_ast.data();
0084 }
0085 
0086 
0087 void ParseJob::run(ThreadWeaver::JobPointer /*self*/, ThreadWeaver::Thread* /*thread*/)
0088 {
0089     if ( abortRequested() || ICore::self()->shuttingDown() ) {
0090         return abortJob();
0091     }
0092     
0093     qCDebug(KDEV_PYTHON) << " ====> PARSING ====> parsing file " << document().toUrl() << "; has priority" << parsePriority();
0094 
0095     {
0096         QMutexLocker l(&Helper::projectPathLock);
0097         Helper::projectSearchPaths.clear();
0098         foreach  (IProject* project, ICore::self()->projectController()->projects() ) {
0099             Helper::projectSearchPaths.append(QUrl::fromLocalFile(project->path().path()));
0100         }
0101     }
0102     
0103     // lock the URL so no other parse job can run on this document
0104     QReadLocker parselock(languageSupport()->parseLock());
0105     UrlParseLock urlLock(document());
0106 
0107     readContents();
0108     
0109     if ( !(minimumFeatures() & TopDUContext::ForceUpdate || minimumFeatures() & Rescheduled) ) {
0110         DUChainReadLocker lock(DUChain::lock());
0111         static const IndexedString langString("python");
0112         foreach(const ParsingEnvironmentFilePointer &file, DUChain::self()->allEnvironmentFiles(document())) {
0113             if ( file->language() != langString ) {
0114                 continue;
0115             }
0116             if ( ! file->needsUpdate() && file->featuresSatisfied(minimumFeatures()) && file->topContext() ) {
0117                 qCDebug(KDEV_PYTHON) << " ====> NOOP    ====> Already up to date:" << document().str();
0118                 setDuChain(file->topContext());
0119                 if ( ICore::self()->languageController()->backgroundParser()->trackerForUrl(document()) ) {
0120                     lock.unlock();
0121                     highlightDUChain();
0122                 }
0123                 return;
0124             }
0125             break;
0126         }
0127     }
0128     
0129     ReferencedTopDUContext toUpdate = nullptr;
0130     {
0131         DUChainReadLocker lock;
0132         toUpdate = DUChainUtils::standardContextForUrl(document().toUrl());
0133     }
0134     if ( toUpdate ) {
0135         translateDUChainToRevision(toUpdate);
0136         toUpdate->setRange(RangeInRevision(0, 0, INT_MAX, INT_MAX));
0137     }
0138     
0139     m_currentSession = new ParseSession();
0140     m_currentSession->setContents(QString::fromUtf8(contents().contents));
0141     m_currentSession->setCurrentDocument(document());
0142     
0143     // call the python API and the AST transformer to populate the syntax tree
0144     QPair<CodeAst::Ptr, bool> parserResults = m_currentSession->parse();
0145     m_ast = parserResults.first;
0146 
0147     auto editor = QSharedPointer<PythonEditorIntegrator>(new PythonEditorIntegrator(m_currentSession.data()));
0148     // if parsing succeeded, continue and do semantic analysis
0149     if ( parserResults.second )
0150     {
0151         // set up the declaration builder, it gets the parsePriority so it can re-schedule imported files with a better priority
0152         DeclarationBuilder builder(editor.data(), parsePriority());
0153         builder.setCurrentlyParsedDocument(document());
0154         builder.setFutureModificationRevision(contents().modification);
0155 
0156         // Run the declaration builder. If necessary, it will run itself again.
0157         m_duContext = builder.build(document(), m_ast.data(), toUpdate.data());
0158         if ( abortRequested() ) {
0159             return abortJob();
0160         }
0161         
0162         setDuChain(m_duContext);
0163         
0164         // gather uses of variables and functions on the document
0165         UseBuilder usebuilder(editor.data(), builder.missingModules());
0166         usebuilder.setCurrentlyParsedDocument(document());
0167         usebuilder.buildUses(m_ast.data());
0168         
0169         // check whether any unresolved imports were encountered
0170         bool needsReparse = ! builder.unresolvedImports().isEmpty();
0171         qCDebug(KDEV_PYTHON) << "Document needs update because of unresolved identifiers: " << needsReparse;
0172         if ( needsReparse ) {
0173             // check whether one of the imports is queued for parsing, this is to avoid deadlocks
0174             // it's also ok if the duchain is now available (and thus has been parsed before already)
0175             bool dependencyInQueue = false;
0176             DUChainWriteLocker lock;
0177             foreach ( const IndexedString& url, builder.unresolvedImports() ) {
0178                 dependencyInQueue = KDevelop::ICore::self()->languageController()->backgroundParser()->isQueued(url);
0179                 dependencyInQueue = dependencyInQueue || DUChain::self()->chainForDocument(url);
0180                 if ( dependencyInQueue ) {
0181                     break;
0182                 }
0183             }
0184             // we check whether this document already has been re-scheduled once and abort if that is the case
0185             // this prevents infinite loops in case something goes wrong (optimally, shouldn't reach here if
0186             // the document was already rescheduled, but there's many cases where this might still happen)
0187             if ( ! ( minimumFeatures() & Rescheduled ) && dependencyInQueue ) {
0188                 constexpr TopDUContext::Features features{TopDUContext::ForceUpdate};
0189                 KDevelop::ICore::self()->languageController()->backgroundParser()->addDocument(document(),
0190                                      static_cast<TopDUContext::Features>(features | Rescheduled), parsePriority(),
0191                                      nullptr, ParseJob::FullSequentialProcessing);
0192             }
0193         }
0194         
0195         // some internal housekeeping work
0196         {
0197             DUChainWriteLocker lock(DUChain::lock());
0198             m_duContext->setFeatures(minimumFeatures());
0199             ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile();
0200             parsingEnvironmentFile->setModificationRevision(contents().modification);
0201             DUChain::self()->updateContextEnvironment(m_duContext, parsingEnvironmentFile.data());
0202         }
0203         
0204         qCDebug(KDEV_PYTHON) << "---- Parsing Succeeded ----";
0205         
0206         if ( abortRequested() ) {
0207             return abortJob();
0208         }
0209         
0210         // start the code highlighter if parsing was successful.
0211         highlightDUChain();
0212     }
0213     else {
0214         // No syntax tree was received from the parser, the expected reason for this is a syntax error in the document.
0215         qWarning() << "---- Parsing FAILED ----";
0216         DUChainWriteLocker lock;
0217         m_duContext = toUpdate.data();
0218         // if there's already a chain for the document, do some cleanup.
0219         if ( m_duContext ) {
0220 //             m_duContext->parsingEnvironmentFile()->clearModificationRevisions(); // TODO why?
0221             ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile();
0222             parsingEnvironmentFile->setModificationRevision(contents().modification);
0223             m_duContext->clearProblems();
0224         }
0225         // otherwise, create a new, empty top context for the file. This serves as a placeholder until
0226         // the syntax is fixed; for example, it prevents the document from being reparsed again until it is modified.
0227         else {
0228             ParsingEnvironmentFile* file = new ParsingEnvironmentFile(document());
0229             static const IndexedString langString("python");
0230             file->setLanguage(langString);
0231             m_duContext = new TopDUContext(document(), RangeInRevision(0, 0, INT_MAX, INT_MAX), file);
0232             m_duContext->setType(DUContext::Global);
0233             DUChain::self()->addDocumentChain(m_duContext);
0234             Q_ASSERT(m_duContext->type() == DUContext::Global);
0235         }
0236         
0237         setDuChain(m_duContext);
0238     }
0239     
0240     if ( abortRequested() ) {
0241         return abortJob();
0242     }
0243     
0244     // The parser might have given us some syntax errors, which are now added to the document.
0245     DUChainWriteLocker lock;
0246     foreach ( const ProblemPointer& p, m_currentSession->m_problems ) {
0247         m_duContext->addProblem(p);
0248     }
0249 
0250     // If enabled, and if the document is open, do PEP8 checking.
0251     eventuallyDoPEP8Checking(m_duContext);
0252 
0253     if ( minimumFeatures() & TopDUContext::AST ) {
0254         DUChainWriteLocker lock;
0255         m_currentSession->ast = m_ast;
0256         m_duContext->setAst(QExplicitlySharedDataPointer<IAstContainer>(m_currentSession.data()));
0257     }
0258     
0259     setDuChain(m_duContext);
0260     DUChain::self()->emitUpdateReady(document(), duChain());
0261 }
0262 
0263 ControlFlowGraph* ParseJob::controlFlowGraph()
0264 {
0265     return nullptr;
0266 }
0267 
0268 DataAccessRepository* ParseJob::dataAccessInformation()
0269 {
0270     return nullptr;
0271 }
0272 
0273 void ParseJob::eventuallyDoPEP8Checking(TopDUContext* topContext)
0274 {
0275     KConfig config("kdevpythonsupportrc");
0276     KConfigGroup configGroup = config.group("pep8");
0277     if ( !PEP8KCModule::isPep8Enabled(configGroup) ) {
0278         return;
0279     }
0280     auto ls = static_cast<Python::LanguageSupport*>(languageSupport());
0281     QMetaObject::invokeMethod(ls, "updateStyleChecking", Q_ARG(KDevelop::ReferencedTopDUContext, topContext));
0282 }
0283 
0284 }
0285 
0286 #include "moc_pythonparsejob.cpp"
0287 
0288 // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on