File indexing completed on 2024-03-24 16:05:55
0001 /* 0002 SPDX-FileCopyrightText: 2007 Andreas Pakulat <apaku@gmx.de> 0003 SPDX-FileCopyrightText: 2007 Piyush verma <piyush.verma@gmail.com> 0004 SPDX-FileCopyrightText: 2010-2013 Sven Brauch <svenbrauch@googlemail.com> 0005 0006 SPDX-License-Identifier: GPL-2.0-or-later 0007 */ 0008 0009 #include "pythonparsejob.h" 0010 0011 #include "pythondebug.h" 0012 #include "pythonhighlighting.h" 0013 #include "pythoneditorintegrator.h" 0014 #include "dumpchain.h" 0015 #include "parsesession.h" 0016 #include "pythonlanguagesupport.h" 0017 #include "declarationbuilder.h" 0018 #include "usebuilder.h" 0019 #include "kshell.h" 0020 #include "duchain/helpers.h" 0021 #include "pep8kcm/kcm_pep8.h" 0022 0023 #include <language/duchain/duchainlock.h> 0024 #include <language/duchain/duchain.h> 0025 #include <language/duchain/topducontext.h> 0026 #include <language/duchain/dumpdotgraph.h> 0027 #include <serialization/indexedstring.h> 0028 #include <language/duchain/duchainutils.h> 0029 #include <language/backgroundparser/urlparselock.h> 0030 #include <language/backgroundparser/backgroundparser.h> 0031 #include <language/highlighting/codehighlighting.h> 0032 #include <language/interfaces/iastcontainer.h> 0033 #include <language/checks/controlflowgraph.h> 0034 #include <language/checks/dataaccessrepository.h> 0035 #include <interfaces/icore.h> 0036 #include <interfaces/iprojectcontroller.h> 0037 #include <interfaces/ilanguagecontroller.h> 0038 #include <interfaces/idocumentcontroller.h> 0039 #include <util/foregroundlock.h> 0040 #include <project/projectmodel.h> 0041 #include <util/path.h> 0042 0043 #include <ktexteditor/document.h> 0044 0045 #include <QReadLocker> 0046 #include <QThread> 0047 #include <QCoreApplication> 0048 #include <QProcess> 0049 #include <QTemporaryFile> 0050 #include <QDebug> 0051 #include <KConfigGroup> 0052 0053 #include <custom-definesandincludes/idefinesandincludesmanager.h> 0054 0055 using namespace KDevelop; 0056 0057 namespace Python 0058 { 0059 0060 ParseJob::ParseJob(const IndexedString &url, ILanguageSupport* languageSupport) 0061 : KDevelop::ParseJob(url, languageSupport) 0062 , m_ast(nullptr) 0063 , m_duContext(nullptr) 0064 { 0065 IDefinesAndIncludesManager* iface = IDefinesAndIncludesManager::manager(); 0066 auto project = ICore::self()->projectController()->findProjectForUrl(url.toUrl()); 0067 if ( project ) { 0068 foreach (Path path, iface->includes(project->projectItem(), IDefinesAndIncludesManager::UserDefined)) { 0069 m_cachedCustomIncludes.append(path.toUrl()); 0070 } 0071 QMutexLocker lock(&Helper::cacheMutex); 0072 Helper::cachedCustomIncludes[project] = m_cachedCustomIncludes; 0073 } 0074 } 0075 0076 ParseJob::~ParseJob() 0077 { 0078 } 0079 0080 CodeAst *ParseJob::ast() const 0081 { 0082 Q_ASSERT( isFinished() && m_ast ); 0083 return m_ast.data(); 0084 } 0085 0086 0087 void ParseJob::run(ThreadWeaver::JobPointer /*self*/, ThreadWeaver::Thread* /*thread*/) 0088 { 0089 if ( abortRequested() || ICore::self()->shuttingDown() ) { 0090 return abortJob(); 0091 } 0092 0093 qCDebug(KDEV_PYTHON) << " ====> PARSING ====> parsing file " << document().toUrl() << "; has priority" << parsePriority(); 0094 0095 { 0096 QMutexLocker l(&Helper::projectPathLock); 0097 Helper::projectSearchPaths.clear(); 0098 foreach (IProject* project, ICore::self()->projectController()->projects() ) { 0099 Helper::projectSearchPaths.append(QUrl::fromLocalFile(project->path().path())); 0100 } 0101 } 0102 0103 // lock the URL so no other parse job can run on this document 0104 QReadLocker parselock(languageSupport()->parseLock()); 0105 UrlParseLock urlLock(document()); 0106 0107 readContents(); 0108 0109 if ( !(minimumFeatures() & TopDUContext::ForceUpdate || minimumFeatures() & Rescheduled) ) { 0110 DUChainReadLocker lock(DUChain::lock()); 0111 static const IndexedString langString("python"); 0112 foreach(const ParsingEnvironmentFilePointer &file, DUChain::self()->allEnvironmentFiles(document())) { 0113 if ( file->language() != langString ) { 0114 continue; 0115 } 0116 if ( ! file->needsUpdate() && file->featuresSatisfied(minimumFeatures()) && file->topContext() ) { 0117 qCDebug(KDEV_PYTHON) << " ====> NOOP ====> Already up to date:" << document().str(); 0118 setDuChain(file->topContext()); 0119 if ( ICore::self()->languageController()->backgroundParser()->trackerForUrl(document()) ) { 0120 lock.unlock(); 0121 highlightDUChain(); 0122 } 0123 return; 0124 } 0125 break; 0126 } 0127 } 0128 0129 ReferencedTopDUContext toUpdate = nullptr; 0130 { 0131 DUChainReadLocker lock; 0132 toUpdate = DUChainUtils::standardContextForUrl(document().toUrl()); 0133 } 0134 if ( toUpdate ) { 0135 translateDUChainToRevision(toUpdate); 0136 toUpdate->setRange(RangeInRevision(0, 0, INT_MAX, INT_MAX)); 0137 } 0138 0139 m_currentSession = new ParseSession(); 0140 m_currentSession->setContents(QString::fromUtf8(contents().contents)); 0141 m_currentSession->setCurrentDocument(document()); 0142 0143 // call the python API and the AST transformer to populate the syntax tree 0144 QPair<CodeAst::Ptr, bool> parserResults = m_currentSession->parse(); 0145 m_ast = parserResults.first; 0146 0147 auto editor = QSharedPointer<PythonEditorIntegrator>(new PythonEditorIntegrator(m_currentSession.data())); 0148 // if parsing succeeded, continue and do semantic analysis 0149 if ( parserResults.second ) 0150 { 0151 // set up the declaration builder, it gets the parsePriority so it can re-schedule imported files with a better priority 0152 DeclarationBuilder builder(editor.data(), parsePriority()); 0153 builder.setCurrentlyParsedDocument(document()); 0154 builder.setFutureModificationRevision(contents().modification); 0155 0156 // Run the declaration builder. If necessary, it will run itself again. 0157 m_duContext = builder.build(document(), m_ast.data(), toUpdate.data()); 0158 if ( abortRequested() ) { 0159 return abortJob(); 0160 } 0161 0162 setDuChain(m_duContext); 0163 0164 // gather uses of variables and functions on the document 0165 UseBuilder usebuilder(editor.data(), builder.missingModules()); 0166 usebuilder.setCurrentlyParsedDocument(document()); 0167 usebuilder.buildUses(m_ast.data()); 0168 0169 // check whether any unresolved imports were encountered 0170 bool needsReparse = ! builder.unresolvedImports().isEmpty(); 0171 qCDebug(KDEV_PYTHON) << "Document needs update because of unresolved identifiers: " << needsReparse; 0172 if ( needsReparse ) { 0173 // check whether one of the imports is queued for parsing, this is to avoid deadlocks 0174 // it's also ok if the duchain is now available (and thus has been parsed before already) 0175 bool dependencyInQueue = false; 0176 DUChainWriteLocker lock; 0177 foreach ( const IndexedString& url, builder.unresolvedImports() ) { 0178 dependencyInQueue = KDevelop::ICore::self()->languageController()->backgroundParser()->isQueued(url); 0179 dependencyInQueue = dependencyInQueue || DUChain::self()->chainForDocument(url); 0180 if ( dependencyInQueue ) { 0181 break; 0182 } 0183 } 0184 // we check whether this document already has been re-scheduled once and abort if that is the case 0185 // this prevents infinite loops in case something goes wrong (optimally, shouldn't reach here if 0186 // the document was already rescheduled, but there's many cases where this might still happen) 0187 if ( ! ( minimumFeatures() & Rescheduled ) && dependencyInQueue ) { 0188 constexpr TopDUContext::Features features{TopDUContext::ForceUpdate}; 0189 KDevelop::ICore::self()->languageController()->backgroundParser()->addDocument(document(), 0190 static_cast<TopDUContext::Features>(features | Rescheduled), parsePriority(), 0191 nullptr, ParseJob::FullSequentialProcessing); 0192 } 0193 } 0194 0195 // some internal housekeeping work 0196 { 0197 DUChainWriteLocker lock(DUChain::lock()); 0198 m_duContext->setFeatures(minimumFeatures()); 0199 ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile(); 0200 parsingEnvironmentFile->setModificationRevision(contents().modification); 0201 DUChain::self()->updateContextEnvironment(m_duContext, parsingEnvironmentFile.data()); 0202 } 0203 0204 qCDebug(KDEV_PYTHON) << "---- Parsing Succeeded ----"; 0205 0206 if ( abortRequested() ) { 0207 return abortJob(); 0208 } 0209 0210 // start the code highlighter if parsing was successful. 0211 highlightDUChain(); 0212 } 0213 else { 0214 // No syntax tree was received from the parser, the expected reason for this is a syntax error in the document. 0215 qWarning() << "---- Parsing FAILED ----"; 0216 DUChainWriteLocker lock; 0217 m_duContext = toUpdate.data(); 0218 // if there's already a chain for the document, do some cleanup. 0219 if ( m_duContext ) { 0220 // m_duContext->parsingEnvironmentFile()->clearModificationRevisions(); // TODO why? 0221 ParsingEnvironmentFilePointer parsingEnvironmentFile = m_duContext->parsingEnvironmentFile(); 0222 parsingEnvironmentFile->setModificationRevision(contents().modification); 0223 m_duContext->clearProblems(); 0224 } 0225 // otherwise, create a new, empty top context for the file. This serves as a placeholder until 0226 // the syntax is fixed; for example, it prevents the document from being reparsed again until it is modified. 0227 else { 0228 ParsingEnvironmentFile* file = new ParsingEnvironmentFile(document()); 0229 static const IndexedString langString("python"); 0230 file->setLanguage(langString); 0231 m_duContext = new TopDUContext(document(), RangeInRevision(0, 0, INT_MAX, INT_MAX), file); 0232 m_duContext->setType(DUContext::Global); 0233 DUChain::self()->addDocumentChain(m_duContext); 0234 Q_ASSERT(m_duContext->type() == DUContext::Global); 0235 } 0236 0237 setDuChain(m_duContext); 0238 } 0239 0240 if ( abortRequested() ) { 0241 return abortJob(); 0242 } 0243 0244 // The parser might have given us some syntax errors, which are now added to the document. 0245 DUChainWriteLocker lock; 0246 foreach ( const ProblemPointer& p, m_currentSession->m_problems ) { 0247 m_duContext->addProblem(p); 0248 } 0249 0250 // If enabled, and if the document is open, do PEP8 checking. 0251 eventuallyDoPEP8Checking(m_duContext); 0252 0253 if ( minimumFeatures() & TopDUContext::AST ) { 0254 DUChainWriteLocker lock; 0255 m_currentSession->ast = m_ast; 0256 m_duContext->setAst(QExplicitlySharedDataPointer<IAstContainer>(m_currentSession.data())); 0257 } 0258 0259 setDuChain(m_duContext); 0260 DUChain::self()->emitUpdateReady(document(), duChain()); 0261 } 0262 0263 ControlFlowGraph* ParseJob::controlFlowGraph() 0264 { 0265 return nullptr; 0266 } 0267 0268 DataAccessRepository* ParseJob::dataAccessInformation() 0269 { 0270 return nullptr; 0271 } 0272 0273 void ParseJob::eventuallyDoPEP8Checking(TopDUContext* topContext) 0274 { 0275 KConfig config("kdevpythonsupportrc"); 0276 KConfigGroup configGroup = config.group("pep8"); 0277 if ( !PEP8KCModule::isPep8Enabled(configGroup) ) { 0278 return; 0279 } 0280 auto ls = static_cast<Python::LanguageSupport*>(languageSupport()); 0281 QMetaObject::invokeMethod(ls, "updateStyleChecking", Q_ARG(KDevelop::ReferencedTopDUContext, topContext)); 0282 } 0283 0284 } 0285 0286 #include "moc_pythonparsejob.cpp" 0287 0288 // kate: space-indent on; indent-width 4; tab-width 4; replace-tabs on; auto-insert-doxygen on