parser/tests/pyasttest.cpp

0001 /*
0002     SPDX-FileCopyrightText: 2012 Sven Brauch svenbrauch @googlemail.com
0003
0004     SPDX-License-Identifier: GPL-2.0-or-later
0005 */
0006
0007 #include <language/duchain/topducontext.h>
0008 #include <language/codegen/coderepresentation.h>
0009 #include <tests/autotestshell.h>
0010 #include <tests/testcore.h>
0011 #include <language/duchain/duchain.h>
0012 #include <language/duchain/types/functiontype.h>
0013 #include <language/duchain/aliasdeclaration.h>
0014
0015 #include "parsesession.h"
0016 #include "pythoneditorintegrator.h"
0017 #include "kdevpythonversion.h"
0018 #include "declarationbuilder.h"
0019 #include "usebuilder.h"
0020 #include "astdefaultvisitor.h"
0021 #include "expressionvisitor.h"
0022 #include "contextbuilder.h"
0023 #include "astbuilder.h"
0024
0025 #include "duchain/helpers.h"
0026
0027 #include "pyasttest.h"
0028 #include "../astbuilder.h"
0029 #include "../parserdebug.h"
0030
0031 #include <ktexteditor_version.h>
0032
0033 #include <QDebug>
0034 #include <QtTest>
0035
0036 using namespace Python;
0037
0038 QTEST_MAIN(PyAstTest)
0039
0040 PyAstTest::PyAstTest(QObject* parent): QObject(parent)
0041 {
0042     initShell();
0043 }
0044
0045 void PyAstTest::initShell()
0046 {
0047     AutoTestShell::init();
0048     TestCore* core = new TestCore();
0049     core->initialize(KDevelop::Core::NoUi);
0050     DUChain::self()->disablePersistentStorage();
0051     KDevelop::CodeRepresentation::setDiskChangesForbidden(true);
0052 }
0053
0054 CodeAst::Ptr PyAstTest::getAst(QString code)
0055 {
0056     QSharedPointer<AstBuilder> builder(new AstBuilder);
0057     CodeAst::Ptr result = builder->parse(QUrl("<empty>"), code);
0058     qDebug() << result->dump();
0059     return result;
0060 }
0061
0062 class VerifyVisitor : public AstDefaultVisitor {
0063 public:
0064     VerifyVisitor() : AstDefaultVisitor(), m_nodecount(0) { };
0065     void visitNode(Ast* node) override {
0066         m_nodecount += 1;
0067         QVERIFY(! node || node->astType < Ast::LastAstType);
0068         AstDefaultVisitor::visitNode(node);
0069     };
0070     void visitName(NameAst* node) override {
0071         QVERIFY(! node->identifier->value.isNull());
0072         AstDefaultVisitor::visitName(node);
0073     };
0074     void visitCode(CodeAst* node) override {
0075         AstDefaultVisitor::visitCode(node);
0076         qDebug() << "done, nodes visited:" << m_nodecount;
0077     };
0078     int m_nodecount;
0079 };
0080
0081 void PyAstTest::testCode(QString code)
0082 {
0083     CodeAst::Ptr ast = getAst(code);
0084     VerifyVisitor v;
0085     v.visitCode(ast.data());
0086 }
0087
0088 void PyAstTest::testExceptionHandlers()
0089 {
0090     QString code = "try: pass\n"
0091                    "except FooBar as baz: pass\n"
0092                    "except Cat as baz: pass\n"
0093                    "except Dawg as asdf: pass\n";
0094     CodeAst::Ptr ast = getAst(code);
0095     VerifyVisitor v;
0096     v.visitCode(ast.data());
0097     QCOMPARE(ast->body.size(), 1);
0098     QVERIFY(ast->body.first()->astType == Ast::TryAstType);
0099     TryAst* try_ = static_cast<TryAst*>(ast->body.first());
0100     QCOMPARE(try_->handlers.size(), 3);
0101     foreach ( ExceptionHandlerAst* handler, try_->handlers ) {
0102         QVERIFY(handler->name);
0103         QCOMPARE(handler->name->astType, Ast::IdentifierAstType);
0104     }
0105 }
0106
0107 void PyAstTest::testStatements()
0108 {
0109     QFETCH(QString, code);
0110     testCode(code);
0111 }
0112
0113 void PyAstTest::testStatements_data()
0114 {
0115     QTest::addColumn<QString>("code");
0116     QTest::newRow("assign_int") << "a = 3";
0117     QTest::newRow("funcdef") << "def myfun(): pass";
0118 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0)
0119     QTest::newRow("asyncfuncdef") << "async def myfun(): pass";
0120     QTest::newRow("asyncfuncdef2") << "async    def myfun(): pass";
0121     QTest::newRow("asyncfuncdef_await") << "async def myfun(): await 3";
0122 #endif
0123     QTest::newRow("funcdef_args") << "def myfun(arg1, arg2): pass";
0124     QTest::newRow("funcdef_vararg") << "def myfun(arg1, *arg): pass";
0125     QTest::newRow("funcdef_kwarg") << "def myfun(**arg): pass";
0126     QTest::newRow("classdef_inheritance") << "class myclass(parent): pass";
0127     QTest::newRow("return") << "return 3";
0128     QTest::newRow("for") << "for i in 1, 2, 3: pass";
0129     QTest::newRow("while") << "while True: pass";
0130     QTest::newRow("if") << "if True: pass";
0131     QTest::newRow("ifElse") << "if True: pass\nelse:pass";
0132     QTest::newRow("with") << "with x as y: pass";
0133     QTest::newRow("raise") << "raise Exception";
0134     QTest::newRow("tryexcept") << "try:pass\nexcept:pass";
0135     QTest::newRow("tryexceptfinally") << "try:pass\nexcept:pass\nfinally:pass";
0136     QTest::newRow("assert") << "assert false";
0137     QTest::newRow("import") << "import foobar";
0138     QTest::newRow("importfrom") << "from foobar import bazbang";
0139     QTest::newRow("global") << "global x";
0140     QTest::newRow("break") << "while True: break";
0141     QTest::newRow("continue") << "while True: continue";
0142     QTest::newRow("pass") << "pass";
0143     QTest::newRow("nonlocal") << "nonlocal x";
0144     QTest::newRow("delete") << "del x";
0145     QTest::newRow("augassign_plus") << "a += b";
0146     QTest::newRow("augassign_minus") << "a -= b";
0147     QTest::newRow("augassign_mul") << "a *= b";
0148     QTest::newRow("augassign_div") << "a /= b";
0149     QTest::newRow("augassign_or") << "a &= b";
0150     QTest::newRow("augassign_and") << "a |= b";
0151     QTest::newRow("augassign_xor") << "a ^= b";
0152 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0)
0153     QTest::newRow("augassign_matmul") << "a @= b";
0154 #endif
0155 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0)
0156     QTest::newRow("varannotation1") << "primes: List[int] = []";
0157     QTest::newRow("varannotation2") << "captain: str  # Note: no initial value!";
0158 #endif
0159
0160 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 10, 0)
0161     QTest::newRow("match_value") << "match x:\n case 1:\n  pass";
0162     QTest::newRow("match_singleton") << "match x:\n case True:\n  pass";
0163     QTest::newRow("match_sequence") << "match x:\n case [a, b]:\n  pass";
0164     QTest::newRow("match_mapping") << "match x:\n case {'type': 'update'}:\n  pass";
0165     // TODO: Mapping rest
0166     QTest::newRow("match_class") << "class Node:\n __match_args__=('type',)\ntype=1\nmatch x:\n case Node():\n  pass";
0167     QTest::newRow("match_star") << "match x:\n case [a, b, *c]:\n  pass";
0168     QTest::newRow("match_as") << "match x:\n case a:\n  pass";
0169     QTest::newRow("match_as_empty") << "match x:\n case _:\n  pass";
0170     QTest::newRow("match_or") << "match x:\n case a|b:\n  pass";
0171     QTest::newRow("match_guard") << "match x:\n case [a, b, c] if c != 0:\n  pass";
0172 #endif
0173
0174 }
0175
0176 void PyAstTest::testSlices()
0177 {
0178     QFETCH(QString, code);
0179     testCode(code);
0180 }
0181
0182 void PyAstTest::testSlices_data()
0183 {
0184     QTest::addColumn<QString>("code");
0185     QTest::newRow("slice1") << "x[1]";
0186     QTest::newRow("slice2") << "x[2:3]";
0187     QTest::newRow("slice3") << "x[::]";
0188     QTest::newRow("slice4") << "x[1:2:3]";
0189     QTest::newRow("slice_ellipsis") << "myList[1:2, ..., 0]";
0190 }
0191
0192 void PyAstTest::testOther()
0193 {
0194     QFETCH(QString, code);
0195     testCode(code);
0196 }
0197
0198 void PyAstTest::testOther_data()
0199 {
0200     QTest::addColumn<QString>("code");
0201     QTest::newRow("args") << "foo(bar, baz, *bang, **baa)";
0202     QTest::newRow("kws") << "foo(bar=baz, bang=3)";
0203     QTest::newRow("excpthandler") << "try:pass\nexcept foo as bar: pass";
0204     QTest::newRow("alias") << "import foo as bar";
0205 }
0206
0207 void PyAstTest::testExpressions()
0208 {
0209     QFETCH(QString, code);
0210     testCode(code);
0211 }
0212
0213 void PyAstTest::testExpressions_data()
0214 {
0215     QTest::addColumn<QString>("code");
0216
0217     QTest::newRow("boolop") << "b or c";
0218     QTest::newRow("binop") << "b ^ c";
0219     QTest::newRow("unop") << "not a";
0220     QTest::newRow("lambda") << "lambda x: y";
0221     QTest::newRow("ifexpr") << "3 if 4 else 5";
0222     QTest::newRow("dict") << "{}";
0223     QTest::newRow("set") << "(3, 5)";
0224     QTest::newRow("listcomp") << "[x for x in y]";
0225     QTest::newRow("setcomp") << "(x for x in y)";
0226     QTest::newRow("dictcomp") << "{x:y for x, y in z}";
0227     QTest::newRow("comp") << "x < y";
0228     QTest::newRow("number") << "3";
0229     QTest::newRow("string") << "\"foo\"";
0230     QTest::newRow("bytes") << "b\"bytes\"";
0231     QTest::newRow("yield") << "yield x";
0232     QTest::newRow("name") << "foo";
0233     QTest::newRow("call") << "foo()";
0234     QTest::newRow("attribute") << "foo.bar";
0235     QTest::newRow("attribute_nontoplevel") << "while True: foo.bar";
0236     QTest::newRow("subscript") << "foo[3]";
0237     QTest::newRow("starred") << "*[1, 2, 3 ,4]";
0238     QTest::newRow("list") << "[]";
0239     QTest::newRow("tuple") << "()";
0240     QTest::newRow("None") << "None";
0241     QTest::newRow("False") << "False";
0242     QTest::newRow("True") << "True";
0243     QTest::newRow("Set") << "{1, 2, 3}";
0244     QTest::newRow("SetComp") << "{1 for x in [1, 2, 3]}";
0245     QTest::newRow("ExtSlice") << "A[1:3,3:5]";
0246     QTest::newRow("FString") << "f\"hi {a.b}\"";
0247     QTest::newRow("Formatted") << "'%s %s' % ('one', 'two')";
0248
0249 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 6, 0)
0250     QTest::newRow("async_generator") << "async def foo(): result = [i async for i in aiter() if i % 2]";
0251     QTest::newRow("await_generator") << "async def foo(): result = [await fun() for fun in funcs]";
0252
0253     QTest::newRow("underscore_literals") << "0x_FF_FF_FF_FF";
0254     QTest::newRow("formatted_string_literal") << "f\"He said his name is {name}.\"";
0255 #endif
0256 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 5, 0)
0257     QTest::newRow("dstar_unpack") << "ext_map = {\n"
0258         "       **{ext: self.obj_extension for ext in self.src_extensions},\n"
0259         "       **{ext: self.res_extension for ext in self._rc_extensions + self._mc_extensions},\n"
0260     "}";
0261     QTest::newRow("async_for") << "[i async for i in foo()]";
0262     QTest::newRow("async_for_stmt") << "async for x in y: pass";
0263     QTest::newRow("async_with_stmt") << "async with x as y: pass";
0264 #endif
0265 #if PYTHON_VERSION >= QT_VERSION_CHECK(3, 8, 0)
0266     QTest::newRow("assignment_expr_1") << "a = (b := 10)";
0267     QTest::newRow("assignment_expr_2") << "a = [q for z in (1, 2, 3) if (q := 2*z)]";
0268     QTest::newRow("positional_params") << "def foo(a, b, /, c, d, *, e): pass";
0269 #endif
0270 }
0271
0272 void PyAstTest::testCorrectedFuncRanges()
0273 {
0274     QFETCH(QString, code);
0275     QFETCH(KTextEditor::Range, range);
0276
0277     CodeAst::Ptr ast = getAst(code);
0278     QVERIFY(ast);
0279     foreach ( Ast* node, ast->body ) {
0280         if ( node->astType != Ast::FunctionDefinitionAstType ) {
0281             continue;
0282         }
0283         FunctionDefinitionAst* func = static_cast<FunctionDefinitionAst*>(node);
0284         QVERIFY(func->name);
0285         qCDebug(KDEV_PYTHON_PARSER) << func->name->range() << range;
0286         QCOMPARE(func->name->range(), range);
0287     }
0288 }
0289
0290 void PyAstTest::testCorrectedFuncRanges_data()
0291 {
0292     QTest::addColumn<QString>("code");
0293     QTest::addColumn<KTextEditor::Range>("range");
0294
0295     QTest::newRow("decorator") << "@decorate\ndef func(arg): pass" << KTextEditor::Range(1, 4, 1, 7);
0296     QTest::newRow("decorator_arg") << "@decorate(yomama=3)\ndef func(arg): pass" << KTextEditor::Range(1, 4, 1, 7);
0297     QTest::newRow("two_decorators") << "@decorate2\n@decorate\ndef func(arg): pass" << KTextEditor::Range(2, 4, 2, 7);
0298     QTest::newRow("decorate_class") << "class foo:\n @decorate2\n @decorate\n def func(arg): pass" << KTextEditor::Range(3, 5, 3, 8);
0299 }
0300
0301 void PyAstTest::testNewPython3()
0302 {
0303     QFETCH(QString, code);
0304     testCode(code);
0305 }
0306
0307 void PyAstTest::testNewPython3_data()
0308 {
0309     QTest::addColumn<QString>("code");
0310     QTest::newRow("funcannotation1") << "def haul(item: Haulable, *vargs: PackAnimal) -> Distance: pass";
0311     QTest::newRow("funcannotation2") << "def foo() -> expr: pass";
0312     QTest::newRow("funcannotation3") << "def foo(a: 'x', b: 5 + 6, c: list) -> max(2, 9): pass";
0313     QTest::newRow("kwonly1") << "def compare(a, b, *, key=None): pass";
0314     QTest::newRow("kwonly2") << "def foo(a: 'x', b: 5 + 6, c: list) -> max(2, 9): pass";
0315     QTest::newRow("listunpack") << "(a, *rest, b) = range(5)";
0316     QTest::newRow("metaclass") << "class C(metaclass=M): pass";
0317     QTest::newRow("exception_chain") << "raise SecondaryException() from primary_exception";
0318     QTest::newRow("yield_from") << "def foo(): yield from [1, 2, 3]";
0319 }
0320
0321 void PyAstTest::testClass()
0322 {
0323     testCode("class c: pass");
0324 }
0325
0326 #include "moc_pyasttest.cpp"