File indexing completed on 2024-04-28 11:20:52
0001 /* 0002 SPDX-License-Identifier: GPL-2.0-or-later 0003 SPDX-FileCopyrightText: 2019 Sirgienko Nikita <warquark@gmail.com> 0004 */ 0005 0006 #include <QtTest> 0007 #include <QDebug> 0008 #include <KLocalizedString> 0009 #include <QMovie> 0010 #include <KZip> 0011 #include <KActionCollection> 0012 0013 #include "worksheet_test.h" 0014 #include "../worksheet.h" 0015 #include "../session.h" 0016 #include "../worksheetentry.h" 0017 #include "../worksheetview.h" 0018 #include "../textentry.h" 0019 #include "../markdownentry.h" 0020 #include "../commandentry.h" 0021 #include "../latexentry.h" 0022 #include "../lib/backend.h" 0023 #include "../lib/expression.h" 0024 #include "../lib/result.h" 0025 #include "../lib/textresult.h" 0026 #include "../lib/imageresult.h" 0027 #include "../lib/latexresult.h" 0028 #include "../lib/animationresult.h" 0029 #include "../lib/mimeresult.h" 0030 #include "../lib/htmlresult.h" 0031 0032 #include "config-cantor-test.h" 0033 0034 static const QString dataPath = QString::fromLocal8Bit(PATH_TO_TEST_NOTEBOOKS)+QLatin1String("/"); 0035 0036 void WorksheetTest::initTestCase() 0037 { 0038 const QStringList& backends = Cantor::Backend::listAvailableBackends(); 0039 if (backends.isEmpty()) 0040 { 0041 QString reason = i18n("Testing of worksheets requires a functioning backends"); 0042 QSKIP( reason.toStdString().c_str(), SkipAll ); 0043 } 0044 } 0045 0046 Worksheet* WorksheetTest::loadWorksheet(const QString& name) 0047 { 0048 Worksheet* w = new Worksheet(Cantor::Backend::getBackend(QLatin1String("maxima")), nullptr, false); 0049 new WorksheetView(w, nullptr); 0050 w->load(dataPath + name); 0051 KActionCollection* collection = new KActionCollection(nullptr, QString()); 0052 w->setActionCollection(collection); 0053 return w; 0054 } 0055 0056 QString WorksheetTest::plainMarkdown(WorksheetEntry* markdownEntry) 0057 { 0058 QString plain; 0059 0060 if (markdownEntry->type() == MarkdownEntry::Type) 0061 { 0062 QString text = markdownEntry->toPlain(QString(), QLatin1String("\n"), QLatin1String("\n")); 0063 text.remove(0,1); 0064 text.chop(2); 0065 plain = text; 0066 } 0067 0068 return plain; 0069 } 0070 0071 QString WorksheetTest::plainText(WorksheetEntry* textEntry) 0072 { 0073 QString plain; 0074 0075 if (textEntry->type() == TextEntry::Type) 0076 { 0077 QString text = textEntry->toPlain(QString(), QLatin1String("\n"), QLatin1String("\n")); 0078 text.remove(0,1); 0079 text.chop(2); 0080 plain = text; 0081 } 0082 0083 return plain; 0084 } 0085 0086 QString WorksheetTest::plainLatex(WorksheetEntry* latexEntry) 0087 { 0088 QString plain; 0089 0090 if (latexEntry->type() == LatexEntry::Type) 0091 { 0092 QString text = latexEntry->toPlain(QString(), QLatin1String("\n"), QLatin1String("\n")); 0093 text.remove(0,1); 0094 text.chop(2); 0095 plain = text; 0096 } 0097 0098 return plain; 0099 } 0100 0101 int WorksheetTest::entriesCount(Worksheet* worksheet) 0102 { 0103 int count = 0; 0104 WorksheetEntry* entry = worksheet->firstEntry(); 0105 while (entry) 0106 { 0107 count++; 0108 entry = entry->next(); 0109 } 0110 return count; 0111 } 0112 0113 Cantor::Expression * WorksheetTest::expression(WorksheetEntry* entry) 0114 { 0115 CommandEntry* command = dynamic_cast<CommandEntry*>(entry); 0116 if (command) 0117 return command->expression(); 0118 else 0119 return nullptr; 0120 } 0121 0122 QString WorksheetTest::plainCommand(WorksheetEntry* commandEntry) 0123 { 0124 QString plain; 0125 0126 if (commandEntry->type() == CommandEntry::Type) 0127 { 0128 plain = commandEntry->toPlain(QString(), QString(), QString()); 0129 } 0130 0131 return plain; 0132 } 0133 0134 void WorksheetTest::testMarkdown(WorksheetEntry* &entry, const QString& content) 0135 { 0136 WorksheetEntry* current = entry; 0137 QVERIFY(current); 0138 entry = entry->next(); 0139 QCOMPARE(current->type(), (int)MarkdownEntry::Type); 0140 QCOMPARE(plainMarkdown(current), content); 0141 } 0142 0143 void WorksheetTest::testTextEntry(WorksheetEntry *& entry, const QString& content) 0144 { 0145 WorksheetEntry* current = entry; 0146 QVERIFY(current); 0147 entry = entry->next(); 0148 QCOMPARE(current->type(), (int)TextEntry::Type); 0149 QCOMPARE(plainText(current), content); 0150 } 0151 0152 void WorksheetTest::testCommandEntry(WorksheetEntry *& entry, int id, const QString& content) 0153 { 0154 WorksheetEntry* current = entry; 0155 QVERIFY(current); 0156 entry = entry->next(); 0157 QCOMPARE(current->type(), (int)CommandEntry::Type); 0158 QCOMPARE(plainCommand(current), content); 0159 QVERIFY(expression(current)); 0160 QCOMPARE(expression(current)->id(), id); 0161 QCOMPARE(expression(current)->results().size(), 0); 0162 } 0163 0164 void WorksheetTest::testCommandEntry(WorksheetEntry* entry, int id, int resultsCount, const QString& content) 0165 { 0166 QVERIFY(entry); 0167 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0168 QCOMPARE(plainCommand(entry), content); 0169 QVERIFY(expression(entry)); 0170 QCOMPARE(expression(entry)->id(), id); 0171 QCOMPARE(expression(entry)->results().size(), resultsCount); 0172 } 0173 0174 void WorksheetTest::testLatexEntry(WorksheetEntry *& entry, const QString& content) 0175 { 0176 WorksheetEntry* current = entry; 0177 QVERIFY(current); 0178 entry = entry->next(); 0179 QCOMPARE(current->type(), (int)LatexEntry::Type); 0180 QCOMPARE(plainLatex(current), content); 0181 } 0182 0183 void WorksheetTest::testImageResult(WorksheetEntry* entry, int index) 0184 { 0185 QVERIFY(expression(entry)); 0186 QVERIFY(expression(entry)->results().size() > index); 0187 QCOMPARE(expression(entry)->results().at(index)->type(), (int)Cantor::ImageResult::Type); 0188 QVERIFY(expression(entry)->results().at(index)->data().value<QImage>().isNull() == false); 0189 } 0190 0191 void WorksheetTest::testTextResult(WorksheetEntry* entry, int index, const QString& content) 0192 { 0193 QVERIFY(expression(entry)); 0194 QVERIFY(expression(entry)->results().size() > index); 0195 QCOMPARE(expression(entry)->results().at(index)->type(), (int)Cantor::TextResult::Type); 0196 Cantor::TextResult* result = static_cast<Cantor::TextResult*>(expression(entry)->results().at(index)); 0197 QVERIFY(result->format() == Cantor::TextResult::PlainTextFormat); 0198 QCOMPARE(result->plain(), content); 0199 } 0200 0201 void WorksheetTest::testHtmlResult(WorksheetEntry* entry, int index, const QString& content) 0202 { 0203 QVERIFY(expression(entry)); 0204 QVERIFY(expression(entry)->results().size() > index); 0205 QCOMPARE(expression(entry)->results().at(index)->type(), (int)Cantor::HtmlResult::Type); 0206 Cantor::HtmlResult* result = static_cast<Cantor::HtmlResult*>(expression(entry)->results().at(index)); 0207 QCOMPARE(result->plain(), content); 0208 } 0209 0210 void WorksheetTest::testHtmlResult(WorksheetEntry* entry, int index, const QString& plain, const QString& html) 0211 { 0212 QVERIFY(expression(entry)); 0213 QVERIFY(expression(entry)->results().size() > index); 0214 QCOMPARE(expression(entry)->results().at(index)->type(), (int)Cantor::HtmlResult::Type); 0215 Cantor::HtmlResult* result = static_cast<Cantor::HtmlResult*>(expression(entry)->results().at(index)); 0216 QCOMPARE(result->data().toString(), html); 0217 QCOMPARE(result->plain(), plain); 0218 } 0219 0220 void WorksheetTest::waitForSignal(QObject* sender, const char* signal) 0221 { 0222 QTimer timeout( this ); 0223 timeout.setSingleShot( true ); 0224 0225 QEventLoop loop; 0226 connect( sender, signal, &loop, SLOT(quit()) ); 0227 connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit); 0228 timeout.start( 25000 ); 0229 loop.exec(); 0230 } 0231 0232 0233 void WorksheetTest::testJupyter1() 0234 { 0235 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 0236 if (backend && backend->isEnabled() == false) 0237 QSKIP("Skip, because python backend don't available", SkipSingle); 0238 0239 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("Lecture-2B-Single-Atom-Lasing.ipynb"))); 0240 0241 qDebug() << w->firstEntry(); 0242 QCOMPARE(entriesCount(w.data()), 41); 0243 0244 WorksheetEntry* entry = w->firstEntry(); 0245 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0246 QCOMPARE(plainMarkdown(entry), QLatin1String("# QuTiP lecture: Single-Atom-Lasing")); 0247 0248 entry = entry->next(); 0249 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0250 QCOMPARE(plainMarkdown(entry), QLatin1String( 0251 "Author: J. R. Johansson (robert@riken.jp), http://dml.riken.jp/~rob/\n" 0252 "\n" 0253 "The latest version of this [IPython notebook](http://ipython.org/ipython-doc/dev/interactive/htmlnotebook.html) lecture is available at [http://github.com/jrjohansson/qutip-lectures](http://github.com/jrjohansson/qutip-lectures).\n" 0254 "\n" 0255 "The other notebooks in this lecture series are indexed at [http://jrjohansson.github.com](http://jrjohansson.github.com)." 0256 )); 0257 0258 entry = entry->next(); 0259 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0260 QCOMPARE(plainCommand(entry), QLatin1String( 0261 "# setup the matplotlib graphics library and configure it to show \n" 0262 "# figures inline in the notebook\n" 0263 "%matplotlib inline\n" 0264 "import matplotlib.pyplot as plt\n" 0265 "import numpy as np" 0266 )); 0267 QVERIFY(expression(entry)); 0268 QCOMPARE(expression(entry)->id(), 1); 0269 QCOMPARE(expression(entry)->results().size(), 0); 0270 0271 entry = entry->next(); 0272 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0273 QCOMPARE(plainCommand(entry), QLatin1String( 0274 "# make qutip available in the rest of the notebook\n" 0275 "from qutip import *\n" 0276 "\n" 0277 "from IPython.display import Image" 0278 )); 0279 QVERIFY(expression(entry)); 0280 QCOMPARE(expression(entry)->id(), 2); 0281 QCOMPARE(expression(entry)->results().size(), 0); 0282 0283 entry = entry->next(); 0284 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0285 QCOMPARE(plainMarkdown(entry), QLatin1String( 0286 "# Introduction and model\n" 0287 "\n" 0288 "Consider a single atom coupled to a single cavity mode, as illustrated in the figure below. If there atom excitation rate $\\Gamma$ exceeds the relaxation rate, a population inversion can occur in the atom, and if coupled to the cavity the atom can then act as a photon pump on the cavity." 0289 )); 0290 0291 entry = entry->next(); 0292 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0293 QCOMPARE(plainCommand(entry), QLatin1String( 0294 "Image(filename='images/schematic-lasing-model.png')" 0295 )); 0296 QVERIFY(expression(entry)); 0297 QCOMPARE(expression(entry)->id(), 3); 0298 QCOMPARE(expression(entry)->results().size(), 1); 0299 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0300 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0301 0302 entry = entry->next(); 0303 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0304 QCOMPARE(plainMarkdown(entry), QLatin1String( 0305 "The coherent dynamics in this model is described by the Hamiltonian\n" 0306 "\n" 0307 "$H = \\hbar \\omega_0 a^\\dagger a + \\frac{1}{2}\\hbar\\omega_a\\sigma_z + \\hbar g\\sigma_x(a^\\dagger + a)$\n" 0308 "\n" 0309 "where $\\omega_0$ is the cavity energy splitting, $\\omega_a$ is the atom energy splitting and $g$ is the atom-cavity interaction strength.\n" 0310 "\n" 0311 "In addition to the coherent dynamics the following incoherent processes are also present: \n" 0312 "\n" 0313 "1. $\\kappa$ relaxation and thermal excitations of the cavity, \n" 0314 "2. $\\Gamma$ atomic excitation rate (pumping process).\n" 0315 "\n" 0316 "The Lindblad master equation for the model is:\n" 0317 "\n" 0318 "$\\frac{d}{dt}\\rho = -i[H, \\rho] + \\Gamma\\left(\\sigma_+\\rho\\sigma_- - \\frac{1}{2}\\sigma_-\\sigma_+\\rho - \\frac{1}{2}\\rho\\sigma_-\\sigma_+\\right)\n" 0319 "+ \\kappa (1 + n_{\\rm th}) \\left(a\\rho a^\\dagger - \\frac{1}{2}a^\\dagger a\\rho - \\frac{1}{2}\\rho a^\\dagger a\\right)\n" 0320 "+ \\kappa n_{\\rm th} \\left(a^\\dagger\\rho a - \\frac{1}{2}a a^\\dagger \\rho - \\frac{1}{2}\\rho a a^\\dagger\\right)$\n" 0321 "\n" 0322 "in units where $\\hbar = 1$.\n" 0323 "\n" 0324 "References:\n" 0325 "\n" 0326 " * [Yi Mu, C.M. Savage, Phys. Rev. A 46, 5944 (1992)](http://dx.doi.org/10.1103/PhysRevA.46.5944)\n" 0327 "\n" 0328 " * [D.A. Rodrigues, J. Imbers, A.D. Armour, Phys. Rev. Lett. 98, 067204 (2007)](http://dx.doi.org/10.1103/PhysRevLett.98.067204)\n" 0329 "\n" 0330 " * [S. Ashhab, J.R. Johansson, A.M. Zagoskin, F. Nori, New J. Phys. 11, 023030 (2009)](http://dx.doi.org/10.1088/1367-2630/11/2/023030)" 0331 )); 0332 0333 entry = entry->next(); 0334 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0335 QCOMPARE(plainMarkdown(entry), QLatin1String("### Problem parameters")); 0336 0337 entry = entry->next(); 0338 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0339 QCOMPARE(plainCommand(entry), QLatin1String( 0340 "w0 = 1.0 * 2 * pi # cavity frequency\n" 0341 "wa = 1.0 * 2 * pi # atom frequency\n" 0342 "g = 0.05 * 2 * pi # coupling strength\n" 0343 "\n" 0344 "kappa = 0.04 # cavity dissipation rate\n" 0345 "gamma = 0.00 # atom dissipation rate\n" 0346 "Gamma = 0.35 # atom pump rate\n" 0347 "\n" 0348 "N = 50 # number of cavity fock states\n" 0349 "n_th_a = 0.0 # avg number of thermal bath excitation\n" 0350 "\n" 0351 "tlist = np.linspace(0, 150, 101)" 0352 )); 0353 QVERIFY(expression(entry)); 0354 QCOMPARE(expression(entry)->id(), 5); 0355 QCOMPARE(expression(entry)->results().size(), 0); 0356 0357 entry = entry->next(); 0358 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0359 QCOMPARE(plainMarkdown(entry), QLatin1String("### Setup the operators, the Hamiltonian and initial state")); 0360 0361 entry = entry->next(); 0362 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0363 QCOMPARE(plainCommand(entry), QLatin1String( 0364 "# initial state\n" 0365 "psi0 = tensor(basis(N,0), basis(2,0)) # start without excitations\n" 0366 "\n" 0367 "# operators\n" 0368 "a = tensor(destroy(N), qeye(2))\n" 0369 "sm = tensor(qeye(N), destroy(2))\n" 0370 "sx = tensor(qeye(N), sigmax())\n" 0371 "\n" 0372 "# Hamiltonian\n" 0373 "H = w0 * a.dag() * a + wa * sm.dag() * sm + g * (a.dag() + a) * sx" 0374 )); 0375 QVERIFY(expression(entry)); 0376 QCOMPARE(expression(entry)->id(), 6); 0377 QCOMPARE(expression(entry)->results().size(), 0); 0378 0379 entry = entry->next(); 0380 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0381 QCOMPARE(plainCommand(entry), QLatin1String("H")); 0382 QVERIFY(expression(entry)); 0383 QCOMPARE(expression(entry)->id(), 7); 0384 QCOMPARE(expression(entry)->results().size(), 1); 0385 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::LatexResult::Type); 0386 { 0387 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->result()); 0388 QCOMPARE(result->code(), QLatin1String( 0389 "Quantum object: dims = [[50, 2], [50, 2]], shape = [100, 100], type = oper, isherm = True\\begin{equation*}\\left(\\begin{array}{*{11}c}0.0 & 0.0 & 0.0 & 0.314 & 0.0 & \\cdots & 0.0 & 0.0 & 0.0 & 0.0 & 0.0\\\\0.0 & 6.283 & 0.314 & 0.0 & 0.0 & \\cdots & 0.0 & 0.0 & 0.0 & 0.0 & 0.0\\\\0.0 & 0.314 & 6.283 & 0.0 & 0.0 & \\cdots & 0.0 & 0.0 & 0.0 & 0.0 & 0.0\\\\0.314 & 0.0 & 0.0 & 12.566 & 0.444 & \\cdots & 0.0 & 0.0 & 0.0 & 0.0 & 0.0\\\\0.0 & 0.0 & 0.0 & 0.444 & 12.566 & \\cdots & 0.0 & 0.0 & 0.0 & 0.0 & 0.0\\\\\\vdots & \\vdots & \\vdots & \\vdots & \\vdots & \\ddots & \\vdots & \\vdots & \\vdots & \\vdots & \\vdots\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.0 & \\cdots & 301.593 & 2.177 & 0.0 & 0.0 & 0.0\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.0 & \\cdots & 2.177 & 301.593 & 0.0 & 0.0 & 2.199\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.0 & \\cdots & 0.0 & 0.0 & 307.876 & 2.199 & 0.0\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.0 & \\cdots & 0.0 & 0.0 & 2.199 & 307.876 & 0.0\\\\0.0 & 0.0 & 0.0 & 0.0 & 0.0 & \\cdots & 0.0 & 2.199 & 0.0 & 0.0 & 314.159\\\\\\end{array}\\right)\\end{equation*}" 0390 )); 0391 QCOMPARE(result->plain(), QLatin1String( 0392 "Quantum object: dims = [[50, 2], [50, 2]], shape = [100, 100], type = oper, isherm = True\n" 0393 "Qobj data =\n" 0394 "[[ 0. 0. 0. ..., 0. 0. 0. ]\n" 0395 " [ 0. 6.28318531 0.31415927 ..., 0. 0. 0. ]\n" 0396 " [ 0. 0.31415927 6.28318531 ..., 0. 0. 0. ]\n" 0397 " ..., \n" 0398 " [ 0. 0. 0. ..., 307.87608005\n" 0399 " 2.19911486 0. ]\n" 0400 " [ 0. 0. 0. ..., 2.19911486\n" 0401 " 307.87608005 0. ]\n" 0402 " [ 0. 0. 0. ..., 0. 0.\n" 0403 " 314.15926536]]" 0404 )); 0405 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 0406 } 0407 0408 entry = entry->next(); 0409 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0410 QCOMPARE(plainMarkdown(entry), QLatin1String("### Create a list of collapse operators that describe the dissipation")); 0411 0412 entry = entry->next(); 0413 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0414 QCOMPARE(plainCommand(entry), QLatin1String( 0415 "# collapse operators\n" 0416 "c_ops = []\n" 0417 "\n" 0418 "rate = kappa * (1 + n_th_a)\n" 0419 "if rate > 0.0:\n" 0420 " c_ops.append(sqrt(rate) * a)\n" 0421 "\n" 0422 "rate = kappa * n_th_a\n" 0423 "if rate > 0.0:\n" 0424 " c_ops.append(sqrt(rate) * a.dag())\n" 0425 "\n" 0426 "rate = gamma\n" 0427 "if rate > 0.0:\n" 0428 " c_ops.append(sqrt(rate) * sm)\n" 0429 "\n" 0430 "rate = Gamma\n" 0431 "if rate > 0.0:\n" 0432 " c_ops.append(sqrt(rate) * sm.dag())" 0433 )); 0434 QVERIFY(expression(entry)); 0435 QCOMPARE(expression(entry)->id(), 8); 0436 QCOMPARE(expression(entry)->results().size(), 0); 0437 0438 entry = entry->next(); 0439 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0440 QCOMPARE(plainMarkdown(entry), QLatin1String( 0441 "### Evolve the system\n" 0442 "\n" 0443 "Here we evolve the system with the Lindblad master equation solver, and we request that the expectation values of the operators $a^\\dagger a$ and $\\sigma_+\\sigma_-$ are returned by the solver by passing the list `[a.dag()*a, sm.dag()*sm]` as the fifth argument to the solver." 0444 )); 0445 0446 entry = entry->next(); 0447 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0448 QCOMPARE(plainCommand(entry), QLatin1String( 0449 "opt = Odeoptions(nsteps=2000) # allow extra time-steps \n" 0450 "output = mesolve(H, psi0, tlist, c_ops, [a.dag() * a, sm.dag() * sm], options=opt)" 0451 )); 0452 QVERIFY(expression(entry)); 0453 QCOMPARE(expression(entry)->id(), 9); 0454 QCOMPARE(expression(entry)->results().size(), 0); 0455 0456 entry = entry->next(); 0457 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0458 QCOMPARE(plainMarkdown(entry), QLatin1String( 0459 "## Visualize the results\n" 0460 "\n" 0461 "Here we plot the excitation probabilities of the cavity and the atom (these expectation values were calculated by the `mesolve` above)." 0462 )); 0463 0464 entry = entry->next(); 0465 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0466 QCOMPARE(plainCommand(entry), QLatin1String( 0467 "n_c = output.expect[0]\n" 0468 "n_a = output.expect[1]\n" 0469 "\n" 0470 "fig, axes = plt.subplots(1, 1, figsize=(8,6))\n" 0471 "\n" 0472 "axes.plot(tlist, n_c, label=\"Cavity\")\n" 0473 "axes.plot(tlist, n_a, label=\"Atom excited state\")\n" 0474 "axes.set_xlim(0, 150)\n" 0475 "axes.legend(loc=0)\n" 0476 "axes.set_xlabel('Time')\n" 0477 "axes.set_ylabel('Occupation probability');" 0478 )); 0479 QVERIFY(expression(entry)); 0480 QCOMPARE(expression(entry)->id(), 10); 0481 QCOMPARE(expression(entry)->results().size(), 1); 0482 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0483 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0484 0485 entry = entry->next(); 0486 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0487 QCOMPARE(plainMarkdown(entry), QLatin1String("## Steady state: cavity fock-state distribution and wigner function")); 0488 0489 entry = entry->next(); 0490 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0491 QCOMPARE(plainCommand(entry), QLatin1String("rho_ss = steadystate(H, c_ops)")); 0492 QVERIFY(expression(entry)); 0493 QCOMPARE(expression(entry)->id(), 11); 0494 QCOMPARE(expression(entry)->results().size(), 0); 0495 0496 entry = entry->next(); 0497 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0498 QCOMPARE(plainCommand(entry), QLatin1String( 0499 "fig, axes = plt.subplots(1, 2, figsize=(12,6))\n" 0500 "\n" 0501 "xvec = np.linspace(-5,5,200)\n" 0502 "\n" 0503 "rho_cavity = ptrace(rho_ss, 0)\n" 0504 "W = wigner(rho_cavity, xvec, xvec)\n" 0505 "wlim = abs(W).max()\n" 0506 "\n" 0507 "axes[1].contourf(xvec, xvec, W, 100, norm=mpl.colors.Normalize(-wlim,wlim), cmap=plt.get_cmap('RdBu'))\n" 0508 "axes[1].set_xlabel(r'Im $\\alpha$', fontsize=18)\n" 0509 "axes[1].set_ylabel(r'Re $\\alpha$', fontsize=18)\n" 0510 "\n" 0511 "axes[0].bar(arange(0, N), real(rho_cavity.diag()), color=\"blue\", alpha=0.6)\n" 0512 "axes[0].set_ylim(0, 1)\n" 0513 "axes[0].set_xlim(0, N)\n" 0514 "axes[0].set_xlabel('Fock number', fontsize=18)\n" 0515 "axes[0].set_ylabel('Occupation probability', fontsize=18);" 0516 )); 0517 QVERIFY(expression(entry)); 0518 QCOMPARE(expression(entry)->id(), 13); 0519 QCOMPARE(expression(entry)->results().size(), 1); 0520 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0521 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0522 0523 entry = entry->next(); 0524 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0525 QCOMPARE(plainMarkdown(entry), QLatin1String("## Cavity fock-state distribution and Wigner function as a function of time")); 0526 0527 entry = entry->next(); 0528 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0529 QCOMPARE(plainCommand(entry), QLatin1String( 0530 "tlist = np.linspace(0, 25, 5)\n" 0531 "output = mesolve(H, psi0, tlist, c_ops, [], options=Odeoptions(nsteps=5000))" 0532 )); 0533 QVERIFY(expression(entry)); 0534 QCOMPARE(expression(entry)->id(), 14); 0535 QCOMPARE(expression(entry)->results().size(), 0); 0536 0537 entry = entry->next(); 0538 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0539 QCOMPARE(plainCommand(entry), QLatin1String( 0540 "rho_ss_sublist = output.states\n" 0541 "\n" 0542 "xvec = np.linspace(-5,5,200)\n" 0543 "\n" 0544 "fig, axes = plt.subplots(2, len(rho_ss_sublist), figsize=(3*len(rho_ss_sublist), 6))\n" 0545 "\n" 0546 "for idx, rho_ss in enumerate(rho_ss_sublist):\n" 0547 "\n" 0548 " # trace out the cavity density matrix\n" 0549 " rho_ss_cavity = ptrace(rho_ss, 0)\n" 0550 " \n" 0551 " # calculate its wigner function\n" 0552 " W = wigner(rho_ss_cavity, xvec, xvec)\n" 0553 " \n" 0554 " # plot its wigner function\n" 0555 " wlim = abs(W).max()\n" 0556 " axes[0,idx].contourf(xvec, xvec, W, 100, norm=mpl.colors.Normalize(-wlim,wlim), cmap=plt.get_cmap('RdBu'))\n" 0557 " axes[0,idx].set_title(r'$t = %.1f$' % tlist[idx])\n" 0558 " \n" 0559 " # plot its fock-state distribution\n" 0560 " axes[1,idx].bar(arange(0, N), real(rho_ss_cavity.diag()), color=\"blue\", alpha=0.8)\n" 0561 " axes[1,idx].set_ylim(0, 1)\n" 0562 " axes[1,idx].set_xlim(0, 15)" 0563 )); 0564 QVERIFY(expression(entry)); 0565 QCOMPARE(expression(entry)->id(), 15); 0566 QCOMPARE(expression(entry)->results().size(), 1); 0567 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0568 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0569 0570 entry = entry->next(); 0571 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0572 QCOMPARE(plainMarkdown(entry), QLatin1String( 0573 "## Steady state average photon occupation in cavity as a function of pump rate\n" 0574 "\n" 0575 "References:\n" 0576 "\n" 0577 " * [S. Ashhab, J.R. Johansson, A.M. Zagoskin, F. Nori, New J. Phys. 11, 023030 (2009)](http://dx.doi.org/10.1088/1367-2630/11/2/023030)" 0578 )); 0579 0580 entry = entry->next(); 0581 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0582 QCOMPARE(plainCommand(entry), QLatin1String( 0583 "def calulcate_avg_photons(N, Gamma):\n" 0584 " \n" 0585 " # collapse operators\n" 0586 " c_ops = []\n" 0587 "\n" 0588 " rate = kappa * (1 + n_th_a)\n" 0589 " if rate > 0.0:\n" 0590 " c_ops.append(sqrt(rate) * a)\n" 0591 "\n" 0592 " rate = kappa * n_th_a\n" 0593 " if rate > 0.0:\n" 0594 " c_ops.append(sqrt(rate) * a.dag())\n" 0595 "\n" 0596 " rate = gamma\n" 0597 " if rate > 0.0:\n" 0598 " c_ops.append(sqrt(rate) * sm)\n" 0599 "\n" 0600 " rate = Gamma\n" 0601 " if rate > 0.0:\n" 0602 " c_ops.append(sqrt(rate) * sm.dag())\n" 0603 " \n" 0604 " # Ground state and steady state for the Hamiltonian: H = H0 + g * H1\n" 0605 " rho_ss = steadystate(H, c_ops)\n" 0606 " \n" 0607 " # cavity photon number\n" 0608 " n_cavity = expect(a.dag() * a, rho_ss)\n" 0609 " \n" 0610 " # cavity second order coherence function\n" 0611 " g2_cavity = expect(a.dag() * a.dag() * a * a, rho_ss) / (n_cavity ** 2)\n" 0612 "\n" 0613 " return n_cavity, g2_cavity" 0614 )); 0615 QVERIFY(expression(entry)); 0616 QCOMPARE(expression(entry)->id(), 16); 0617 QCOMPARE(expression(entry)->results().size(), 0); 0618 0619 entry = entry->next(); 0620 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0621 QCOMPARE(plainCommand(entry), QLatin1String( 0622 "Gamma_max = 2 * (4*g**2) / kappa\n" 0623 "Gamma_vec = np.linspace(0.1, Gamma_max, 50)\n" 0624 "\n" 0625 "n_avg_vec = []\n" 0626 "g2_vec = []\n" 0627 "\n" 0628 "for Gamma in Gamma_vec:\n" 0629 " n_avg, g2 = calulcate_avg_photons(N, Gamma)\n" 0630 " n_avg_vec.append(n_avg)\n" 0631 " g2_vec.append(g2)" 0632 )); 0633 QVERIFY(expression(entry)); 0634 QCOMPARE(expression(entry)->id(), 17); 0635 QCOMPARE(expression(entry)->results().size(), 0); 0636 0637 entry = entry->next(); 0638 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0639 QCOMPARE(plainCommand(entry), QLatin1String( 0640 "fig, axes = plt.subplots(1, 1, figsize=(12,6))\n" 0641 "\n" 0642 "axes.plot(Gamma_vec * kappa / (4*g**2), n_avg_vec, color=\"blue\", alpha=0.6, label=\"numerical\")\n" 0643 "\n" 0644 "axes.set_xlabel(r'$\\Gamma\\kappa/(4g^2)$', fontsize=18)\n" 0645 "axes.set_ylabel(r'Occupation probability $\\langle n \\rangle$', fontsize=18)\n" 0646 "axes.set_xlim(0, 2);" 0647 )); 0648 QVERIFY(expression(entry)); 0649 QCOMPARE(expression(entry)->id(), 18); 0650 QCOMPARE(expression(entry)->results().size(), 1); 0651 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0652 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0653 0654 entry = entry->next(); 0655 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0656 QCOMPARE(plainCommand(entry), QLatin1String( 0657 "fig, axes = plt.subplots(1, 1, figsize=(12,6))\n" 0658 "\n" 0659 "axes.plot(Gamma_vec * kappa / (4*g**2), g2_vec, color=\"blue\", alpha=0.6, label=\"numerical\")\n" 0660 "\n" 0661 "axes.set_xlabel(r'$\\Gamma\\kappa/(4g^2)$', fontsize=18)\n" 0662 "axes.set_ylabel(r'$g^{(2)}(0)$', fontsize=18)\n" 0663 "axes.set_xlim(0, 2)\n" 0664 "axes.text(0.1, 1.1, \"Lasing regime\", fontsize=16)\n" 0665 "axes.text(1.5, 1.8, \"Thermal regime\", fontsize=16);" 0666 )); 0667 QVERIFY(expression(entry)); 0668 QCOMPARE(expression(entry)->id(), 19); 0669 QCOMPARE(expression(entry)->results().size(), 1); 0670 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0671 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0672 0673 entry = entry->next(); 0674 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0675 QCOMPARE(plainMarkdown(entry), QLatin1String( 0676 "Here we see that lasing is suppressed for $\\Gamma\\kappa/(4g^2) > 1$. \n" 0677 "\n" 0678 "\n" 0679 "Let's look at the fock-state distribution at $\\Gamma\\kappa/(4g^2) = 0.5$ (lasing regime) and $\\Gamma\\kappa/(4g^2) = 1.5$ (suppressed regime):" 0680 )); 0681 0682 entry = entry->next(); 0683 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0684 QCOMPARE(plainMarkdown(entry), QLatin1String( 0685 "### Case 1: $\\Gamma\\kappa/(4g^2) = 0.5$" 0686 )); 0687 0688 entry = entry->next(); 0689 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0690 QCOMPARE(plainCommand(entry), QLatin1String( 0691 "Gamma = 0.5 * (4*g**2) / kappa" 0692 )); 0693 QVERIFY(expression(entry)); 0694 QCOMPARE(expression(entry)->id(), 20); 0695 QCOMPARE(expression(entry)->results().size(), 0); 0696 0697 entry = entry->next(); 0698 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0699 QCOMPARE(plainCommand(entry), QLatin1String( 0700 "c_ops = [sqrt(kappa * (1 + n_th_a)) * a, sqrt(kappa * n_th_a) * a.dag(), sqrt(gamma) * sm, sqrt(Gamma) * sm.dag()]\n" 0701 "\n" 0702 "rho_ss = steadystate(H, c_ops)" 0703 )); 0704 QVERIFY(expression(entry)); 0705 QCOMPARE(expression(entry)->id(), 21); 0706 QCOMPARE(expression(entry)->results().size(), 0); 0707 0708 entry = entry->next(); 0709 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0710 QCOMPARE(plainCommand(entry), QLatin1String( 0711 "fig, axes = plt.subplots(1, 2, figsize=(16,6))\n" 0712 "\n" 0713 "xvec = np.linspace(-10,10,200)\n" 0714 "\n" 0715 "rho_cavity = ptrace(rho_ss, 0)\n" 0716 "W = wigner(rho_cavity, xvec, xvec)\n" 0717 "wlim = abs(W).max()\n" 0718 "axes[1].contourf(xvec, xvec, W, 100, norm=mpl.colors.Normalize(-wlim,wlim), cmap=plt.get_cmap('RdBu'))\n" 0719 "axes[1].set_xlabel(r'Im $\\alpha$', fontsize=18)\n" 0720 "axes[1].set_ylabel(r'Re $\\alpha$', fontsize=18)\n" 0721 "\n" 0722 "axes[0].bar(arange(0, N), real(rho_cavity.diag()), color=\"blue\", alpha=0.6)\n" 0723 "axes[0].set_xlabel(r'$n$', fontsize=18)\n" 0724 "axes[0].set_ylabel(r'Occupation probability', fontsize=18)\n" 0725 "axes[0].set_ylim(0, 1)\n" 0726 "axes[0].set_xlim(0, N);" 0727 )); 0728 QVERIFY(expression(entry)); 0729 QCOMPARE(expression(entry)->id(), 22); 0730 QCOMPARE(expression(entry)->results().size(), 1); 0731 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0732 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0733 0734 entry = entry->next(); 0735 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0736 QCOMPARE(plainMarkdown(entry), QLatin1String( 0737 "### Case 2: $\\Gamma\\kappa/(4g^2) = 1.5$" 0738 )); 0739 0740 entry = entry->next(); 0741 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0742 QCOMPARE(plainCommand(entry), QLatin1String( 0743 "Gamma = 1.5 * (4*g**2) / kappa" 0744 )); 0745 QVERIFY(expression(entry)); 0746 QCOMPARE(expression(entry)->id(), 23); 0747 QCOMPARE(expression(entry)->results().size(), 0); 0748 0749 entry = entry->next(); 0750 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0751 QCOMPARE(plainCommand(entry), QLatin1String( 0752 "c_ops = [sqrt(kappa * (1 + n_th_a)) * a, sqrt(kappa * n_th_a) * a.dag(), sqrt(gamma) * sm, sqrt(Gamma) * sm.dag()]\n" 0753 "\n" 0754 "rho_ss = steadystate(H, c_ops)" 0755 )); 0756 QVERIFY(expression(entry)); 0757 QCOMPARE(expression(entry)->id(), 24); 0758 QCOMPARE(expression(entry)->results().size(), 0); 0759 0760 entry = entry->next(); 0761 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0762 QCOMPARE(plainCommand(entry), QLatin1String( 0763 "fig, axes = plt.subplots(1, 2, figsize=(16,6))\n" 0764 "\n" 0765 "xvec = np.linspace(-10,10,200)\n" 0766 "\n" 0767 "rho_cavity = ptrace(rho_ss, 0)\n" 0768 "W = wigner(rho_cavity, xvec, xvec)\n" 0769 "wlim = abs(W).max()\n" 0770 "axes[1].contourf(xvec, xvec, W, 100, norm=mpl.colors.Normalize(-wlim,wlim), cmap=plt.get_cmap('RdBu'))\n" 0771 "axes[1].set_xlabel(r'Im $\\alpha$', fontsize=18)\n" 0772 "axes[1].set_ylabel(r'Re $\\alpha$', fontsize=18)\n" 0773 "\n" 0774 "axes[0].bar(arange(0, N), real(rho_cavity.diag()), color=\"blue\", alpha=0.6)\n" 0775 "axes[0].set_xlabel(r'$n$', fontsize=18)\n" 0776 "axes[0].set_ylabel(r'Occupation probability', fontsize=18)\n" 0777 "axes[0].set_ylim(0, 1)\n" 0778 "axes[0].set_xlim(0, N);" 0779 )); 0780 QVERIFY(expression(entry)); 0781 QCOMPARE(expression(entry)->id(), 26); 0782 QCOMPARE(expression(entry)->results().size(), 1); 0783 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::ImageResult::Type); 0784 QVERIFY(expression(entry)->result()->data().value<QImage>().isNull() == false); 0785 0786 entry = entry->next(); 0787 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0788 QCOMPARE(plainMarkdown(entry), QLatin1String( 0789 "Too large pumping rate $\\Gamma$ kills the lasing process: reversed threshold." 0790 )); 0791 0792 entry = entry->next(); 0793 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 0794 QCOMPARE(plainMarkdown(entry), QLatin1String( 0795 "### Software version" 0796 )); 0797 0798 entry = entry->next(); 0799 QCOMPARE(entry->type(), (int)CommandEntry::Type); 0800 QCOMPARE(plainCommand(entry), QLatin1String( 0801 "from qutip.ipynbtools import version_table\n" 0802 "\n" 0803 "version_table()" 0804 )); 0805 QVERIFY(expression(entry)); 0806 QCOMPARE(expression(entry)->id(), 27); 0807 QCOMPARE(expression(entry)->results().size(), 1); 0808 testHtmlResult(entry, 0, QString::fromUtf8( 0809 "<IPython.core.display.HTML at 0x7f2f2d5a0048>" 0810 ), QString::fromUtf8( 0811 "<table><tr><th>Software</th><th>Version</th></tr><tr><td>IPython</td><td>2.0.0</td></tr><tr><td>OS</td><td>posix [linux]</td></tr><tr><td>Python</td><td>3.4.1 (default, Jun 9 2014, 17:34:49) \n" 0812 "[GCC 4.8.3]</td></tr><tr><td>QuTiP</td><td>3.0.0.dev-5a88aa8</td></tr><tr><td>Numpy</td><td>1.8.1</td></tr><tr><td>matplotlib</td><td>1.3.1</td></tr><tr><td>Cython</td><td>0.20.1post0</td></tr><tr><td>SciPy</td><td>0.13.3</td></tr><tr><td colspan='2'>Thu Jun 26 14:28:35 2014 JST</td></tr></table>" 0813 )); 0814 0815 QCOMPARE(entry->next(), nullptr); 0816 } 0817 0818 void WorksheetTest::testJupyter2() 0819 { 0820 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 0821 if (backend && backend->isEnabled() == false) 0822 QSKIP("Skip, because python backend don't available", SkipSingle); 0823 0824 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("AEC.04 - Evolutionary Strategies and Covariance Matrix Adaptation.ipynb"))); 0825 0826 QCOMPARE(w->isReadOnly(), false); 0827 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 0828 0829 WorksheetEntry* entry = w->firstEntry(); 0830 0831 testMarkdown(entry, QLatin1String( 0832 "<div align='left' style=\"width:400px;height:120px;overflow:hidden;\">\n" 0833 "<a href='http://www.uff.br'>\n" 0834 "<img align='left' style='display: block;height: 92%' src='https://github.com/lmarti/jupyter_custom/raw/master/imgs/uff.png' alt='UFF logo' title='UFF logo'/>\n" 0835 "</a>\n" 0836 "<a href='http://www.ic.uff.br'>\n" 0837 "<img align='left' style='display: block;height: 100%' src='https://github.com/lmarti/jupyter_custom/raw/master/imgs/logo-ic.png' alt='IC logo' title='IC logo'/>\n" 0838 "</a>\n" 0839 "</div>" 0840 )); 0841 0842 testMarkdown(entry, QString::fromLocal8Bit( 0843 "# Understanding evolutionary strategies and covariance matrix adaptation\n" 0844 "\n" 0845 "## Luis MartÃ, [IC](http://www.ic.uff.br)/[UFF](http://www.uff.br)\n" 0846 "\n" 0847 "[http://lmarti.com](http://lmarti.com); [lmarti@ic.uff.br](mailto:lmarti@ic.uff.br) \n" 0848 "\n" 0849 "[Advanced Evolutionary Computation: Theory and Practice](http://lmarti.com/aec-2014) " 0850 )); 0851 0852 testMarkdown(entry, QString::fromLocal8Bit( 0853 "The notebook is better viewed rendered as slides. You can convert it to slides and view them by:\n" 0854 "- using [nbconvert](http://ipython.org/ipython-doc/1/interactive/nbconvert.html) with a command like:\n" 0855 " ```bash\n" 0856 " $ ipython nbconvert --to slides --post serve <this-notebook-name.ipynb>\n" 0857 " ```\n" 0858 "- installing [Reveal.js - Jupyter/IPython Slideshow Extension](https://github.com/damianavila/live_reveal)\n" 0859 "- using the online [IPython notebook slide viewer](https://slideviewer.herokuapp.com/) (some slides of the notebook might not be properly rendered).\n" 0860 "\n" 0861 "This and other related IPython notebooks can be found at the course github repository:\n" 0862 "* [https://github.com/lmarti/evolutionary-computation-course](https://github.com/lmarti/evolutionary-computation-course)" 0863 )); 0864 0865 testCommandEntry(entry, 1, QLatin1String( 0866 "import numpy as np\n" 0867 "import matplotlib.pyplot as plt\n" 0868 "import matplotlib.colors as colors\n" 0869 "from matplotlib import cm \n" 0870 "from mpl_toolkits.mplot3d import axes3d\n" 0871 "from scipy.stats import norm, multivariate_normal\n" 0872 "import math\n" 0873 "\n" 0874 "%matplotlib inline\n" 0875 "%config InlineBackend.figure_format = 'retina'\n" 0876 "plt.rc('text', usetex=True)\n" 0877 "plt.rc('font', family='serif')\n" 0878 "plt.rcParams['text.latex.preamble'] ='\\\\usepackage{libertine}\\n\\\\usepackage[utf8]{inputenc}'\n" 0879 "\n" 0880 "import seaborn\n" 0881 "seaborn.set(style='whitegrid')\n" 0882 "seaborn.set_context('notebook')" 0883 )); 0884 0885 testMarkdown(entry, QString::fromLocal8Bit( 0886 "### Statistics recap\n" 0887 "\n" 0888 "* [Random variable](http://en.wikipedia.org/wiki/Random_variable): a variable whose value is subject to variations due to __chance__. A random variable can take on a set of possible different values, each with an associated probability, in contrast to other mathematical variables.\n" 0889 "\n" 0890 "* [Probability distribution](http://en.wikipedia.org/wiki/Probability_distribution): mathematical function describing the possible values of a random variable and their associated probabilities.\n" 0891 "\n" 0892 "* [Probability density function (pdf)](http://en.wikipedia.org/wiki/Probability_density_function) of a __continuous random variable__ is a function that describes the relative likelihood for this random variable to take on a given value. \n" 0893 " * The probability of the random variable falling within a particular range of values is given by the integral of this variable’s density over that range.\n" 0894 " * The probability density function is nonnegative everywhere, and its integral over the entire space is equal to one.\n" 0895 " \n" 0896 "<img src='http://upload.wikimedia.org/wikipedia/commons/2/25/The_Normal_Distribution.svg' width='50%' align='center'/>\n" 0897 " " 0898 )); 0899 0900 testMarkdown(entry, QLatin1String( 0901 "### [Moments](http://en.wikipedia.org/wiki/Moment_(mathematics)\n" 0902 "\n" 0903 "The probability distribution of a random variable is often characterised by a small number of parameters, which also have a practical interpretation.\n" 0904 "\n" 0905 "* [Mean](http://en.wikipedia.org/wiki/Mean) (a.k.a expected value) refers to one measure of the central tendency either of a probability distribution or of the random variable characterized by that distribution.\n" 0906 " * population mean: $\\mu = \\operatorname{E}[X]$.\n" 0907 " * estimation of sample mean: $\\bar{x}$.\n" 0908 "* [Standard deviation](http://en.wikipedia.org/wiki/Standard_deviation) measures the amount of variation or dispersion from the mean.\n" 0909 " * population deviation:\n" 0910 " $$\n" 0911 "\\sigma = \\sqrt{\\operatorname E[X^2]-(\\operatorname E[X])^2} = \\sqrt{\\frac{1}{N} \\sum_{i=1}^N (x_i - \\mu)^2}.\n" 0912 "$$\n" 0913 " * unbiased estimator:\n" 0914 " $$ \n" 0915 " s^2 = \\frac{1}{N-1} \\sum_{i=1}^N (x_i - \\overline{x})^2.\n" 0916 " $$" 0917 )); 0918 0919 testMarkdown(entry, QLatin1String("### Two samples")); 0920 0921 testCommandEntry(entry, 2, QLatin1String( 0922 "sample1 = np.random.normal(0, 0.5, 1000)\n" 0923 "sample2 = np.random.normal(1,1,500)" 0924 )); 0925 0926 testCommandEntry(entry, 3, QLatin1String( 0927 "def plot_normal_sample(sample, mu, sigma):\n" 0928 " 'Plots an histogram and the normal distribution corresponding to the parameters.'\n" 0929 " x = np.linspace(mu - 4*sigma, mu + 4*sigma, 100)\n" 0930 " plt.plot(x, norm.pdf(x, mu, sigma), 'b', lw=2)\n" 0931 " plt.hist(sample, 30, normed=True, alpha=0.2)\n" 0932 " plt.annotate('3$\\sigma$', \n" 0933 " xy=(mu + 3*sigma, 0), xycoords='data',\n" 0934 " xytext=(0, 100), textcoords='offset points',\n" 0935 " fontsize=15,\n" 0936 " arrowprops=dict(arrowstyle=\"->\",\n" 0937 " connectionstyle=\"arc,angleA=180,armA=20,angleB=90,armB=15,rad=7\"))\n" 0938 " plt.annotate('-3$\\sigma$', \n" 0939 " xy=(mu -3*sigma, 0), xycoords='data', \n" 0940 " xytext=(0, 100), textcoords='offset points',\n" 0941 " fontsize=15,\n" 0942 " arrowprops=dict(arrowstyle=\"->\",\n" 0943 " connectionstyle=\"arc,angleA=180,armA=20,angleB=90,armB=15,rad=7\"))" 0944 )); 0945 0946 testCommandEntry(entry, 4, 2, QLatin1String( 0947 "plt.figure(figsize=(11,4))\n" 0948 "plt.subplot(121)\n" 0949 "plot_normal_sample(sample1, 0, 0.5)\n" 0950 "plt.title('Sample 1: $\\mu=0$, $\\sigma=0.5$')\n" 0951 "plt.subplot(122)\n" 0952 "plot_normal_sample(sample2, 1, 1)\n" 0953 "plt.title('Sample 2: $\\mu=1$, $\\sigma=1$')\n" 0954 "plt.tight_layout();" 0955 )); 0956 testTextResult(entry, 0, QLatin1String( 0957 "/usr/local/lib/python3.6/dist-packages/matplotlib/axes/_axes.py:6462: UserWarning: The 'normed' kwarg is deprecated, and has been replaced by the 'density' kwarg.\n" 0958 " warnings.warn(\"The 'normed' kwarg is deprecated, and has been \"" 0959 )); 0960 testImageResult(entry, 1); 0961 entry = entry->next(); 0962 0963 testCommandEntry(entry, 5, 1, QLatin1String( 0964 "print('Sample 1; estimated mean:', sample1.mean(), ' and std. dev.: ', sample1.std())\n" 0965 "print('Sample 2; estimated mean:', sample2.mean(), ' and std. dev.: ', sample2.std())" 0966 )); 0967 testTextResult(entry, 0, QLatin1String( 0968 "Sample 1; estimated mean: 0.007446590585087637 and std. dev.: 0.5083158965764596\n" 0969 "Sample 2; estimated mean: 0.969635147915706 and std. dev.: 1.0213164282805647" 0970 )); 0971 entry = entry->next(); 0972 0973 testMarkdown(entry, QLatin1String( 0974 "[Covariance](http://en.wikipedia.org/wiki/Covariance) is a measure of how much two random variables change together. \n" 0975 "$$\n" 0976 "\\operatorname{cov}(X,Y) = \\operatorname{E}{\\big[(X - \\operatorname{E}[X])(Y - \\operatorname{E}[Y])\\big]},\n" 0977 "$$\n" 0978 "$$\n" 0979 "\\operatorname{cov}(X,X) = s(X),\n" 0980 "$$\n" 0981 "\n" 0982 "* The sign of the covariance therefore shows the tendency in the linear relationship between the variables. \n" 0983 "* The magnitude of the covariance is not easy to interpret. \n" 0984 "* The normalized version of the covariance, the correlation coefficient, however, shows by its magnitude the strength of the linear relation." 0985 )); 0986 0987 testMarkdown(entry, QLatin1String("### Understanding covariance")); 0988 0989 testCommandEntry(entry, 6, QLatin1String( 0990 "sample_2d = np.array(list(zip(sample1, np.ones(len(sample1))))).T" 0991 )); 0992 0993 testCommandEntry(entry, 7, 1, QLatin1String( 0994 "plt.scatter(sample_2d[0,:], sample_2d[1,:], marker='x');" 0995 )); 0996 testImageResult(entry, 0); 0997 entry = entry->next(); 0998 0999 testCommandEntry(entry, 8, 1, QLatin1String( 1000 "np.cov(sample_2d) # computes covariance between the two components of the sample" 1001 )); 1002 testTextResult(entry, 0, QLatin1String( 1003 "array([[0.25864369, 0. ],\n" 1004 " [0. , 0. ]])" 1005 )); 1006 entry = entry->next(); 1007 1008 testMarkdown(entry, QLatin1String( 1009 "As the sample is only distributed along one axis, the covariance does not detects any relationship between them." 1010 )); 1011 1012 testMarkdown(entry, QLatin1String( 1013 "What happens when we rotate the sample?" 1014 )); 1015 1016 testCommandEntry(entry, 9, QLatin1String( 1017 "def rotate_sample(sample, angle=-45):\n" 1018 " 'Rotates a sample by `angle` degrees.'\n" 1019 " theta = (angle/180.) * np.pi\n" 1020 " rot_matrix = np.array([[np.cos(theta), -np.sin(theta)], \n" 1021 " [np.sin(theta), np.cos(theta)]])\n" 1022 " return sample.T.dot(rot_matrix).T" 1023 )); 1024 1025 testCommandEntry(entry, 10, QLatin1String( 1026 "rot_sample_2d = rotate_sample(sample_2d)" 1027 )); 1028 1029 testCommandEntry(entry, 11, 1, QLatin1String( 1030 "plt.scatter(rot_sample_2d[0,:], rot_sample_2d[1,:], marker='x');" 1031 )); 1032 testImageResult(entry, 0); 1033 entry = entry->next(); 1034 1035 testCommandEntry(entry, 12, 1, QLatin1String( 1036 "np.cov(rot_sample_2d)" 1037 )); 1038 testTextResult(entry, 0, QLatin1String( 1039 "array([[0.12932185, 0.12932185],\n" 1040 " [0.12932185, 0.12932185]])" 1041 )); 1042 entry = entry->next(); 1043 1044 testMarkdown(entry, QLatin1String( 1045 "### A two-dimensional normally-distributed variable" 1046 )); 1047 1048 testCommandEntry(entry, 13, 2, QLatin1String( 1049 "mu = [0,1]\n" 1050 "cov = [[1,0],[0,0.2]] # diagonal covariance, points lie on x or y-axis\n" 1051 "sample = np.random.multivariate_normal(mu,cov,1000).T\n" 1052 "plt.scatter(sample[0], sample[1], marker='x', alpha=0.29)\n" 1053 "\n" 1054 "estimated_mean = sample.mean(axis=1)\n" 1055 "estimated_cov = np.cov(sample)\n" 1056 "e_x,e_y = np.random.multivariate_normal(estimated_mean,estimated_cov,500).T\n" 1057 "\n" 1058 "plt.plot(e_x,e_y,'rx', alpha=0.47)\n" 1059 "x, y = np.mgrid[-4:4:.01, -1:3:.01]\n" 1060 "pos = np.empty(x.shape + (2,))\n" 1061 "pos[:, :, 0] = x; pos[:, :, 1] = y\n" 1062 "rv = multivariate_normal(estimated_mean, estimated_cov)\n" 1063 "plt.contour(x, y, rv.pdf(pos), cmap=cm.viridis_r, lw=4)\n" 1064 "plt.axis('equal');" 1065 )); 1066 testTextResult(entry, 0, QLatin1String( 1067 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'lw'\n" 1068 " s)" 1069 )); 1070 testImageResult(entry, 1); 1071 entry = entry->next(); 1072 1073 testMarkdown(entry, QLatin1String( 1074 "### This is better understood in 3D" 1075 )); 1076 1077 testCommandEntry(entry, 14, 1, QLatin1String( 1078 "fig = plt.figure(figsize=(11,5))\n" 1079 "ax = fig.gca(projection='3d')\n" 1080 "ax.plot_surface(x, y, rv.pdf(pos), cmap=cm.viridis_r, rstride=30, cstride=10, linewidth=1, alpha=0.47)\n" 1081 "ax.plot_wireframe(x, y, rv.pdf(pos), linewidth=0.47, alpha=0.47)\n" 1082 "ax.scatter(e_x, e_y, 0.4, marker='.', alpha=0.47)\n" 1083 "ax.axis('tight');" 1084 )); 1085 testImageResult(entry, 0); 1086 entry = entry->next(); 1087 1088 testMarkdown(entry, QLatin1String( 1089 "Again, what happens if we rotate the sample?" 1090 )); 1091 1092 testCommandEntry(entry, 15, QLatin1String( 1093 "rot_sample = rotate_sample(sample)\n" 1094 "estimated_mean = rot_sample.mean(axis=1)\n" 1095 "estimated_cov = np.cov(rot_sample)\n" 1096 "e_x,e_y = np.random.multivariate_normal(estimated_mean,estimated_cov,500).T" 1097 )); 1098 1099 testCommandEntry(entry, 16, 1, QLatin1String( 1100 "fig = plt.figure(figsize=(11,4))\n" 1101 "plt.subplot(121)\n" 1102 "plt.scatter(rot_sample[0,:], rot_sample[1,:], marker='x', alpha=0.7)\n" 1103 "plt.title('\"Original\" data')\n" 1104 "plt.axis('equal')\n" 1105 "plt.subplot(122)\n" 1106 "plt.scatter(e_x, e_y, marker='o', color='g', alpha=0.7)\n" 1107 "plt.title('Sampled data')\n" 1108 "plt.axis('equal');" 1109 )); 1110 testImageResult(entry, 0); 1111 entry = entry->next(); 1112 1113 testMarkdown(entry, QLatin1String( 1114 "Covariance captures the dependency and can model disposition of the \"original\" sample." 1115 )); 1116 1117 testCommandEntry(entry, 17, QLatin1String( 1118 "x, y = np.mgrid[-4:4:.01, -3:3:.01]\n" 1119 "pos = np.empty(x.shape + (2,))\n" 1120 "pos[:, :, 0] = x; pos[:, :, 1] = y\n" 1121 "rv = multivariate_normal(estimated_mean, estimated_cov)" 1122 )); 1123 1124 testCommandEntry(entry, 18, 1, QLatin1String( 1125 "fig = plt.figure(figsize=(11,5))\n" 1126 "ax = fig.gca(projection='3d')\n" 1127 "ax.plot_surface(x, y, rv.pdf(pos), cmap=cm.viridis_r, rstride=30, cstride=10, linewidth=1, alpha=0.47)\n" 1128 "ax.plot_wireframe(x, y, rv.pdf(pos), linewidth=0.47, alpha=0.47)\n" 1129 "ax.scatter(e_x, e_y, 0.4, marker='.', alpha=0.47)\n" 1130 "ax.axis('tight');" 1131 )); 1132 testImageResult(entry, 0); 1133 entry = entry->next(); 1134 1135 testMarkdown(entry, QLatin1String( 1136 "# Evolutionary Strategies\n" 1137 "\n" 1138 "We will be using DEAP again to present some of the ES main concepts." 1139 )); 1140 1141 testCommandEntry(entry, 19, QLatin1String( 1142 "import array, random, time, copy\n" 1143 "\n" 1144 "from deap import base, creator, benchmarks, tools, algorithms\n" 1145 "\n" 1146 "random.seed(42) # Fixing a random seed: You should not do this in practice." 1147 )); 1148 1149 testMarkdown(entry, QLatin1String( 1150 "Before we dive into the discussion lets code some support functions." 1151 )); 1152 1153 testCommandEntry(entry, 20, QLatin1String( 1154 "def plot_problem_3d(problem, bounds, resolution=100., \n" 1155 " cmap=cm.viridis_r, rstride=10, cstride=10, \n" 1156 " linewidth=0.15, alpha=0.65, ax=None):\n" 1157 " 'Plots a given deap benchmark problem in 3D mesh.'\n" 1158 " (minx,miny),(maxx,maxy) = bounds\n" 1159 " x_range = np.arange(minx, maxx, (maxx-minx)/resolution)\n" 1160 " y_range = np.arange(miny, maxy, (maxy-miny)/resolution)\n" 1161 " \n" 1162 " X, Y = np.meshgrid(x_range, y_range)\n" 1163 " Z = np.zeros((len(x_range), len(y_range)))\n" 1164 " \n" 1165 " for i in range(len(x_range)):\n" 1166 " for j in range(len(y_range)):\n" 1167 " Z[i,j] = problem((x_range[i], y_range[j]))[0]\n" 1168 " \n" 1169 " if not ax:\n" 1170 " fig = plt.figure(figsize=(11,6))\n" 1171 " ax = fig.gca(projection='3d')\n" 1172 " \n" 1173 " cset = ax.plot_surface(X, Y, Z, cmap=cmap, rstride=rstride, cstride=cstride, linewidth=linewidth, alpha=alpha)" 1174 )); 1175 1176 testCommandEntry(entry, 21, QLatin1String( 1177 "def plot_problem_controur(problem, bounds, optimum=None,\n" 1178 " resolution=100., cmap=cm.viridis_r, \n" 1179 " rstride=1, cstride=10, linewidth=0.15,\n" 1180 " alpha=0.65, ax=None):\n" 1181 " 'Plots a given deap benchmark problem as a countour plot'\n" 1182 " (minx,miny),(maxx,maxy) = bounds\n" 1183 " x_range = np.arange(minx, maxx, (maxx-minx)/resolution)\n" 1184 " y_range = np.arange(miny, maxy, (maxy-miny)/resolution)\n" 1185 " \n" 1186 " X, Y = np.meshgrid(x_range, y_range)\n" 1187 " Z = np.zeros((len(x_range), len(y_range)))\n" 1188 " \n" 1189 " for i in range(len(x_range)):\n" 1190 " for j in range(len(y_range)):\n" 1191 " Z[i,j] = problem((x_range[i], y_range[j]))[0]\n" 1192 " \n" 1193 " if not ax:\n" 1194 " fig = plt.figure(figsize=(6,6))\n" 1195 " ax = fig.gca()\n" 1196 " ax.set_aspect('equal')\n" 1197 " ax.autoscale(tight=True)\n" 1198 " \n" 1199 " cset = ax.contourf(X, Y, Z, cmap=cmap, rstride=rstride, cstride=cstride, linewidth=linewidth, alpha=alpha)\n" 1200 " \n" 1201 " if optimum:\n" 1202 " ax.plot(optimum[0], optimum[1], 'bx', linewidth=4, markersize=15)" 1203 )); 1204 1205 testCommandEntry(entry, 22, QLatin1String( 1206 "def plot_cov_ellipse(pos, cov, volume=.99, ax=None, fc='lightblue', ec='darkblue', alpha=1, lw=1):\n" 1207 " ''' Plots an ellipse that corresponds to a bivariate normal distribution.\n" 1208 " Adapted from http://www.nhsilbert.net/source/2014/06/bivariate-normal-ellipse-plotting-in-python/'''\n" 1209 " from scipy.stats import chi2\n" 1210 " from matplotlib.patches import Ellipse\n" 1211 "\n" 1212 " def eigsorted(cov):\n" 1213 " vals, vecs = np.linalg.eigh(cov)\n" 1214 " order = vals.argsort()[::-1]\n" 1215 " return vals[order], vecs[:,order]\n" 1216 "\n" 1217 " if ax is None:\n" 1218 " ax = plt.gca()\n" 1219 "\n" 1220 " vals, vecs = eigsorted(cov)\n" 1221 " theta = np.degrees(np.arctan2(*vecs[:,0][::-1]))\n" 1222 "\n" 1223 " kwrg = {'facecolor':fc, 'edgecolor':ec, 'alpha':alpha, 'linewidth':lw}\n" 1224 "\n" 1225 " # Width and height are \"full\" widths, not radius\n" 1226 " width, height = 2 * np.sqrt(chi2.ppf(volume,2)) * np.sqrt(vals)\n" 1227 " ellip = Ellipse(xy=pos, width=width, height=height, angle=theta, **kwrg)\n" 1228 " ax.add_artist(ellip)" 1229 )); 1230 1231 testMarkdown(entry, QLatin1String( 1232 "### Why benchmarks (test) functions?\n" 1233 "\n" 1234 "In applied mathematics, [test functions](http://en.wikipedia.org/wiki/Test_functions_for_optimization), also known as artificial landscapes, are useful to evaluate characteristics of optimization algorithms, such as:\n" 1235 "\n" 1236 "* Velocity of convergence.\n" 1237 "* Precision.\n" 1238 "* Robustness.\n" 1239 "* General performance.\n" 1240 "\n" 1241 "DEAP has a number of test problems already implemented. See http://deap.readthedocs.org/en/latest/api/benchmarks.html" 1242 )); 1243 1244 testMarkdown(entry, QLatin1String( 1245 "### [Bohachevsky benchmark problem](http://deap.readthedocs.org/en/latest/api/benchmarks.html#deap.benchmarks.bohachevsky)\n" 1246 "\n" 1247 "$$\\text{minimize } f(\\mathbf{x}) = \\sum_{i=1}^{N-1}(x_i^2 + 2x_{i+1}^2 - 0.3\\cos(3\\pi x_i) - 0.4\\cos(4\\pi x_{i+1}) + 0.7), \\mathbf{x}\\in \\left[-100,100\\right]^n,$$\n" 1248 "\n" 1249 "> Optimum in $\\mathbf{x}=\\mathbf{0}$, $f(\\mathbf{x})=0$." 1250 )); 1251 1252 testCommandEntry(entry, 23, QLatin1String( 1253 "current_problem = benchmarks.bohachevsky" 1254 )); 1255 1256 testCommandEntry(entry, 24, 1, QLatin1String( 1257 "plot_problem_3d(current_problem, ((-10,-10), (10,10)))" 1258 )); 1259 testImageResult(entry, 0); 1260 entry = entry->next(); 1261 1262 testMarkdown(entry, QLatin1String( 1263 "The Bohachevsky problem has many local optima." 1264 )); 1265 1266 testCommandEntry(entry, 25, 1, QLatin1String( 1267 "plot_problem_3d(current_problem, ((-2.5,-2.5), (2.5,2.5)))" 1268 )); 1269 testImageResult(entry, 0); 1270 entry = entry->next(); 1271 1272 testCommandEntry(entry, 26, 2, QLatin1String( 1273 "ax = plt.figure().gca()\n" 1274 "plot_problem_controur(current_problem, ((-2.5,-2.5), (2.5,2.5)), optimum=(0,0), ax=ax)\n" 1275 "ax.set_aspect('equal')" 1276 )); 1277 testTextResult(entry, 0, QLatin1String( 1278 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1279 " s)" 1280 )); 1281 testImageResult(entry, 1); 1282 entry = entry->next(); 1283 1284 testMarkdown(entry, QLatin1String( 1285 "## ($\\mu$,$\\lambda$) evolutionary strategy\n" 1286 "\n" 1287 "Some basic initialization parameters." 1288 )); 1289 1290 testCommandEntry(entry, 27, QLatin1String( 1291 "search_space_dims = 2 # we want to plot the individuals so this must be 2\n" 1292 "\n" 1293 "MIN_VALUE, MAX_VALUE = -10., 10.\n" 1294 "MIN_STRAT, MAX_STRAT = 0.0000001, 1. " 1295 )); 1296 1297 testCommandEntry(entry, 28, QLatin1String( 1298 "# We are facing a minimization problem\n" 1299 "creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0,))\n" 1300 "\n" 1301 "# Evolutionary strategies need a location (mean)\n" 1302 "creator.create(\"Individual\", array.array, typecode='d', \n" 1303 " fitness=creator.FitnessMin, strategy=None)\n" 1304 "# ...and a value of the strategy parameter.\n" 1305 "creator.create(\"Strategy\", array.array, typecode=\"d\")" 1306 )); 1307 1308 testMarkdown(entry, QLatin1String( 1309 "Evolutionary strategy individuals are more complex than those we have seen so far.\n" 1310 "\n" 1311 "They need a custom creation/initialization function." 1312 )); 1313 1314 testCommandEntry(entry, 29, QLatin1String( 1315 "def init_univariate_es_ind(individual_class, strategy_class,\n" 1316 " size, min_value, max_value, \n" 1317 " min_strat, max_strat):\n" 1318 " ind = individual_class(random.uniform(min_value, max_value) \n" 1319 " for _ in range(size))\n" 1320 " # we modify the instance to include the strategy in run-time.\n" 1321 " ind.strategy = strategy_class(random.uniform(min_strat, max_strat) for _ in range(size))\n" 1322 " return ind" 1323 )); 1324 1325 testCommandEntry(entry, 30, QLatin1String( 1326 "toolbox = base.Toolbox() \n" 1327 "toolbox.register(\"individual\", init_univariate_es_ind, \n" 1328 " creator.Individual, \n" 1329 " creator.Strategy,\n" 1330 " search_space_dims, \n" 1331 " MIN_VALUE, MAX_VALUE, \n" 1332 " MIN_STRAT, MAX_STRAT)\n" 1333 "toolbox.register(\"population\", tools.initRepeat, list, \n" 1334 " toolbox.individual)" 1335 )); 1336 1337 testMarkdown(entry, QLatin1String( 1338 "How does an individual and a population looks like?" 1339 )); 1340 1341 testCommandEntry(entry, 31, QLatin1String( 1342 "ind = toolbox.individual()\n" 1343 "pop = toolbox.population(n=3)" 1344 )); 1345 1346 testCommandEntry(entry, 32, QLatin1String( 1347 "def plot_individual(individual, ax=None):\n" 1348 " 'Plots an ES indiviual as center and 3*sigma ellipsis.'\n" 1349 " cov = np.eye(len(individual)) * individual.strategy\n" 1350 " plot_cov_ellipse(individual, cov, volume=0.99, alpha=0.56, ax=ax)\n" 1351 " if ax:\n" 1352 " ax.scatter(individual[0], individual[1], \n" 1353 " marker='+', color='k', zorder=100)\n" 1354 " else:\n" 1355 " plt.scatter(individual[0], individual[1], \n" 1356 " marker='+', color='k', zorder=100)\n" 1357 "\n" 1358 " \n" 1359 "def plot_population(pop, gen=None, max_gen=None, ax=None):\n" 1360 " if gen:\n" 1361 " plt.subplot(max_gen, 1, gen)\n" 1362 " \n" 1363 " for ind in pop:\n" 1364 " plot_individual(ind, ax)" 1365 )); 1366 1367 qDebug() << "command entry 33"; 1368 testCommandEntry(entry, 33, 2, QString::fromUtf8( 1369 "plot_problem_controur(current_problem, ((-10,-10), (10,10)), optimum=(0,0))\n" 1370 "plot_individual(ind)" 1371 )); 1372 testTextResult(entry, 0, QString::fromUtf8( 1373 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1374 " s)" 1375 )); 1376 testImageResult(entry, 1); 1377 entry = entry->next(); 1378 1379 qDebug() << "command entry 34"; 1380 testCommandEntry(entry, 34, 2, QString::fromUtf8( 1381 "plot_problem_controur(current_problem, ((-10,-10), (10,10)), optimum=(0,0))\n" 1382 "plot_population(pop)" 1383 )); 1384 testTextResult(entry, 0, QString::fromUtf8( 1385 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1386 " s)" 1387 )); 1388 testImageResult(entry, 1); 1389 entry = entry->next(); 1390 1391 testMarkdown(entry, QString::fromUtf8( 1392 "### Mutation of an evolution strategy individual according to its strategy attribute. \n" 1393 "First the strategy is mutated according to an extended log normal rule, \n" 1394 "$$\n" 1395 "\\boldsymbol{\\sigma}_t = \\exp(\\tau_0 \\mathcal{N}_0(0, 1)) \\left[ \\sigma_{t-1, 1}\\exp(\\tau\n" 1396 "\\mathcal{N}_1(0, 1)), \\ldots, \\sigma_{t-1, n} \\exp(\\tau\n" 1397 "\\mathcal{N}_n(0, 1))\\right],\n" 1398 "$$\n" 1399 "with \n" 1400 "$$\\tau_0 =\n" 1401 "\\frac{c}{\\sqrt{2n}}\\text{ and }\\tau = \\frac{c}{\\sqrt{2\\sqrt{n}}},\n" 1402 "$$\n" 1403 "\n" 1404 "the individual is mutated by a normal distribution of mean 0 and standard deviation of $\\boldsymbol{\\sigma}_{t}$ (its current strategy). \n" 1405 "\n" 1406 "A recommended choice is $c=1$ when using a $(10,100)$ evolution strategy." 1407 )); 1408 1409 qDebug() << "command entry 35"; 1410 testCommandEntry(entry, 35, QString::fromUtf8( 1411 "toolbox.register(\"mutate\", tools.mutESLogNormal, c=1, indpb=0.1)" 1412 )); 1413 1414 testMarkdown(entry, QString::fromUtf8( 1415 "Blend crossover on both, the individual and the strategy." 1416 )); 1417 1418 qDebug() << "command entry 36"; 1419 testCommandEntry(entry, 36, QString::fromUtf8( 1420 "toolbox.register(\"mate\", tools.cxESBlend, alpha=0.1)\n" 1421 "toolbox.register(\"evaluate\", current_problem)\n" 1422 "toolbox.register(\"select\", tools.selBest)" 1423 )); 1424 1425 qDebug() << "command entry 37"; 1426 testCommandEntry(entry, 37, QString::fromUtf8( 1427 "mu_es, lambda_es = 3,21\n" 1428 "\n" 1429 "pop = toolbox.population(n=mu_es)\n" 1430 "hof = tools.HallOfFame(1)\n" 1431 "\n" 1432 "pop_stats = tools.Statistics(key=copy.deepcopy)\n" 1433 "pop_stats.register('pop', copy.deepcopy) # -- copies the populations themselves\n" 1434 " \n" 1435 "pop, logbook = algorithms.eaMuCommaLambda(pop, toolbox, mu=mu_es, lambda_=lambda_es, \n" 1436 " cxpb=0.6, mutpb=0.3, ngen=40, stats=pop_stats, halloffame=hof, verbose=False)" 1437 )); 1438 1439 testMarkdown(entry, QString::fromUtf8( 1440 "### The final population" 1441 )); 1442 1443 qDebug() << "command entry 38"; 1444 testCommandEntry(entry, 38, 2, QString::fromUtf8( 1445 "plot_problem_controur(current_problem, ((-10,-10), (10,10)), optimum=(0,0))\n" 1446 "plot_population(pop)" 1447 )); 1448 testTextResult(entry, 0, QString::fromUtf8( 1449 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1450 " s)" 1451 )); 1452 testImageResult(entry, 1); 1453 entry = entry->next(); 1454 1455 testMarkdown(entry, QString::fromUtf8( 1456 "The plot (most probably) shows a \"dark blue\" ellipse as all individuals are overlapping. " 1457 )); 1458 1459 testMarkdown(entry, QString::fromUtf8( 1460 "Let's see how the evolutionary process took place in animated form." 1461 )); 1462 1463 qDebug() << "command entry 39"; 1464 testCommandEntry(entry, 39, QString::fromUtf8( 1465 "from matplotlib import animation\n" 1466 "from IPython.display import HTML" 1467 )); 1468 1469 qDebug() << "command entry 40"; 1470 testCommandEntry(entry, 40, QString::fromUtf8( 1471 "def animate(i):\n" 1472 " 'Updates all plots to match frame _i_ of the animation.'\n" 1473 " ax.clear()\n" 1474 " plot_problem_controur(current_problem, ((-10.1,-10.1), (10.1,10.1)), optimum=(0,0), ax=ax)\n" 1475 " plot_population(logbook[i]['pop'], ax=ax)\n" 1476 " ax.set_title('$t=$' +str(i))\n" 1477 " return []" 1478 )); 1479 1480 qDebug() << "command entry 41"; 1481 testCommandEntry(entry, 41, 1, QString::fromUtf8( 1482 "fig = plt.figure(figsize=(5,5))\n" 1483 "ax = fig.gca()\n" 1484 "anim = animation.FuncAnimation(fig, animate, frames=len(logbook), interval=300, blit=True)\n" 1485 "plt.close()" 1486 )); 1487 testTextResult(entry, 0, QString::fromUtf8( 1488 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1489 " s)" 1490 )); 1491 entry = entry->next(); 1492 1493 qDebug() << "command entry 42"; 1494 testCommandEntry(entry, 42, 2, QString::fromUtf8( 1495 "HTML(anim.to_html5_video())" 1496 )); 1497 testTextResult(entry, 0, QString::fromUtf8( 1498 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1499 " s)\n" 1500 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1501 " s)\n" 1502 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1503 " s)\n" 1504 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1505 " s)\n" 1506 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1507 " s)\n" 1508 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1509 " s)\n" 1510 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1511 " s)\n" 1512 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1513 " s)\n" 1514 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1515 " s)\n" 1516 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1517 " s)\n" 1518 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1519 " s)\n" 1520 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1521 " s)\n" 1522 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1523 " s)\n" 1524 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1525 " s)\n" 1526 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1527 " s)\n" 1528 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1529 " s)\n" 1530 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1531 " s)\n" 1532 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1533 " s)\n" 1534 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1535 " s)\n" 1536 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1537 " s)\n" 1538 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1539 " s)\n" 1540 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1541 " s)\n" 1542 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1543 " s)\n" 1544 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1545 " s)\n" 1546 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1547 " s)\n" 1548 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1549 " s)\n" 1550 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1551 " s)\n" 1552 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1553 " s)\n" 1554 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1555 " s)\n" 1556 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1557 " s)\n" 1558 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1559 " s)\n" 1560 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1561 " s)\n" 1562 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1563 " s)\n" 1564 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1565 " s)\n" 1566 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1567 " s)\n" 1568 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1569 " s)\n" 1570 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1571 " s)\n" 1572 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1573 " s)\n" 1574 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1575 " s)\n" 1576 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1577 " s)\n" 1578 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1579 " s)" 1580 )); 1581 testHtmlResult(entry, 1, QString::fromUtf8( 1582 "<IPython.core.display.HTML object>" 1583 )); 1584 entry = entry->next(); 1585 1586 testMarkdown(entry, QString::fromUtf8( 1587 "How the population progressed as the evolution proceeded?" 1588 )); 1589 1590 qDebug() << "command entry 43"; 1591 testCommandEntry(entry, 43, QString::fromUtf8( 1592 "pop = toolbox.population(n=mu_es)\n" 1593 "\n" 1594 "stats = tools.Statistics(lambda ind: ind.fitness.values)\n" 1595 "stats.register(\"avg\", np.mean)\n" 1596 "stats.register(\"std\", np.std)\n" 1597 "stats.register(\"min\", np.min)\n" 1598 "stats.register(\"max\", np.max)\n" 1599 " \n" 1600 "pop, logbook = algorithms.eaMuCommaLambda(pop, toolbox, \n" 1601 " mu=mu_es, lambda_=lambda_es, \n" 1602 " cxpb=0.6, mutpb=0.3, \n" 1603 " ngen=40, stats=stats, \n" 1604 " verbose=False)" 1605 )); 1606 1607 qDebug() << "command entry 44"; 1608 testCommandEntry(entry, 44, 1, QString::fromUtf8( 1609 "plt.figure(1, figsize=(7, 4))\n" 1610 "plt.plot(logbook.select('avg'), 'b-', label='Avg. fitness')\n" 1611 "plt.fill_between(range(len(logbook)), logbook.select('max'), logbook.select('min'), facecolor='blue', alpha=0.47)\n" 1612 "plt.plot(logbook.select('std'), 'm--', label='Std. deviation')\n" 1613 "plt.legend(frameon=True)\n" 1614 "plt.ylabel('Fitness'); plt.xlabel('Iterations');" 1615 )); 1616 testImageResult(entry, 0); 1617 entry = entry->next(); 1618 1619 testMarkdown(entry, QString::fromUtf8( 1620 "What happens if we increase $\\mu$ and $\\lambda$?" 1621 )); 1622 1623 qDebug() << "command entry 45"; 1624 testCommandEntry(entry, 45, 1, QString::fromUtf8( 1625 "mu_es, lambda_es = 10,100\n" 1626 "pop, logbook = algorithms.eaMuCommaLambda(toolbox.population(n=mu_es), toolbox, mu=mu_es, lambda_=lambda_es, \n" 1627 " cxpb=0.6, mutpb=0.3, ngen=40, stats=stats, halloffame=hof, verbose=False)\n" 1628 "plt.figure(1, figsize=(7, 4))\n" 1629 "plt.plot(logbook.select('avg'), 'b-', label='Avg. fitness')\n" 1630 "plt.fill_between(range(len(logbook)), logbook.select('max'), logbook.select('min'), facecolor='blue', alpha=0.47)\n" 1631 "plt.plot(logbook.select('std'), 'm--', label='Std. deviation')\n" 1632 "plt.legend(frameon=True)\n" 1633 "plt.ylabel('Fitness'); plt.xlabel('Iterations');" 1634 )); 1635 testImageResult(entry, 0); 1636 entry = entry->next(); 1637 1638 testMarkdown(entry, QString::fromUtf8( 1639 "# Covariance Matrix Adaptation Evolutionary Strategy" 1640 )); 1641 1642 testMarkdown(entry, QString::fromUtf8( 1643 "* In an evolution strategy, new candidate solutions are sampled according to a multivariate normal distribution in the $\\mathbb{R}^n$. \n" 1644 "* Recombination amounts to selecting a new mean value for the distribution. \n" 1645 "* Mutation amounts to adding a random vector, a perturbation with zero mean. \n" 1646 "* Pairwise dependencies between the variables in the distribution are represented by a covariance matrix. \n" 1647 "\n" 1648 "### The covariance matrix adaptation (CMA) is a method to update the covariance matrix of this distribution. \n" 1649 "\n" 1650 "> This is particularly useful, if the objective function $f()$ is ill-conditioned." 1651 )); 1652 1653 testMarkdown(entry, QString::fromUtf8( 1654 "### CMA-ES features\n" 1655 "\n" 1656 "* Adaptation of the covariance matrix amounts to learning a second order model of the underlying objective function.\n" 1657 "* This is similar to the approximation of the inverse Hessian matrix in the Quasi-Newton method in classical optimization. \n" 1658 "* In contrast to most classical methods, fewer assumptions on the nature of the underlying objective function are made. \n" 1659 "* *Only the ranking between candidate solutions is exploited* for learning the sample distribution and neither derivatives nor even the function values themselves are required by the method." 1660 )); 1661 1662 qDebug() << "command entry 46"; 1663 testCommandEntry(entry, 46, QString::fromUtf8( 1664 "from deap import cma" 1665 )); 1666 1667 testMarkdown(entry, QString::fromUtf8( 1668 "A similar setup to the previous one." 1669 )); 1670 1671 qDebug() << "command entry 47"; 1672 testCommandEntry(entry, 47, 1, QString::fromUtf8( 1673 "creator.create(\"Individual\", list, fitness=creator.FitnessMin)\n" 1674 "toolbox = base.Toolbox()\n" 1675 "toolbox.register(\"evaluate\", current_problem)" 1676 )); 1677 testTextResult(entry, 0, QString::fromUtf8( 1678 "/home/mmmm1998/.local/lib/python3.6/site-packages/deap/creator.py:141: RuntimeWarning: A class named 'Individual' has already been created and it will be overwritten. Consider deleting previous creation of that class or rename it.\n" 1679 " RuntimeWarning)" 1680 )); 1681 entry = entry->next(); 1682 1683 testMarkdown(entry, QString::fromUtf8( 1684 "We will place our start point by hand at $(5,5)$." 1685 )); 1686 1687 qDebug() << "command entry 48"; 1688 testCommandEntry(entry, 48, 1, QString::fromUtf8( 1689 "cma_es = cma.Strategy(centroid=[5.0]*search_space_dims, sigma=5.0, lambda_=5*search_space_dims)\n" 1690 "toolbox.register(\"generate\", cma_es.generate, creator.Individual)\n" 1691 "toolbox.register(\"update\", cma_es.update)\n" 1692 "\n" 1693 "hof = tools.HallOfFame(1)\n" 1694 "stats = tools.Statistics(lambda ind: ind.fitness.values)\n" 1695 "stats.register(\"avg\", np.mean)\n" 1696 "stats.register(\"std\", np.std)\n" 1697 "stats.register(\"min\", np.min)\n" 1698 "stats.register(\"max\", np.max)\n" 1699 "\n" 1700 "# The CMA-ES algorithm converge with good probability with those settings\n" 1701 "pop, logbook = algorithms.eaGenerateUpdate(toolbox, ngen=60, stats=stats, \n" 1702 " halloffame=hof, verbose=False)\n" 1703 " \n" 1704 "print(\"Best individual is %s, fitness: %s\" % (hof[0], hof[0].fitness.values))" 1705 )); 1706 testTextResult(entry, 0, QString::fromUtf8( 1707 "Best individual is [-2.524016407520609e-08, -4.0857988576506457e-08], fitness: (6.517009154549669e-14,)" 1708 )); 1709 entry = entry->next(); 1710 1711 qDebug() << "command entry 49"; 1712 testCommandEntry(entry, 49, 1, QString::fromUtf8( 1713 "plt.figure(1, figsize=(7, 4))\n" 1714 "plt.plot(logbook.select('avg'), 'b-', label='Avg. fitness')\n" 1715 "plt.fill_between(range(len(logbook)), logbook.select('max'), logbook.select('min'), facecolor='blue', alpha=0.47)\n" 1716 "plt.plot(logbook.select('std'), 'm--', label='Std. deviation')\n" 1717 "plt.legend(frameon=True)\n" 1718 "plt.ylabel('Fitness'); plt.xlabel('Iterations');" 1719 )); 1720 testImageResult(entry, 0); 1721 entry = entry->next(); 1722 1723 testMarkdown(entry, QString::fromUtf8( 1724 "### OK, but wouldn't it be nice to have an animated plot of how CMA-ES progressed? \n" 1725 "\n" 1726 "* We need to do some coding to make this animation work.\n" 1727 "* We are going to create a class named `PlotableStrategy` that inherits from `deap.cma.Strategy`. This class logs the features we need to make the plots as evolution takes place. That is, for every iteration we store:\n" 1728 " * Current centroid and covariance ellipsoid.\n" 1729 " * Updated centroid and covariance.\n" 1730 " * Sampled individuals.\n" 1731 " * Evolution path.\n" 1732 " \n" 1733 "_Note_: I think that DEAP's implementation of CMA-ES has the drawback of storing information that should be stored as part of \"individuals\". I leave this for an afternoon hack." 1734 )); 1735 1736 qDebug() << "command entry 50"; 1737 testCommandEntry(entry, 50, QString::fromUtf8( 1738 "from math import sqrt, log, exp\n" 1739 "class PlotableStrategy(cma.Strategy):\n" 1740 " \"\"\"This is a modification of deap.cma.Strategy class.\n" 1741 " We store the execution data in order to plot it.\n" 1742 " **Note:** This class should not be used for other uses than\n" 1743 " the one it is meant for.\"\"\"\n" 1744 " \n" 1745 " def __init__(self, centroid, sigma, **kargs):\n" 1746 " \"\"\"Does the original initialization and then reserves \n" 1747 " the space for the statistics.\"\"\"\n" 1748 " super(PlotableStrategy, self).__init__(centroid, sigma, **kargs)\n" 1749 " \n" 1750 " self.stats_centroids = []\n" 1751 " self.stats_new_centroids = []\n" 1752 " self.stats_covs = []\n" 1753 " self.stats_new_covs = []\n" 1754 " self.stats_offspring = []\n" 1755 " self.stats_offspring_weights = []\n" 1756 " self.stats_ps = []\n" 1757 " \n" 1758 " def update(self, population):\n" 1759 " \"\"\"Update the current covariance matrix strategy from the\n" 1760 " *population*.\n" 1761 " \n" 1762 " :param population: A list of individuals from which to update the\n" 1763 " parameters.\n" 1764 " \"\"\"\n" 1765 " # -- store current state of the algorithm\n" 1766 " self.stats_centroids.append(copy.deepcopy(self.centroid))\n" 1767 " self.stats_covs.append(copy.deepcopy(self.C))\n" 1768 " \n" 1769 " \n" 1770 " population.sort(key=lambda ind: ind.fitness, reverse=True)\n" 1771 " \n" 1772 " # -- store sorted offspring\n" 1773 " self.stats_offspring.append(copy.deepcopy(population))\n" 1774 " \n" 1775 " old_centroid = self.centroid\n" 1776 " self.centroid = np.dot(self.weights, population[0:self.mu])\n" 1777 " \n" 1778 " # -- store new centroid\n" 1779 " self.stats_new_centroids.append(copy.deepcopy(self.centroid))\n" 1780 " \n" 1781 " c_diff = self.centroid - old_centroid\n" 1782 " \n" 1783 " \n" 1784 " # Cumulation : update evolution path\n" 1785 " self.ps = (1 - self.cs) * self.ps \\\n" 1786 " + sqrt(self.cs * (2 - self.cs) * self.mueff) / self.sigma \\\n" 1787 " * np.dot(self.B, (1. / self.diagD) \\\n" 1788 " * np.dot(self.B.T, c_diff))\n" 1789 " \n" 1790 " # -- store new evol path\n" 1791 " self.stats_ps.append(copy.deepcopy(self.ps))\n" 1792 " \n" 1793 " hsig = float((np.linalg.norm(self.ps) / \n" 1794 " sqrt(1. - (1. - self.cs)**(2. * (self.update_count + 1.))) / self.chiN\n" 1795 " < (1.4 + 2. / (self.dim + 1.))))\n" 1796 " \n" 1797 " self.update_count += 1\n" 1798 " \n" 1799 " self.pc = (1 - self.cc) * self.pc + hsig \\\n" 1800 " * sqrt(self.cc * (2 - self.cc) * self.mueff) / self.sigma \\\n" 1801 " * c_diff\n" 1802 " \n" 1803 " # Update covariance matrix\n" 1804 " artmp = population[0:self.mu] - old_centroid\n" 1805 " self.C = (1 - self.ccov1 - self.ccovmu + (1 - hsig) \\\n" 1806 " * self.ccov1 * self.cc * (2 - self.cc)) * self.C \\\n" 1807 " + self.ccov1 * np.outer(self.pc, self.pc) \\\n" 1808 " + self.ccovmu * np.dot((self.weights * artmp.T), artmp) \\\n" 1809 " / self.sigma**2\n" 1810 " \n" 1811 " # -- store new covs\n" 1812 " self.stats_new_covs.append(copy.deepcopy(self.C))\n" 1813 " \n" 1814 " self.sigma *= np.exp((np.linalg.norm(self.ps) / self.chiN - 1.) \\\n" 1815 " * self.cs / self.damps)\n" 1816 " \n" 1817 " self.diagD, self.B = np.linalg.eigh(self.C)\n" 1818 " indx = np.argsort(self.diagD)\n" 1819 " \n" 1820 " self.cond = self.diagD[indx[-1]]/self.diagD[indx[0]]\n" 1821 " \n" 1822 " self.diagD = self.diagD[indx]**0.5\n" 1823 " self.B = self.B[:, indx]\n" 1824 " self.BD = self.B * self.diagD" 1825 )); 1826 1827 testMarkdown(entry, QString::fromUtf8( 1828 "It is now possible to use/test our new class." 1829 )); 1830 1831 qDebug() << "command entry 51"; 1832 testCommandEntry(entry, 51, QString::fromUtf8( 1833 "toolbox = base.Toolbox()\n" 1834 "toolbox.register(\"evaluate\", current_problem)" 1835 )); 1836 1837 qDebug() << "command entry 52"; 1838 testCommandEntry(entry, 52, QString::fromUtf8( 1839 "max_gens = 40\n" 1840 "cma_es = PlotableStrategy(centroid=[5.0]*search_space_dims, sigma=1.0, lambda_=5*search_space_dims)\n" 1841 "toolbox.register(\"generate\", cma_es.generate, creator.Individual)\n" 1842 "toolbox.register(\"update\", cma_es.update)\n" 1843 "\n" 1844 "# The CMA-ES algorithm converge with good probability with those settings\n" 1845 "a = algorithms.eaGenerateUpdate(toolbox, ngen=max_gens, verbose=False)" 1846 )); 1847 1848 testMarkdown(entry, QString::fromUtf8( 1849 "Me can now code the `animate_cma_es()` function." 1850 )); 1851 1852 qDebug() << "command entry 53"; 1853 testCommandEntry(entry, 53, QString::fromUtf8( 1854 "norm=colors.Normalize(vmin=np.min(cma_es.weights), vmax=np.max(cma_es.weights))\n" 1855 "sm = cm.ScalarMappable(norm=norm, cmap=plt.get_cmap('gray'))" 1856 )); 1857 1858 qDebug() << "command entry 54"; 1859 testCommandEntry(entry, 54, QString::fromUtf8( 1860 "def animate_cma_es(gen):\n" 1861 " ax.cla()\n" 1862 " plot_problem_controur(current_problem, ((-11,-11), (11,11)), optimum=(0,0), ax=ax)\n" 1863 " \n" 1864 " plot_cov_ellipse(cma_es.stats_centroids[gen], cma_es.stats_covs[gen], volume=0.99, alpha=0.29, ax=ax)\n" 1865 " ax.plot(cma_es.stats_centroids[gen][0], cma_es.stats_centroids[gen][1], 'ro', markeredgecolor = 'none', ms=10)\n" 1866 " \n" 1867 " plot_cov_ellipse(cma_es.stats_new_centroids[gen], cma_es.stats_new_covs[gen], volume=0.99, \n" 1868 " alpha=0.29, fc='green', ec='darkgreen', ax=ax)\n" 1869 " ax.plot(cma_es.stats_new_centroids[gen][0], cma_es.stats_new_centroids[gen][1], 'go', markeredgecolor = 'none', ms=10)\n" 1870 " \n" 1871 " for i in range(gen+1):\n" 1872 " if i == 0:\n" 1873 " ax.plot((0,cma_es.stats_ps[i][0]),\n" 1874 " (0,cma_es.stats_ps[i][1]), 'b--')\n" 1875 " else:\n" 1876 " ax.plot((cma_es.stats_ps[i-1][0],cma_es.stats_ps[i][0]),\n" 1877 " (cma_es.stats_ps[i-1][1],cma_es.stats_ps[i][1]),'b--')\n" 1878 " \n" 1879 " for i,ind in enumerate(cma_es.stats_offspring[gen]):\n" 1880 " if i < len(cma_es.weights):\n" 1881 " color = sm.to_rgba(cma_es.weights[i])\n" 1882 " else:\n" 1883 " color= sm.to_rgba(norm.vmin)\n" 1884 " ax.plot(ind[0], ind[1], 'o', color = color, ms=5, markeredgecolor = 'none')\n" 1885 " \n" 1886 " ax.set_ylim((-10,10))\n" 1887 " ax.set_xlim((-10,10))\n" 1888 " ax.set_title('$t=$' +str(gen))\n" 1889 " return []" 1890 )); 1891 1892 testMarkdown(entry, QString::fromUtf8( 1893 "### CMA-ES progress " 1894 )); 1895 1896 qDebug() << "command entry 55"; 1897 testCommandEntry(entry, 55, 1, QString::fromUtf8( 1898 "fig = plt.figure(figsize=(6,6))\n" 1899 "ax = fig.gca()\n" 1900 "anim = animation.FuncAnimation(fig, animate_cma_es, frames=max_gens, interval=300, blit=True)\n" 1901 "plt.close()" 1902 )); 1903 testTextResult(entry, 0, QString::fromUtf8( 1904 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1905 " s)" 1906 )); 1907 entry = entry->next(); 1908 1909 qDebug() << "command entry 56"; 1910 testCommandEntry(entry, 56, 2, QString::fromUtf8( 1911 "HTML(anim.to_html5_video())" 1912 )); 1913 testTextResult(entry, 0, QString::fromUtf8( 1914 "/usr/local/lib/python3.6/dist-packages/matplotlib/contour.py:960: UserWarning: The following kwargs were not used by contour: 'rstride', 'cstride', 'linewidth'\n" 1915 " s)" 1916 )); 1917 testHtmlResult(entry, 1, QString::fromUtf8( 1918 "<IPython.core.display.HTML object>" 1919 )); 1920 entry = entry->next(); 1921 1922 testMarkdown(entry, QString::fromUtf8( 1923 "* Current centroid and covariance: **red**.\n" 1924 "* Updated centroid and covariance: **green**. \n" 1925 "* Sampled individuals: **shades of gray representing their corresponding weight**.\n" 1926 "* Evolution path: **blue line starting in (0,0)**. " 1927 )); 1928 1929 testMarkdown(entry, QString::fromUtf8( 1930 "## Homework\n" 1931 "\n" 1932 "1. Make an animated plot with the covariance update process. You can rely on the notebook of the previous demonstration class.\n" 1933 "2. Compare ES, CMA-ES and a genetic algortihm.\n" 1934 "2. How do you think that evolutionary strategies and CMA-ES should be modified in order to cope with combinatorial problems?\n" 1935 "3. How can evolution strategies be improved?\n" 1936 )); 1937 1938 testMarkdown(entry, QString::fromUtf8( 1939 "<hr/>\n" 1940 "<div class=\"container-fluid\">\n" 1941 " <div class='well'>\n" 1942 " <div class=\"row\">\n" 1943 " <div class=\"col-md-3\" align='center'>\n" 1944 " <img align='center'alt=\"Creative Commons License\" style=\"border-width:0\" src=\"https://i.creativecommons.org/l/by-nc-sa/4.0/88x31.png\"/>\n" 1945 " </div>\n" 1946 " <div class=\"col-md-9\">\n" 1947 " This work is licensed under a [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International License](http://creativecommons.org/licenses/by-nc-sa/4.0/).\n" 1948 " </div>\n" 1949 " </div>\n" 1950 " </div>\n" 1951 "</div>" 1952 )); 1953 1954 qDebug() << "command entry 58"; 1955 testCommandEntry(entry, 58, 1, QString::fromUtf8( 1956 "# To install run: pip install version_information\n" 1957 "%load_ext version_information\n" 1958 "%version_information scipy, numpy, matplotlib, seaborn, deap" 1959 )); 1960 testHtmlResult(entry, 0, QString::fromLatin1( 1961 "Software versions\n" 1962 "Python 3.6.7 64bit [GCC 8.2.0]\n" 1963 "IPython 6.3.1\n" 1964 "OS Linux 4.15.0 50 generic x86_64 with Ubuntu 18.04 bionic\n" 1965 "scipy 1.1.0\n" 1966 "numpy 1.14.5\n" 1967 "matplotlib 2.2.2\n" 1968 "seaborn 0.9.0\n" 1969 "deap 1.2\n" 1970 "Wed May 29 19:31:25 2019 MSK" 1971 ), QString::fromLatin1( 1972 "<table><tr><th>Software</th><th>Version</th></tr><tr><td>Python</td><td>3.6.7 64bit [GCC 8.2.0]</td></tr><tr><td>IPython</td><td>6.3.1</td></tr><tr><td>OS</td><td>Linux 4.15.0 50 generic x86_64 with Ubuntu 18.04 bionic</td></tr><tr><td>scipy</td><td>1.1.0</td></tr><tr><td>numpy</td><td>1.14.5</td></tr><tr><td>matplotlib</td><td>2.2.2</td></tr><tr><td>seaborn</td><td>0.9.0</td></tr><tr><td>deap</td><td>1.2</td></tr><tr><td colspan='2'>Wed May 29 19:31:25 2019 MSK</td></tr></table>" 1973 )); 1974 entry = entry->next(); 1975 1976 qDebug() << "command entry 59"; 1977 testCommandEntry(entry, 59, 1, QString::fromUtf8( 1978 "# this code is here for cosmetic reasons\n" 1979 "from IPython.core.display import HTML\n" 1980 "from urllib.request import urlopen\n" 1981 "HTML(urlopen('https://raw.githubusercontent.com/lmarti/jupyter_custom/master/custom.include').read().decode('utf-8'))" 1982 )); 1983 testHtmlResult(entry, 0, QString::fromUtf8( 1984 "<IPython.core.display.HTML object>" 1985 )); 1986 entry = entry->next(); 1987 1988 testMarkdown(entry, QString::fromUtf8( 1989 " " 1990 )); 1991 1992 QCOMPARE(entry, nullptr); 1993 } 1994 1995 void WorksheetTest::testJupyter3() 1996 { 1997 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 1998 if (backend && backend->isEnabled() == false) 1999 QSKIP("Skip, because python backend don't available", SkipSingle); 2000 2001 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("Population_Genetics.ipynb"))); 2002 2003 QCOMPARE(w->isReadOnly(), false); 2004 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 2005 2006 WorksheetEntry* entry = w->firstEntry(); 2007 2008 testMarkdown(entry, QString::fromUtf8( 2009 "## Population Genetics in *an* RNA World\n" 2010 "\n" 2011 "In order to study population genetics, we first need a model of a population. And even before that, we need to define what we mean by *population*. Populations can be defined on many levels and with many diffferent criteria. For our purposes, we will simply say that a population is a set of individuals sharing a common environment. And because this is population *genetics* we can think of individuals as entities comprising of specific genes or chromosomes. \n" 2012 "\n" 2013 "So where do we get a population from? As you may have discussed in previous workshops, there are very large datasets containing sequencing information from different populations. So we could download one of these datasets and perform some analysis on it. But I find this can be dry and tedious. So why download data when we can simply create our own?\n" 2014 "\n" 2015 "In this workshop we're going to be creating and studying our own \"artificial\" populations to illustrate some important population genetics concepts and methodologies. Not only will this help you learn population genetics, but you will get a lot more programming practice than if we were to simply parse data files and go from there. \n" 2016 "\n" 2017 "More specifically, we're going to build our own RNA world.\n" 2018 "\n" 2019 "As you may know, RNA is widely thought to be the first self replicating life-form to arise x billion years ago. One of the strongest arguments for this theory is that RNA is able to carry information in its nucleotides like DNA, and like protein, it is able to adopt higher order structures to catalyze reactions, such as self replication. So it is likely, and there is growing evidence that this is the case, that the first form of replicating life was RNA. And because of this dual property of RNA as an information vessel as well as a structural/functional element we can use RNA molecules to build very nice population models. \n" 2020 "\n" 2021 "So in this notebook, I'll be walking you through building genetic populations, simulating their evolution, and using statistics and other mathematical tools for understanding key properties of populations.\n" 2022 "\n" 2023 "### Building an RNA population\n" 2024 "\n" 2025 "As we saw earlier, RNA has the nice property of posessing a strong mapping between information carrying (sequence) and function (structure). This is analogous to what is known in evolutionary terms as a genotype and a phenotype. With these properties, we have everything we need to model a population, and simulate its evolution.\n" 2026 "\n" 2027 "#### RNA sequence-structure\n" 2028 "\n" 2029 "We can think of the genotype as a sequence $s$ consisting of letters/nucleotides from the alphabet $\\{U,A,C,G\\}$. The corresponding phenotype $\\omega$ is the secondary structure of $s$ which can be thought of as a pairing between nucleotides in the primary sequence that give rise to a 2D architecture. Because it has been shown that the function of many biomolecules, including RNA, is driven by structure this gives us a good proxy for phenotype. \n" 2030 "\n" 2031 "Below is an example of what an RNA secondary structure, or pairing, looks like." 2032 )); 2033 2034 qDebug() << "command entry 1"; 2035 testCommandEntry(entry, 1, 1, QString::fromUtf8( 2036 "### 1\n" 2037 "\n" 2038 "from IPython.display import Image\n" 2039 "#This will load an image of an RNA secondary structure\n" 2040 "Image(url='http://www.tbi.univie.ac.at/~pkerp/forgi/_images/1y26_ss.png')" 2041 )); 2042 testHtmlResult(entry, 0, QString::fromLatin1( 2043 "<IPython.core.display.Image object>" 2044 ), QString::fromLatin1( 2045 "<img src=\"http://www.tbi.univie.ac.at/~pkerp/forgi/_images/1y26_ss.png\"/>" 2046 )); 2047 entry = entry->next(); 2048 2049 testMarkdown(entry, QString::fromUtf8( 2050 "As you can see, unparied positions are forming loop-like structures, and paired positions are forming stem-like structures. It is this spatial arrangement of nucleotides that drives RNA's function. Therefore, another sequence that adopts a similar shape, is likely to behave in a similar manner. Another thing to notice is that, although in reality this is often not the case, in general we only allow pairs between $\\{C,G\\}$ and $\\{A, U\\}$ nucleotides, most modern approaches allow for non-canonical pairings and you will find some examples of this in the above structure.\n" 2051 "\n" 2052 "*How do we go from a sequence to a structure?*\n" 2053 "\n" 2054 "So a secondary structure is just a list of pairings between positions. How do we get the optimal pairing?\n" 2055 "\n" 2056 "The algorithm we're going to be using in our simulations is known as the Nussinov Algorithm. The Nussinov algorithm is one of the first and simplest attempts at predicting RNA structure. Because bonds tend to stabilize RNA, the algorithm tries to maximize the number of pairs in the structure and return that as its solution. Current approaches achieve more accurate solutions by using energy models based one experimental values to then obtain a structure that minimizes free energy. But since we're not really concerned with the accuracy of our predictions, Nussinov is a good entry point. Furthermore, the main algorithmic concepts are the same between Nussinov and state of the art RNA structure prediction algorithms. I implemented the algorithm in a separate file called `fold.py` that we can import and use its functions. I'm not going to go into detail here on how the algorithm works because it is beyond the scope of this workshop but there is a bonus exercise at the end if you're curious.\n" 2057 "\n" 2058 "You can predict a secondary structure by calling `nussinov()` with a sequence string and it will return a tuple in the form `(structure, pairs)`." 2059 )); 2060 2061 qDebug() << "command entry 2"; 2062 testCommandEntry(entry, 2, 1, QString::fromUtf8( 2063 "import numpy as np\n" 2064 "from fold import nussinov\n" 2065 "\n" 2066 "sequence_to_fold = \"ACCCGAUGUUAUAUAUACCU\"\n" 2067 "struc = nussinov(sequence_to_fold)\n" 2068 "print(sequence_to_fold)\n" 2069 "print(struc[0])" 2070 )); 2071 testTextResult(entry, 0, QString::fromUtf8( 2072 "ACCCGAUGUUAUAUAUACCU\n" 2073 "(...(..(((....).))))" 2074 )); 2075 entry = entry->next(); 2076 2077 testMarkdown(entry, QString::fromUtf8( 2078 "You will see a funny dot-bracket string in the output. This is a representation of the structure of an RNA. Quite simply, a matching parir of parentheses (open and close) correspond to the nucleotides at those positions being paried. Whereas, a dot means that that position is unpaired in the structure. Feel free to play around with the input sequence to get a better understanding of the notation.\n" 2079 "\n" 2080 "So that's enough about RNA structure prediction. Let's move on to building our populations.\n" 2081 "\n" 2082 "### Fitness of a sequence: Target Structure\n" 2083 "\n" 2084 "Now that we have a good way of getting a phenotype (secondary structure), we need a way to evaluate the fitness of that phenotype. If we think in real life terms, fitness is the ability of a genotype to replicate into the next generation. If you have a gene carrying a mutation that causes some kind of disease, your fitness is decreased and you have a lower chance of contributing offspring to the next generation. On a molecular level the same concept applies. A molecule needs to accomplish a certain function, i.e. bind to some other molecule or send some kind of signal. And as we've seen before, the most important factor that determines how well it can carry out this function is its structure. So we can imagine that a certain structure, we can call this a 'target' structure, is required in order to accomplish a certain function. So a sequence that folds correctly to a target structure is seen as having a greater fitness than one that does not. Since we've encoded structures as simple dot-bracket strings, we can easily compare structures and thus evaluate the fitness between a given structure and the target, or 'correct' structure. \n" 2085 "\n" 2086 "There are many ways to compare structures $w_{1}$ and $w_{2}$, but we're going to use one of the simplest ways, which is base-pair distance. This is just the number of pairs in $w_{1}$ that are not in $w_{2}$. Again, this is beyond the scope of this workshop so I'll just give you the code for it and if you would like to know more you can ask me." 2087 )); 2088 2089 qDebug() << "command entry 3"; 2090 testCommandEntry(entry, 3, 1, QString::fromUtf8( 2091 "### 3\n" 2092 "\n" 2093 "#ss_to_bp() and bp_distance() by Vladimir Reinharz.\n" 2094 "def ss_to_bp(ss):\n" 2095 " bps = set()\n" 2096 " l = []\n" 2097 " for i, x in enumerate(ss):\n" 2098 " if x == '(':\n" 2099 " l.append(i)\n" 2100 " elif x == ')':\n" 2101 " bps.add((l.pop(), i))\n" 2102 " return bps\n" 2103 "\n" 2104 "def bp_distance(w1, w2):\n" 2105 " \"\"\"\n" 2106 " return base pair distance between structures w1 and w1. \n" 2107 " w1 and w1 are lists of tuples representing pairing indices.\n" 2108 " \"\"\"\n" 2109 " return len(set(w1).symmetric_difference(set(w2)))\n" 2110 "\n" 2111 "#let's fold two sequences\n" 2112 "w1 = nussinov(\"CCAAAAGG\")\n" 2113 "w2 = nussinov(\"ACAAAAGA\")\n" 2114 "\n" 2115 "print(w1)\n" 2116 "print(w2)\n" 2117 "\n" 2118 "#give the list of pairs to bp_distance and see what the distance is.\n" 2119 "print(bp_distance(w1[-1], w2[-1]))" 2120 )); 2121 testTextResult(entry, 0, QString::fromUtf8( 2122 "('((....))', [(0, 7), (1, 6)])\n" 2123 "('.(....).', [(1, 6)])\n" 2124 "1" 2125 )); 2126 entry = entry->next(); 2127 2128 testMarkdown(entry, QString::fromUtf8( 2129 "## Defining a cell: a little bit of Object Oriented Programming (OOP)\n" 2130 "\n" 2131 "Since we're going to be playing aroudn with sequences and structures and fitness values a lot, it's best to package it all nicely into an object. As you'll have seen with Vlad, objects are just a nice way of grouping data into an easily accessible form. \n" 2132 "\n" 2133 "We're trying to simulate evolution on a very simple kind of organism, or cell. It contains two copies of a RNA gene, each with a corresponding structure. " 2134 )); 2135 2136 qDebug() << "command entry 4"; 2137 testCommandEntry(entry, 4, 1, QString::fromUtf8( 2138 "### 4\n" 2139 "class Cell:\n" 2140 " def __init__(self, seq_1, struc_1, seq_2, struc_2):\n" 2141 " self.sequence_1 = seq_1\n" 2142 " self.sequence_2 = seq_2\n" 2143 " self.structure_1 = struc_1\n" 2144 " self.structure_2 = struc_2\n" 2145 " \n" 2146 "#for now just try initializing a Cell with made up sequences and structures\n" 2147 "cell = Cell(\"AACCCCUU\", \"((.....))\", \"GGAAAACA\", \"(....).\")\n" 2148 "print(cell.sequence_1, cell.structure_2, cell.sequence_1, cell.structure_2)" 2149 )); 2150 testTextResult(entry, 0, QString::fromUtf8( 2151 "AACCCCUU (....). AACCCCUU (....)." 2152 )); 2153 entry = entry->next(); 2154 2155 testMarkdown(entry, QString::fromUtf8( 2156 "## Populations of Cells\n" 2157 "\n" 2158 "Now we've defined a 'Cell'. Since a population is a collection of individuals our populations will naturally consist of **lists** of 'Cell' objects, each with their own sequences. Here we initialize all the Cells with random sequences and add them to the 'population' list." 2159 )); 2160 2161 qDebug() << "command entry 5"; 2162 testCommandEntry(entry, 5, QString::fromUtf8( 2163 "### 5\n" 2164 "import random\n" 2165 "\n" 2166 "def populate(target, pop_size=100):\n" 2167 " \n" 2168 " population = []\n" 2169 "\n" 2170 " for i in range(pop_size):\n" 2171 " #get a random sequence to start with\n" 2172 " sequence = \"\".join([random.choice(\"AUCG\") for _ in range(len(target))])\n" 2173 " #use nussinov to get the secondary structure for the sequence\n" 2174 " structure = nussinov(sequence)\n" 2175 " #add a new Cell object to the population list\n" 2176 " new_cell = Cell(sequence, structure, sequence, structure)\n" 2177 " new_cell.id = i\n" 2178 " new_cell.parent = i\n" 2179 " population.append(new_cell)\n" 2180 " \n" 2181 " return population" 2182 )); 2183 2184 testMarkdown(entry, QString::fromUtf8( 2185 "Try creating a new population and printing the first 10 sequences and structures (in dot-bracket) on the first chromosome!" 2186 )); 2187 2188 qDebug() << "command entry 6"; 2189 testCommandEntry(entry, 6, 1, QString::fromUtf8( 2190 "### 6\n" 2191 "target = \"(.(((....).).).)....\"\n" 2192 "pop = populate(target, pop_size=100)\n" 2193 "for p in pop[:10]:\n" 2194 " print(p.id, p.sequence_1, p.structure_1[0], p.sequence_2, p.structure_2[0])" 2195 )); 2196 testTextResult(entry, 0, QString::fromUtf8( 2197 "0 GGACGGAGCAUUAUCUGCUA (((...(....).))..).. GGACGGAGCAUUAUCUGCUA (((...(....).))..)..\n" 2198 "1 ACAAUCGCCUCACUCACGUU (.(..((..(.....))))) ACAAUCGCCUCACUCACGUU (.(..((..(.....)))))\n" 2199 "2 UCCGUUCUAUUAGUUCAUAG .(..(((.....)...).)) UCCGUUCUAUUAGUUCAUAG .(..(((.....)...).))\n" 2200 "3 CAAACCUUGUUCGUAAUACA .(....)((((....).))) CAAACCUUGUUCGUAAUACA .(....)((((....).)))\n" 2201 "4 GCAUCGAGUGCGCGGCAUAA ((..((....)).).).... GCAUCGAGUGCGCGGCAUAA ((..((....)).).)....\n" 2202 "5 GAAUUCUGAGAUCAUACUCG (((((.....)..))..)). GAAUUCUGAGAUCAUACUCG (((((.....)..))..)).\n" 2203 "6 GGAACCGUAGGCUUUGCAAG (.(((....)..))..)... GGAACCGUAGGCUUUGCAAG (.(((....)..))..)...\n" 2204 "7 GCAAAAGACAGCCCGCAUCA ((....).)((....).).. GCAAAAGACAGCCCGCAUCA ((....).)((....).)..\n" 2205 "8 GGGUACCGACAACGGAGCUC ((.(.(((....)))).).) GGGUACCGACAACGGAGCUC ((.(.(((....)))).).)\n" 2206 "9 CUCUUAUUUCACUUAGCUGU (.(((.....)...))..). CUCUUAUUUCACUUAGCUGU (.(((.....)...))..)." 2207 )); 2208 entry = entry->next(); 2209 2210 testMarkdown(entry, QString::fromUtf8( 2211 "## The Fitness of a Cell\n" 2212 "\n" 2213 "Now that we can store populatoins of cells, we need a way to evaluate the fitness of a given Cell. Recall that a Cell is simply an object that contains two RNA sequences (think of it as two copies of a gene on each chromosome). \n" 2214 "\n" 2215 "So we simply need to loop through each Cell in a population and compute base pair distance to the target structure. However, simply using base-pair distance is not a very good measure of fitness. There are two reasons for this: \n" 2216 "\n" 2217 "1. We want fitness to represent a *probability* that a cell will reproduce, and base pair distance is an integer.\n" 2218 "2. We want this probability to be a *relative* measure. That is, we want to be the fitness to be proportional to how good a cell is with respect to all others in the population. This touches on an important principle in evolution where we only need to be 'better' than the competition and not good in some absolute measure. For example, if you and I are being chased by a bear. In order to survive, I only need to be faster than you, and not necessarily some absolute level of fitness.\n" 2219 "\n" 2220 "In order to get a probability (number between 0 and 1) we use the following equation to define the fitness of a structure $\\omega$ on a target structure $T$:\n" 2221 "\n" 2222 "$$P(\\omega, T) = N^{-1} exp(\\frac{-\\beta \\texttt{dist}(\\omega, T)}{\\texttt{len}(\\omega)})$$\n" 2223 "\n" 2224 "$$N = \\sum_{i \\in Pop}{P(\\omega_i, T})$$\n" 2225 "\n" 2226 "Here, the $N$ is what gives us the 'relative' measure because we divide the fitness of the Cell by the sum of the fitness of every other Cell. \n" 2227 "\n" 2228 "Let's take a quick look at how this function behaves if we plot different base pair distance values.\n" 2229 "\n" 2230 "What is the effect of the parameter $\\beta$? Try plotting the same function but with different values of $\\beta$." 2231 )); 2232 2233 qDebug() << "command entry 8"; 2234 testCommandEntry(entry, 8, 2, QString::fromUtf8( 2235 "%matplotlib inline\n" 2236 "import matplotlib.pyplot as plt\n" 2237 "import math\n" 2238 "import seaborn as sns\n" 2239 "\n" 2240 "target_length = 50\n" 2241 "beta = -2\n" 2242 "\n" 2243 "plt.plot([math.exp(beta * (bp_dist / float(target_length))) for bp_dist in range(target_length)])\n" 2244 "plt.xlabel(\"Base pair distance to target structure\")\n" 2245 "plt.ylabel(\"P(w, T)\")" 2246 )); 2247 testTextResult(entry, 0, QString::fromLatin1("Text(0,0.5,'P(w, T)')")); 2248 testImageResult(entry, 1); 2249 entry = entry->next(); 2250 2251 testMarkdown(entry, QString::fromUtf8( 2252 "As you can see, it's a very simple function that evaluates to 1 (highest fitness) if the base pair distance is 0, and decreases as the structures get further and further away from the target. I didn't include the $N$ in the plotting as it will be a bit more annoying to compute, but it is simply a scaling factor so the shape and main idea won't be different.\n" 2253 "\n" 2254 "Now we can use this function to get a fitness value for each Cell in our population." 2255 )); 2256 2257 qDebug() << "command entry 9"; 2258 testCommandEntry(entry, 9, 1, QString::fromUtf8( 2259 "### 7\n" 2260 "\n" 2261 "def compute_fitness(population, target, beta=-2):\n" 2262 " \"\"\"\n" 2263 " Assigns a fitness and bp_distance value to each cell in the population.\n" 2264 " \"\"\"\n" 2265 " #store the fitness values of each cell\n" 2266 " tot = []\n" 2267 " #iterate through each cell\n" 2268 " for cell in population:\n" 2269 " \n" 2270 " #calculate the bp_distance of each chromosome using the cell's structure\n" 2271 " bp_distance_1 = bp_distance(cell.structure_1[-1], ss_to_bp(target))\n" 2272 " bp_distance_2 = bp_distance(cell.structure_2[-1], ss_to_bp(target))\n" 2273 " \n" 2274 " #use the bp_distances and the above fitness equation to calculate the fitness of each chromosome\n" 2275 " fitness_1 = math.exp((beta * bp_distance_1 / float(len(cell.sequence_1))))\n" 2276 " fitness_2 = math.exp((beta * bp_distance_2 / float(len(cell.sequence_2))))\n" 2277 "\n" 2278 " #get the fitness of the whole cell by multiplying the fitnesses of each chromosome\n" 2279 " cell.fitness = fitness_1 * fitness_2\n" 2280 " \n" 2281 " #store the bp_distance of each chromosome.\n" 2282 " cell.bp_distance_1 = bp_distance_1\n" 2283 " cell.bp_distance_2 = bp_distance_2\n" 2284 " \n" 2285 " \n" 2286 " #add the cell's fitness value to the list of all fitness values (used for normalization later)\n" 2287 " tot.append(cell.fitness)\n" 2288 "\n" 2289 " #normalization factor is sum of all fitness values in population\n" 2290 " norm = np.sum(tot)\n" 2291 " #divide all fitness values by the normalization factor.\n" 2292 " for cell in population:\n" 2293 " cell.fitness = cell.fitness / norm\n" 2294 "\n" 2295 " return None\n" 2296 "\n" 2297 "compute_fitness(pop, target)\n" 2298 "for cell in pop[:10]:\n" 2299 " print(cell.fitness, cell.bp_distance_1, cell.bp_distance_2)" 2300 )); 2301 testTextResult(entry, 0, QString::fromUtf8( 2302 "0.013612068231863143 6 6\n" 2303 "0.007470461436952334 9 9\n" 2304 "0.009124442203822766 8 8\n" 2305 "0.007470461436952334 9 9\n" 2306 "0.013612068231863143 6 6\n" 2307 "0.007470461436952334 9 9\n" 2308 "0.02030681957427158 4 4\n" 2309 "0.009124442203822766 8 8\n" 2310 "0.006116296518116008 10 10\n" 2311 "0.009124442203822766 8 8" 2312 )); 2313 entry = entry->next(); 2314 2315 testMarkdown(entry, QString::fromUtf8( 2316 "## Introducing diversity: Mutations\n" 2317 "\n" 2318 "Evolution would go nowhere without random mutations. While mutations are technically just random errors in the copying of genetic material, they are essential in the process of evolution. This is because they introduce novel diversity to populatons, which with a low frequency can be beneficial. And when a beneficial mutation arises (i.e. a mutation that increases fitness, or replication probability) it quickly takes over the population and the populatioin as a whole has a higher fitness.\n" 2319 "\n" 2320 "Implementing mutations in our model will be quite straightforward. Since mutations happen at the genotype/sequence level, we simply have to iterate through our strings of nucleotides (sequences) and randomly introduce changes." 2321 )); 2322 2323 qDebug() << "command entry 10"; 2324 testCommandEntry(entry, 10, 1, QString::fromUtf8( 2325 "def mutate(sequence, mutation_rate=0.001):\n" 2326 " \"\"\"Takes a sequence and mutates bases with probability mutation_rate\"\"\"\n" 2327 " \n" 2328 " #start an empty string to store the mutated sequence\n" 2329 " new_sequence = \"\"\n" 2330 " #boolean storing whether or not the sequence got mutated\n" 2331 " mutated = False\n" 2332 " #go through every bp in the sequence\n" 2333 " for bp in sequence:\n" 2334 " #generate a random number between 0 and 1\n" 2335 " r = random.random()\n" 2336 " #if r is below mutation rate, introduce a mutation\n" 2337 " if r < mutation_rate:\n" 2338 " #add a randomly sampled nucleotide to the new sequence\n" 2339 " new_sequence = new_sequence + random.choice(\"aucg\")\n" 2340 " mutated = True\n" 2341 " else:\n" 2342 " #if the mutation condition did not get met, copy the current bp to the new sequence\n" 2343 " new_sequence = new_sequence + bp\n" 2344 " \n" 2345 " return (new_sequence, mutated)\n" 2346 "\n" 2347 "sequence_to_mutate = 'AAAAGGAGUGUGUAUGU'\n" 2348 "print(sequence_to_mutate)\n" 2349 "print(mutate(sequence_to_mutate, mutation_rate=0.5))" 2350 )); 2351 testTextResult(entry, 0, QString::fromUtf8( 2352 "AAAAGGAGUGUGUAUGU\n" 2353 "('AcAAGgAuUGUuaAaGa', True)" 2354 )); 2355 entry = entry->next(); 2356 2357 testMarkdown(entry, QString::fromUtf8( 2358 "## Selection\n" 2359 "\n" 2360 "The final process in this evolution model is selection. Once you have populations with a diverse range of fitnesses, we need to select the fittest individuals and let them replicate and contribute offspring to the next generation. In real populations this is just the process of reproduction. If you're fit enough you will be likely to reproduce more than another individual who is not as well suited to the environment.\n" 2361 "\n" 2362 "In order to represent this process in our model, we will use the fitness values that we assigned to each Cell earlier and use that to select replicating Cells. This is equivalent to sampling from a population with the sampling being weighted by the fitness of each Cell. Thankfully, `numpy.random.choice` comes to the rescue here. Once we have sampled enough Cells to build our next generation, we introduce mutations and compute the fitness values of the new generation." 2363 )); 2364 2365 qDebug() << "command entry 11"; 2366 testCommandEntry(entry, 11, 1, QString::fromUtf8( 2367 "def selection(population, target, mutation_rate=0.001, beta=-2):\n" 2368 " \"\"\"\n" 2369 " Returns a new population with offspring of the input population\n" 2370 " \"\"\"\n" 2371 "\n" 2372 " #select the sequences that will be 'parents' and contribute to the next generation\n" 2373 " parents = np.random.choice(population, len(population), p=[rna.fitness for rna in population], replace=True)\n" 2374 "\n" 2375 " #build the next generation using the parents list\n" 2376 " next_generation = [] \n" 2377 " for i, p in enumerate(parents):\n" 2378 " new_cell = Cell(p.sequence_1, p.structure_1, p.sequence_2, p.structure_2)\n" 2379 " new_cell.id = i\n" 2380 " new_cell.parent = p.id\n" 2381 " \n" 2382 " next_generation.append(new_cell)\n" 2383 "\n" 2384 " #introduce mutations in next_generation sequeneces and re-fold when a mutation occurs\n" 2385 " for rna in next_generation: \n" 2386 " mutated_sequence_1, mutated_1 = mutate(rna.sequence_1, mutation_rate=mutation_rate)\n" 2387 " mutated_sequence_2, mutated_2 = mutate(rna.sequence_2, mutation_rate=mutation_rate)\n" 2388 " \n" 2389 " if mutated_1:\n" 2390 " rna.sequence_1 = mutated_sequence_1\n" 2391 " rna.structure_1 = nussinov(mutated_sequence_1)\n" 2392 " if mutated_2:\n" 2393 " rna.sequence_2 = mutated_sequence_2\n" 2394 " rna.structure_2 = nussinov(mutated_sequence_2)\n" 2395 " else:\n" 2396 " continue\n" 2397 "\n" 2398 " #update fitness values for the new generation\n" 2399 " compute_fitness(next_generation, target, beta=beta)\n" 2400 "\n" 2401 " return next_generation\n" 2402 "\n" 2403 "next_gen = selection(pop, target)\n" 2404 "for cell in next_gen[:10]:\n" 2405 " print(cell.sequence_1)" 2406 )); 2407 testTextResult(entry, 0, QString::fromUtf8( 2408 "GAGCUUUAAACUAAUCUAAU\n" 2409 "GCAAAAGACAGCCaGCAUCA\n" 2410 "UUUCUUUUUCCCCCCCGAUG\n" 2411 "AAGCCCUAGGUAUGUUGUAG\n" 2412 "AAGAAGUACCCAUACAGAUG\n" 2413 "CUAAGACGACUUUUAGUUCA\n" 2414 "ACCUGCCAUCAUCACCAGAC\n" 2415 "AGAAUUGCUGUUCUCUAUCU\n" 2416 "GCGGAUCAUACUCCAAGUCG\n" 2417 "GAGCUUUAAACUAAUCUAAU" 2418 )); 2419 entry = entry->next(); 2420 2421 testMarkdown(entry, QString::fromUtf8( 2422 "## Gathering information on our populations\n" 2423 "\n" 2424 "Here we simply store some statistics (in a dictionary) on the population at each generation such as the average base pair distance and the average fitness of the populations. No coding to do here, it's not a very interesting function but feel free to give it a look." 2425 )); 2426 2427 qDebug() << "command entry 12"; 2428 testCommandEntry(entry, 12, QString::fromUtf8( 2429 "def record_stats(pop, population_stats):\n" 2430 " \"\"\"\n" 2431 " Takes a population list and a dictionary and updates it with stats on the population.\n" 2432 " \"\"\"\n" 2433 " generation_bp_distance_1 = [rna.bp_distance_1 for rna in pop]\n" 2434 " generation_bp_distance_2 = [rna.bp_distance_2 for rna in pop]\n" 2435 "\n" 2436 " mean_bp_distance_1 = np.mean(generation_bp_distance_1)\n" 2437 " mean_bp_distance_2 = np.mean(generation_bp_distance_2)\n" 2438 " \n" 2439 " mean_fitness = np.mean([rna.fitness for rna in pop])\n" 2440 "\n" 2441 "\n" 2442 " population_stats.setdefault('mean_bp_distance_1', []).append(mean_bp_distance_1)\n" 2443 " population_stats.setdefault('mean_bp_distance_2', []).append(mean_bp_distance_2)\n" 2444 " \n" 2445 " population_stats.setdefault('mean_fitness', []).append(mean_fitness)\n" 2446 " \n" 2447 " return None" 2448 )); 2449 2450 testMarkdown(entry, QString::fromUtf8( 2451 "## And finally.... evolution\n" 2452 "\n" 2453 "We can put all the above parts together in a simple function that does the following:\n" 2454 "\n" 2455 "1. start a new population and compute its fitness\n" 2456 "2. repeat the following for the desired number of generations:\n" 2457 " 1. record statistics on population\n" 2458 " 2. perform selection+mutation\n" 2459 " 3. store new population\n" 2460 "\n" 2461 "And that's it! We have an evolutionary reactor!" 2462 )); 2463 2464 qDebug() << "command entry 13"; 2465 testCommandEntry(entry, 13, QString::fromUtf8( 2466 "def evolve(target, generations=10, pop_size=100, mutation_rate=0.001, beta=-2):\n" 2467 " \"\"\"\n" 2468 " Takes target structure and sets up initial population, performs selection and iterates for desired generations.\n" 2469 " \"\"\"\n" 2470 " #store list of all populations throughotu generations [[cells from generation 1], [cells from gen. 2]...]\n" 2471 " populations = []\n" 2472 " #start a dictionary that will hold some stats on the populations.\n" 2473 " population_stats = {}\n" 2474 " \n" 2475 " #get a starting population\n" 2476 " initial_population = populate(target, pop_size=pop_size)\n" 2477 " #compute fitness of initial population\n" 2478 " compute_fitness(initial_population, target)\n" 2479 "\n" 2480 " #set current_generation to initial population.\n" 2481 " current_generation = initial_population\n" 2482 "\n" 2483 " #iterate the selection process over the desired number of generations\n" 2484 " for i in range(generations):\n" 2485 "\n" 2486 " #let's get some stats on the structures in the populations \n" 2487 " record_stats(current_generation, population_stats)\n" 2488 " \n" 2489 " #add the current generation to our list of populations.\n" 2490 " populations.append(current_generation)\n" 2491 "\n" 2492 " #select the next generation\n" 2493 " new_gen = selection(current_generation, target, mutation_rate=mutation_rate, beta=beta)\n" 2494 " #set current generation to be the generation we just obtained.\n" 2495 " current_generation = new_gen \n" 2496 " \n" 2497 " return (populations, population_stats)" 2498 )); 2499 2500 testMarkdown(entry, QString::fromUtf8( 2501 "Try a run of the `evolve()` function." 2502 )); 2503 2504 qDebug() << "command entry 14"; 2505 testCommandEntry(entry, 14, QString::fromUtf8( 2506 "pops, pops_stats = evolve(\"(((....)))\", generations=20, pop_size=1000, mutation_rate=0.005, beta=-2)" 2507 )); 2508 2509 testMarkdown(entry, QString::fromUtf8( 2510 "Let's see if it actually worked by plotting the average base pair distance as a function of generations for both genes in each cell. We should expect a gradual decrease as the populations get closer to the target structure." 2511 )); 2512 2513 qDebug() << "command entry 15"; 2514 testCommandEntry(entry, 15, 1, QString::fromUtf8( 2515 "def evo_plot(pops_stats):\n" 2516 " \"\"\"\n" 2517 " Plot base pair distance for each chromosome over generations.\n" 2518 " \"\"\"\n" 2519 " for m in ['mean_bp_distance_1', 'mean_bp_distance_2']:\n" 2520 " plt.plot(pops_stats[m], label=m)\n" 2521 " plt.legend()\n" 2522 " plt.xlabel(\"Generations\")\n" 2523 " plt.ylabel(\"Mean Base Pair Distance\")\n" 2524 " \n" 2525 "evo_plot(pops_stats)" 2526 )); 2527 testImageResult(entry, 0); 2528 entry = entry->next(); 2529 2530 testMarkdown(entry, QString::fromUtf8( 2531 "You should see a nice drop in base pair distance! Another way of visualizing this is by plotting a histogram of the base pair distance of all Cells in the initial population versus the final population." 2532 )); 2533 2534 qDebug() << "command entry 16"; 2535 testCommandEntry(entry, 16, 1, QString::fromUtf8( 2536 "def bp_distance_distributions(pops):\n" 2537 " \"\"\"\n" 2538 " Plots histograms of base pair distance in initial and final populations.\n" 2539 " \"\"\"\n" 2540 " #plot bp_distance_1 for rnas in first population\n" 2541 " g = sns.distplot([rna.bp_distance_1 for rna in pops[0]], label='initial population')\n" 2542 " #plot bp_distance_1 for rnas in first population\n" 2543 " g = sns.distplot([rna.bp_distance_1 for rna in pops[-1]], label='final population')\n" 2544 " g.set(xlabel='Mean Base Pair Distance')\n" 2545 " g.legend()\n" 2546 "bp_distance_distributions(pops)" 2547 )); 2548 testImageResult(entry, 0); 2549 entry = entry->next(); 2550 2551 testMarkdown(entry, QString::fromUtf8( 2552 "## Studying our evolved sequences with some Population Genetics tools\n" 2553 "\n" 2554 "Now that we've generated some sequences, we can analyze them!\n" 2555 "\n" 2556 "So after several rounds of selection, what do we get? We have a bunch of different sequences. We would like a way to characterize this diversity. One important tool for doing this is by making what is known as phylogenetic trees. \n" 2557 "\n" 2558 "Phylogenetic trees tell us about which groups of similar sequences are present and how they are likely related in evolutionary time. \n" 2559 "\n" 2560 "There are several ways of building phylogenetic trees using BioPython. Here we will go over one type and I'll leave another one as an exercise.\n" 2561 "\n" 2562 "### UPGMA (Unweighted Pair Group Method with Arithmetic Means)\n" 2563 "\n" 2564 "This is basically a clustering method based on the distance (or number of differences) between every pair of sequences. It assumes that sequences that are more similar are more likely to be related than the other way around. \n" 2565 "\n" 2566 "For $N$ sequences, the algorithm builds an $NxN$ matrix that stores the distance between each sequence to every other sequence. The algorithm goes through this matrix and finds the pair of sequences that is most similar and merges it into a 'cluster' or in tree terms, connects them to a common node. This process is repeated until all the sequences have been assigned to a group. Refer to the wikipedia article on [UPGMA](https://en.wikipedia.org/wiki/UPGMA) for a more detailed explanation. \n" 2567 "" 2568 )); 2569 2570 qDebug() << "command entry 18"; 2571 testCommandEntry(entry, 18, QString::fromUtf8( 2572 "from Bio import SeqIO\n" 2573 "from Bio.Seq import Seq\n" 2574 "from Bio.SeqRecord import SeqRecord\n" 2575 "from Bio import AlignIO" 2576 )); 2577 2578 qDebug() << "command entry 19"; 2579 testCommandEntry(entry, 19, QString::fromUtf8( 2580 "sequences = []\n" 2581 "#let's take the first 10 sequences of our population to keep things simple\n" 2582 "for seq in pops[-1][:10]:\n" 2583 " #store each sequence in the sequences list as a SeqRecord object\n" 2584 " sequences.append(SeqRecord(Seq(seq.sequence_1), id=str(seq.id)))\n" 2585 " \n" 2586 "\n" 2587 "#write our sequences to fasta format\n" 2588 "with open(\"seq.fasta\", \"w+\") as f:\n" 2589 " SeqIO.write(sequences, f, \"fasta\")" 2590 )); 2591 2592 testMarkdown(entry, QString::fromUtf8( 2593 "The UPGMA algorithm requires a `MultipleSeqAlignment` object to build the distance matrix. So now that we have the `seq.fasta` file, we can give it to an online multiple sequence alignment tool. We can do this through BioPython but it requires some installation and setup so we will skip that for now. Go to the [MUSCLE Web Server](http://www.ebi.ac.uk/Tools/msa/muscle/) and give it the `seq.fasta` file. It will take a few seconds and it will give you an alignment and click *Download Alignment File*, copy paste the whole thing to a new file called `aln.clustal`. This is the alignment we will use to build our tree." 2594 )); 2595 2596 qDebug() << "command entry 21"; 2597 testCommandEntry(entry, 21, QString::fromUtf8( 2598 "#open the alignmnent file\n" 2599 "with open(\"aln.clustal\", \"r\") as aln:\n" 2600 " #use AlignIO to read the alignment file in 'clustal' format\n" 2601 " alignment = AlignIO.read(aln, \"clustal\")" 2602 )); 2603 2604 qDebug() << "command entry 22"; 2605 testCommandEntry(entry, 22, 1, QString::fromUtf8( 2606 "from Bio.Phylo.TreeConstruction import DistanceCalculator\n" 2607 "\n" 2608 "#calculate the distance matrix\n" 2609 "calculator = DistanceCalculator('identity')\n" 2610 "#adds distance matrix to the calculator object and returns it\n" 2611 "dm = calculator.get_distance(alignment)\n" 2612 "print(dm)\n" 2613 "" 2614 )); 2615 testTextResult(entry, 0, QString::fromUtf8( 2616 "8\t0\n" 2617 "7\t0.6\t0\n" 2618 "0\t0.8\t0.6\t0\n" 2619 "3\t1.0\t0.7\t0.4\t0\n" 2620 "6\t1.0\t0.7\t0.4\t0.09999999999999998\t0\n" 2621 "9\t0.8\t1.0\t0.5\t0.7\t0.6\t0\n" 2622 "1\t0.9\t0.9\t0.8\t0.9\t0.8\t0.4\t0\n" 2623 "2\t0.9\t0.9\t0.8\t0.9\t0.8\t0.4\t0.0\t0\n" 2624 "4\t0.9\t0.9\t0.9\t1.0\t0.9\t0.5\t0.09999999999999998\t0.09999999999999998\t0\n" 2625 "5\t0.9\t0.9\t0.9\t1.0\t0.9\t0.5\t0.09999999999999998\t0.09999999999999998\t0.0\t0\n" 2626 "\t8\t7\t0\t3\t6\t9\t1\t2\t4\t5" 2627 )); 2628 entry = entry->next(); 2629 2630 qDebug() << "command entry 23"; 2631 testCommandEntry(entry, 23, QString::fromUtf8( 2632 "from Bio.Phylo.TreeConstruction import DistanceTreeConstructor\n" 2633 "\n" 2634 "#initialize a DistanceTreeConstructor object based on our distance calculator object\n" 2635 "constructor = DistanceTreeConstructor(calculator)\n" 2636 "\n" 2637 "#build the tree\n" 2638 "upgma_tree = constructor.build_tree(alignment)" 2639 )); 2640 2641 qDebug() << "command entry 24"; 2642 testCommandEntry(entry, 24, 1, QString::fromUtf8( 2643 "from Bio import Phylo\n" 2644 "import pylab\n" 2645 "#draw the tree\n" 2646 "Phylo.draw(upgma_tree)" 2647 )); 2648 testImageResult(entry, 0); 2649 entry = entry->next(); 2650 2651 testMarkdown(entry, QString::fromUtf8( 2652 "## Introducing mating to the model\n" 2653 "\n" 2654 "The populations we generated evolved asexually. This means that individuals do not mate or exchange genetic information. So to make our simulation a bit more interesting let's let the Cells mate. This is going to require a few small changes in the `selection()` function. Previously, when we selected sequences to go into the next generation we just let them provide one offspring which was a copy of itself and introduced mutations. Now instead of choosing one Cell at a time, we will randomly choose two 'parents' that will mate. When they mate, each parent will contribute one of its chromosomes to the child. We'll repeat this process until we have filled the next generation." 2655 )); 2656 2657 qDebug() << "command entry 25"; 2658 testCommandEntry(entry, 25, 1, QString::fromUtf8( 2659 "def selection_with_mating(population, target, mutation_rate=0.001, beta=-2):\n" 2660 " next_generation = []\n" 2661 " \n" 2662 " counter = 0\n" 2663 " while len(next_generation) < len(population):\n" 2664 " #select two parents based on their fitness\n" 2665 " parents_pair = np.random.choice(population, 2, p=[rna.fitness for rna in population], replace=False)\n" 2666 " \n" 2667 " #take the sequence and structure from the first parent's first chromosome and give it to the child\n" 2668 " child_chrom_1 = (parents_pair[0].sequence_1, parents_pair[0].structure_1)\n" 2669 "\n" 2670 " #do the same for the child's second chromosome and the second parent.\n" 2671 " child_chrom_2 = (parents_pair[1].sequence_2, parents_pair[1].structure_2)\n" 2672 "\n" 2673 "\n" 2674 " #initialize the new child Cell witht he new chromosomes.\n" 2675 " child_cell = Cell(child_chrom_1[0], child_chrom_1[1], child_chrom_2[0], child_chrom_2[1])\n" 2676 "\n" 2677 " #give the child and id and store who its parents are\n" 2678 " child_cell.id = counter\n" 2679 " child_cell.parent_1 = parents_pair[0].id\n" 2680 " child_cell.parent_2 = parents_pair[1].id\n" 2681 "\n" 2682 " #add the child to the new generation\n" 2683 " next_generation.append(child_cell)\n" 2684 " \n" 2685 " counter = counter + 1\n" 2686 " \n" 2687 " \n" 2688 " #introduce mutations in next_generation sequeneces and re-fold when a mutation occurs (same as before)\n" 2689 " for rna in next_generation: \n" 2690 " mutated_sequence_1, mutated_1 = mutate(rna.sequence_1, mutation_rate=mutation_rate)\n" 2691 " mutated_sequence_2, mutated_2 = mutate(rna.sequence_2, mutation_rate=mutation_rate)\n" 2692 "\n" 2693 " if mutated_1:\n" 2694 " rna.sequence_1 = mutated_sequence_1\n" 2695 " rna.structure_1 = nussinov(mutated_sequence_1)\n" 2696 " if mutated_2:\n" 2697 " rna.sequence_2 = mutated_sequence_2\n" 2698 " rna.structure_2 = nussinov(mutated_sequence_2)\n" 2699 " else:\n" 2700 " continue\n" 2701 "\n" 2702 " #update fitness values for the new generation\n" 2703 " compute_fitness(next_generation, target, beta=beta)\n" 2704 "\n" 2705 " return next_generation \n" 2706 "\n" 2707 "#run a small test to make sure it works\n" 2708 "next_gen = selection_with_mating(pop, target)\n" 2709 "for cell in next_gen[:10]:\n" 2710 " print(cell.sequence_1)" 2711 )); 2712 testTextResult(entry, 0, QString::fromUtf8( 2713 "CGUACCUGAAAAGCUAACUA\n" 2714 "GGAACCGUAGGCUUUGCAAG\n" 2715 "ACCUGCCAUCAUCACCAGAC\n" 2716 "UAGAGGUAGAAUUGUAGGCU\n" 2717 "GAUUCCGCGCGAAUACCGCG\n" 2718 "GCAUCGAGUGCGCGGCAUAA\n" 2719 "UAAUAAAAAGGUGCUGAUAU\n" 2720 "GAUUCCGCGCGAAUACCGCG\n" 2721 "UCACUAAACUCCUCGACUAC\n" 2722 "AUGAUCAUGGUGAGCAGUUU" 2723 )); 2724 entry = entry->next(); 2725 2726 testMarkdown(entry, QString::fromUtf8( 2727 "Now we just have to update our `evolution()` function to call the new `selection_with_mating()` function." 2728 )); 2729 2730 qDebug() << "command entry 26"; 2731 testCommandEntry(entry, 26, QString::fromUtf8( 2732 "def evolve_with_mating(target, generations=10, pop_size=100, mutation_rate=0.001, beta=-2):\n" 2733 " populations = []\n" 2734 " population_stats = {}\n" 2735 " \n" 2736 " initial_population = populate(target, pop_size=pop_size)\n" 2737 " compute_fitness(initial_population, target)\n" 2738 " \n" 2739 " current_generation = initial_population\n" 2740 "\n" 2741 " #iterate the selection process over the desired number of generations\n" 2742 " for i in range(generations):\n" 2743 " #let's get some stats on the structures in the populations \n" 2744 " record_stats(current_generation, population_stats)\n" 2745 " \n" 2746 " #add the current generation to our list of populations.\n" 2747 " populations.append(current_generation)\n" 2748 "\n" 2749 " #select the next generation, but this time with mutations\n" 2750 " new_gen = selection_with_mating(current_generation, target, mutation_rate=mutation_rate, beta=beta)\n" 2751 " current_generation = new_gen \n" 2752 " \n" 2753 " return (populations, population_stats)" 2754 )); 2755 2756 testMarkdown(entry, QString::fromUtf8( 2757 "Try out the new evolution model!" 2758 )); 2759 2760 qDebug() << "command entry 28"; 2761 testCommandEntry(entry, 28, 1, QString::fromUtf8( 2762 "pops_mating, pops_stats_mating = evolve_with_mating(\"(((....)))\", generations=20, pop_size=1000, beta=0)\n" 2763 "\n" 2764 "evo_plot(pops_stats_mating)" 2765 )); 2766 testImageResult(entry, 0); 2767 entry = entry->next(); 2768 2769 testMarkdown(entry, QString::fromUtf8( 2770 "## Hardy Weinberg Equilibrium\n" 2771 "\n" 2772 "When we are presented with data from a population we don't know much about. It is often useful to try to learn whether there are any evolutionary or behavioural influences that are shaping population dynamics. This could be in the form of selective pressure, mating preference, genetic drift, mutations, gene flow, etc. So in order to detect if something like this is happening we need to develop a test. This is where Hardy Weinberg comes in. \n" 2773 "\n" 2774 "The Hardy Weinberg equilibrium states that \"allele and genotype frequencies remain constant in the absence of other evolutionary influences. (such as the ones we mentioned above)\" - Wikipedia.\n" 2775 "\n" 2776 "So if we can measure allele/genotype frequencies (which we can do because we have sequences), we can see whether the HW principle holds true. If it does not, then we can do more digging to see what could be happening to shift populations away from equilibrium.\n" 2777 "\n" 2778 "In order to do this we need to define an 'allele'. An allele (for our purproses) will be a locus (position in a sequence) that can take one of two states, a *reference* state or an *alternate* state. For example, we can look at locus number **5** (position 5 in our RNA sequences) and call reference **C**, and alternate **G**. If we are in HW we can predict the frequency of each allele in our population.\n" 2779 "\n" 2780 "To simplify our notation we will call the alternate allele *A* and the reference allele *a*. We can write the probability of each allele as $p_{A} + p_{a} = 1$. Since we are dealing with diploid populations, each individual will have two copies of each locus so it can be $p_{AA}, p{Aa}, p{aA}, p{aa}$. By simple probability laws we can get an expression for the probability of each genotype based on the probabilities of the single loci $p_{a}$ and $p_{A}$.\n" 2781 "\n" 2782 "$$p_{aa}\\simeq p_{a}^2$$\n" 2783 "\n" 2784 "$$p_{AA}\\simeq p_{A}^2$$\n" 2785 "\n" 2786 "$$p_{Aa,~aA} \\simeq 2 p_{a} p_{A}.$$\n" 2787 "\n" 2788 "Since it is hard to know what the true probability of observing either $p_{a}$ and $p_{A}$ we can estimate this probability from our data as follows:\n" 2789 "\n" 2790 "$$\\hat p_a=\\frac{2N_{aa}+N_{aA}}{2N}=1-\\hat p_A.$$\n" 2791 "\n" 2792 "Where $N$ denotes the number of each genotype that we observe in our sequences. \n" 2793 "\n" 2794 "Based on these estimates we can expect the following frequencies for each genotype: \n" 2795 "\n" 2796 "$N_{aa}\\simeq e_{aa}=N \\hat p_a^2$\n" 2797 "\n" 2798 "$N_{AA}\\simeq e_{AA}= N \\hat p_{A}^2$\n" 2799 "\n" 2800 "$N_{Aa,~aA} \\simeq e_{Aa} = 2 N \\hat p_{a} \\hat p_{A}.$\n" 2801 "\n" 2802 "Now we have expected values, and observed values. We need a test to determine whether we have a significant departure from the hypothesis of Hardy Weinberg equilibrium. The statistical test that is commonly used is known as the $\\chi^{2}$ test. If you take a look at the equation you'll see that the statistic simply takes the squared difference between our observed value and the expected value (divided by expected) and sums this for each possible genotype. The reason we take the squared difference is because we want to deal only with positive values, hence the name $\\chi^{2}$.\n" 2803 "\n" 2804 "$$X^2= \\frac{(N_{aa}-e_{aa})^2}{e_{aa}}+ \\frac{(N_{Aa}-e_{Aa})^2}{e_{Aa}}+ \\frac{(N_{AA}-e_{AA})^2}{e_{AA}}.$$\n" 2805 "\n" 2806 "The first thing we need to do is get alleles from our sequence data. This boils down to going through each sequence at the position of interest and counting the number of $AA$, $Aa$, $aa$ we get.\n" 2807 "\n" 2808 "\n" 2809 "\\** the sections on Hardy Weinberg and F-statistics are adapted from Simon Gravel's HGEN 661 Notes" 2810 )); 2811 2812 qDebug() << "command entry 29"; 2813 testCommandEntry(entry, 29, QString::fromUtf8( 2814 "def allele_finder(pop, locus, ref, alt):\n" 2815 " genotypes = []\n" 2816 " for p in pop:\n" 2817 " #get the nucleotide at the locus from the first chromosome \n" 2818 " locus_1 = p.sequence_1[locus].upper()\n" 2819 " #same for the second\n" 2820 " locus_2 = p.sequence_2[locus].upper()\n" 2821 " \n" 2822 " #check that it is either ref or alt, we don't care about other alleles for now.\n" 2823 " if locus_1 in (ref, alt) and locus_2 in (ref, alt):\n" 2824 " #if the alelle is ref, store a value of 1 in allele_1, and 0 otherwise\n" 2825 " allele_1 = int(locus_1 == ref)\n" 2826 " #same for the second allele\n" 2827 " allele_2 = int(locus_2 == ref)\n" 2828 " \n" 2829 " #add allele to our list of alleles as a tuple. \n" 2830 " genotypes.append((allele_1, allele_2))\n" 2831 " return genotypes" 2832 )); 2833 2834 qDebug() << "command entry 30"; 2835 testCommandEntry(entry, 30, 1, QString::fromUtf8( 2836 "pop_hw, stats_hw = evolve_with_mating(\"(((....)))\", pop_size=1000, generations=10, beta=0, mutation_rate=0.005)\n" 2837 "alleles = allele_finder(pop_hw[-1], 5, 'C', 'G')\n" 2838 "print(alleles[:10])" 2839 )); 2840 testTextResult(entry, 0, QString::fromUtf8( 2841 "[(0, 0), (0, 0), (0, 1), (1, 0), (0, 1), (1, 0), (1, 0), (1, 0), (1, 0), (0, 1)]" 2842 )); 2843 entry = entry->next(); 2844 2845 testMarkdown(entry, QString::fromUtf8( 2846 "Now that we have alleles represented in the right form, we can see if our population is at Hardy Weinberg equilibrium using the $\\chi_{2}$ test and the equations above." 2847 )); 2848 2849 qDebug() << "command entry 31"; 2850 testCommandEntry(entry, 31, 1, QString::fromUtf8( 2851 "from scipy import stats\n" 2852 "from scipy.stats import chi2\n" 2853 "\n" 2854 "def hardy_weinberg_chi2_test(alleles):\n" 2855 " \n" 2856 " #store counts for N_AA, N_Aa/aA, N_aa\n" 2857 " hom_ref_count = 0\n" 2858 " het_count = 0\n" 2859 " hom_alt_count = 0\n" 2860 " \n" 2861 " #each allele in the list alleles is in the form (0,0) or (0,1) or (1,0) or (1,1)\n" 2862 " #count how many of each type we have\n" 2863 " for a in alleles:\n" 2864 " if (a[0]==0 and a[1]==0):\n" 2865 " hom_ref_count += 1\n" 2866 " elif ((a[0]==0 and a[1]==1) or (a[0]==1 and a[1]==0)):\n" 2867 " het_count += 1\n" 2868 " elif (a[0]==1 and a[1]==1):\n" 2869 " hom_alt_count += 1\n" 2870 " else:\n" 2871 " continue\n" 2872 " \n" 2873 " #total number of genotypes: N\n" 2874 " genotype_count = hom_ref_count + het_count + hom_alt_count\n" 2875 "\n" 2876 " #estimate p_a, p_A\n" 2877 " alt_counts = (2 * hom_alt_count) + het_count\n" 2878 " ref_counts = (2 * hom_ref_count) + het_count\n" 2879 " \n" 2880 " \n" 2881 " #get expectations e_AA, e_aA,Aa, e_aa\n" 2882 " hom_ref_expectation = ref_counts**2 / (4.*genotype_count) # the expected number of homozygote references \n" 2883 " het_expectation = ref_counts * alt_counts / (2.*genotype_count) # the expected number of hets \n" 2884 " hom_alt_expectation = alt_counts**2 / (4.*genotype_count) # the expected number of homozygote nonreferences \n" 2885 "\n" 2886 " #store observed values in list in the form [N_AA, N_aA,Aa, N_aa]\n" 2887 " observations = [hom_ref_count, het_count, hom_alt_count]\n" 2888 " #store expected values in the same form\n" 2889 " expectations = [hom_ref_expectation, het_expectation, hom_alt_expectation]\n" 2890 " \n" 2891 " #start a dictionary that will store our results.\n" 2892 " statistics = {\n" 2893 " 'hom_ref': (hom_ref_count, hom_ref_expectation),\n" 2894 " 'het': (het_count, het_expectation),\n" 2895 " 'hom_alt': (hom_alt_count, hom_alt_expectation), \n" 2896 " 'ref_counts': ref_counts, \n" 2897 " 'alt_counts': alt_counts,\n" 2898 " 'genotype_count': genotype_count\n" 2899 " }\n" 2900 "\n" 2901 " #call scipy function for chi2 test.\n" 2902 " chi_2_statistic = stats.chisquare(observations, f_exp=expectations, ddof=1, axis=0)\n" 2903 " \n" 2904 " #return chi2 and statistics dictionary\n" 2905 " return (chi_2_statistic, statistics)\n" 2906 "\n" 2907 "hardy_weinberg_chi2_test(alleles)" 2908 )); 2909 testTextResult(entry, 0, QString::fromLatin1( 2910 "(Power_divergenceResult(statistic=0.001476611280908458, pvalue=0.9693474730942313),\n" 2911 " {'hom_ref': (81, 81.14693877551021),\n" 2912 " 'het': (120, 119.70612244897958),\n" 2913 " 'hom_alt': (44, 44.14693877551021),\n" 2914 " 'ref_counts': 282,\n" 2915 " 'alt_counts': 208,\n" 2916 " 'genotype_count': 245})" 2917 )); 2918 entry = entry->next(); 2919 2920 testMarkdown(entry, QString::fromUtf8( 2921 "Can we say that our population is at equilibrium? Can you find parameters for `evolution_with_mating()` that will give us populations outside of the HW equilibrium?" 2922 )); 2923 2924 testMarkdown(entry, QString::fromUtf8( 2925 "## A brief interlude on the p-value\n" 2926 "\n" 2927 "Let's take a minute to understand what the p-value means. The p-value is a probability. Specifically, it is the probability of observing a value equal to or more extreme than that our statistic given the test distribution. So in our case, it is the probability of observing a $X^2$ greater than or equal to the one the test gives us under a $\\chi^2$ distribution. When this value is very small, it suggests that it is unlikely that we are sampling from our assumed 'null' distribution and that some other alternate distribution is the true distribution. So a low p-value here would be evidence against the neutral Hardy Weinberg model and would suggest that our population is experiencing some influences such as mating preference, selection, mutation etc.\n" 2928 "\n" 2929 "A lot of research bases its conclusions solely on p-value and it is important to be very wary of this bad practice. It has become a bad convention that people say a p-value lower than some arbitrary threshold means one's findings are significant. However, very often the p-value does not give us the whole story and we need to know about things like sample size, size of impact, reproducibility, power of the test, etc. (check this out [American Statistical Association statement on p-values](http://www.nature.com/news/statisticians-issue-warning-over-misuse-of-p-values-1.19503), [p-hacking](http://fivethirtyeight.com/features/science-isnt-broken/#part1), and [this](http://allendowney.blogspot.ca/2016/06/there-is-still-only-one-test.html))\n" 2930 "\n" 2931 "Let's just visualize this very quickly using the $\\chi^{2}_{1}$ distribution. You will see that the p-value corresponds to the shaded red area under the curve. That area is the probability of observing a value as extreme or more than the one we found. When that is a very small area, we can be more confident that our assumption of HW is false." 2932 )); 2933 2934 qDebug() << "command entry 32"; 2935 testCommandEntry(entry, 32, 2, QString::fromUtf8( 2936 "#number of samples to take from the x2 distribution.\n" 2937 "number_of_samples = 1000\n" 2938 "\n" 2939 "range_points = 2000\n" 2940 "range_start = 0\n" 2941 "\n" 2942 "degrees_of_freedom = 1\n" 2943 "\n" 2944 "range_end = chi2.ppf(1-1./number_of_samples, degrees_of_freedom)\n" 2945 " \n" 2946 "x_range = np.linspace(range_start, range_end, range_points) \n" 2947 "plt.plot(x_range, chi2.pdf(x_range, degrees_of_freedom))\n" 2948 "\n" 2949 "#find the index value of our statistic value. you can put in different values here.\n" 2950 "statistic = 0.5\n" 2951 "\n" 2952 "#find the index in x_range corresponding to the statistic value (within 0.01)\n" 2953 "point = 0\n" 2954 "for i, nb in enumerate(x_range):\n" 2955 " if nb < statistic + .01 and nb > statistic - .01:\n" 2956 " point = i\n" 2957 "\n" 2958 "#fill area under the curve representing p-value\n" 2959 "plt.fill_between(x_range[point:], chi2.pdf(x_range, degrees_of_freedom)[point:], alpha=0.3, color=\"red\")\n" 2960 "\n" 2961 "plt.xlabel(\"X-statistic\")\n" 2962 "plt.ylabel(r\"$\\chi^2_%d$\" % degrees_of_freedom)\n" 2963 "" 2964 )); 2965 testTextResult(entry, 0, QString::fromUtf8( 2966 "Text(0,0.5,'$\\\\chi^2_1$')" 2967 )); 2968 testImageResult(entry, 1); 2969 entry = entry->next(); 2970 2971 testMarkdown(entry, QString::fromUtf8( 2972 "## Population structure: F-statistics\n" 2973 "\n" 2974 "The last topic we'll cover is F-statistics. \n" 2975 "\n" 2976 "Once we find that our population strays from the HW condition we can begin to ask why that is the case. Often this deviation from the expected allele frequencies under HW is due to mating preference. Hardy Weinberg assumes that all individuals in a population have an equal probability of mating with any other individual (random mating). However, when certain individuals prefer to mate with specific others (in real populations this can be due to culture, race, geographic barriers, etc.), you get what is known as population structure. Population structure means that we begin to see *sub-populations* within our total population where individuals prefer to mate within their sub-population. This biased mating will result in a higher number of homozygotes than we would expect under Hardy-Weinberg equilibrium. Simply because mating preferences will tend to drive populations toward similar genotypes. So if this is the case, and no other factors are biasing allele dynamics, within sub-populations we should have Hardy-Weinberg like conditions. \n" 2977 "\n" 2978 "For example, if Raptors fans prefer to mate with other Raptors fans, then when we consider only Raptors fans, we should observe random mating. Simply because if the mating preference criterion is 'being a Raptor's fan' then any Raptor's fan will be equally likely to mate with any other Raptor's fan so we have Hardy Weinberg again.\n" 2979 "\n" 2980 "Let's express this in quantities we can measure.\n" 2981 "\n" 2982 "From before we calculated the observed and expected number of heterozygotes in a population. Let's call these $\\hat H$ and $H_{IT}$ respectively. $\\hat H$ is just the count of heterozygotes, and $H_{IT}$ is the same as the expected number of heterozygotes we calculated earlier.\n" 2983 "\n" 2984 "We define a quantity $e_{IT}$ as a measure of the 'excess heterozygosity' in the population when we consider all individuals $I$ in the total population $T$. $e_{IT} > 1$ when we have more heterozygotes than we expect under HW. And $0 < e_{IT} < 1$ if we have less heterozygotes than we would expect under HW.\n" 2985 "\n" 2986 "\n" 2987 "$$e_{IT}=\\frac{\\mbox{observed proportion of hets}}{\\mbox{expected proportion of hets}}=\\frac{ H_{obs}}{H_{IT}}$$\n" 2988 "\n" 2989 "We use $e_{IT}$ to define the statistic $F_{IT}$\n" 2990 "\n" 2991 "$$F_{IT}=1-e_{IT}$$\n" 2992 "\n" 2993 "So $F_{IT} > 0$ when we have a lack of heterozygotes and $F_{IT} < 0$ when we have an excess of heterozygotes. $F_{IT} = 0$ under random mating.\n" 2994 "\n" 2995 "When we have a subpropulation $S$ we can calculate the equivalent quantity but instead of considering heterozygosity in the whole population we only take a sub-population into account.\n" 2996 "\n" 2997 "$$e_{IS} = \\frac{H_{obs}}{H_{IS}}$$\n" 2998 "\n" 2999 "And lastly, we have $F_{ST}$. This one is not as intuitive to derive so I'm not including the derivation here. But basically it measure the excess heterozygosity in the total population due to the presence of two subpopulations with allele frequencies $p_{1}$ and $p_{2}$.\n" 3000 "\n" 3001 "$$F_{ST}= \\frac{(p_1-p_2)^2}{4 p (1-p)}$$" 3002 )); 3003 3004 qDebug() << "command entry 33"; 3005 testCommandEntry(entry, 33, QString::fromUtf8( 3006 "def F_statistics(total_pop, sub_pop_1, sub_pop_2): \n" 3007 " \"\"\"\n" 3008 " Uses definitions above and allele counts from two sub-populations and a total population to compute F-statistics.\n" 3009 " \"\"\"\n" 3010 " #recall that the input dictionaries each contain a tuple in the form(observed, expected) for each genotype\n" 3011 " f_IT = 1 - total_pop['het'][0] / (1. * total_pop['het'][1])\n" 3012 " \n" 3013 " \n" 3014 " f_IS_1 = 1 - sub_pop_1['het'][0] / (1. * sub_pop_1['het'][1])\n" 3015 " f_IS_2 = 1 - sub_pop_2['het'][0] / (1. * sub_pop_2['het'][1]) \n" 3016 " \n" 3017 " p1 = sub_pop_1['ref_counts'] / (1. * sub_pop_1['genotype_count'])\n" 3018 " p2 = sub_pop_2['ref_counts'] / (1. * sub_pop_2['genotype_count'])\n" 3019 " \n" 3020 " p = total_pop['ref_counts'] / (1. * total_pop['genotype_count'])\n" 3021 " \n" 3022 " f_ST = ((p1 - p2) ** 2) / (4.0 * p * (1 - p)) \n" 3023 " \n" 3024 " F_dict = {\n" 3025 " 'f_IT': f_IT,\n" 3026 " 'f_IS_1': f_IS_1,\n" 3027 " 'f_IS_2': f_IS_2,\n" 3028 " 'f_ST': f_ST\n" 3029 " }\n" 3030 " \n" 3031 " return F_dict" 3032 )); 3033 3034 testMarkdown(entry, QString::fromUtf8( 3035 "Let's get some data for our F-tests. First we need to evolve two populations indepenently of each other, to simulate isolated mating. Then to simulate the total population we combine the two sub-populations. We then use our `allele_finder()` function to get all the alleles, and the `hardy_weinberg_chi_2_test()` function to get our expected and observed counts. Finally we plug those into the `f_statistics()` function." 3036 )); 3037 3038 qDebug() << "command entry 34"; 3039 testCommandEntry(entry, 34, QString::fromUtf8( 3040 "generation = -1\n" 3041 "\n" 3042 "#run two independent simulations\n" 3043 "sub_pop_1, sub_pop_1_stats= evolve_with_mating(\"(((....)))\", pop_size=1000, generations=15, beta=-1, mutation_rate=0.005)\n" 3044 "sub_pop_2, sub_pop_2_stats= evolve_with_mating(\"(((....)))\", pop_size=1000, generations=15, beta=-1, mutation_rate=0.005)" 3045 )); 3046 3047 qDebug() << "command entry 35"; 3048 testCommandEntry(entry, 35, 1, QString::fromUtf8( 3049 "#merge the two populations into a total population.\n" 3050 "total_pop = sub_pop_1[generation] + sub_pop_2[generation]\n" 3051 "\n" 3052 "\n" 3053 "#choose a reference and alternate allele\n" 3054 "ref_allele = \"A\"\n" 3055 "alt_allele = \"G\"\n" 3056 "\n" 3057 "#choose the position of the locus of interest.\n" 3058 "locus = 1\n" 3059 "\n" 3060 "#get list of alleles for each population\n" 3061 "total_pop_alleles = allele_finder(total_pop, locus, ref_allele, alt_allele)\n" 3062 "sub_pop_1_alleles = allele_finder(sub_pop_1[generation],locus, ref_allele, alt_allele)\n" 3063 "sub_pop_2_alleles = allele_finder(sub_pop_2[generation],locus, ref_allele, alt_allele)\n" 3064 "\n" 3065 "#get homo/het expectations using hardy weinberg function\n" 3066 "total_pop_counts = hardy_weinberg_chi2_test(total_pop_alleles)[1]\n" 3067 "sub_pop_1_counts = hardy_weinberg_chi2_test(sub_pop_1_alleles)[1]\n" 3068 "sub_pop_2_counts = hardy_weinberg_chi2_test(sub_pop_2_alleles)[1]\n" 3069 "\n" 3070 "#call f-statistics function\n" 3071 "f_statistics = F_statistics(total_pop_counts, sub_pop_1_counts, sub_pop_2_counts)\n" 3072 "print(f_statistics)" 3073 )); 3074 testTextResult(entry, 0, QString::fromUtf8( 3075 "{'f_IT': 0.054216581725422874, 'f_IS_1': 0.037559168553200184, 'f_IS_2': -0.08899167437557809, 'f_ST': -0.3885918781521351}" 3076 )); 3077 entry = entry->next(); 3078 3079 testMarkdown(entry, QString::fromUtf8( 3080 "Try playing with different evolution parameters and see the effect on the different F-statistics. This workshop is a work in progress so there may be some biases in our simulation scheme that can make for come confusing F-statistics. If you come up with anything interesting I would love to know about it." 3081 )); 3082 3083 testMarkdown(entry, QString::fromUtf8( 3084 "## Exercises / Extra Material\n" 3085 "\n" 3086 "### Programming Exercises\n" 3087 "\n" 3088 "i. *Heatmap of mutation rates vs. population sizes.* (short) \n" 3089 "\n" 3090 "Make a heatmap that plots the base pair distance of the average base pair distance of the population at generation `-1` for mutation rates $\\mu = \\{0, 0.001, 0.01, 0.1, 0.5\\}$ and population sizes $N=\\{10, 100, 1000, 10000\\}$. The resulting heatmap will be `5x4` dimensions. You may choose how many generations to evolve your populations, just plot the last one in the heatmap." 3091 )); 3092 3093 qDebug() << "command entry 36"; 3094 testCommandEntry(entry, 36, 2, QString::fromUtf8( 3095 "#lists of mutation rates and population sizes to test\n" 3096 "mutation_rates = [0, 0.001, 0.01, 0.1, 0.5]\n" 3097 "population_sizes = [10, 100, 1000, 10000]\n" 3098 "\n" 3099 "#number of generations to run each simulation\n" 3100 "generations = 1\n" 3101 "#target structure\n" 3102 "target = \"(.((....)))\"\n" 3103 "\n" 3104 "#list to store our results\n" 3105 "bp_distances = []\n" 3106 "\n" 3107 "#nested for loop to go through each combination of mutation rates and population sizes.\n" 3108 "for m in mutation_rates:\n" 3109 " #list to store the population size results for current mutation rate.\n" 3110 " bp_distances_by_pop_size = []\n" 3111 " #try each population size\n" 3112 " for p in population_sizes:\n" 3113 " #call evolve() with m and p \n" 3114 " pop, pop_stats = evolve(target, mutation_rate=m, pop_size=p, generations=generations)\n" 3115 " #add bp_distance of chromosome 1 at generation -1 (last generation) to bp_distances_by_pop_size\n" 3116 " bp_distances_by_pop_size.append(pop_stats['mean_bp_distance_1'][-1])\n" 3117 " #add to global list once all combinations of current mutation rate and population sizes.\n" 3118 " bp_distances.append(bp_distances_by_pop_size)\n" 3119 " \n" 3120 "#use bp_distances matrxi to make a heatmap\n" 3121 "sns.heatmap(bp_distances)\n" 3122 "\n" 3123 "#labels\n" 3124 "plt.xlabel(\"Population Size\")\n" 3125 "#xticks/yticks takes a list of numbers that specify the position of the ticks and a list with the tick labels\n" 3126 "plt.xticks([i + .5 for i in range(len(population_sizes))], population_sizes)\n" 3127 "plt.ylabel(\"Mutation Rate\")\n" 3128 "plt.yticks([i + .5 for i in range(len(mutation_rates))], mutation_rates)" 3129 )); 3130 testTextResult(entry, 0, QString::fromLatin1( 3131 "([<matplotlib.axis.YTick at 0x7fa02ba18ac8>,\n" 3132 " <matplotlib.axis.YTick at 0x7fa02ba183c8>,\n" 3133 " <matplotlib.axis.YTick at 0x7fa02ba59a58>,\n" 3134 " <matplotlib.axis.YTick at 0x7fa02acbe978>,\n" 3135 " <matplotlib.axis.YTick at 0x7fa02acc5048>],\n" 3136 " <a list of 5 Text yticklabel objects>)" 3137 )); 3138 testImageResult(entry, 1); 3139 entry = entry->next(); 3140 3141 testMarkdown(entry, QString::fromUtf8( 3142 "ii. *Introduce mating preferences within a population.* (medium length) \n" 3143 "\n" 3144 "Modify the `selection_with_mating()` function to allow for mating preferences within a population. In our example above we were just running two independent simulations to study barriers to gene flow. But now you will implement mating preferences within a single simulation. Your function will assign each Cell a new attribute called `self.preference` which will take a string value denoting the mating type the current cell prefers to mate with. For example we can have a population with three mating types: $\\{A, B, C\\}$. Your function will randomly assign preferences to each cell in the initial population. We will define a preference between types $A$ and $B$ as the probability that two cells of those given types will mate if selected. \n" 3145 "\n" 3146 "$$\n" 3147 "preferences(A,B,C) = \n" 3148 "\\begin{bmatrix}\n" 3149 " 0.7 & 0.1 & 0.2 \\\\\n" 3150 " 0.1 & 0.9 & 0 \\\\\n" 3151 " 0.2 & 0 & 0.8 \\\\\n" 3152 "\\end{bmatrix}\n" 3153 "$$\n" 3154 "\n" 3155 "Once you selected two potential parents for mating (as we did earlier) you will use the matrix to evaluate whether or not the two parents will mate and contribute an offspring to the next generation. " 3156 )); 3157 3158 qDebug() << "command entry 37"; 3159 testCommandEntry(entry, 37, 1, QString::fromUtf8( 3160 "def populate_with_preferences(target, preference_types, pop_size=100):\n" 3161 " \n" 3162 " population = []\n" 3163 "\n" 3164 " for i in range(pop_size):\n" 3165 " #get a random sequence to start with\n" 3166 " sequence = \"\".join([random.choice(\"AUCG\") for _ in range(len(target))])\n" 3167 " #use nussinov to get the secondary structure for the sequence\n" 3168 " structure = nussinov(sequence)\n" 3169 " #add a new Cell object to the population list\n" 3170 " new_cell = Cell(sequence, structure, sequence, structure)\n" 3171 " new_cell.id = i\n" 3172 " new_cell.parent = i\n" 3173 " \n" 3174 " #assign preference\n" 3175 " new_cell.preference = random.choice(preference_types)\n" 3176 " population.append(new_cell)\n" 3177 " \n" 3178 " return population\n" 3179 "\n" 3180 "def selection_with_mating_preference(population, target, preference_matrix, preference_types, mutation_rate=0.001, beta=-2):\n" 3181 " next_generation = []\n" 3182 " \n" 3183 " counter = 0\n" 3184 " while len(next_generation) < len(population):\n" 3185 " #select two parents based on their fitness\n" 3186 " parents_pair = np.random.choice(population, 2, p=[rna.fitness for rna in population], replace=False)\n" 3187 " \n" 3188 " #look up probabilty of mating in the preference_matrix\n" 3189 " mating_probability = preference_matrix[parents_pair[0].preference][parents_pair[1].preference]\n" 3190 " \n" 3191 " r = random.random()\n" 3192 " #if random number below mating_probability, mate the Cells as before\n" 3193 " if r < mating_probability:\n" 3194 " #take the sequence and structure from the first parent's first chromosome and give it to the child\n" 3195 " child_chrom_1 = (parents_pair[0].sequence_1, parents_pair[0].structure_1)\n" 3196 "\n" 3197 " #do the same for the child's second chromosome and the second parent.\n" 3198 " child_chrom_2 = (parents_pair[1].sequence_2, parents_pair[1].structure_2)\n" 3199 "\n" 3200 "\n" 3201 " #initialize the new child Cell witht he new chromosomes.\n" 3202 " child_cell = Cell(child_chrom_1[0], child_chrom_1[1], child_chrom_2[0], child_chrom_2[1])\n" 3203 "\n" 3204 " #give the child and id and store who its parents are\n" 3205 " child_cell.id = counter\n" 3206 " child_cell.parent_1 = parents_pair[0].id\n" 3207 " child_cell.parent_2 = parents_pair[1].id\n" 3208 " \n" 3209 " #give the child a random preference\n" 3210 " child_cell.preference = random.choice(preference_types)\n" 3211 "\n" 3212 " #add the child to the new generation\n" 3213 " next_generation.append(child_cell)\n" 3214 "\n" 3215 " counter = counter + 1\n" 3216 " \n" 3217 " \n" 3218 " #introduce mutations in next_generation sequeneces and re-fold when a mutation occurs (same as before)\n" 3219 " for rna in next_generation: \n" 3220 " mutated_sequence_1, mutated_1 = mutate(rna.sequence_1, mutation_rate=mutation_rate)\n" 3221 " mutated_sequence_2, mutated_2 = mutate(rna.sequence_2, mutation_rate=mutation_rate)\n" 3222 "\n" 3223 " if mutated_1:\n" 3224 " rna.sequence_1 = mutated_sequence_1\n" 3225 " rna.structure_1 = nussinov(mutated_sequence_1)\n" 3226 " if mutated_2:\n" 3227 " rna.sequence_2 = mutated_sequence_2\n" 3228 " rna.structure_2 = nussinov(mutated_sequence_2)\n" 3229 " else:\n" 3230 " continue\n" 3231 "\n" 3232 " #update fitness values for the new generation\n" 3233 " compute_fitness(next_generation, target, beta=beta)\n" 3234 "\n" 3235 " return next_generation \n" 3236 "\n" 3237 "\n" 3238 "def evolve_with_mating_preferences(target, preference_types, preference_matrix,\\\n" 3239 " generations=10, pop_size=100, mutation_rate=0.001, beta=-2):\n" 3240 " populations = []\n" 3241 " population_stats = {}\n" 3242 " \n" 3243 " initial_population = populate_with_preferences(target, preference_types, pop_size=pop_size)\n" 3244 " compute_fitness(initial_population, target)\n" 3245 " \n" 3246 " current_generation = initial_population\n" 3247 "\n" 3248 " #iterate the selection process over the desired number of generations\n" 3249 " for i in range(generations):\n" 3250 " #let's get some stats on the structures in the populations \n" 3251 " record_stats(current_generation, population_stats)\n" 3252 " \n" 3253 " #add the current generation to our list of populations.\n" 3254 " populations.append(current_generation)\n" 3255 "\n" 3256 " #select the next generation, but this time with mutations\n" 3257 " new_gen = selection_with_mating_preference(current_generation, target, preference_matrix, \\\n" 3258 " preference_types, mutation_rate=mutation_rate, beta=beta)\n" 3259 " current_generation = new_gen \n" 3260 " \n" 3261 " return (populations, population_stats)\n" 3262 "\n" 3263 "\n" 3264 "\n" 3265 "#run a small test to make sure it works\n" 3266 "target = \".(((....)))\"\n" 3267 "#for convenience, let's give the preference types integer values in sequential order\n" 3268 "preference_types = [0,1,2]\n" 3269 "\n" 3270 "preference_matrix = np.array([[0.7, 0.1, 0.2],[0.1, 0.9, 0],[0.2, 0, 0.8]])\n" 3271 " \n" 3272 "pops, pop_stats = evolve_with_mating_preferences(target, preference_types, preference_matrix)\n" 3273 "\n" 3274 "for cell in pops[-1][:10]:\n" 3275 " print(cell.sequence_1)" 3276 )); 3277 testTextResult(entry, 0, QString::fromUtf8( 3278 "UAUCUCUAGAA\n" 3279 "UCAAACGGUUU\n" 3280 "CCUAGACUUUC\n" 3281 "UAUCUCUAGAA\n" 3282 "CCUAGACUUUC\n" 3283 "GGCAaUGGUGC\n" 3284 "GGCAaUGGUGC\n" 3285 "CGGUGCCAUGG\n" 3286 "CCCGGUUACGU\n" 3287 "CGGGGAGUUUU" 3288 )); 3289 entry = entry->next(); 3290 3291 testMarkdown(entry, QString::fromUtf8( 3292 "### Population Genetics / Bioinformatics Exercises\n" 3293 "\n" 3294 "*Exercise 1. Make a tree using maximum parsimony.*\n" 3295 "\n" 3296 "We saw how to make trees using a distance score. Another popular method is known as the maximum parsimony approach. I won't go into too much detail on this since we are short on time, but I will give a quick intro and we'll look at how ot make a tree using maximum parsimony.\n" 3297 "\n" 3298 "This approach is based on the principle of parsimony, which states that the simplest explanation for our data is the most likely to be true. So given an alignment, we assume that the best tree is the one that minimizes the number of changes, or mutations. This is often a reasonable assumption to make since mutation rates in real populations are generally low, and things like back-mutations (e.g. A --> C --> A) are unlikely. Computing the tree that that maximizes parsimony directly is a difficult task, but evaluating the parsimony score of a tree given the tree is easy. So this approach basically generates many random trees for the data and scores them based on parsimony keeping the most parsimonious tree. Take a look at [the biopython manual to work through this example](http://biopython.org/wiki/Phylo), and [this one](http://biopython.org/DIST/docs/api/Bio.Phylo.TreeConstruction.ParsimonyTreeConstructor-class.html).\n" 3299 "\n" 3300 "Since we already have an alignment (`aln.clustal`) we will just re-use it and make a maximum parsimony tree instead. " 3301 )); 3302 3303 qDebug() << "command entry 38"; 3304 testCommandEntry(entry, 38, 1, QString::fromUtf8( 3305 "from Bio.Phylo.TreeConstruction import *\n" 3306 "\n" 3307 "#open our alignment file (or make a new one if you want)\n" 3308 "with open('aln.clustal', 'r') as align:\n" 3309 " aln = AlignIO.read(align, 'clustal')\n" 3310 "\n" 3311 "#create a parsimony scorer object\n" 3312 "scorer = ParsimonyScorer()\n" 3313 "#the searcher object will search through possible trees and score them.\n" 3314 "searcher = NNITreeSearcher(scorer)\n" 3315 "\n" 3316 "#takes our searcher object and a seed tree (upgma_tree) to find the best tree\n" 3317 "constructor = ParsimonyTreeConstructor(searcher, upgma_tree)\n" 3318 "\n" 3319 "#build the tree \n" 3320 "parsimony_tree = constructor.build_tree(aln)\n" 3321 "\n" 3322 "#draw the tree\n" 3323 "Phylo.draw(parsimony_tree)" 3324 )); 3325 testImageResult(entry, 0); 3326 entry = entry->next(); 3327 3328 testMarkdown(entry, QString::fromUtf8( 3329 "*Exercise 2. Bootstrapping*\n" 3330 "\n" 3331 "We just saw two methods of growing phylogenetic trees given an alignment. However, as we saw with the maximum parsimony approach, there can be many different trees for a single data set. How do we know our tree is a good representation of the data? By 'good' here we will instead use the word 'robust'. Is the tree we use too sensitive to the particularities of the data we gave it? If we make a small change in the sequence will we get a very different tree? Normally these problems would be addressed by re-sampling and seeing if we obtain similar results. But we can't really re-sample evolution. It happened once and we can't make it happen again. So we use something called *bootstrapping* which is a technique often used in statistics where instead of generating new data, you re-sample from your present data.\n" 3332 "\n" 3333 "So we have a multiple sequence alignment with $M$ sequences (rows) each with sequences of length $N$ nucleotides (columns). For each row, we can randomly sample $N$ nucleotides with replacement to make a new 'bootstrapped' sequence also of length $N$. Think of it as a kind of shuffling of the data. This gives us a whole new alignment that we can again use to make a new tree.\n" 3334 "\n" 3335 "This process is repeated many times to obtain many trees. The differences in topology (shape/structure) of the trees we obtained are assessed. If after this shuffling/perturbations we still get similar enough looking trees we can say that our final tree is robust to small changes in the data. ([some more reading on this](http://projecteuclid.org/download/pdf_1/euclid.ss/1063994979))\n" 3336 "\n" 3337 "Let's run a small example of this using the bootstrapping functions in `BioPython`." 3338 )); 3339 3340 qDebug() << "command entry 39"; 3341 testCommandEntry(entry, 39, 1, QString::fromUtf8( 3342 "from Bio.Phylo.Consensus import *\n" 3343 "\n" 3344 "#open our alignment file.\n" 3345 "with open('aln.clustal', 'r') as align:\n" 3346 " aln = AlignIO.read(align, 'clustal')\n" 3347 "\n" 3348 "#take 5 bootstrap samples from our alignment\n" 3349 "bootstraps = bootstrap(aln,5)\n" 3350 "\n" 3351 "#let's print each new alignment in clustal format. you should see 5 different alignments.\n" 3352 "for b in bootstraps:\n" 3353 " print(b.format('clustal'))" 3354 )); 3355 testTextResult(entry, 0, QString::fromUtf8( 3356 "CLUSTAL X (1.81) multiple sequence alignment\n" 3357 "\n" 3358 "\n" 3359 "8 GAAGAGACAC\n" 3360 "7 GGGGGAGCGC\n" 3361 "0 GCCACAGCGC\n" 3362 "3 ACCACAGUGU\n" 3363 "6 CCCACAGUGU\n" 3364 "9 CCCACUAGAG\n" 3365 "1 CCCUCUCGCG\n" 3366 "2 CCCUCUCGCG\n" 3367 "4 CUUUUUCGCG\n" 3368 "5 CUUUUUCGCG\n" 3369 " \n" 3370 "\n" 3371 "\n" 3372 "\n" 3373 "CLUSTAL X (1.81) multiple sequence alignment\n" 3374 "\n" 3375 "\n" 3376 "8 GGAUUUCUUA\n" 3377 "7 GAGCCCCCCG\n" 3378 "0 AAGGGGCGGG\n" 3379 "3 AAGGGGUGGG\n" 3380 "6 AAGGGGUGGG\n" 3381 "9 AUAUUUGUUA\n" 3382 "1 UUCUUUGUUC\n" 3383 "2 UUCUUUGUUC\n" 3384 "4 UUCUUUGUUC\n" 3385 "5 UUCUUUGUUC\n" 3386 " \n" 3387 "\n" 3388 "\n" 3389 "\n" 3390 "CLUSTAL X (1.81) multiple sequence alignment\n" 3391 "\n" 3392 "\n" 3393 "8 UCUUGAGAUC\n" 3394 "7 CCCGGGGGGC\n" 3395 "0 GCGAGGGCAC\n" 3396 "3 GUGCAGACCU\n" 3397 "6 GUGCCGCCCU\n" 3398 "9 UGUACACCAG\n" 3399 "1 UGUGCCCCGG\n" 3400 "2 UGUGCCCCGG\n" 3401 "4 UGUGCCCUGG\n" 3402 "5 UGUGCCCUGG\n" 3403 " \n" 3404 "\n" 3405 "\n" 3406 "\n" 3407 "CLUSTAL X (1.81) multiple sequence alignment\n" 3408 "\n" 3409 "\n" 3410 "8 GGUGCGCAUG\n" 3411 "7 GACGCAUGGG\n" 3412 "0 AAGGUAAGAA\n" 3413 "3 AAGAUAUGCA\n" 3414 "6 AAGCUAUGCA\n" 3415 "9 AUUCUUAAAA\n" 3416 "1 UUUCAUACGU\n" 3417 "2 UUUCAUACGU\n" 3418 "4 UUUCAUACGU\n" 3419 "5 UUUCAUACGU\n" 3420 " \n" 3421 "\n" 3422 "\n" 3423 "\n" 3424 "CLUSTAL X (1.81) multiple sequence alignment\n" 3425 "\n" 3426 "\n" 3427 "8 GCUAGGAACC\n" 3428 "7 GCGGGGGGCU\n" 3429 "0 AUACGGGGCA\n" 3430 "3 AUCCAAGGUU\n" 3431 "6 AUCCCCGGUU\n" 3432 "9 AUACCCAAGA\n" 3433 "1 UAGCCCCCGA\n" 3434 "2 UAGCCCCCGA\n" 3435 "4 UAGUCCCCGA\n" 3436 "5 UAGUCCCCGA" 3437 )); 3438 entry = entry->next(); 3439 3440 qDebug() << "command entry 40"; 3441 testCommandEntry(entry, 40, 5, QString::fromUtf8( 3442 "#now we want to use the bootstrapping to make new trees based on the new samples. we'll go back to making UPGMA trees.\n" 3443 "\n" 3444 "#start a calculator that uses sequence identity to calculate differences\n" 3445 "calculator = DistanceCalculator('identity')\n" 3446 "#start a distance tree constructor object \n" 3447 "constructor = DistanceTreeConstructor(calculator)\n" 3448 "#generate 5 bootstrap UPGMA trees\n" 3449 "trees = bootstrap_trees(aln, 5, constructor)\n" 3450 "\n" 3451 "#let's look at the trees. (if you have few samples, short sequences the trees might look very similar)\n" 3452 "for t in trees:\n" 3453 " Phylo.draw(t)" 3454 )); 3455 testImageResult(entry, 0); 3456 testImageResult(entry, 1); 3457 testImageResult(entry, 2); 3458 testImageResult(entry, 3); 3459 testImageResult(entry, 4); 3460 entry = entry->next(); 3461 3462 qDebug() << "command entry 41"; 3463 testCommandEntry(entry, 41, 1, QString::fromUtf8( 3464 "#biopython gives us a useful function that puts all this together by bootstrapping trees and making a 'consensus' tree.\n" 3465 "consensus_tree = bootstrap_consensus(aln, 100, constructor, majority_consensus)\n" 3466 "Phylo.draw(consensus_tree)" 3467 )); 3468 testImageResult(entry, 0); 3469 entry = entry->next(); 3470 3471 testMarkdown(entry, QString::fromUtf8( 3472 "*Exercise 3. T-tests*\n" 3473 "\n" 3474 "Similarly to the $\\chi^{2}$ test we saw for testing deviations from HW equilibrium, we can use a T-test to compare differences in means between two independent samples. We can use this to revisit a the first programming question in the exercsies section. Does mutation rate and population size have an effect on the fitness of populations? We can translate this question to, is there a difference in the mean base pair distance between populations under different mutation and population size regimes?\n" 3475 "\n" 3476 "Scipy has a very useful function that implements the T-test called `scipy.stats.ttest_ind`. Run two independent simulations (with different mutation rates) and compute the difference in mean bp distance between the two at their final generation. Store the populations in two different variables. Give a list of `bp_distance_1` values for each memeber of the population to `ttest_ind()`. \n" 3477 "\n" 3478 "Make sure to read teh `ttest_ind()` documentation, particularly about the argumetn `equal_var`. What should we set it to?" 3479 )); 3480 3481 qDebug() << "command entry 42"; 3482 testCommandEntry(entry, 42, 1, QString::fromUtf8( 3483 "import collections\n" 3484 "\n" 3485 "target = \"..(((....).))\"\n" 3486 "\n" 3487 "#run two simulations\n" 3488 "hi_mut_pop, hi_mut_stats = evolve(target, generations=5, pop_size=1000, mutation_rate=0.5)\n" 3489 "lo_mut_pop, hi_mut_stats = evolve(target, generations=5, pop_size=1000, mutation_rate=0.05)\n" 3490 "\n" 3491 "#store lits of base pair distances for each population at last generation.\n" 3492 "hi_bps = [p.bp_distance_1 for p in hi_mut_pop[-1]]\n" 3493 "lo_bps = [p.bp_distance_1 for p in lo_mut_pop[-1]]\n" 3494 "\n" 3495 "#run the \n" 3496 "stats.ttest_ind(hi_bps, lo_bps, equal_var=False)" 3497 )); 3498 testTextResult(entry, 0, QString::fromLatin1( 3499 "Ttest_indResult(statistic=1.3671266704990508, pvalue=0.17188793847221653)" 3500 )); 3501 entry = entry->next(); 3502 3503 testMarkdown(entry, QString::fromUtf8( 3504 "### Bonus! (difficult programming exercise)\n" 3505 "1. *Nussinov Algorithm (Only try this if you are feeling brave and are done with the other exercises or are interested in getting a taste of Computer Science. It is beyond the scope of this workshop.)*\n" 3506 "\n" 3507 "There are several approaches for solving this problem, we will look at the simplest one here which is known as the Nussinov Algorithm. This algorithm is a popular example of a class of algorithms know as dynamic programming algorithms. The main idea behind these algorithms is that we can break down the problem into many subproblems which are easier to compute than the full problem. Once we have obtained the solution for the subproblems, we can retrieve the solution to the full problem by doing something called a backtrace (more on the backtrace later). \n" 3508 "\n" 3509 "Here, the problem is obtaining the optimal pairing on a string of nucleotides. In order to know how good our structure is, we assign a score to it. One possible scoring scheme could be adding 1 to the score per paired set of nucleotides, and 0 otherwise. So in other words, we want a pairing that will give us the highest possible score. We can write this quantity as $OPT(i, j)$ where $i$ and $j$ are the indices of the sequence between which we obtain the pairing score. Our algorithm is therefore going to compute a folding score for all substrings bound by $i$ and $j$ and store the value in what is known as a dynamic programming table. Our dynamic programming table will be a $N$ x $N$ array where $N$ is the length of our sequence. So now that we have a way of measuring how good a structure is, we need a way to evaluate scores given a subsequence. To do this, we set some rules on the structure of an RNA sequence:\n" 3510 "\n" 3511 "\n" 3512 "If $i$ and $j$ form a pair:\n" 3513 "1. The pair $i$ and $j$ must form a valid watson-crick pair.\n" 3514 "2. $i < j-4$. This ensures that bonding is not happening between positions that are too close to each other, which would produce steric clashes.\n" 3515 "3. If pair $(i,j)$ and $(k, l)$ are in the structure, then $i < k < j < l$. This ensures that there is no crossing over of pairs which would result in pseudoknots.\n" 3516 "4. No base appears in more than one pair.\n" 3517 "\n" 3518 "Using these rules we can begin to build our algorithm. The first part of our algorithm needs to take as input indices $i$ and $j$ and return the value $OPT(i,j)$ which is the optimal score of a structure between $i$ and $j$. We start by thinking about values of $i$ and $j$ for which we can immediately know the solution, this is known as a 'base case'. This is a case where the solution is known and no further recursion is required. Once the algorithm reaches the base case, it can return a solution and propagate it upward to the first recursive call. So once we have reached $i$ and $j$ that are too close to form a structure (rule number 2), we know that the score is 0. \n" 3519 "\n" 3520 "Otherwise, we must weigh the possibility of forming a pair or not forming a pair. If $i$ and $j$ are unpaired, then $OPT(i,j)$ is just $OPT(i, j-1)$ since the score will not increase for unpaired indices. \n" 3521 "\n" 3522 "The other case is that $i$ is paired to some index $t$ on the interval $[i,j]$. We then add 1 to the score and consider the structure formed before and after the pairing between $i$ and $t$. We can write these two cases as $OPT(i, t-1)$ and $OPT(t+1, j)$. But how do we know which $t$ to pair $i$ with? Well we simply try all possible values of $t$ within the allowed range and choose the best one. \n" 3523 "\n" 3524 "All of this can be summed up as follows:\n" 3525 "\n" 3526 "$$ OPT(i,j) = max\\begin{cases}\n" 3527 " OPT(i, j-1) \\quad \\text{If $i$ and $j$ are not paired with each other.}\\\\\n" 3528 " max(1 + OPT(i, t-1) + OPT(t+1, j)) \\quad \\text{Where we try all values of $t$ < j - 4}\n" 3529 " \\end{cases}$$\n" 3530 "\n" 3531 "\n" 3532 "We can now use this recursion to fill our dynamic programming table. Once we have filled the table with scores, we can retrieve the optimal folding by a process called backtracking. We won't go into detail on how this works, but the main idea is that we can start by looking at the entry containing the score for the full sequence $OPT[0][N]$. We can then look at adjacent entries and deduce which case (pairing or not pairing) resulted in the current value. We can continue like this for the full table until we have retrieved the full structure." 3533 )); 3534 3535 qDebug() << "command entry 43"; 3536 testCommandEntry(entry, 43, 1, QString::fromUtf8( 3537 "min_loop_length = 4\n" 3538 "\n" 3539 "def pair_check(tup):\n" 3540 " if tup in [('A', 'U'), ('U', 'A'), ('C', 'G'), ('G', 'C')]:\n" 3541 " return True\n" 3542 " return False\n" 3543 "\n" 3544 "def OPT(i,j, sequence):\n" 3545 " \"\"\" returns the score of the optimal pairing between indices i and j\"\"\"\n" 3546 " #base case: no pairs allowed when i and j are less than 4 bases apart\n" 3547 " if i >= j-min_loop_length:\n" 3548 " return 0\n" 3549 " else:\n" 3550 " #i and j can either be paired or not be paired, if not paired then the optimal score is OPT(i,j-1)\n" 3551 " unpaired = OPT(i, j-1, sequence)\n" 3552 "\n" 3553 " #check if j can be involved in a pairing with a position t\n" 3554 " pairing = [1 + OPT(i, t-1, sequence) + OPT(t+1, j-1, sequence) for t in range(i, j-4)\\\n" 3555 " if pair_check((sequence[t], sequence[j]))]\n" 3556 " if not pairing:\n" 3557 " pairing = [0]\n" 3558 " paired = max(pairing)\n" 3559 "\n" 3560 "\n" 3561 " return max(unpaired, paired)\n" 3562 "\n" 3563 "\n" 3564 "def traceback(i, j, structure, DP, sequence):\n" 3565 " #in this case we've gone through the whole sequence. Nothing to do.\n" 3566 " if j <= i:\n" 3567 " return\n" 3568 " #if j is unpaired, there will be no change in score when we take it out, so we just recurse to the next index\n" 3569 " elif DP[i][j] == DP[i][j-1]:\n" 3570 " traceback(i, j-1, structure, DP, sequence)\n" 3571 " #hi\n" 3572 " else:\n" 3573 " #try pairing j with a matching index k to its left.\n" 3574 " for k in [b for b in range(i, j-min_loop_length) if pair_check((sequence[b], sequence[j]))]:\n" 3575 " #if the score at i,j is the result of adding 1 from pairing (j,k) and whatever score\n" 3576 " #comes from the substructure to its left (i, k-1) and to its right (k+1, j-1)\n" 3577 " if k-1 < 0:\n" 3578 " if DP[i][j] == DP[k+1][j-1] + 1:\n" 3579 " structure.append((k,j))\n" 3580 " traceback(k+1, j-1, structure, DP, sequence)\n" 3581 " break\n" 3582 " elif DP[i][j] == DP[i][k-1] + DP[k+1][j-1] + 1:\n" 3583 " #add the pair (j,k) to our list of pairs\n" 3584 " structure.append((k,j))\n" 3585 " #move the recursion to the two substructures formed by this pairing\n" 3586 " traceback(i, k-1, structure, DP, sequence)\n" 3587 " traceback(k+1, j-1, structure, DP, sequence)\n" 3588 " break\n" 3589 "\n" 3590 "def write_structure(sequence, structure):\n" 3591 " dot_bracket = [\".\" for _ in range(len(sequence))]\n" 3592 " for s in structure:\n" 3593 " dot_bracket[min(s)] = \"(\"\n" 3594 " dot_bracket[max(s)] = \")\"\n" 3595 " return \"\".join(dot_bracket)\n" 3596 "\n" 3597 "\n" 3598 "#initialize matrix with zeros where can't have pairings\n" 3599 "def initialize(N):\n" 3600 " #NxN matrix that stores the scores of the optimal pairings.\n" 3601 " DP = np.empty((N,N))\n" 3602 " DP[:] = np.NAN\n" 3603 " for k in range(0, min_loop_length):\n" 3604 " for i in range(N-k):\n" 3605 " j = i + k\n" 3606 " DP[i][j] = 0\n" 3607 " return DP\n" 3608 "\n" 3609 "def nussinov(sequence):\n" 3610 " N = len(sequence)\n" 3611 " DP = initialize(N)\n" 3612 " structure = []\n" 3613 "\n" 3614 " #fill the DP matrix\n" 3615 " for k in range(min_loop_length, N):\n" 3616 " for i in range(N-k):\n" 3617 " j = i + k\n" 3618 " DP[i][j] = OPT(i,j, sequence)\n" 3619 "\n" 3620 " #copy values to lower triangle to avoid null references\n" 3621 " for i in range(N):\n" 3622 " for j in range(0, i):\n" 3623 " DP[i][j] = DP[j][i]\n" 3624 "\n" 3625 "\n" 3626 " traceback(0,N-1, structure, DP, sequence)\n" 3627 " return (sequence, write_structure(sequence, structure))\n" 3628 "\n" 3629 "print(nussinov(\"ACCCGAUGUUAUAUAUACCU\"))" 3630 )); 3631 testTextResult(entry, 0, QString::fromUtf8( 3632 "('ACCCGAUGUUAUAUAUACCU', '(...(..(((....).))))')" 3633 )); 3634 entry = entry->next(); 3635 3636 QCOMPARE(entry, nullptr); 3637 } 3638 3639 void WorksheetTest::testJupyter4() 3640 { 3641 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 3642 if (backend && backend->isEnabled() == false) 3643 QSKIP("Skip, because python backend don't available", SkipSingle); 3644 3645 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("A Reaction-Diffusion Equation Solver in Python with Numpy.ipynb"))); 3646 3647 QCOMPARE(w->isReadOnly(), false); 3648 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 3649 3650 WorksheetEntry* entry = w->firstEntry(); 3651 3652 testMarkdown(entry, QString::fromUtf8( 3653 "This notebook demonstrates how IPython notebooks can be used to discuss the theory and implementation of numerical algorithms on one page.\n" 3654 "\n" 3655 "With `ipython nbconvert --to markdown name.ipynb` a notebook like this one can be made into a \n" 3656 "[blog post](http://georg.io/2013/12/Crank_Nicolson) in one easy step. To display the graphics in your resultant blog post use,\n" 3657 "for instance, your [Dropbox Public folder](https://www.dropbox.com/help/16/en) that you can \n" 3658 "[activate here](https://www.dropbox.com/enable_public_folder)." 3659 )); 3660 3661 testMarkdown(entry, QString::fromUtf8( 3662 "# The Crank-Nicolson Method" 3663 )); 3664 3665 testMarkdown(entry, QString::fromUtf8( 3666 "The [Crank-Nicolson method](http://en.wikipedia.org/wiki/Crank%E2%80%93Nicolson_method) is a well-known finite difference method for the\n" 3667 "numerical integration of the heat equation and closely related partial differential equations.\n" 3668 "\n" 3669 "We often resort to a Crank-Nicolson (CN) scheme when we integrate numerically reaction-diffusion systems in one space dimension\n" 3670 "\n" 3671 "$$\\frac{\\partial u}{\\partial t} = D \\frac{\\partial^2 u}{\\partial x^2} + f(u),$$\n" 3672 "\n" 3673 "$$\\frac{\\partial u}{\\partial x}\\Bigg|_{x = 0, L} = 0,$$\n" 3674 "\n" 3675 "where $u$ is our concentration variable, $x$ is the space variable, $D$ is the diffusion coefficient of $u$, $f$ is the reaction term,\n" 3676 "and $L$ is the length of our one-dimensional space domain.\n" 3677 "\n" 3678 "Note that we use [Neumann boundary conditions](http://en.wikipedia.org/wiki/Neumann_boundary_condition) and specify that the solution\n" 3679 "$u$ has zero space slope at the boundaries, effectively prohibiting entrance or exit of material at the boundaries (no-flux boundary conditions)." 3680 )); 3681 3682 testMarkdown(entry, QString::fromUtf8( 3683 "## Finite Difference Methods" 3684 )); 3685 3686 testMarkdown(entry, QString::fromUtf8( 3687 "Many fantastic textbooks and tutorials have been written about finite difference methods, for instance a free textbook by\n" 3688 "[Lloyd Trefethen](http://people.maths.ox.ac.uk/trefethen/pdetext.html).\n" 3689 "\n" 3690 "Here we describe a few basic aspects of finite difference methods.\n" 3691 "\n" 3692 "The above reaction-diffusion equation describes the time evolution of variable $u(x,t)$ in one space dimension ($u$ is a line concentration).\n" 3693 "If we knew an analytic expression for $u(x,t)$ then we could plot $u$ in a two-dimensional coordinate system with axes $t$ and $x$.\n" 3694 "\n" 3695 "To approximate $u(x,t)$ numerically we discretize this two-dimensional coordinate system resulting, in the simplest case, in a\n" 3696 "two-dimensional [regular grid](http://en.wikipedia.org/wiki/Regular_grid).\n" 3697 "This picture is employed commonly when constructing finite differences methods, see for instance \n" 3698 "[Figure 3.2.1 of Trefethen](http://people.maths.ox.ac.uk/trefethen/3all.pdf).\n" 3699 "\n" 3700 "Let us discretize both time and space as follows:\n" 3701 "\n" 3702 "$$t_n = n \\Delta t,~ n = 0, \\ldots, N-1,$$\n" 3703 "\n" 3704 "$$x_j = j \\Delta x,~ j = 0, \\ldots, J-1,$$\n" 3705 "\n" 3706 "where $N$ and $J$ are the number of discrete time and space points in our grid respectively.\n" 3707 "$\\Delta t$ and $\\Delta x$ are the time step and space step respectively and defined as follows:\n" 3708 "\n" 3709 "$$\\Delta t = T / N,$$\n" 3710 "\n" 3711 "$$\\Delta x = L / J,$$\n" 3712 "\n" 3713 "where $T$ is the point in time up to which we will integrate $u$ numerically.\n" 3714 "\n" 3715 "Our ultimate goal is to construct a numerical method that allows us to approximate the unknonwn analytic solution $u(x,t)$\n" 3716 "reasonably well in these discrete grid points.\n" 3717 "\n" 3718 "That is we want construct a method that computes values $U(j \\Delta x, n \\Delta t)$ (note: capital $U$) so that\n" 3719 "\n" 3720 "$$U(j \\Delta x, n \\Delta t) \\approx u(j \\Delta x, n \\Delta t)$$\n" 3721 "\n" 3722 "As a shorthand we will write $U_j^n = U(j \\Delta x, n \\Delta t)$ and $(j,n)$ to refer to grid point $(j \\Delta x, n \\Delta t)$." 3723 )); 3724 3725 testMarkdown(entry, QString::fromUtf8( 3726 "## The Crank-Nicolson Stencil" 3727 )); 3728 3729 testMarkdown(entry, QString::fromUtf8( 3730 "Based on the two-dimensional grid we construct we then approximate the operators of our reaction-diffusion system.\n" 3731 "\n" 3732 "For instance, to approximate the time derivative on the left-hand side in grid point $(j,n)$ we use the values of $U$ in two specific grid points:\n" 3733 "\n" 3734 "$$\\frac{\\partial u}{\\partial t}\\Bigg|_{x = j \\Delta x, t = n \\Delta t} \\approx \\frac{U_j^{n+1} - U_j^n}{\\Delta t}.$$\n" 3735 "\n" 3736 "We can think of this scheme as a stencil that we superimpose on our $(x,t)$-grid and this particular stencil is\n" 3737 "commonly referred to as [forward difference](http://en.wikipedia.org/wiki/Finite_difference#Forward.2C_backward.2C_and_central_differences).\n" 3738 "\n" 3739 "The spatial part of the [Crank-Nicolson stencil](http://journals.cambridge.org/abstract_S0305004100023197)\n" 3740 "(or see [Table 3.2.2 of Trefethen](http://people.maths.ox.ac.uk/trefethen/3all.pdf))\n" 3741 "for the heat equation ($u_t = u_{xx}$) approximates the \n" 3742 "[Laplace operator](http://en.wikipedia.org/wiki/Laplace_operator) of our equation and takes the following form\n" 3743 "\n" 3744 "$$\\frac{\\partial^2 u}{\\partial x^2}\\Bigg|_{x = j \\Delta x, t = n \\Delta t} \\approx \\frac{1}{2 \\Delta x^2} \\left( U_{j+1}^n - 2 U_j^n + U_{j-1}^n + U_{j+1}^{n+1} - 2 U_j^{n+1} + U_{j-1}^{n+1}\\right).$$\n" 3745 "\n" 3746 "To approximate $f(u(j \\Delta x, n \\Delta t))$ we write simply $f(U_j^n)$.\n" 3747 "\n" 3748 "These approximations define the stencil for our numerical method as pictured on [Wikipedia](http://en.wikipedia.org/wiki/Crank%E2%80%93Nicolson_method).\n" 3749 "\n" 3750 "![SVG](https://dl.dropboxusercontent.com/u/129945779/georgio/CN-stencil.svg)\n" 3751 "\n" 3752 "Applying this stencil to grid point $(j,n)$ gives us the following approximation of our reaction-diffusion equation:\n" 3753 "\n" 3754 "$$\\frac{U_j^{n+1} - U_j^n}{\\Delta t} = \\frac{D}{2 \\Delta x^2} \\left( U_{j+1}^n - 2 U_j^n + U_{j-1}^n + U_{j+1}^{n+1} - 2 U_j^{n+1} + U_{j-1}^{n+1}\\right) + f(U_j^n).$$" 3755 )); 3756 3757 testMarkdown(entry, QString::fromUtf8( 3758 "## Reordering Stencil into Linear System" 3759 )); 3760 3761 testMarkdown(entry, QString::fromUtf8( 3762 "Let us define $\\sigma = \\frac{D \\Delta t}{2 \\Delta x^2}$ and reorder the above approximation of our reaction-diffusion equation:\n" 3763 "\n" 3764 "$$-\\sigma U_{j-1}^{n+1} + (1+2\\sigma) U_j^{n+1} -\\sigma U_{j+1}^{n+1} = \\sigma U_{j-1}^n + (1-2\\sigma) U_j^n + \\sigma U_{j+1}^n + \\Delta t f(U_j^n).$$\n" 3765 "\n" 3766 "This equation makes sense for space indices $j = 1,\\ldots,J-2$ but it does not make sense for indices $j=0$ and $j=J-1$ (on the boundaries):\n" 3767 "\n" 3768 "$$j=0:~-\\sigma U_{-1}^{n+1} + (1+2\\sigma) U_0^{n+1} -\\sigma U_{1}^{n+1} = \\sigma U_{-1}^n + (1-2\\sigma) U_0^n + \\sigma U_{1}^n + \\Delta t f(U_0^n),$$\n" 3769 "\n" 3770 "$$j=J-1:~-\\sigma U_{J-2}^{n+1} + (1+2\\sigma) U_{J-1}^{n+1} -\\sigma U_{J}^{n+1} = \\sigma U_{J-2}^n + (1-2\\sigma) U_{J-1}^n + \\sigma U_{J}^n + \\Delta t f(U_{J-1}^n).$$\n" 3771 "\n" 3772 "The problem here is that the values $U_{-1}^n$ and $U_J^n$ lie outside our grid.\n" 3773 "\n" 3774 "However, we can work out what these values should equal by considering our Neumann boundary condition.\n" 3775 "Let us discretize our boundary condition at $j=0$ with the \n" 3776 "[backward difference](http://en.wikipedia.org/wiki/Finite_difference#Forward.2C_backward.2C_and_central_differences) and\n" 3777 "at $j=J-1$ with the\n" 3778 "[forward difference](http://en.wikipedia.org/wiki/Finite_difference#Forward.2C_backward.2C_and_central_differences):\n" 3779 "\n" 3780 "$$\\frac{U_1^n - U_0^n}{\\Delta x} = 0,$$\n" 3781 "\n" 3782 "$$\\frac{U_J^n - U_{J-1}^n}{\\Delta x} = 0.$$\n" 3783 "\n" 3784 "These two equations make it clear that we need to amend our above numerical approximation for\n" 3785 "$j=0$ with the identities $U_0^n = U_1^n$ and $U_0^{n+1} = U_1^{n+1}$, and\n" 3786 "for $j=J-1$ with the identities $U_{J-1}^n = U_J^n$ and $U_{J-1}^{n+1} = U_J^{n+1}$.\n" 3787 "\n" 3788 "Let us reinterpret our numerical approximation of the line concentration of $u$ in a fixed point in time as a vector $\\mathbf{U}^n$:\n" 3789 "\n" 3790 "$$\\mathbf{U}^n = \n" 3791 "\\begin{bmatrix} U_0^n \\\\ \\vdots \\\\ U_{J-1}^n \\end{bmatrix}.$$\n" 3792 "\n" 3793 "Using this notation we can now write our above approximation for a fixed point in time, $t = n \\Delta t$, compactly as a linear system:\n" 3794 "\n" 3795 "$$\n" 3796 "\\begin{bmatrix}\n" 3797 "1+\\sigma & -\\sigma & 0 & 0 & 0 & \\cdots & 0 & 0 & 0 & 0\\\\\n" 3798 "-\\sigma & 1+2\\sigma & -\\sigma & 0 & 0 & \\cdots & 0 & 0 & 0 & 0 \\\\\n" 3799 "0 & -\\sigma & 1+2\\sigma & -\\sigma & \\cdots & 0 & 0 & 0 & 0 & 0 \\\\\n" 3800 "0 & 0 & \\ddots & \\ddots & \\ddots & \\ddots & 0 & 0 & 0 & 0 \\\\\n" 3801 "0 & 0 & 0 & 0 & 0 & 0 & 0 & -\\sigma & 1+2\\sigma & -\\sigma \\\\\n" 3802 "0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & -\\sigma & 1+\\sigma\n" 3803 "\\end{bmatrix}\n" 3804 "\\begin{bmatrix}\n" 3805 "U_0^{n+1} \\\\\n" 3806 "U_1^{n+1} \\\\\n" 3807 "U_2^{n+1} \\\\\n" 3808 "\\vdots \\\\\n" 3809 "U_{J-2}^{n+1} \\\\\n" 3810 "U_{J-1}^{n+1}\n" 3811 "\\end{bmatrix} =\n" 3812 "\\begin{bmatrix}\n" 3813 "1-\\sigma & \\sigma & 0 & 0 & 0 & \\cdots & 0 & 0 & 0 & 0\\\\\n" 3814 "\\sigma & 1-2\\sigma & \\sigma & 0 & 0 & \\cdots & 0 & 0 & 0 & 0 \\\\\n" 3815 "0 & \\sigma & 1-2\\sigma & \\sigma & \\cdots & 0 & 0 & 0 & 0 & 0 \\\\\n" 3816 "0 & 0 & \\ddots & \\ddots & \\ddots & \\ddots & 0 & 0 & 0 & 0 \\\\\n" 3817 "0 & 0 & 0 & 0 & 0 & 0 & 0 & \\sigma & 1-2\\sigma & \\sigma \\\\\n" 3818 "0 & 0 & 0 & 0 & 0 & 0 & 0 & 0 & \\sigma & 1-\\sigma\n" 3819 "\\end{bmatrix}\n" 3820 "\\begin{bmatrix}\n" 3821 "U_0^{n} \\\\\n" 3822 "U_1^{n} \\\\\n" 3823 "U_2^{n} \\\\\n" 3824 "\\vdots \\\\\n" 3825 "U_{J-2}^{n} \\\\\n" 3826 "U_{J-1}^{n}\n" 3827 "\\end{bmatrix} +\n" 3828 "\\begin{bmatrix}\n" 3829 "\\Delta t f(U_0^n) \\\\\n" 3830 "\\Delta t f(U_1^n) \\\\\n" 3831 "\\Delta t f(U_2^n) \\\\\n" 3832 "\\vdots \\\\\n" 3833 "\\Delta t f(U_{J-2}^n) \\\\\n" 3834 "\\Delta t f(U_{J-1}^n)\n" 3835 "\\end{bmatrix}.\n" 3836 "$$\n" 3837 "\n" 3838 "Note that since our numerical integration starts with a well-defined initial condition at $n=0$, $\\mathbf{U}^0$, the\n" 3839 "vector $\\mathbf{U}^{n+1}$ on the left-hand side is the only unknown in this system of linear equations.\n" 3840 "\n" 3841 "Thus, to integrate numerically our reaction-diffusion system from time point $n$ to $n+1$ we need to solve numerically for vector $\\mathbf{U}^{n+1}$.\n" 3842 "\n" 3843 "Let us call the matrix on the left-hand side $A$, the one on the right-hand side $B$,\n" 3844 "and the vector on the right-hand side $\\mathbf{f}^n$.\n" 3845 "Using this notation we can write the above system as\n" 3846 "\n" 3847 "$$A \\mathbf{U}^{n+1} = B \\mathbf{U}^n + f^n.$$\n" 3848 "\n" 3849 "In this linear equation, matrices $A$ and $B$ are defined by our problem: we need to specify these matrices once for our\n" 3850 "problem and incorporate our boundary conditions in them.\n" 3851 "Vector $\\mathbf{f}^n$ is a function of $\\mathbf{U}^n$ and so needs to be reevaluated in every time point $n$.\n" 3852 "We also need to carry out one matrix-vector multiplication every time point, $B \\mathbf{U}^n$, and\n" 3853 "one vector-vector addition, $B \\mathbf{U}^n + f^n$.\n" 3854 "\n" 3855 "The most expensive numerical operation is inversion of matrix $A$ to solve for $\\mathbf{U}^{n+1}$, however we may\n" 3856 "get away with doing this only once and store the inverse of $A$ as $A^{-1}$:\n" 3857 "\n" 3858 "$$\\mathbf{U}^{n+1} = A^{-1} \\left( B \\mathbf{U}^n + f^n \\right).$$" 3859 )); 3860 3861 testMarkdown(entry, QString::fromUtf8( 3862 "## A Crank-Nicolson Example in Python" 3863 )); 3864 3865 testMarkdown(entry, QString::fromUtf8( 3866 "Let us apply the CN method to a two-variable reaction-diffusion system that was introduced by \n" 3867 "[Mori *et al.*](http://www.sciencedirect.com/science/article/pii/S0006349508704442):\n" 3868 "\n" 3869 "$$\\frac{\\partial u}{\\partial t} = D_u \\frac{\\partial^2 u}{\\partial x^2} + f(u,v),$$\n" 3870 "\n" 3871 "$$\\frac{\\partial v}{\\partial t} = D_v \\frac{\\partial^2 v}{\\partial x^2} - f(u,v),$$\n" 3872 "\n" 3873 "with Neumann boundary conditions\n" 3874 "\n" 3875 "$$\\frac{\\partial u}{\\partial x}\\Bigg|_{x=0,L} = 0,$$\n" 3876 "\n" 3877 "$$\\frac{\\partial v}{\\partial x}\\Bigg|_{x=0,L} = 0.$$\n" 3878 "\n" 3879 "The variables of this system, $u$ and $v$, represent the concetrations of the active form and its inactive form respectively.\n" 3880 "The reaction term $f(u,v)$ describes the interchange (activation and inactivation) between these two states of the protein.\n" 3881 "A particular property of this system is that the inactive has much greater diffusivity that the active form, $D_v \\gg D_u$.\n" 3882 "\n" 3883 "Using the CN method to integrate this system numerically, we need to set up two separate approximations\n" 3884 "\n" 3885 "$$A_u \\mathbf{U}^{n+1} = B_u \\mathbf{U}^n + \\mathbf{f}^n,$$\n" 3886 "\n" 3887 "$$A_v \\mathbf{V}^{n+1} = B_v \\mathbf{V}^n - \\mathbf{f}^n,$$\n" 3888 "\n" 3889 "with two different $\\sigma$ terms, $\\sigma_u = \\frac{D_u \\Delta t}{2 \\Delta x^2}$ and $\\sigma_v = \\frac{D_v \\Delta t}{2 \\Delta x^2}$." 3890 )); 3891 3892 testMarkdown(entry, QString::fromUtf8( 3893 "### Import Packages" 3894 )); 3895 3896 testMarkdown(entry, QString::fromUtf8( 3897 "For the matrix-vector multiplication, vector-vector addition, and matrix inversion that we will need to carry\n" 3898 "out we will use the Python library [NumPy](http://www.numpy.org/).\n" 3899 "To visualize our numerical solutions, we will use [pyplot](http://matplotlib.org/api/pyplot_api.html)." 3900 )); 3901 3902 qDebug() << "command entry 1"; 3903 testCommandEntry(entry, 1, QString::fromUtf8( 3904 "import numpy\n" 3905 "from matplotlib import pyplot" 3906 )); 3907 3908 testMarkdown(entry, QString::fromUtf8( 3909 "Numpy allows us to truncate the numerical values of matrices and vectors to improve their display with \n" 3910 "[`set_printoptions`](http://docs.scipy.org/doc/numpy/reference/generated/numpy.set_printoptions.html)." 3911 )); 3912 3913 qDebug() << "command entry 2"; 3914 testCommandEntry(entry, 2, QString::fromUtf8( 3915 "numpy.set_printoptions(precision=3)" 3916 )); 3917 3918 testMarkdown(entry, QString::fromUtf8( 3919 "### Specify Grid" 3920 )); 3921 3922 testMarkdown(entry, QString::fromUtf8( 3923 "Our one-dimensional domain has unit length and we define `J = 100` equally spaced\n" 3924 "grid points in this domain.\n" 3925 "This divides our domain into `J-1` subintervals, each of length `dx`." 3926 )); 3927 3928 qDebug() << "command entry 3"; 3929 testCommandEntry(entry, 3, QString::fromUtf8( 3930 "L = 1.\n" 3931 "J = 100\n" 3932 "dx = float(L)/float(J-1)\n" 3933 "x_grid = numpy.array([j*dx for j in range(J)])" 3934 )); 3935 3936 testMarkdown(entry, QString::fromUtf8( 3937 "Equally, we define `N = 1000` equally spaced grid points on our time domain of length `T = 200` thus dividing our time domain into `N-1` intervals of length `dt`." 3938 )); 3939 3940 qDebug() << "command entry 4"; 3941 testCommandEntry(entry, 4, QString::fromUtf8( 3942 "T = 200\n" 3943 "N = 1000\n" 3944 "dt = float(T)/float(N-1)\n" 3945 "t_grid = numpy.array([n*dt for n in range(N)])" 3946 )); 3947 3948 testMarkdown(entry, QString::fromUtf8( 3949 "### Specify System Parameters and the Reaction Term" 3950 )); 3951 3952 testMarkdown(entry, QString::fromUtf8( 3953 "We choose our parameter values based on the work by\n" 3954 "[Mori *et al.*](http://www.sciencedirect.com/science/article/pii/S0006349508704442)." 3955 )); 3956 3957 qDebug() << "command entry 5"; 3958 testCommandEntry(entry, 5, QString::fromUtf8( 3959 "D_v = float(10.)/float(100.)\n" 3960 "D_u = 0.01 * D_v\n" 3961 "\n" 3962 "k0 = 0.067\n" 3963 "f = lambda u, v: dt*(v*(k0 + float(u*u)/float(1. + u*u)) - u)\n" 3964 "g = lambda u, v: -f(u,v)\n" 3965 " \n" 3966 "sigma_u = float(D_u*dt)/float((2.*dx*dx))\n" 3967 "sigma_v = float(D_v*dt)/float((2.*dx*dx))\n" 3968 "\n" 3969 "total_protein = 2.26" 3970 )); 3971 3972 testMarkdown(entry, QString::fromUtf8( 3973 "### Specify the Initial Condition" 3974 )); 3975 3976 testMarkdown(entry, QString::fromUtf8( 3977 "As discussed by\n" 3978 "[Mori *et al.*](http://www.sciencedirect.com/science/article/pii/S0006349508704442),\n" 3979 "we can expect to observe interesting behaviour in the steady state of this system\n" 3980 "if we choose a heterogeneous initial condition for $u$.\n" 3981 "\n" 3982 "Here, we initialize $u$ with a step-like heterogeneity:" 3983 )); 3984 3985 qDebug() << "command entry 7"; 3986 testCommandEntry(entry, 7, QString::fromUtf8( 3987 "no_high = 10\n" 3988 "U = numpy.array([0.1 for i in range(no_high,J)] + [2. for i in range(0,no_high)])\n" 3989 "V = numpy.array([float(total_protein-dx*sum(U))/float(J*dx) for i in range(0,J)])" 3990 )); 3991 3992 testMarkdown(entry, QString::fromUtf8( 3993 "Note that we make certain that total protein amounts equal a certain value,\n" 3994 "`total_protein`.\n" 3995 "The importance of this was discussed by \n" 3996 "[Walther *et al.*](http://link.springer.com/article/10.1007%2Fs11538-012-9766-5).\n" 3997 "\n" 3998 "Let us plot our initial condition for confirmation:" 3999 )); 4000 4001 qDebug() << "command entry 9"; 4002 testCommandEntry(entry, 9, 1, QString::fromUtf8( 4003 "pyplot.ylim((0., 2.1))\n" 4004 "pyplot.xlabel('x'); pyplot.ylabel('concentration')\n" 4005 "pyplot.plot(x_grid, U)\n" 4006 "pyplot.plot(x_grid, V)\n" 4007 "pyplot.show()" 4008 )); 4009 testImageResult(entry, 0); 4010 entry = entry->next(); 4011 4012 testMarkdown(entry, QString::fromUtf8( 4013 "The blue curve is the initial condition for $U$, stored in Python variable `U`,\n" 4014 "and the green curve is the initial condition for $V$ stored in `V`." 4015 )); 4016 4017 testMarkdown(entry, QString::fromUtf8( 4018 "### Create Matrices" 4019 )); 4020 4021 testMarkdown(entry, QString::fromUtf8( 4022 "The matrices that we need to construct are all tridiagonal so they are easy to\n" 4023 "construct with \n" 4024 "[`numpy.diagflat`](http://docs.scipy.org/doc/numpy/reference/generated/numpy.diagflat.html)." 4025 )); 4026 4027 qDebug() << "command entry 10"; 4028 testCommandEntry(entry, 10, QString::fromUtf8( 4029 "A_u = numpy.diagflat([-sigma_u for i in range(J-1)], -1) +\\\n" 4030 " numpy.diagflat([1.+sigma_u]+[1.+2.*sigma_u for i in range(J-2)]+[1.+sigma_u]) +\\\n" 4031 " numpy.diagflat([-sigma_u for i in range(J-1)], 1)\n" 4032 " \n" 4033 "B_u = numpy.diagflat([sigma_u for i in range(J-1)], -1) +\\\n" 4034 " numpy.diagflat([1.-sigma_u]+[1.-2.*sigma_u for i in range(J-2)]+[1.-sigma_u]) +\\\n" 4035 " numpy.diagflat([sigma_u for i in range(J-1)], 1)\n" 4036 " \n" 4037 "A_v = numpy.diagflat([-sigma_v for i in range(J-1)], -1) +\\\n" 4038 " numpy.diagflat([1.+sigma_v]+[1.+2.*sigma_v for i in range(J-2)]+[1.+sigma_v]) +\\\n" 4039 " numpy.diagflat([-sigma_v for i in range(J-1)], 1)\n" 4040 " \n" 4041 "B_v = numpy.diagflat([sigma_v for i in range(J-1)], -1) +\\\n" 4042 " numpy.diagflat([1.-sigma_v]+[1.-2.*sigma_v for i in range(J-2)]+[1.-sigma_v]) +\\\n" 4043 " numpy.diagflat([sigma_v for i in range(J-1)], 1)" 4044 )); 4045 4046 testMarkdown(entry, QString::fromUtf8( 4047 "To confirm, this is what `A_u` looks like:" 4048 )); 4049 4050 qDebug() << "command entry 11"; 4051 testCommandEntry(entry, 11, 1, QString::fromUtf8( 4052 "print A_u" 4053 )); 4054 testTextResult(entry, 0, QString::fromUtf8( 4055 "[[ 1.981 -0.981 0. ... 0. 0. 0. ]\n" 4056 " [-0.981 2.962 -0.981 ... 0. 0. 0. ]\n" 4057 " [ 0. -0.981 2.962 ... 0. 0. 0. ]\n" 4058 " ...\n" 4059 " [ 0. 0. 0. ... 2.962 -0.981 0. ]\n" 4060 " [ 0. 0. 0. ... -0.981 2.962 -0.981]\n" 4061 " [ 0. 0. 0. ... 0. -0.981 1.981]]" 4062 )); 4063 entry = entry->next(); 4064 4065 testMarkdown(entry, QString::fromUtf8( 4066 "### Solve the System Iteratively" 4067 )); 4068 4069 testMarkdown(entry, QString::fromUtf8( 4070 "To advance our system by one time step, we need to do one matrix-vector multiplication followed by one vector-vector addition on the right hand side.\n" 4071 "\n" 4072 "To facilitate this, we rewrite our reaction term so that it accepts concentration vectors $\\mathbf{U}^n$ and $\\mathbf{V}^n$ as arguments\n" 4073 "and returns vector $\\mathbf{f}^n$.\n" 4074 "\n" 4075 "As a reminder, this is our non-vectorial definition of $f$\n" 4076 "\n" 4077 " f = lambda u, v: v*(k0 + float(u*u)/float(1. + u*u)) - u" 4078 )); 4079 4080 qDebug() << "command entry 12"; 4081 testCommandEntry(entry, 12, QString::fromUtf8( 4082 "f_vec = lambda U, V: numpy.multiply(dt, numpy.subtract(numpy.multiply(V, \n" 4083 " numpy.add(k0, numpy.divide(numpy.multiply(U,U), numpy.add(1., numpy.multiply(U,U))))), U))" 4084 )); 4085 4086 testMarkdown(entry, QString::fromUtf8( 4087 "Let us make certain that this produces the same values as our non-vectorial `f`:" 4088 )); 4089 4090 qDebug() << "command entry 13"; 4091 testCommandEntry(entry, 13, 1, QString::fromUtf8( 4092 "print f(U[0], V[0])" 4093 )); 4094 testTextResult(entry, 0, QString::fromUtf8( 4095 "0.009961358982745121" 4096 )); 4097 entry = entry->next(); 4098 4099 qDebug() << "command entry 14"; 4100 testCommandEntry(entry, 14, 1, QString::fromUtf8( 4101 "print f(U[-1], V[-1])" 4102 )); 4103 testTextResult(entry, 0, QString::fromUtf8( 4104 "-0.06238322322322325" 4105 )); 4106 entry = entry->next(); 4107 4108 qDebug() << "command entry 15"; 4109 testCommandEntry(entry, 15, 1, QString::fromUtf8( 4110 "print f_vec(U, V)" 4111 )); 4112 testTextResult(entry, 0, QString::fromUtf8( 4113 "[ 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4114 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4115 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4116 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4117 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4118 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4119 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4120 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4121 " 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01 0.01\n" 4122 " -0.062 -0.062 -0.062 -0.062 -0.062 -0.062 -0.062 -0.062 -0.062 -0.062]" 4123 )); 4124 entry = entry->next(); 4125 4126 testMarkdown(entry, QString::fromUtf8( 4127 "Accounting for rounding of the displayed values due to the `set_printoptions` we set above, we\n" 4128 "can see that `f` and `f_vec` generate the same values for our initial condition at both ends of our domain." 4129 )); 4130 4131 testMarkdown(entry, QString::fromUtf8( 4132 "We will use [`numpy.linalg.solve`](http://docs.scipy.org/doc/numpy/reference/generated/numpy.linalg.solve.html) to solve\n" 4133 "our linear system each time step." 4134 )); 4135 4136 testMarkdown(entry, QString::fromUtf8( 4137 "While we integrate our system over time we will record both `U` and `V` at each\n" 4138 "time step in `U_record` and `V_record` respectively so that we can plot\n" 4139 "our numerical solutions over time." 4140 )); 4141 4142 qDebug() << "command entry 16"; 4143 testCommandEntry(entry, 16, QString::fromUtf8( 4144 "U_record = []\n" 4145 "V_record = []\n" 4146 "\n" 4147 "U_record.append(U)\n" 4148 "V_record.append(V)\n" 4149 "\n" 4150 "for ti in range(1,N):\n" 4151 " U_new = numpy.linalg.solve(A_u, B_u.dot(U) + f_vec(U,V))\n" 4152 " V_new = numpy.linalg.solve(A_v, B_v.dot(V) - f_vec(U,V))\n" 4153 " \n" 4154 " U = U_new\n" 4155 " V = V_new\n" 4156 " \n" 4157 " U_record.append(U)\n" 4158 " V_record.append(V)" 4159 )); 4160 4161 testMarkdown(entry, QString::fromUtf8( 4162 "### Plot the Numerical Solution" 4163 )); 4164 4165 testMarkdown(entry, QString::fromUtf8( 4166 "Let us take a look at the numerical solution we attain after `N` time steps." 4167 )); 4168 4169 qDebug() << "command entry 18"; 4170 testCommandEntry(entry, 18, 1, QString::fromUtf8( 4171 "pyplot.ylim((0., 2.1))\n" 4172 "pyplot.xlabel('x'); pyplot.ylabel('concentration')\n" 4173 "pyplot.plot(x_grid, U)\n" 4174 "pyplot.plot(x_grid, V)\n" 4175 "pyplot.show()" 4176 )); 4177 testImageResult(entry, 0); 4178 entry = entry->next(); 4179 4180 testMarkdown(entry, QString::fromUtf8( 4181 "And here is a [kymograph](http://en.wikipedia.org/wiki/Kymograph) of the values of `U`.\n" 4182 "This plot shows concisely the behaviour of `U` over time and we can clear observe the wave-pinning\n" 4183 "behaviour described by [Mori *et al.*](http://www.sciencedirect.com/science/article/pii/S0006349508704442).\n" 4184 "Furthermore, we observe that this wave pattern is stable for about 50 units of time and we therefore\n" 4185 "conclude that this wave pattern is a stable steady state of our system." 4186 )); 4187 4188 qDebug() << "command entry 21"; 4189 testCommandEntry(entry, 21, 1, QString::fromUtf8( 4190 "U_record = numpy.array(U_record)\n" 4191 "V_record = numpy.array(V_record)\n" 4192 "\n" 4193 "fig, ax = pyplot.subplots()\n" 4194 "pyplot.xlabel('x'); pyplot.ylabel('t')\n" 4195 "heatmap = ax.pcolor(x_grid, t_grid, U_record, vmin=0., vmax=1.2)" 4196 )); 4197 testImageResult(entry, 0); 4198 entry = entry->next(); 4199 4200 QCOMPARE(entry, nullptr); 4201 } 4202 4203 void WorksheetTest::testJupyter5() 4204 { 4205 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 4206 if (backend && backend->isEnabled() == false) 4207 QSKIP("Skip, because python backend don't available", SkipSingle); 4208 4209 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("Automata and Computability using Jupyter.ipynb"))); 4210 4211 QCOMPARE(w->isReadOnly(), false); 4212 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 4213 4214 WorksheetEntry* entry = w->firstEntry(); 4215 4216 testMarkdown(entry, QString::fromUtf8( 4217 "# Jove helps teach models of computation using Jupyter \n" 4218 "\n" 4219 "Included are modules on:\n" 4220 "\n" 4221 "* Sets, strings and languages\n" 4222 "* Language operations\n" 4223 "* Construction of and operations on DFA and NFA\n" 4224 "* Regular expression parsing and automata inter-conversion\n" 4225 "* Derivate-based parsing\n" 4226 "* Pushdown automata\n" 4227 "* The construction of parsers using context-free productions, including\n" 4228 " a full lexer/parser for Jove's own markdown syntax\n" 4229 "* Studies of parsing: ambiguity, associativity, precedence\n" 4230 "* Turing machines (including one for the Collatz problem)\n" 4231 "\n" 4232 "For a complete Jove top-level reference, kindly refer to https://github.com/ganeshutah/Jove from where you can download and obtain Jove. You can also visit this Github link now and poke around (the NBViewer will display the contents).\n" 4233 "\n" 4234 "Once you are in the top-level Gallery link we provide, feel free to explore the hierarchy of modules found there.\n" 4235 "\n" 4236 "These notebooks should give you an idea of the contents.\n" 4237 "\n" 4238 "* [DFA Illustrations (has a Youtube)](http://nbviewer.jupyter.org/github/ganeshutah/Jove/blob/master/notebooks/tutorial/DFAUnit2.ipynb)\n" 4239 "\n" 4240 "* [Regular Operations](http://nbviewer.jupyter.org/github/ganeshutah/Jove/blob/master/notebooks/driver/Drive_AllRegularOps.ipynb)\n" 4241 "\n" 4242 "* [PDA Operations](http://nbviewer.jupyter.org/github/ganeshutah/Jove/blob/master/notebooks/driver/Drive_PDA_Based_Parsing.ipynb)\n" 4243 "\n" 4244 "* [TM Operations](http://nbviewer.jupyter.org/github/ganeshutah/Jove/blob/master/notebooks/driver/Drive_TM.ipynb)" 4245 )); 4246 4247 qDebug() << "command entry 1"; 4248 testCommandEntry(entry, 1, 1, QString::fromUtf8( 4249 "from IPython.display import YouTubeVideo\n" 4250 "YouTubeVideo('dGcLHtYLgDU')" 4251 )); 4252 testHtmlResult(entry, 0, QString::fromLatin1( 4253 "<IPython.lib.display.YouTubeVideo at 0x7fa7a1ee4c50>" 4254 )); 4255 entry = entry->next(); 4256 4257 qDebug() << "command entry 2"; 4258 testCommandEntry(entry, 2, 1, QString::fromUtf8( 4259 "import sys\n" 4260 "sys.path[0:0] = ['/home/mmmm1998/Документы/Репозитории/Jove','/home/mmmm1998/Документы/Репозитории/Jove/3rdparty'] # Put these at the head of the search path\n" 4261 "from jove.DotBashers import *\n" 4262 "from jove.Def_DFA import *\n" 4263 "from jove.Def_NFA import *\n" 4264 "from jove.Def_RE2NFA import *\n" 4265 "from jove.Def_NFA2RE import *\n" 4266 "from jove.Def_md2mc import *" 4267 )); 4268 testTextResult(entry, 0, QString::fromUtf8( 4269 "You may use any of these help commands:\n" 4270 "help(ResetStNum)\n" 4271 "help(NxtStateStr)\n" 4272 "\n" 4273 "You may use any of these help commands:\n" 4274 "help(mkp_dfa)\n" 4275 "help(mk_dfa)\n" 4276 "help(totalize_dfa)\n" 4277 "help(addtosigma_delta)\n" 4278 "help(step_dfa)\n" 4279 "help(run_dfa)\n" 4280 "help(accepts_dfa)\n" 4281 "help(comp_dfa)\n" 4282 "help(union_dfa)\n" 4283 "help(intersect_dfa)\n" 4284 "help(pruneUnreach)\n" 4285 "help(iso_dfa)\n" 4286 "help(langeq_dfa)\n" 4287 "help(same_status)\n" 4288 "help(h_langeq_dfa)\n" 4289 "help(fixptDist)\n" 4290 "help(min_dfa)\n" 4291 "help(pairFR)\n" 4292 "help(state_combos)\n" 4293 "help(sepFinNonFin)\n" 4294 "help(bash_eql_classes)\n" 4295 "help(listminus)\n" 4296 "help(bash_1)\n" 4297 "help(mk_rep_eqc)\n" 4298 "help(F_of)\n" 4299 "help(rep_of_s)\n" 4300 "help(q0_of)\n" 4301 "help(Delta_of)\n" 4302 "help(mk_state_eqc_name)\n" 4303 "\n" 4304 "You may use any of these help commands:\n" 4305 "help(mk_nfa)\n" 4306 "help(totalize_nfa)\n" 4307 "help(step_nfa)\n" 4308 "help(run_nfa)\n" 4309 "help(ec_step_nfa)\n" 4310 "help(Eclosure)\n" 4311 "help(Echelp)\n" 4312 "help(accepts_nfa)\n" 4313 "help(nfa2dfa)\n" 4314 "help(n2d)\n" 4315 "help(inSets)\n" 4316 "help(rev_dfa)\n" 4317 "help(min_dfa_brz)\n" 4318 "\n" 4319 "You may use any of these help commands:\n" 4320 "help(re2nfa)\n" 4321 "\n" 4322 "You may use any of these help commands:\n" 4323 "help(RE2Str)\n" 4324 "help(mk_gnfa)\n" 4325 "help(mk_gnfa_from_D)\n" 4326 "help(dfa2nfa)\n" 4327 "help(del_gnfa_states)\n" 4328 "help(gnfa_w_REStr)\n" 4329 "help(del_one_gnfa_state)\n" 4330 "help(Edges_Exist_Via)\n" 4331 "help(choose_state_to_del)\n" 4332 "help(form_alt_RE)\n" 4333 "help(form_concat_RE)\n" 4334 "help(form_kleene_RE)\n" 4335 "\n" 4336 "You may use any of these help commands:\n" 4337 "help(md2mc)\n" 4338 ".. and if you want to dig more, then ..\n" 4339 "help(default_line_attr)\n" 4340 "help(length_ok_input_items)\n" 4341 "help(union_line_attr_list_fld)\n" 4342 "help(extend_rsltdict)\n" 4343 "help(form_delta)\n" 4344 "help(get_machine_components)" 4345 )); 4346 entry = entry->next(); 4347 4348 testMarkdown(entry, QString::fromUtf8( 4349 " # Jove allows you to set problems in markdown and have students solve" 4350 )); 4351 4352 testMarkdown(entry, QString::fromUtf8( 4353 "1) LOdd1Three0 : Set of strings over {0,1} with an odd # of 1s OR exactly three 0s. \n" 4354 "\n" 4355 "* Hint on how to arrive at the language:\n" 4356 "\n" 4357 " - develop NFAs for the two cases and perform their union. Obtain DFA\n" 4358 "\n" 4359 " - develop REs for the two cases and perform the union. \n" 4360 "\n" 4361 " - Testing the creations:\n" 4362 "\n" 4363 " . Come up with language for even # of 1s and separately for \"other than three 0s\". \n" 4364 " \n" 4365 " . Do two intersections. \n" 4366 " \n" 4367 " . Is the language empty?\n" 4368 "\n" 4369 "\n" 4370 "2) Language of strings over {0,1} with exactly two occurrences of 0101 in it.\n" 4371 "\n" 4372 " * Come up with it directly (take overlaps into account, i.e. 010101 has two occurrences in it\n" 4373 "\n" 4374 " * Come up in another way\n" 4375 "\n" 4376 "Notes:\n" 4377 "\n" 4378 "* Most of the problem students will have in this course is interpreting English (technical English)\n" 4379 "\n" 4380 "* So again, read the writeup at the beginning of Module6 (should be ready soon today) and work on using the tool.\n" 4381 "\n" 4382 "\n" 4383 "\n" 4384 "" 4385 )); 4386 4387 testMarkdown(entry, QString::fromUtf8( 4388 "__Solutions__\n" 4389 "\n" 4390 "1) LOdd1Three0 : Set of strings over {0,1} with an odd # of 1s OR exactly three 0s. \n" 4391 "\n" 4392 "* Hint on how to arrive at the language:\n" 4393 "\n" 4394 " - develop NFAs for the two cases and perform their union. Obtain DFA\n" 4395 "\n" 4396 " - develop REs for the two cases and perform the union. \n" 4397 "\n" 4398 " - Testing the creations:\n" 4399 "\n" 4400 " . Come up with language for even # of 1s and separately for \"other than three 0s\". \n" 4401 " \n" 4402 " . Do two intersections. \n" 4403 " \n" 4404 " . Is the language empty?\n" 4405 "\n" 4406 "\n" 4407 "2) Language of strings over {0,1} with exactly two occurrences of 0101 in it.\n" 4408 "\n" 4409 " * Come up with it directly (take overlaps into account, i.e. 010101 has two occurrences in it\n" 4410 "\n" 4411 " * Come up in another way\n" 4412 "\n" 4413 "Notes:\n" 4414 "\n" 4415 "* Most of the problem students will have in this course is interpreting English (technical English)\n" 4416 "\n" 4417 "* So again, read the writeup at the beginning of Module6 (should be ready soon today) and work on using the tool.\n" 4418 "\n" 4419 "\n" 4420 "\n" 4421 "" 4422 )); 4423 4424 qDebug() << "command entry 3"; 4425 testCommandEntry(entry, 3, 1, QString::fromUtf8( 4426 "RE_Odd1s = \"(0* 1 0* (1 0* 1 0)*)*\"\n" 4427 "NFA_Odd1s = re2nfa(RE_Odd1s)\n" 4428 "DO_Odd1s = dotObj_dfa(min_dfa(nfa2dfa(NFA_Odd1s)))\n" 4429 "DO_Odd1s" 4430 )); 4431 testImageResult(entry, 0); 4432 entry = entry->next(); 4433 4434 qDebug() << "command entry 4"; 4435 testCommandEntry(entry, 4, 1, QString::fromUtf8( 4436 "RE_Ex3z = \"1* 0 1* 0 1* 0 1* + (0* 1 0* (1 0* 1 0*)*)\"\n" 4437 "NFA_Ex3z = re2nfa(RE_Ex3z)\n" 4438 "DO_Ex3z = dotObj_dfa(min_dfa(nfa2dfa(NFA_Ex3z)))\n" 4439 "DO_Ex3z" 4440 )); 4441 testImageResult(entry, 0); 4442 entry = entry->next(); 4443 4444 testMarkdown(entry, QString::fromUtf8( 4445 "# Check out all remaining modules of Jove covering these\n" 4446 "\n" 4447 "* Brzozowski derivatives for parsing\n" 4448 "* Brzozowski minimization\n" 4449 "* Context-free parsing\n" 4450 "* (soon to come) [Binary Decision Diagrams; obtain now from software/ at](http://www.cs.utah.edu/fv)\n" 4451 "* (soon to come) Post Correspondence Problem" 4452 )); 4453 4454 testMarkdown(entry, QString::fromUtf8( 4455 "# Brzozowski's minimization defined\n" 4456 "\n" 4457 "It is nothing but these steps done in this order:\n" 4458 "\n" 4459 "* Reverse\n" 4460 "* Determinize\n" 4461 "* Reverse\n" 4462 "* Determinize\n" 4463 "\n" 4464 "Voila! The machine is now minimal!" 4465 )); 4466 4467 qDebug() << "command entry 5"; 4468 testCommandEntry(entry, 5, QString::fromUtf8( 4469 "# The above example, with min_dfa replaced by the rev;det;rev;det\n" 4470 "\n" 4471 "DofNFA_Ex3z = nfa2dfa(re2nfa(\"1* 0 1* 0 1* 0 1* + (0* 1 0* (1 0* 1 0*)*)\"))\n" 4472 "dotObj_dfa(DofNFA_Ex3z)\n" 4473 "dotObj_dfa(DofNFA_Ex3z)\n" 4474 "minDofNFA_Ex3z = nfa2dfa(rev_dfa(nfa2dfa(rev_dfa(DofNFA_Ex3z))))" 4475 )); 4476 4477 qDebug() << "command entry 6"; 4478 testCommandEntry(entry, 6, 1, QString::fromUtf8( 4479 "dotObj_dfa(minDofNFA_Ex3z)" 4480 )); 4481 testImageResult(entry, 0); 4482 entry = entry->next(); 4483 4484 testMarkdown(entry, QString::fromUtf8( 4485 "# What's the largest postage that can't be made using 3,5 and 7 cents?\n" 4486 "\n" 4487 "Answer is 4. Find it out." 4488 )); 4489 4490 qDebug() << "command entry 7"; 4491 testCommandEntry(entry, 7, 1, QString::fromUtf8( 4492 "dotObj_dfa(min_dfa_brz(nfa2dfa(re2nfa(\"(111+11111+1111111)*\"))))" 4493 )); 4494 testImageResult(entry, 0); 4495 entry = entry->next(); 4496 4497 testMarkdown(entry, QString::fromUtf8( 4498 "# Show ambiguity in parsing" 4499 )); 4500 4501 qDebug() << "command entry 8"; 4502 testCommandEntry(entry, 8, 1, QString::fromUtf8( 4503 "# Parsing an arithmetic expression\n" 4504 "pdaEamb = md2mc('''PDA\n" 4505 "!!E -> E * E | E + E | ~E | ( E ) | 2 | 3\n" 4506 "I : '', # ; E# -> M\n" 4507 "M : '', E ; ~E -> M\n" 4508 "M : '', E ; E+E -> M\n" 4509 "M : '', E ; E*E -> M\n" 4510 "M : '', E ; (E) -> M\n" 4511 "M : '', E ; 2 -> M\n" 4512 "M : '', E ; 3 -> M\n" 4513 "M : ~, ~ ; '' -> M\n" 4514 "M : 2, 2 ; '' -> M\n" 4515 "M : 3, 3 ; '' -> M\n" 4516 "M : (, ( ; '' -> M\n" 4517 "M : ), ) ; '' -> M\n" 4518 "M : +, + ; '' -> M\n" 4519 "M : *, * ; '' -> M\n" 4520 "M : '', # ; # -> F\n" 4521 "'''\n" 4522 ")" 4523 )); 4524 testTextResult(entry, 0, QString::fromUtf8( 4525 "Generating LALR tables" 4526 )); 4527 entry = entry->next(); 4528 4529 qDebug() << "command entry 9"; 4530 testCommandEntry(entry, 9, 1, QString::fromUtf8( 4531 "from jove.Def_PDA import *" 4532 )); 4533 testTextResult(entry, 0, QString::fromUtf8( 4534 "You may use any of these help commands:\n" 4535 "help(explore_pda)\n" 4536 "help(run_pda)\n" 4537 "help(classify_l_id_path)\n" 4538 "help(h_run_pda)\n" 4539 "help(interpret_w_eps)\n" 4540 "help(step_pda)\n" 4541 "help(suvivor_id)\n" 4542 "help(term_id)\n" 4543 "help(final_id)\n" 4544 "help(cvt_str_to_sym)\n" 4545 "help(is_surv_id)\n" 4546 "help(subsumed)\n" 4547 "help(is_term_id)\n" 4548 "help(is_final_id)" 4549 )); 4550 entry = entry->next(); 4551 4552 qDebug() << "command entry 10"; 4553 testCommandEntry(entry, 10, 1, QString::fromUtf8( 4554 "explore_pda(\"3+2*3+2*3\", pdaEamb, STKMAX=7)" 4555 )); 4556 4557 testTextResult(entry, 0, QString::fromUtf8( 4558 "*** Exploring wrt STKMAX = 7 ; increase it if needed ***\n" 4559 "String 3+2*3+2*3 accepted by your PDA in 13 ways :-) \n" 4560 "Here are the ways: \n" 4561 "Final state ('F', '', '#')\n" 4562 "Reached as follows:\n" 4563 "-> ('I', '3+2*3+2*3', '#')\n" 4564 "-> ('M', '3+2*3+2*3', 'E#')\n" 4565 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4566 "-> ('M', '3+2*3+2*3', 'E*E*E#')\n" 4567 "-> ('M', '3+2*3+2*3', 'E+E*E*E#')\n" 4568 "-> ('M', '3+2*3+2*3', '3+E*E*E#')\n" 4569 "-> ('M', '+2*3+2*3', '+E*E*E#')\n" 4570 "-> ('M', '2*3+2*3', 'E*E*E#')\n" 4571 "-> ('M', '2*3+2*3', '2*E*E#')\n" 4572 "-> ('M', '*3+2*3', '*E*E#')\n" 4573 "-> ('M', '3+2*3', 'E*E#')\n" 4574 "-> ('M', '3+2*3', 'E+E*E#')\n" 4575 "-> ('M', '3+2*3', '3+E*E#')\n" 4576 "-> ('M', '+2*3', '+E*E#')\n" 4577 "-> ('M', '2*3', 'E*E#')\n" 4578 "-> ('M', '2*3', '2*E#')\n" 4579 "-> ('M', '*3', '*E#')\n" 4580 "-> ('M', '3', 'E#')\n" 4581 "-> ('M', '3', '3#')\n" 4582 "-> ('M', '', '#')\n" 4583 "-> ('F', '', '#') .\n" 4584 "Final state ('F', '', '#')\n" 4585 "Reached as follows:\n" 4586 "-> ('I', '3+2*3+2*3', '#')\n" 4587 "-> ('M', '3+2*3+2*3', 'E#')\n" 4588 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4589 "-> ('M', '3+2*3+2*3', 'E+E*E#')\n" 4590 "-> ('M', '3+2*3+2*3', '3+E*E#')\n" 4591 "-> ('M', '+2*3+2*3', '+E*E#')\n" 4592 "-> ('M', '2*3+2*3', 'E*E#')\n" 4593 "-> ('M', '2*3+2*3', 'E*E*E#')\n" 4594 "-> ('M', '2*3+2*3', '2*E*E#')\n" 4595 "-> ('M', '*3+2*3', '*E*E#')\n" 4596 "-> ('M', '3+2*3', 'E*E#')\n" 4597 "-> ('M', '3+2*3', 'E+E*E#')\n" 4598 "-> ('M', '3+2*3', '3+E*E#')\n" 4599 "-> ('M', '+2*3', '+E*E#')\n" 4600 "-> ('M', '2*3', 'E*E#')\n" 4601 "-> ('M', '2*3', '2*E#')\n" 4602 "-> ('M', '*3', '*E#')\n" 4603 "-> ('M', '3', 'E#')\n" 4604 "-> ('M', '3', '3#')\n" 4605 "-> ('M', '', '#')\n" 4606 "-> ('F', '', '#') .\n" 4607 "Final state ('F', '', '#')\n" 4608 "Reached as follows:\n" 4609 "-> ('I', '3+2*3+2*3', '#')\n" 4610 "-> ('M', '3+2*3+2*3', 'E#')\n" 4611 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4612 "-> ('M', '3+2*3+2*3', 'E+E*E#')\n" 4613 "-> ('M', '3+2*3+2*3', '3+E*E#')\n" 4614 "-> ('M', '+2*3+2*3', '+E*E#')\n" 4615 "-> ('M', '2*3+2*3', 'E*E#')\n" 4616 "-> ('M', '2*3+2*3', '2*E#')\n" 4617 "-> ('M', '*3+2*3', '*E#')\n" 4618 "-> ('M', '3+2*3', 'E#')\n" 4619 "-> ('M', '3+2*3', 'E*E#')\n" 4620 "-> ('M', '3+2*3', 'E+E*E#')\n" 4621 "-> ('M', '3+2*3', '3+E*E#')\n" 4622 "-> ('M', '+2*3', '+E*E#')\n" 4623 "-> ('M', '2*3', 'E*E#')\n" 4624 "-> ('M', '2*3', '2*E#')\n" 4625 "-> ('M', '*3', '*E#')\n" 4626 "-> ('M', '3', 'E#')\n" 4627 "-> ('M', '3', '3#')\n" 4628 "-> ('M', '', '#')\n" 4629 "-> ('F', '', '#') .\n" 4630 "Final state ('F', '', '#')\n" 4631 "Reached as follows:\n" 4632 "-> ('I', '3+2*3+2*3', '#')\n" 4633 "-> ('M', '3+2*3+2*3', 'E#')\n" 4634 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4635 "-> ('M', '3+2*3+2*3', 'E+E*E#')\n" 4636 "-> ('M', '3+2*3+2*3', '3+E*E#')\n" 4637 "-> ('M', '+2*3+2*3', '+E*E#')\n" 4638 "-> ('M', '2*3+2*3', 'E*E#')\n" 4639 "-> ('M', '2*3+2*3', '2*E#')\n" 4640 "-> ('M', '*3+2*3', '*E#')\n" 4641 "-> ('M', '3+2*3', 'E#')\n" 4642 "-> ('M', '3+2*3', 'E+E#')\n" 4643 "-> ('M', '3+2*3', '3+E#')\n" 4644 "-> ('M', '+2*3', '+E#')\n" 4645 "-> ('M', '2*3', 'E#')\n" 4646 "-> ('M', '2*3', 'E*E#')\n" 4647 "-> ('M', '2*3', '2*E#')\n" 4648 "-> ('M', '*3', '*E#')\n" 4649 "-> ('M', '3', 'E#')\n" 4650 "-> ('M', '3', '3#')\n" 4651 "-> ('M', '', '#')\n" 4652 "-> ('F', '', '#') .\n" 4653 "Final state ('F', '', '#')\n" 4654 "Reached as follows:\n" 4655 "-> ('I', '3+2*3+2*3', '#')\n" 4656 "-> ('M', '3+2*3+2*3', 'E#')\n" 4657 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4658 "-> ('M', '3+2*3+2*3', 'E+E*E#')\n" 4659 "-> ('M', '3+2*3+2*3', '3+E*E#')\n" 4660 "-> ('M', '+2*3+2*3', '+E*E#')\n" 4661 "-> ('M', '2*3+2*3', 'E*E#')\n" 4662 "-> ('M', '2*3+2*3', 'E+E*E#')\n" 4663 "-> ('M', '2*3+2*3', 'E*E+E*E#')\n" 4664 "-> ('M', '2*3+2*3', '2*E+E*E#')\n" 4665 "-> ('M', '*3+2*3', '*E+E*E#')\n" 4666 "-> ('M', '3+2*3', 'E+E*E#')\n" 4667 "-> ('M', '3+2*3', '3+E*E#')\n" 4668 "-> ('M', '+2*3', '+E*E#')\n" 4669 "-> ('M', '2*3', 'E*E#')\n" 4670 "-> ('M', '2*3', '2*E#')\n" 4671 "-> ('M', '*3', '*E#')\n" 4672 "-> ('M', '3', 'E#')\n" 4673 "-> ('M', '3', '3#')\n" 4674 "-> ('M', '', '#')\n" 4675 "-> ('F', '', '#') .\n" 4676 "Final state ('F', '', '#')\n" 4677 "Reached as follows:\n" 4678 "-> ('I', '3+2*3+2*3', '#')\n" 4679 "-> ('M', '3+2*3+2*3', 'E#')\n" 4680 "-> ('M', '3+2*3+2*3', 'E*E#')\n" 4681 "-> ('M', '3+2*3+2*3', 'E+E*E#')\n" 4682 "-> ('M', '3+2*3+2*3', 'E+E+E*E#')\n" 4683 "-> ('M', '3+2*3+2*3', '3+E+E*E#')\n" 4684 "-> ('M', '+2*3+2*3', '+E+E*E#')\n" 4685 "-> ('M', '2*3+2*3', 'E+E*E#')\n" 4686 "-> ('M', '2*3+2*3', 'E*E+E*E#')\n" 4687 "-> ('M', '2*3+2*3', '2*E+E*E#')\n" 4688 "-> ('M', '*3+2*3', '*E+E*E#')\n" 4689 "-> ('M', '3+2*3', 'E+E*E#')\n" 4690 "-> ('M', '3+2*3', '3+E*E#')\n" 4691 "-> ('M', '+2*3', '+E*E#')\n" 4692 "-> ('M', '2*3', 'E*E#')\n" 4693 "-> ('M', '2*3', '2*E#')\n" 4694 "-> ('M', '*3', '*E#')\n" 4695 "-> ('M', '3', 'E#')\n" 4696 "-> ('M', '3', '3#')\n" 4697 "-> ('M', '', '#')\n" 4698 "-> ('F', '', '#') .\n" 4699 "Final state ('F', '', '#')\n" 4700 "Reached as follows:\n" 4701 "-> ('I', '3+2*3+2*3', '#')\n" 4702 "-> ('M', '3+2*3+2*3', 'E#')\n" 4703 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4704 "-> ('M', '3+2*3+2*3', 'E*E+E#')\n" 4705 "-> ('M', '3+2*3+2*3', 'E+E*E+E#')\n" 4706 "-> ('M', '3+2*3+2*3', '3+E*E+E#')\n" 4707 "-> ('M', '+2*3+2*3', '+E*E+E#')\n" 4708 "-> ('M', '2*3+2*3', 'E*E+E#')\n" 4709 "-> ('M', '2*3+2*3', '2*E+E#')\n" 4710 "-> ('M', '*3+2*3', '*E+E#')\n" 4711 "-> ('M', '3+2*3', 'E+E#')\n" 4712 "-> ('M', '3+2*3', '3+E#')\n" 4713 "-> ('M', '+2*3', '+E#')\n" 4714 "-> ('M', '2*3', 'E#')\n" 4715 "-> ('M', '2*3', 'E*E#')\n" 4716 "-> ('M', '2*3', '2*E#')\n" 4717 "-> ('M', '*3', '*E#')\n" 4718 "-> ('M', '3', 'E#')\n" 4719 "-> ('M', '3', '3#')\n" 4720 "-> ('M', '', '#')\n" 4721 "-> ('F', '', '#') .\n" 4722 "Final state ('F', '', '#')\n" 4723 "Reached as follows:\n" 4724 "-> ('I', '3+2*3+2*3', '#')\n" 4725 "-> ('M', '3+2*3+2*3', 'E#')\n" 4726 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4727 "-> ('M', '3+2*3+2*3', '3+E#')\n" 4728 "-> ('M', '+2*3+2*3', '+E#')\n" 4729 "-> ('M', '2*3+2*3', 'E#')\n" 4730 "-> ('M', '2*3+2*3', 'E*E#')\n" 4731 "-> ('M', '2*3+2*3', 'E*E*E#')\n" 4732 "-> ('M', '2*3+2*3', '2*E*E#')\n" 4733 "-> ('M', '*3+2*3', '*E*E#')\n" 4734 "-> ('M', '3+2*3', 'E*E#')\n" 4735 "-> ('M', '3+2*3', 'E+E*E#')\n" 4736 "-> ('M', '3+2*3', '3+E*E#')\n" 4737 "-> ('M', '+2*3', '+E*E#')\n" 4738 "-> ('M', '2*3', 'E*E#')\n" 4739 "-> ('M', '2*3', '2*E#')\n" 4740 "-> ('M', '*3', '*E#')\n" 4741 "-> ('M', '3', 'E#')\n" 4742 "-> ('M', '3', '3#')\n" 4743 "-> ('M', '', '#')\n" 4744 "-> ('F', '', '#') .\n" 4745 "Final state ('F', '', '#')\n" 4746 "Reached as follows:\n" 4747 "-> ('I', '3+2*3+2*3', '#')\n" 4748 "-> ('M', '3+2*3+2*3', 'E#')\n" 4749 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4750 "-> ('M', '3+2*3+2*3', '3+E#')\n" 4751 "-> ('M', '+2*3+2*3', '+E#')\n" 4752 "-> ('M', '2*3+2*3', 'E#')\n" 4753 "-> ('M', '2*3+2*3', 'E*E#')\n" 4754 "-> ('M', '2*3+2*3', '2*E#')\n" 4755 "-> ('M', '*3+2*3', '*E#')\n" 4756 "-> ('M', '3+2*3', 'E#')\n" 4757 "-> ('M', '3+2*3', 'E*E#')\n" 4758 "-> ('M', '3+2*3', 'E+E*E#')\n" 4759 "-> ('M', '3+2*3', '3+E*E#')\n" 4760 "-> ('M', '+2*3', '+E*E#')\n" 4761 "-> ('M', '2*3', 'E*E#')\n" 4762 "-> ('M', '2*3', '2*E#')\n" 4763 "-> ('M', '*3', '*E#')\n" 4764 "-> ('M', '3', 'E#')\n" 4765 "-> ('M', '3', '3#')\n" 4766 "-> ('M', '', '#')\n" 4767 "-> ('F', '', '#') .\n" 4768 "Final state ('F', '', '#')\n" 4769 "Reached as follows:\n" 4770 "-> ('I', '3+2*3+2*3', '#')\n" 4771 "-> ('M', '3+2*3+2*3', 'E#')\n" 4772 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4773 "-> ('M', '3+2*3+2*3', '3+E#')\n" 4774 "-> ('M', '+2*3+2*3', '+E#')\n" 4775 "-> ('M', '2*3+2*3', 'E#')\n" 4776 "-> ('M', '2*3+2*3', 'E*E#')\n" 4777 "-> ('M', '2*3+2*3', '2*E#')\n" 4778 "-> ('M', '*3+2*3', '*E#')\n" 4779 "-> ('M', '3+2*3', 'E#')\n" 4780 "-> ('M', '3+2*3', 'E+E#')\n" 4781 "-> ('M', '3+2*3', '3+E#')\n" 4782 "-> ('M', '+2*3', '+E#')\n" 4783 "-> ('M', '2*3', 'E#')\n" 4784 "-> ('M', '2*3', 'E*E#')\n" 4785 "-> ('M', '2*3', '2*E#')\n" 4786 "-> ('M', '*3', '*E#')\n" 4787 "-> ('M', '3', 'E#')\n" 4788 "-> ('M', '3', '3#')\n" 4789 "-> ('M', '', '#')\n" 4790 "-> ('F', '', '#') .\n" 4791 "Final state ('F', '', '#')\n" 4792 "Reached as follows:\n" 4793 "-> ('I', '3+2*3+2*3', '#')\n" 4794 "-> ('M', '3+2*3+2*3', 'E#')\n" 4795 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4796 "-> ('M', '3+2*3+2*3', '3+E#')\n" 4797 "-> ('M', '+2*3+2*3', '+E#')\n" 4798 "-> ('M', '2*3+2*3', 'E#')\n" 4799 "-> ('M', '2*3+2*3', 'E*E#')\n" 4800 "-> ('M', '2*3+2*3', 'E+E*E#')\n" 4801 "-> ('M', '2*3+2*3', 'E*E+E*E#')\n" 4802 "-> ('M', '2*3+2*3', '2*E+E*E#')\n" 4803 "-> ('M', '*3+2*3', '*E+E*E#')\n" 4804 "-> ('M', '3+2*3', 'E+E*E#')\n" 4805 "-> ('M', '3+2*3', '3+E*E#')\n" 4806 "-> ('M', '+2*3', '+E*E#')\n" 4807 "-> ('M', '2*3', 'E*E#')\n" 4808 "-> ('M', '2*3', '2*E#')\n" 4809 "-> ('M', '*3', '*E#')\n" 4810 "-> ('M', '3', 'E#')\n" 4811 "-> ('M', '3', '3#')\n" 4812 "-> ('M', '', '#')\n" 4813 "-> ('F', '', '#') .\n" 4814 "Final state ('F', '', '#')\n" 4815 "Reached as follows:\n" 4816 "-> ('I', '3+2*3+2*3', '#')\n" 4817 "-> ('M', '3+2*3+2*3', 'E#')\n" 4818 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4819 "-> ('M', '3+2*3+2*3', '3+E#')\n" 4820 "-> ('M', '+2*3+2*3', '+E#')\n" 4821 "-> ('M', '2*3+2*3', 'E#')\n" 4822 "-> ('M', '2*3+2*3', 'E+E#')\n" 4823 "-> ('M', '2*3+2*3', 'E*E+E#')\n" 4824 "-> ('M', '2*3+2*3', '2*E+E#')\n" 4825 "-> ('M', '*3+2*3', '*E+E#')\n" 4826 "-> ('M', '3+2*3', 'E+E#')\n" 4827 "-> ('M', '3+2*3', '3+E#')\n" 4828 "-> ('M', '+2*3', '+E#')\n" 4829 "-> ('M', '2*3', 'E#')\n" 4830 "-> ('M', '2*3', 'E*E#')\n" 4831 "-> ('M', '2*3', '2*E#')\n" 4832 "-> ('M', '*3', '*E#')\n" 4833 "-> ('M', '3', 'E#')\n" 4834 "-> ('M', '3', '3#')\n" 4835 "-> ('M', '', '#')\n" 4836 "-> ('F', '', '#') .\n" 4837 "Final state ('F', '', '#')\n" 4838 "Reached as follows:\n" 4839 "-> ('I', '3+2*3+2*3', '#')\n" 4840 "-> ('M', '3+2*3+2*3', 'E#')\n" 4841 "-> ('M', '3+2*3+2*3', 'E+E#')\n" 4842 "-> ('M', '3+2*3+2*3', 'E+E+E#')\n" 4843 "-> ('M', '3+2*3+2*3', '3+E+E#')\n" 4844 "-> ('M', '+2*3+2*3', '+E+E#')\n" 4845 "-> ('M', '2*3+2*3', 'E+E#')\n" 4846 "-> ('M', '2*3+2*3', 'E*E+E#')\n" 4847 "-> ('M', '2*3+2*3', '2*E+E#')\n" 4848 "-> ('M', '*3+2*3', '*E+E#')\n" 4849 "-> ('M', '3+2*3', 'E+E#')\n" 4850 "-> ('M', '3+2*3', '3+E#')\n" 4851 "-> ('M', '+2*3', '+E#')\n" 4852 "-> ('M', '2*3', 'E#')\n" 4853 "-> ('M', '2*3', 'E*E#')\n" 4854 "-> ('M', '2*3', '2*E#')\n" 4855 "-> ('M', '*3', '*E#')\n" 4856 "-> ('M', '3', 'E#')\n" 4857 "-> ('M', '3', '3#')\n" 4858 "-> ('M', '', '#')\n" 4859 "-> ('F', '', '#') ." 4860 )); 4861 entry = entry->next(); 4862 4863 testMarkdown(entry, QString::fromUtf8( 4864 "# Show how to disambiguate" 4865 )); 4866 4867 qDebug() << "command entry 11"; 4868 testCommandEntry(entry, 11, 1, QString::fromUtf8( 4869 "# Parsing an arithmetic expression\n" 4870 "pdaE = md2mc('''PDA\n" 4871 "!!E -> E+T | T\n" 4872 "!!T -> T*F | F\n" 4873 "!!F -> 2 | 3 | ~F | (E)\n" 4874 "I : '', # ; E# -> M\n" 4875 "M : '', E ; E+T -> M\n" 4876 "M : '', E ; T -> M\n" 4877 "M : '', T ; T*F -> M\n" 4878 "M : '', T ; F -> M\n" 4879 "M : '', F ; 2 -> M\n" 4880 "M : '', F ; 3 -> M\n" 4881 "M : '', F ; ~F -> M\n" 4882 "M : '', F ; (E) -> M\n" 4883 "M : ~, ~ ; '' -> M\n" 4884 "M : 2, 2 ; '' -> M\n" 4885 "M : 3, 3 ; '' -> M\n" 4886 "M : (, ( ; '' -> M\n" 4887 "M : ), ) ; '' -> M\n" 4888 "M : +, + ; '' -> M\n" 4889 "M : *, * ; '' -> M\n" 4890 "M : '', # ; # -> F\n" 4891 "'''\n" 4892 ")" 4893 )); 4894 testTextResult(entry, 0, QString::fromUtf8( 4895 "Generating LALR tables" 4896 )); 4897 entry = entry->next(); 4898 4899 qDebug() << "command entry 12"; 4900 testCommandEntry(entry, 12, 1, QString::fromUtf8( 4901 "explore_pda(\"3+2*3+2*3\", pdaE, STKMAX=7)" 4902 )); 4903 testTextResult(entry, 0, QString::fromUtf8( 4904 "*** Exploring wrt STKMAX = 7 ; increase it if needed ***\n" 4905 "String 3+2*3+2*3 accepted by your PDA in 1 ways :-) \n" 4906 "Here are the ways: \n" 4907 "Final state ('F', '', '#')\n" 4908 "Reached as follows:\n" 4909 "-> ('I', '3+2*3+2*3', '#')\n" 4910 "-> ('M', '3+2*3+2*3', 'E#')\n" 4911 "-> ('M', '3+2*3+2*3', 'E+T#')\n" 4912 "-> ('M', '3+2*3+2*3', 'E+T+T#')\n" 4913 "-> ('M', '3+2*3+2*3', 'T+T+T#')\n" 4914 "-> ('M', '3+2*3+2*3', 'F+T+T#')\n" 4915 "-> ('M', '3+2*3+2*3', '3+T+T#')\n" 4916 "-> ('M', '+2*3+2*3', '+T+T#')\n" 4917 "-> ('M', '2*3+2*3', 'T+T#')\n" 4918 "-> ('M', '2*3+2*3', 'T*F+T#')\n" 4919 "-> ('M', '2*3+2*3', 'F*F+T#')\n" 4920 "-> ('M', '2*3+2*3', '2*F+T#')\n" 4921 "-> ('M', '*3+2*3', '*F+T#')\n" 4922 "-> ('M', '3+2*3', 'F+T#')\n" 4923 "-> ('M', '3+2*3', '3+T#')\n" 4924 "-> ('M', '+2*3', '+T#')\n" 4925 "-> ('M', '2*3', 'T#')\n" 4926 "-> ('M', '2*3', 'T*F#')\n" 4927 "-> ('M', '2*3', 'F*F#')\n" 4928 "-> ('M', '2*3', '2*F#')\n" 4929 "-> ('M', '*3', '*F#')\n" 4930 "-> ('M', '3', 'F#')\n" 4931 "-> ('M', '3', '3#')\n" 4932 "-> ('M', '', '#')\n" 4933 "-> ('F', '', '#') ." 4934 )); 4935 entry = entry->next(); 4936 4937 testMarkdown(entry, QString::fromUtf8( 4938 "# And finally, run a Turing Machine with \"dynamic tape allocation\" :-)\n" 4939 "\n" 4940 "* Why not show how TMs are encoded? \n" 4941 "* This markdown gets parsed to build a TM!\n" 4942 "* This TM is for the famous \"3x+1\" problem (Collatz's Problem)" 4943 )); 4944 4945 qDebug() << "command entry 13"; 4946 testCommandEntry(entry, 13, QString::fromUtf8( 4947 "collatz_tm_str = \"\"\"\n" 4948 "TM\n" 4949 "\n" 4950 "i_start : 0; ., R -> i_start !! erase this zero and try to find more\n" 4951 "i_start : 1; 1, R -> goto_lsb !! we have a proper number, go to the lsb\n" 4952 "i_start : .; ., S -> error !! error on no input or input == 0\n" 4953 "\n" 4954 "\n" 4955 "goto_lsb : 0; 0,R | 1; 1,R -> goto_lsb !! scan off the right edge of the number\n" 4956 "goto_lsb : .; .,L -> branch !! take a step back to be on the lsb and start branch\n" 4957 "\n" 4958 "\n" 4959 "branch : 0; .,L -> branch !! number is even, divide by two and re-branch\n" 4960 "branch : 1; 1,L -> check_n_eq_1 !! number is odd, check if it is 1\n" 4961 "\n" 4962 "\n" 4963 "check_n_eq_1 : 0; 0,R | 1; 1,R -> 01_fma !! number wasn't 1, goto 3n+1\n" 4964 "check_n_eq_1 : .; .,R -> f_halt !! number was 1, halt\n" 4965 "\n" 4966 "\n" 4967 "!! carrying 0 we see a 0 so write 0 and carry 0 forward\n" 4968 "00_fma : 0; 0,L -> 00_fma\n" 4969 "\n" 4970 "!! carrying 0 we see a 1 (times 3 is 11) so write 1 and carry 1 forward\n" 4971 "00_fma : 1; 1,L -> 01_fma\n" 4972 "\n" 4973 "!! reached the end of the number, go back to the start\n" 4974 "00_fma : .; .,R -> goto_lsb \n" 4975 "\n" 4976 "\n" 4977 "!! carrying 1 we see a 0 so write 1 and carry 0 forward\n" 4978 "01_fma : 0; 1,L -> 00_fma \n" 4979 "\n" 4980 "!! carrying 1 we see a 1 (times 3 is 11, plus our carry is 100) so write 0 and carry 10 forward\n" 4981 "01_fma : 1; 0,L -> 10_fma \n" 4982 "\n" 4983 "!! reached the end of the number, write our 1 and go back to the start\n" 4984 "01_fma : .; 1,R -> goto_lsb \n" 4985 "\n" 4986 "\n" 4987 "!! carrying 10 we see a 0, so write 0 and carry 1 forward\n" 4988 "10_fma : 0; 0,L -> 01_fma\n" 4989 "\n" 4990 "!! carrying 10 we see a 1 (times 3 is 11, plus our carry is 101), so write 1 and carry 10 forward\n" 4991 "10_fma : 1; 1,L -> 10_fma\n" 4992 "\n" 4993 "!! reached the end of the number, write a 0 from our 10 and carry 1\n" 4994 "10_fma : .; 0,L -> 01_fma\n" 4995 "\n" 4996 "!!\"\"\"\n" 4997 "" 4998 )); 4999 5000 qDebug() << "command entry 14"; 5001 testCommandEntry(entry, 14, 2, QString::fromUtf8( 5002 "# Now show the above TM graphically!\n" 5003 "collatz_tm = md2mc(collatz_tm_str)\n" 5004 "dotObj_tm(collatz_tm, FuseEdges=True)" 5005 )); 5006 testTextResult(entry, 0, QString::fromUtf8( 5007 "Generating LALR tables" 5008 )); 5009 testImageResult(entry, 1); 5010 entry = entry->next(); 5011 5012 qDebug() << "command entry 15"; 5013 testCommandEntry(entry, 15, 1, QString::fromUtf8( 5014 "from jove.Def_TM import *" 5015 )); 5016 testTextResult(entry, 0, QString::fromUtf8( 5017 "You may use any of these help commands:\n" 5018 "help(step_tm)\n" 5019 "help(run_tm)\n" 5020 "help(explore_tm)" 5021 )); 5022 entry = entry->next(); 5023 5024 qDebug() << "command entry 16"; 5025 testCommandEntry(entry, 16, 1, QString::fromUtf8( 5026 "# Will loop if the Collatz (\"3x+1\") program will ever loop!\n" 5027 "explore_tm(collatz_tm, \"0110\", 100)" 5028 )); 5029 testTextResult(entry, 0, QString::fromUtf8( 5030 "Allocating 8 tape cells to the RIGHT!\n" 5031 "Allocating 8 tape cells to the LEFT!\n" 5032 "Detailing the halted configs now.\n" 5033 "Accepted at ('f_halt', 5, '.....1..............', 65)\n" 5034 " via .. \n" 5035 " ->('i_start', 0, '0110', 100)\n" 5036 " ->('i_start', 1, '.110', 99)\n" 5037 " ->('goto_lsb', 2, '.110', 98)\n" 5038 " ->('goto_lsb', 3, '.110', 97)\n" 5039 " ->('goto_lsb', 4, '.110', 96)\n" 5040 " ->('branch', 3, '.110........', 95)\n" 5041 " ->('branch', 2, '.11.........', 94)\n" 5042 " ->('check_n_eq_1', 1, '.11.........', 93)\n" 5043 " ->('01_fma', 2, '.11.........', 92)\n" 5044 " ->('10_fma', 1, '.10.........', 91)\n" 5045 " ->('10_fma', 0, '.10.........', 90)\n" 5046 " ->('01_fma', 7, '........010.........', 89)\n" 5047 " ->('goto_lsb', 8, '.......1010.........', 88)\n" 5048 " ->('goto_lsb', 9, '.......1010.........', 87)\n" 5049 " ->('goto_lsb', 10, '.......1010.........', 86)\n" 5050 " ->('goto_lsb', 11, '.......1010.........', 85)\n" 5051 " ->('branch', 10, '.......1010.........', 84)\n" 5052 " ->('branch', 9, '.......101..........', 83)\n" 5053 " ->('check_n_eq_1', 8, '.......101..........', 82)\n" 5054 " ->('01_fma', 9, '.......101..........', 81)\n" 5055 " ->('10_fma', 8, '.......100..........', 80)\n" 5056 " ->('01_fma', 7, '.......100..........', 79)\n" 5057 " ->('10_fma', 6, '.......000..........', 78)\n" 5058 " ->('01_fma', 5, '......0000..........', 77)\n" 5059 " ->('goto_lsb', 6, '.....10000..........', 76)\n" 5060 " ->('goto_lsb', 7, '.....10000..........', 75)\n" 5061 " ->('goto_lsb', 8, '.....10000..........', 74)\n" 5062 " ->('goto_lsb', 9, '.....10000..........', 73)\n" 5063 " ->('goto_lsb', 10, '.....10000..........', 72)\n" 5064 " ->('branch', 9, '.....10000..........', 71)\n" 5065 " ->('branch', 8, '.....1000...........', 70)\n" 5066 " ->('branch', 7, '.....100............', 69)\n" 5067 " ->('branch', 6, '.....10.............', 68)\n" 5068 " ->('branch', 5, '.....1..............', 67)\n" 5069 " ->('check_n_eq_1', 4, '.....1..............', 66)\n" 5070 " ->('f_halt', 5, '.....1..............', 65)" 5071 )); 5072 entry = entry->next(); 5073 5074 testMarkdown(entry, QString::fromUtf8( 5075 "# END: You have a ton more waiting for your execution pleasure!" 5076 )); 5077 5078 QCOMPARE(entry, nullptr); 5079 } 5080 5081 void WorksheetTest::testJupyter6() 5082 { 5083 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 5084 if (backend && backend->isEnabled() == false) 5085 QSKIP("Skip, because python backend don't available", SkipSingle); 5086 5087 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("Cue Combination with Neural Populations .ipynb"))); 5088 5089 QCOMPARE(w->isReadOnly(), false); 5090 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 5091 5092 WorksheetEntry* entry = w->firstEntry(); 5093 5094 testMarkdown(entry, QString::fromUtf8( 5095 "# Humans and animals integrate multisensory cues near-optimally\n" 5096 "## An intuition for how populations of neurons can perform Bayesian inference" 5097 )); 5098 5099 qDebug() << "command entry 30"; 5100 testCommandEntry(entry, 30, QString::fromUtf8( 5101 "from __future__ import division\n" 5102 "import numpy as np\n" 5103 "from scipy.special import factorial\n" 5104 "import scipy.stats as stats\n" 5105 "import pylab\n" 5106 "import matplotlib.pyplot as plt\n" 5107 "%matplotlib inline\n" 5108 "import seaborn as sns\n" 5109 "sns.set_style(\"darkgrid\")\n" 5110 "import ipywidgets\n" 5111 "from IPython.display import display\n" 5112 "from matplotlib.font_manager import FontProperties\n" 5113 "fontP = FontProperties()\n" 5114 "fontP.set_size('medium')\n" 5115 "%config InlineBackend.figure_format = 'svg'\n" 5116 "\n" 5117 "\n" 5118 "def mean_firing_rate(gain, stimulus, preferred_stimulus, std_tc, baseline):\n" 5119 " # Gaussian tuning curve that determines the mean firing rate (Poisson rate parameter) for a given stimulus\n" 5120 " return baseline + gain*stats.norm.pdf(preferred_stimulus, loc = stimulus, scale = std_tc)\n" 5121 "\n" 5122 "def get_spikes(gain, stimulus, preferred_stimuli, std_tc, baseline):\n" 5123 " # produce a vector of spikes for some population given some stimulus\n" 5124 " lambdas = mean_firing_rate(gain, stimulus, preferred_stimuli, std_tc, baseline)\n" 5125 " return np.random.poisson(lambdas)\n" 5126 " \n" 5127 "def likelihood(stimulus, r, gain, preferred_stimuli, std_tc, baseline):\n" 5128 " # returns p(r|s)\n" 5129 " lambdas = mean_firing_rate(gain, stimulus, preferred_stimuli, std_tc, baseline)\n" 5130 " return np.prod(lambdas**r)\n" 5131 "\n" 5132 "def spikes_and_inference(r_V = True,\n" 5133 " r_A = True,\n" 5134 " show_tuning_curves = False,\n" 5135 " show_spike_count = False,\n" 5136 " show_likelihoods = True,\n" 5137 " true_stimulus = 10,\n" 5138 " number_of_neurons = 40,\n" 5139 " r_V_gain = 15,\n" 5140 " r_A_gain = 75,\n" 5141 " r_V_tuning_curve_sigma = 10,\n" 5142 " r_A_tuning_curve_sigma = 10,\n" 5143 " tuning_curve_baseline = 0,\n" 5144 " joint_likelihood = True,\n" 5145 " r_V_plus_r_A = True,\n" 5146 " cue = False):\n" 5147 " np.random.seed(7)\n" 5148 " max_s = 40\n" 5149 " preferred_stimuli = np.linspace(-max_s*2, max_s*2, number_of_neurons)\n" 5150 " n_hypothesized_s = 250\n" 5151 " hypothesized_s = np.linspace(-max_s, max_s, n_hypothesized_s)\n" 5152 " gains = {'r1': r_V_gain,\n" 5153 " 'r2': r_A_gain,\n" 5154 " 'r1+r2': r_V_gain + r_A_gain}\n" 5155 " sigma_TCs = {'r1': r_V_tuning_curve_sigma,\n" 5156 " 'r2': r_A_tuning_curve_sigma,\n" 5157 " 'r1+r2': (r_V_tuning_curve_sigma + r_A_tuning_curve_sigma)/2}\n" 5158 " spikes = {'r1': get_spikes(gains['r1'], true_stimulus, preferred_stimuli, sigma_TCs['r1'], tuning_curve_baseline),\n" 5159 " 'r2': get_spikes(gains['r2'], true_stimulus, preferred_stimuli, sigma_TCs['r2'], tuning_curve_baseline)}\n" 5160 " spikes['r1+r2'] = spikes['r1'] + spikes['r2']\n" 5161 " active_pops = []\n" 5162 " if r_V: active_pops.append('r1')\n" 5163 " if r_A: active_pops.append('r2')\n" 5164 " if r_V_plus_r_A: active_pops.append('r1+r2')\n" 5165 "\n" 5166 " colors = {'r1': sns.xkcd_rgb['light purple'],\n" 5167 " 'r2': sns.xkcd_rgb['dark pink'],\n" 5168 " 'r1+r2': sns.xkcd_rgb['royal blue'],\n" 5169 " 'joint': sns.xkcd_rgb['gold']}\n" 5170 " nSubplots = show_spike_count + show_tuning_curves + show_likelihoods\n" 5171 " fig, axes = plt.subplots(nSubplots, figsize = (7, 1.5*nSubplots)) # number of subplots according to what's been requested\n" 5172 " if not isinstance(axes, np.ndarray): axes = [axes] # makes axes into a list even if it's just one subplot\n" 5173 " subplot_idx = 0\n" 5174 " \n" 5175 " def plot_true_stimulus_and_legend(subplot_idx):\n" 5176 " axes[subplot_idx].plot(true_stimulus, 0, 'k^', markersize = 12, clip_on = False, label = 'true rattlesnake location')\n" 5177 " axes[subplot_idx].legend(loc = 'center left', bbox_to_anchor = (1, 0.5), prop = fontP)\n" 5178 " \n" 5179 " if show_tuning_curves:\n" 5180 " for neuron in range(number_of_neurons):\n" 5181 " if r_V:\n" 5182 " axes[subplot_idx].plot(hypothesized_s,\n" 5183 " mean_firing_rate(gains['r1'],\n" 5184 " hypothesized_s,\n" 5185 " preferred_stimuli[neuron],\n" 5186 " sigma_TCs['r1'],\n" 5187 " tuning_curve_baseline),\n" 5188 " color = colors['r1'])\n" 5189 " if r_A:\n" 5190 " axes[subplot_idx].plot(hypothesized_s,\n" 5191 " mean_firing_rate(gains['r2'],\n" 5192 " hypothesized_s,\n" 5193 " preferred_stimuli[neuron],\n" 5194 " sigma_TCs['r2'],\n" 5195 " tuning_curve_baseline),\n" 5196 " color = colors['r2'])\n" 5197 " axes[subplot_idx].set_xlabel('location $s$')\n" 5198 " axes[subplot_idx].set_ylabel('mean firing rate\\n(spikes/s)')\n" 5199 " axes[subplot_idx].set_ylim((0, 4))\n" 5200 " axes[subplot_idx].set_xlim((-40, 40))\n" 5201 " axes[subplot_idx].set_yticks(np.linspace(0, 4, 5))\n" 5202 " subplot_idx += 1\n" 5203 "\n" 5204 " if show_spike_count:\n" 5205 " idx = abs(preferred_stimuli) < max_s\n" 5206 " if r_V:\n" 5207 " axes[subplot_idx].plot(preferred_stimuli[idx], spikes['r1'][idx], 'o', color = colors['r1'],\n" 5208 " clip_on = False, label = '$\\mathbf{r}_\\mathrm{V}$',\n" 5209 " markersize=4)\n" 5210 " if r_A:\n" 5211 " axes[subplot_idx].plot(preferred_stimuli[idx], spikes['r2'][idx], 'o', color = colors['r2'],\n" 5212 " clip_on = False, label = '$\\mathbf{r}_\\mathrm{A}$',\n" 5213 " markersize=4)\n" 5214 " if r_V_plus_r_A:\n" 5215 " axes[subplot_idx].plot(preferred_stimuli[idx], spikes['r1+r2'][idx], 'o', color = colors['r1+r2'],\n" 5216 " clip_on = False, label = '$\\mathbf{r}_\\mathrm{V}+\\mathbf{r}_\\mathrm{A}$',\n" 5217 " markersize=8, zorder=1)\n" 5218 " axes[subplot_idx].set_xlabel('preferred location')\n" 5219 " axes[subplot_idx].set_ylabel('spike count')\n" 5220 " axes[subplot_idx].set_ylim((0, 10))\n" 5221 " axes[subplot_idx].set_xlim((-40, 40))\n" 5222 " plot_true_stimulus_and_legend(subplot_idx)\n" 5223 " subplot_idx += 1\n" 5224 "\n" 5225 " if show_likelihoods:\n" 5226 " if cue:\n" 5227 " var = 'c'\n" 5228 " else:\n" 5229 " var = '\\mathbf{r}'\n" 5230 " likelihoods = {}\n" 5231 " \n" 5232 " for population in active_pops:\n" 5233 " likelihoods[population] = np.zeros_like(hypothesized_s)\n" 5234 " for idx, ort in enumerate(hypothesized_s):\n" 5235 " likelihoods[population][idx] = likelihood(ort, spikes[population], gains[population],\n" 5236 " preferred_stimuli, sigma_TCs[population], tuning_curve_baseline)\n" 5237 " likelihoods[population] /= np.sum(likelihoods[population]) # normalize\n" 5238 "\n" 5239 " if r_V:\n" 5240 " axes[subplot_idx].plot(hypothesized_s, likelihoods['r1'], color = colors['r1'],\n" 5241 " linewidth = 2, label = '$p({}_\\mathrm{{V}}|s)$'.format(var))\n" 5242 " if r_A:\n" 5243 " axes[subplot_idx].plot(hypothesized_s, likelihoods['r2'], color = colors['r2'],\n" 5244 " linewidth = 2, label = '$p({}_\\mathrm{{A}}|s)$'.format(var))\n" 5245 " if r_V_plus_r_A:\n" 5246 " axes[subplot_idx].plot(hypothesized_s, likelihoods['r1+r2'], color = colors['r1+r2'],\n" 5247 " linewidth = 2, label = '$p({}_\\mathrm{{V}}+{}_\\mathrm{{A}}|s)$'.format(var, var))\n" 5248 " if joint_likelihood:\n" 5249 " product = likelihoods['r1']*likelihoods['r2']\n" 5250 " product /= np.sum(product)\n" 5251 " axes[subplot_idx].plot(hypothesized_s, product, color = colors['joint'],linewidth = 7,\n" 5252 " label = '$p({}_\\mathrm{{V}}|s)\\ p({}_\\mathrm{{A}}|s)$'.format(var, var), zorder = 1)\n" 5253 "\n" 5254 " axes[subplot_idx].set_xlabel('location $s$')\n" 5255 " axes[subplot_idx].set_ylabel('probability')\n" 5256 " axes[subplot_idx].set_xlim((-40, 40))\n" 5257 " axes[subplot_idx].legend()\n" 5258 " axes[subplot_idx].set_yticks([])\n" 5259 " \n" 5260 " plot_true_stimulus_and_legend(subplot_idx)\n" 5261 " subplot_idx += 1" 5262 )); 5263 5264 testMarkdown(entry, QString::fromUtf8( 5265 "\n" 5266 "\n" 5267 "" 5268 )); 5269 5270 testMarkdown(entry, QString::fromUtf8( 5271 "\n" 5272 "\n" 5273 "" 5274 )); 5275 5276 testMarkdown(entry, QString::fromUtf8( 5277 "<p>We live in a complex environment and must constantly integrate sensory information to interact with the world around us. Inputs from different modalities might not always be congruent with each other, but dissociating the true nature of the stimulus may be a matter of life or death for an organism.</p>\n" 5278 "<img src=\"http://www.wtadler.com/picdrop/rattlesnake.jpg\" width=25% height=25% align=\"left\" style=\"margin: 10px 10px 10px 0px;\" >\n" 5279 "<p>You hear and see evidence of a rattlesnake in tall grass near you. You get an auditory and a visual cue of the snake's location $s$. Both cues are associated with a likelihood function indicating the probability of that cue for all possible locations of the snake. The likelihood function associated with the visual cue, $p(c_\\mathrm{V}|s)$, has high uncertainty, because of the tall grass. The auditory cue is easier to localize, so its associated likelihood function, $p(c_\\mathrm{A}|s)$, is sharper. In accordance with Bayes' Rule, and assuming a flat prior over the snake's location, an optimal estimate of the location of the snake can be computed by multiplying the two likelihoods. This joint likelihood will be between the two cues but closer to the less uncertain cue, and will have less uncertainty than both unimodal likelihood functions.</p>" 5280 )); 5281 5282 qDebug() << "command entry 31"; 5283 testCommandEntry(entry, 31, 1, QString::fromUtf8( 5284 "spikes_and_inference(show_likelihoods = True, r_V_plus_r_A = False, cue = True)" 5285 )); 5286 testImageResult(entry, 0); 5287 entry = entry->next(); 5288 5289 testMarkdown(entry, QString::fromUtf8( 5290 "\n" 5291 "\n" 5292 "" 5293 )); 5294 5295 testMarkdown(entry, QString::fromUtf8( 5296 "\n" 5297 "\n" 5298 "" 5299 )); 5300 5301 testMarkdown(entry, QString::fromUtf8( 5302 "Behavioral experiments have demonstrated that humans perform near-optimal Bayesian inference on ambiguous sensory information (van Beers *et al.*, 1999; Ernst & Banks, 2002; Kording & Wolpert, 2004; Stocker & Simoncelli, 2006). This has been demonstrated in cue combination experiments in which subjects report a near-optimal estimate of the stimulus given two noisy measurements of that stimulus. However, the neural basis for how humans might perform these computations is unclear. \n" 5303 "\n" 5304 "Ma *et. al.* (2006) propose that variance in cortical activity, rather than impairing sensory systems, is an adaptive mechanism to encode uncertainty in sensory measurements. They provide theory showing how the brain might use probabilistic population codes to perform near-optimal cue combination. We will re-derive the theory in here, and demonstrate it by simulating and decoding neural populations.\n" 5305 "\n" 5306 "## Cues can be represented by neural populations\n" 5307 "\n" 5308 "To return to our deadly rattlesnake, let's now assume that $c_\\mathrm{V}$ and $c_\\mathrm{A}$ are represented by populations of neurons $\\mathbf{r}_\\mathrm{V}$ and $\\mathbf{r}_\\mathrm{A}$, respectively. For our math and simulations, we assume that $\\mathbf{r}_\\mathrm{V}$ and $\\mathbf{r}_\\mathrm{A}$ are each composed of $N$ neurons that:\n" 5309 "\n" 5310 "* have independent Poisson variability\n" 5311 "* have regularly spaced Gaussian tuning curves that are identical in mean and variance for neurons with the same index in both populations\n" 5312 "\n" 5313 "The populations may have different gains, $g_\\mathrm{V}$ and $g_\\mathrm{A}$.\n" 5314 "\n" 5315 "These are the tuning curves for the neurons in $\\mathbf{r}_\\mathrm{V}$ (purple) and $\\mathbf{r}_\\mathrm{A}$ (red). Each curve represents the mean firing rate of a single neuron given a location $s$. Each neuron thus has a preferred location, which is where its tuning curve peaks." 5316 )); 5317 5318 qDebug() << "command entry 32"; 5319 testCommandEntry(entry, 32, 1, QString::fromUtf8( 5320 "spikes_and_inference(show_tuning_curves = True, show_likelihoods = False)" 5321 )); 5322 testImageResult(entry, 0); 5323 entry = entry->next(); 5324 5325 testMarkdown(entry, QString::fromUtf8( 5326 "The tuning curves are dense enough that we can also assume that $\\sum_{i=0}^N f_i(s) = k$ (*i.e.*, the sum of the tuning curves in a population is constant.)" 5327 )); 5328 5329 testMarkdown(entry, QString::fromUtf8( 5330 "\n" 5331 "\n" 5332 "" 5333 )); 5334 5335 testMarkdown(entry, QString::fromUtf8( 5336 "\n" 5337 "\n" 5338 "" 5339 )); 5340 5341 testMarkdown(entry, QString::fromUtf8( 5342 "First, we will show how the brain can decode a likelihood over stimulus from neural activity. Then we will ask how the brain can compute joint likelihoods.\n" 5343 "### How can the brain decode $p(\\mathbf{r_\\mathrm{V}}|s)$?\n" 5344 "\n" 5345 "\\begin{align}\n" 5346 "L(s) &= p(\\mathbf{r_\\mathrm{V}}\\ |\\ s) \\tag{1} \\\\ \n" 5347 "&= \\prod_{i=0}^N \\frac{e^{-g_\\mathrm{V}\\ f_i(s)}\\ g_\\mathrm{V}\\ f_i(s)^{r_{\\mathrm{V}i}}}{r_{\\mathrm{V}i}!} \\tag{2} \\\\\n" 5348 "&\\propto \\prod_{i=0}^N e^{-g_\\mathrm{V}\\ f_i(s)}\\ f_i(s)^{r_{\\mathrm{V}i}} \\tag{3} \\\\\n" 5349 "&= e^{-g_\\mathrm{V}\\sum_{i=0}^N f_i(s)} \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}}\\tag{4} \\\\ \n" 5350 "&= e^{-g_\\mathrm{V}k} \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}} \\tag{5} \\\\\n" 5351 "&\\propto \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}} \\tag{6} \\\\\n" 5352 "\\end{align}\n" 5353 "\n" 5354 "### Then what is the joint likelihood $p(\\mathbf{r_\\mathrm{V}}|s)\\ p(\\mathbf{r_\\mathrm{A}}|s)$?\n" 5355 "\n" 5356 "\\begin{align}\n" 5357 "L(s) &= p(\\mathbf{r_\\mathrm{V}}\\ |\\ s)\\ p(\\mathbf{r_\\mathrm{A}}|s) \\tag{7} \\\\\n" 5358 "&\\propto \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}}\\ \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{A}i}} \\tag{8} \\\\\n" 5359 "&= \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}} \\tag{9} \\\\\n" 5360 "\\end{align}\n" 5361 "\n" 5362 "## How can the brain compute the joint likelihood $p(\\mathbf{r}_\\mathrm{V}|s)\\ p(\\mathbf{r}_\\mathrm{A}|s)$?\n" 5363 "The fact that we see neurons from $\\mathbf{r}_\\mathrm{V}$ and $\\mathbf{r}_\\mathrm{A}$ being added on a neuron-by-neuron basis in the exponent above suggests that we could construct a third population vector, $\\mathbf{r}_\\mathrm{V}+\\mathbf{r}_\\mathrm{A}$, and decode that.\n" 5364 "\n" 5365 "### First, we must prove that the sum of two Poisson-distributed random variables $X+Y$ is again Poisson-distributed.\n" 5366 "\\begin{align}\n" 5367 "X &\\sim \\textrm{Poisson}(\\lambda_x) \\textrm{, so } p(X=k)=\\frac{\\lambda_x^k\\ e^{-\\lambda_x}}{k!} \\tag{10} \\\\\n" 5368 "Y &\\sim \\textrm{Poisson}(\\lambda_y) \\textrm{, so } p(X=k)=\\frac{\\lambda_y^k\\ e^{-\\lambda_y}}{k!} \\tag{11} \\\\\n" 5369 "X+Y &\\overset{?}{\\sim} \\textrm{Poisson}(\\lambda_{x+y}) \\textrm{ and, if so, } \\lambda_{x+y}=? \\tag{12} \\\\\n" 5370 "\\end{align}\n" 5371 "\n" 5372 "\\begin{align}\n" 5373 "p(X+Y=n) &= p(X=0)\\ p(Y=n) + p(X=1)\\ p(Y=n-1)\\ +...+\\ p(X=n-1)\\ p(Y = 1) + p(X=n)\\ p(Y=0) \\tag{13} \\\\\n" 5374 "&= \\sum_{k=0}^n p(X=k)\\ p(Y=n-k) \\tag{14} \\\\\n" 5375 "&= \\sum_{k=0}^n \\frac{\\lambda_x^k\\ e^{-\\lambda_x}\\ \\lambda_y^{n-k}\\ e^{-\\lambda_y}}{k!(n-k)!} \\tag{15} \\\\\n" 5376 "&= e^{-(\\lambda_x+\\lambda_y)} \\sum_{k=0}^n \\frac{1}{k!(n-k)!}\\ \\lambda_x^k\\ \\lambda_y^{n-k} \\tag{16} \\\\\n" 5377 "&= e^{-(\\lambda_x+\\lambda_y)} \\frac{1}{n!} \\sum_{k=0}^n \\frac{n!}{k!(n-k)!}\\ \\lambda_x^k\\ \\lambda_y^{n-k} \\tag{17} \\\\\n" 5378 "&= e^{-(\\lambda_x+\\lambda_y)} \\frac{1}{n!} \\sum_{k=0}^n \\binom{n}{k}\\ \\lambda_x^k\\ \\lambda_y^{n-k}\\ [ \\textrm{because} \\frac{n!}{k!(n-k)!}=\\binom{n}{k} ]\\tag{18} \\\\\n" 5379 "&=\\frac{e^{-(\\lambda_x + \\lambda_y)}(\\lambda_x+\\lambda_y)^n}{n!} [ \\textrm{because} \\sum_{k=0}^n \\binom{n}{k}\\ x^ky^{n-k} = (x+y)^n ]\\tag{19} \\\\\n" 5380 "\\end{align}\n" 5381 "\n" 5382 "Therefore, $X + Y \\sim \\mathrm{Poisson}(\\lambda_x + \\lambda_y)$.\n" 5383 "\n" 5384 "## What is $p(\\mathbf{r}_\\mathrm{V}+\\mathbf{r}_\\mathrm{A} | s)$?\n" 5385 "\n" 5386 "In our case:\n" 5387 "\n" 5388 "\\begin{align}\n" 5389 "r_{\\mathrm{V}i} &\\sim \\textrm{Poisson}(g_\\mathrm{V}\\ f_i(s)) \\tag{20} \\\\\n" 5390 "r_{\\mathrm{A}i} &\\sim \\textrm{Poisson}(g_\\mathrm{A}\\ f_i(s)) \\tag{21} \\\\\n" 5391 "r_{\\mathrm{V}i}+r_{\\mathrm{A}i} &\\sim \\textrm{Poisson}((g_\\mathrm{V}+g_\\mathrm{A})\\ f_i(s)) \\tag{22} \\\\\n" 5392 "\\end{align}\n" 5393 "\n" 5394 "\\begin{align}\n" 5395 "L(s)&=p(\\mathbf{r}_\\mathrm{V} + \\mathbf{r}_\\mathrm{A}\\ |\\ s)\n" 5396 "= \\prod_{i=0}^N \\frac{e^{-f_i(s)(g_\\mathrm{V}+g_\\mathrm{A})}\\ (g_\\mathrm{V}+g_\\mathrm{A})\\ f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}}}{(r_{\\mathrm{V}i}+r_{\\mathrm{A}i})!} \\tag{23} \\\\\n" 5397 "&\\propto \\prod_{i=0}^N e^{-f_i(s)(g_\\mathrm{V}+g_\\mathrm{A})}\\ f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}} \\tag{24} \\\\\n" 5398 "&= e^{-(g_\\mathrm{V}+g_\\mathrm{A})\\sum_{i=0}^Nf_i(s)} \\prod_{i=0}^N \\ f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}} \\tag{25} \\\\\n" 5399 "&= e^{-(g_\\mathrm{V}+g_\\mathrm{A})k} \\prod_{i=0}^N \\ f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}} \\tag{26} \\\\\n" 5400 "&\\propto \\prod_{i=0}^N f_i(s)^{r_{\\mathrm{V}i}+r_{\\mathrm{A}i}} \\tag{27} \\\\\n" 5401 "\\end{align}\n" 5402 "\n" 5403 "Since equations $(9)$ and $(27)$ are proportional, we have shown that optimal cue combination can be executed by decoding linear sums of populations." 5404 )); 5405 5406 testMarkdown(entry, QString::fromUtf8( 5407 "$$x = 2$$" 5408 )); 5409 5410 testMarkdown(entry, QString::fromUtf8( 5411 "\n" 5412 "\n" 5413 "" 5414 )); 5415 5416 testMarkdown(entry, QString::fromUtf8( 5417 "\n" 5418 "\n" 5419 "" 5420 )); 5421 5422 testMarkdown(entry, QString::fromUtf8( 5423 "## Simulation\n" 5424 "Here are the spike counts (during 1 s) from the two populations on one trial. Depicted in blue is a third population vector that is the sum of $\\mathbf{r}_\\mathrm{V}$ and $\\mathbf{r}_\\mathrm{A}$." 5425 )); 5426 5427 qDebug() << "command entry 33"; 5428 testCommandEntry(entry, 33, 1, QString::fromUtf8( 5429 "spikes_and_inference(show_spike_count = True, show_likelihoods = False)" 5430 )); 5431 testImageResult(entry, 0); 5432 entry = entry->next(); 5433 5434 testMarkdown(entry, QString::fromUtf8( 5435 "\n" 5436 "\n" 5437 "" 5438 )); 5439 5440 testMarkdown(entry, QString::fromUtf8( 5441 "\n" 5442 "\n" 5443 "" 5444 )); 5445 5446 testMarkdown(entry, QString::fromUtf8( 5447 "Here are the decoded likelihoods for each population alone $(6)$, the joint likelihood $(9)$, and the likelihood for the summed population $(27)$. Note that the joint likelihood (gold) is less uncertain than either unimodal likelihood. Also note that it is identical to the likelihood for the summed population (blue)." 5448 )); 5449 5450 qDebug() << "command entry 34"; 5451 testCommandEntry(entry, 34, 1, QString::fromUtf8( 5452 "spikes_and_inference()" 5453 )); 5454 testImageResult(entry, 0); 5455 entry = entry->next(); 5456 5457 testMarkdown(entry, QString::fromUtf8( 5458 "\n" 5459 "\n" 5460 "" 5461 )); 5462 5463 testMarkdown(entry, QString::fromUtf8( 5464 "\n" 5465 "\n" 5466 "" 5467 )); 5468 5469 testMarkdown(entry, QString::fromUtf8( 5470 "Here, we break the assumption that the two populations have the same tuning curve width. Note that the joint likelihood (gold) is no longer identical to the likelihood for the summed population (blue)." 5471 )); 5472 5473 qDebug() << "command entry 35"; 5474 testCommandEntry(entry, 35, 1, QString::fromUtf8( 5475 "spikes_and_inference(r_V_tuning_curve_sigma = 7, r_A_tuning_curve_sigma = 10)" 5476 )); 5477 testImageResult(entry, 0); 5478 entry = entry->next(); 5479 5480 testMarkdown(entry, QString::fromUtf8( 5481 "\n" 5482 "\n" 5483 "" 5484 )); 5485 5486 testMarkdown(entry, QString::fromUtf8( 5487 "\n" 5488 "\n" 5489 "" 5490 )); 5491 5492 testMarkdown(entry, QString::fromUtf8( 5493 "Now you can play interactively with the parameters of the simulation using these sliders, and watch the decoded likelihoods shift around. Every time you change a parameter, new sets of spikes are generated and used to infer $s$.\n" 5494 "\n" 5495 "For the simulation to be interactive, you'll have to download this notebook." 5496 )); 5497 5498 qDebug() << "command entry 36"; 5499 testCommandEntry(entry, 36, 1, QString::fromUtf8( 5500 "i = ipywidgets.interactive(spikes_and_inference,\n" 5501 " true_stimulus = (-40, 40, .1),\n" 5502 " number_of_neurons = (2, 200, 1),\n" 5503 " r_V_gain = (0, 100, 1),\n" 5504 " r_A_gain = (0, 100, 1),\n" 5505 " r_V_tuning_curve_sigma = (0.1, 50, .1),\n" 5506 " r_A_tuning_curve_sigma = (0.1, 50, .1),\n" 5507 " tuning_curve_baseline = (0, 20, .1));\n" 5508 "display(ipywidgets.VBox(i.children[2:-1]))" 5509 )); 5510 testImageResult(entry, 0); 5511 entry = entry->next(); 5512 5513 testMarkdown(entry, QString::fromUtf8( 5514 "\n" 5515 "\n" 5516 "" 5517 )); 5518 5519 testMarkdown(entry, QString::fromUtf8( 5520 "\n" 5521 "\n" 5522 "" 5523 )); 5524 5525 testMarkdown(entry, QString::fromUtf8( 5526 "## Conclusion\n" 5527 "\n" 5528 "It has been shown behaviorally that humans perform near-optimal Bayesian inference on ambiguous sensory information. As suggested by Ma *et. al.* (2006) and shown here, it is possible that the brain does this operation by simply performing linear combinations of populations of Poisson neurons receiving various sensory input. Cortical neurons may be particularly well suited for this task because they have Poisson-like firing rates, displaying reliable variability from trial to trial (Tolhurst, Movshon & Dean, 1982; Softky & Koch, 1993).\n" 5529 "\n" 5530 "High levels of noise in these populations might at first be difficult to reconcile considering highly precise behavioral data. However, variability in neural populations might be direcly representative of uncertainty in environmental stimuli. Variability in cortical populations would then be critical for precise neural coding.\n" 5531 "\n" 5532 "## References\n" 5533 "\n" 5534 "* Ernst MO, Banks MS. (2002). Humans integrate visual and haptic information in a statistically optimal fashion. *Nature.*\n" 5535 "* Körding KP, Wolpert DM. (2004). Bayesian integration in sensorimotor learning. *Nature.*\n" 5536 "* Ma WJ, Beck JM, Latham PE, Pouget A. (2006). Bayesian inference with probabilistic population codes. *Nature Neuroscience.*\n" 5537 "* Softky WR, Koch C. (1993). The highly irregular firing of cortical cells is inconsistent with temporal integration of random EPSPs. *Journal of Neuroscience.*\n" 5538 "* Stocker AA, Simoncelli EP. (2006). Noise characteristics and prior expectations in human visual speed perception. *Nature Neuroscience.*\n" 5539 "* Tolhurst, DJ, Movshon JA, Dean AF. (1983). The statistical reliability of signals in single neurons in cat and monkey visual cortex. *Vision Research.*\n" 5540 "* van Beers RJ, Sittig AC, Gon JJ. (1999). Integration of proprioceptive and visual position-information: An experimentally supported model. *Journal of Neurophysiology.*" 5541 )); 5542 5543 QCOMPARE(entry, nullptr); 5544 } 5545 5546 void WorksheetTest::testJupyter7() 5547 { 5548 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 5549 if (backend && backend->isEnabled() == false) 5550 QSKIP("Skip, because python backend don't available", SkipSingle); 5551 5552 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("Transformation2D.ipynb"))); 5553 5554 QCOMPARE(w->isReadOnly(), false); 5555 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 5556 5557 WorksheetEntry* entry = w->firstEntry(); 5558 5559 testMarkdown(entry, QString::fromUtf8( 5560 "# Rigid-body transformations in a plane (2D)\n" 5561 "\n" 5562 "> Marcos Duarte \n" 5563 "> Laboratory of Biomechanics and Motor Control ([http://demotu.org/](http://demotu.org/)) \n" 5564 "> Federal University of ABC, Brazil" 5565 )); 5566 5567 testMarkdown(entry, QString::fromUtf8( 5568 "The kinematics of a rigid body is completely described by its pose, i.e., its position and orientation in space (and the corresponding changes are translation and rotation). The translation and rotation of a rigid body are also known as rigid-body transformations (or simply, rigid transformations).\n" 5569 "\n" 5570 "Remember that in physics, a [rigid body](https://en.wikipedia.org/wiki/Rigid_body) is a model (an idealization) for a body in which deformation is neglected, i.e., the distance between every pair of points in the body is considered constant. Consequently, the position and orientation of a rigid body can be completely described by a corresponding coordinate system attached to it. For instance, two (or more) coordinate systems can be used to represent the same rigid body at two (or more) instants or two (or more) rigid bodies in space.\n" 5571 "\n" 5572 "Rigid-body transformations are used in motion analysis (e.g., of the human body) to describe the position and orientation of each segment (using a local (anatomical) coordinate system defined for each segment) in relation to a global coordinate system fixed at the laboratory. Furthermore, one can define an additional coordinate system called technical coordinate system also fixed at the rigid body but not based on anatomical landmarks. In this case, the position of the technical markers is first described in the laboratory coordinate system, and then the technical coordinate system is calculated to recreate the anatomical landmarks position in order to finally calculate the original anatomical coordinate system (and obtain its unknown position and orientation through time).\n" 5573 "\n" 5574 "In what follows, we will study rigid-body transformations by looking at the transformations between two coordinate systems. For simplicity, let's first analyze planar (two-dimensional) rigid-body transformations and later we will extend these concepts to three dimensions (where the study of rotations are more complicated)." 5575 )); 5576 5577 testMarkdown(entry, QString::fromUtf8( 5578 "## Affine transformations\n" 5579 "\n" 5580 "Translation and rotation are two examples of [affine transformations](https://en.wikipedia.org/wiki/Affine_transformation). Affine transformations preserve straight lines, but not necessarily the distance between points. Other examples of affine transformations are scaling, shear, and reflection. The figure below illustrates different affine transformations in a plane. Note that a 3x3 matrix is shown on top of each transformation; these matrices are known as the transformation matrices and are the mathematical representation of the physical transformations. Next, we will study how to use this approach to describe the translation and rotation of a rigid-body. \n" 5581 "<br>\n" 5582 "<figure><img src='https://upload.wikimedia.org/wikipedia/commons/thumb/2/2c/2D_affine_transformation_matrix.svg/360px-2D_affine_transformation_matrix.svg.png' alt='Affine transformations'/> <figcaption><center><i>Figure. Examples of affine transformations in a plane applied to a square (with the letter <b>F</b> in it) and the corresponding transformation matrices (<a href=\"https://en.wikipedia.org/wiki/Affine_transformation\">image from Wikipedia</a>).</i></center></figcaption> </figure>" 5583 )); 5584 5585 testMarkdown(entry, QString::fromUtf8( 5586 "## Translation\n" 5587 "\n" 5588 "In a two-dimensional space, two coordinates and one angle are sufficient to describe the pose of the rigid body, totalizing three degrees of freedom for a rigid body. Let's see first the transformation for translation, then for rotation, and combine them at last.\n" 5589 "\n" 5590 "A pure two-dimensional translation of a coordinate system in relation to other coordinate system and the representation of a point in these two coordinate systems are illustrated in the figure below (remember that this is equivalent to describing a translation between two rigid bodies). \n" 5591 "<br>\n" 5592 "<figure><img src='./../images/translation2D.png' alt='translation 2D'/> <figcaption><center><i>Figure. A point in two-dimensional space represented in two coordinate systems (Global and local), with one system translated.</i></center></figcaption> </figure>\n" 5593 "\n" 5594 "The position of point $\\mathbf{P}$ originally described in the local coordinate system but now described in the Global coordinate system in vector form is:\n" 5595 "\n" 5596 "$$ \\mathbf{P_G} = \\mathbf{L_G} + \\mathbf{P_l} $$\n" 5597 "\n" 5598 "Or for each component:\n" 5599 "\n" 5600 "$$ \\mathbf{P_X} = \\mathbf{L_X} + \\mathbf{P}_x $$\n" 5601 "\n" 5602 "$$ \\mathbf{P_Y} = \\mathbf{L_Y} + \\mathbf{P}_y $$\n" 5603 "\n" 5604 "And in matrix form is:\n" 5605 "\n" 5606 "$$\n" 5607 "\\begin{bmatrix}\n" 5608 "\\mathbf{P_X} \\\\\n" 5609 "\\mathbf{P_Y} \n" 5610 "\\end{bmatrix} =\n" 5611 "\\begin{bmatrix}\n" 5612 "\\mathbf{L_X} \\\\\n" 5613 "\\mathbf{L_Y} \n" 5614 "\\end{bmatrix} +\n" 5615 "\\begin{bmatrix}\n" 5616 "\\mathbf{P}_x \\\\\n" 5617 "\\mathbf{P}_y \n" 5618 "\\end{bmatrix}\n" 5619 "$$\n" 5620 "\n" 5621 "Because position and translation can be treated as vectors, the inverse operation, to describe the position at the local coordinate system in terms of the Global coordinate system, is simply:\n" 5622 "\n" 5623 "$$ \\mathbf{P_l} = \\mathbf{P_G} -\\mathbf{L_G} $$\n" 5624 "<br>\n" 5625 "$$ \\begin{bmatrix}\n" 5626 "\\mathbf{P}_x \\\\\n" 5627 "\\mathbf{P}_y \n" 5628 "\\end{bmatrix} =\n" 5629 "\\begin{bmatrix}\n" 5630 "\\mathbf{P_X} \\\\\n" 5631 "\\mathbf{P_Y} \n" 5632 "\\end{bmatrix} - \n" 5633 "\\begin{bmatrix}\n" 5634 "\\mathbf{L_X} \\\\\n" 5635 "\\mathbf{L_Y} \n" 5636 "\\end{bmatrix} $$\n" 5637 "\n" 5638 "From classical mechanics, this transformation is an example of [Galilean transformation](http://en.wikipedia.org/wiki/Galilean_transformation). \n" 5639 "\n" 5640 "For example, if the local coordinate system is translated by $\\mathbf{L_G}=[2, 3]$ in relation to the Global coordinate system, a point with coordinates $\\mathbf{P_l}=[4, 5]$ at the local coordinate system will have the position $\\mathbf{P_G}=[6, 8]$ at the Global coordinate system:" 5641 )); 5642 5643 qDebug() << "command entry 1"; 5644 testCommandEntry(entry, 1, QString::fromUtf8( 5645 "# Import the necessary libraries\n" 5646 "import numpy as np" 5647 )); 5648 5649 qDebug() << "command entry 2"; 5650 testCommandEntry(entry, 2, 1, QString::fromUtf8( 5651 "LG = np.array([2, 3]) # (Numpy 1D array with 2 elements)\n" 5652 "Pl = np.array([4, 5])\n" 5653 "PG = LG + Pl\n" 5654 "PG" 5655 )); 5656 testTextResult(entry, 0, QString::fromLatin1( 5657 "array([6, 8])" 5658 )); 5659 entry = entry->next(); 5660 5661 testMarkdown(entry, QString::fromUtf8( 5662 "This operation also works if we have more than one data point (NumPy knows how to handle vectors with different dimensions):" 5663 )); 5664 5665 qDebug() << "command entry 3"; 5666 testCommandEntry(entry, 3, 1, QString::fromUtf8( 5667 "Pl = np.array([[4, 5], [6, 7], [8, 9]]) # 2D array with 3 rows and two columns\n" 5668 "PG = LG + Pl\n" 5669 "PG" 5670 )); 5671 testTextResult(entry, 0, QString::fromLatin1( 5672 "array([[ 6, 8],\n" 5673 " [ 8, 10],\n" 5674 " [10, 12]])" 5675 )); 5676 entry = entry->next(); 5677 5678 testMarkdown(entry, QString::fromUtf8( 5679 "## Rotation\n" 5680 "\n" 5681 "A pure two-dimensional rotation of a coordinate system in relation to other coordinate system and the representation of a point in these two coordinate systems are illustrated in the figure below (remember that this is equivalent to describing a rotation between two rigid bodies). The rotation is around an axis orthogonal to this page, not shown in the figure (for a three-dimensional coordinate system the rotation would be around the $\\mathbf{Z}$ axis). \n" 5682 "<br>\n" 5683 "<figure><img src='./../images/rotation2D.png' alt='rotation 2D'/> <figcaption><center><i>Figure. A point in the two-dimensional space represented in two coordinate systems (Global and local), with one system rotated in relation to the other around an axis orthogonal to both coordinate systems.</i></center></figcaption> </figure>\n" 5684 "\n" 5685 "Consider we want to express the position of point $\\mathbf{P}$ in the Global coordinate system in terms of the local coordinate system knowing only the coordinates at the local coordinate system and the angle of rotation between the two coordinate systems. \n" 5686 "\n" 5687 "There are different ways of deducing that, we will see three of these methods next. " 5688 )); 5689 5690 testMarkdown(entry, QString::fromUtf8( 5691 "### 1. Using trigonometry\n" 5692 "\n" 5693 "From figure below, the coordinates of point $\\mathbf{P}$ in the Global coordinate system can be determined finding the sides of the triangles marked in red. \n" 5694 "<br>\n" 5695 "<figure><img src='./../images/rotation2Db.png' alt='rotation 2D'/> <figcaption><center><i>Figure. The coordinates of a point at the Global coordinate system in terms of the coordinates of this point at the local coordinate system.</i></center></figcaption> </figure>\n" 5696 "\n" 5697 "Then: \n" 5698 "\n" 5699 "$$ \\mathbf{P_X} = \\mathbf{P}_x \\cos \\alpha - \\mathbf{P}_y \\sin \\alpha $$\n" 5700 "\n" 5701 "$$ \\mathbf{P_Y} = \\mathbf{P}_x \\sin \\alpha + \\mathbf{P}_y \\cos \\alpha $$ \n" 5702 "\n" 5703 "The equations above can be expressed in matrix form:\n" 5704 "\n" 5705 "$$\n" 5706 "\\begin{bmatrix} \n" 5707 "\\mathbf{P_X} \\\\\n" 5708 "\\mathbf{P_Y} \n" 5709 "\\end{bmatrix} =\n" 5710 "\\begin{bmatrix}\n" 5711 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 5712 "\\sin\\alpha & \\cos\\alpha \n" 5713 "\\end{bmatrix} \\begin{bmatrix}\n" 5714 "\\mathbf{P}_x \\\\\n" 5715 "\\mathbf{P}_y \n" 5716 "\\end{bmatrix} $$\n" 5717 "\n" 5718 "Or simply:\n" 5719 "\n" 5720 "$$ \\mathbf{P_G} = \\mathbf{R_{Gl}}\\mathbf{P_l} $$\n" 5721 "\n" 5722 "Where $\\mathbf{R_{Gl}}$ is the rotation matrix that rotates the coordinates from the local to the Global coordinate system:\n" 5723 "\n" 5724 "$$ \\mathbf{R_{Gl}} = \\begin{bmatrix}\n" 5725 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 5726 "\\sin\\alpha & \\cos\\alpha \n" 5727 "\\end{bmatrix} $$\n" 5728 "\n" 5729 "So, given any position at the local coordinate system, with the rotation matrix above we are able to determine the position at the Global coordinate system. Let's check that before looking at other methods to obtain this matrix. \n" 5730 "\n" 5731 "For instance, consider a local coordinate system rotated by $45^o$ in relation to the Global coordinate system, a point in the local coordinate system with position $\\mathbf{P_l}=[1, 1]$ will have the following position at the Global coordinate system:" 5732 )); 5733 5734 qDebug() << "command entry 4"; 5735 testCommandEntry(entry, 4, 1, QString::fromUtf8( 5736 "RGl = np.array([[np.cos(np.pi/4), -np.sin(np.pi/4)], [np.sin(np.pi/4), np.cos(np.pi/4)]])\n" 5737 "Pl = np.array([[1, 1]]).T # transpose the array for correct matrix multiplication\n" 5738 "PG = np.dot(RGl, Pl) # the function dot() is used for matrix multiplication of arrays\n" 5739 "np.around(PG, 4) # round the number due to floating-point arithmetic errors" 5740 )); 5741 testTextResult(entry, 0, QString::fromLatin1( 5742 "array([[0. ],\n" 5743 " [1.4142]])" 5744 )); 5745 entry = entry->next(); 5746 5747 testMarkdown(entry, QString::fromUtf8( 5748 "We have rounded the number to 4 decimal places due to [floating-point arithmetic errors in the computation](http://floating-point-gui.de). \n" 5749 "\n" 5750 "And if we have the points [1,1], [0,1], [1,0] at the local coordinate system, their positions at the Global coordinate system are:" 5751 )); 5752 5753 qDebug() << "command entry 5"; 5754 testCommandEntry(entry, 5, 1, QString::fromUtf8( 5755 "Pl = np.array([[1, 1], [0, 1], [1, 0]]).T # transpose array for matrix multiplication\n" 5756 "PG = np.dot(RGl, Pl) # the function dot() is used for matrix multiplication with arrays\n" 5757 "np.around(PG, 4) # round the number due to floating point arithmetic errors" 5758 )); 5759 testTextResult(entry, 0, QString::fromLatin1( 5760 "array([[ 0. , -0.7071, 0.7071],\n" 5761 " [ 1.4142, 0.7071, 0.7071]])" 5762 )); 5763 entry = entry->next(); 5764 5765 testMarkdown(entry, QString::fromUtf8( 5766 "We have done all the calculations using the array function in NumPy. A [NumPy array is different than a matrix](http://www.scipy.org/NumPy_for_Matlab_Users), if we want to use explicit matrices in NumPy, the calculation above will be:" 5767 )); 5768 5769 qDebug() << "command entry 6"; 5770 testCommandEntry(entry, 6, 1, QString::fromUtf8( 5771 "RGl = np.mat([[np.cos(np.pi/4), -np.sin(np.pi/4)], [np.sin(np.pi/4), np.cos(np.pi/4)]])\n" 5772 "Pl = np.mat([[1, 1], [0,1], [1, 0]]).T # 2x3 matrix\n" 5773 "PG = RGl*Pl # matrix multiplication in NumPy\n" 5774 "np.around(PG, 4) # round the number due to floating point arithmetic errors" 5775 )); 5776 testTextResult(entry, 0, QString::fromLatin1( 5777 "array([[ 0. , -0.7071, 0.7071],\n" 5778 " [ 1.4142, 0.7071, 0.7071]])" 5779 )); 5780 entry = entry->next(); 5781 5782 testMarkdown(entry, QString::fromUtf8( 5783 "Both array and matrix types work in NumPy, but you should choose only one type and not mix them; the array is preferred because it is [the standard vector/matrix/tensor type of NumPy](http://www.scipy.org/NumPy_for_Matlab_Users)." 5784 )); 5785 5786 testMarkdown(entry, QString::fromUtf8( 5787 "### 2. Using direction cosines\n" 5788 "\n" 5789 "Another way to determine the rotation matrix is to use the concept of direction cosine. \n" 5790 "\n" 5791 "> Direction cosines are the cosines of the angles between any two vectors. \n" 5792 "\n" 5793 "For the present case with two coordinate systems, they are the cosines of the angles between each axis of one coordinate system and each axis of the other coordinate system. The figure below illustrates the directions angles between the two coordinate systems, expressing the local coordinate system in terms of the Global coordinate system. \n" 5794 "<br>\n" 5795 "<figure><img src='./../images/directioncosine2D.png' alt='direction angles 2D'/> <figcaption><center><i>Figure. Definition of direction angles at the two-dimensional space.</i></center></figcaption> </figure> \n" 5796 "<br>\n" 5797 "$$ \\mathbf{R_{Gl}} = \\begin{bmatrix}\n" 5798 "\\cos\\mathbf{X}x & \\cos\\mathbf{X}y \\\\\n" 5799 "\\cos\\mathbf{Y}x & \\cos\\mathbf{Y}y \n" 5800 "\\end{bmatrix} = \n" 5801 "\\begin{bmatrix}\n" 5802 "\\cos(\\alpha) & \\cos(90^o+\\alpha) \\\\\n" 5803 "\\cos(90^o-\\alpha) & \\cos(\\alpha)\n" 5804 "\\end{bmatrix} = \n" 5805 "\\begin{bmatrix}\n" 5806 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 5807 "\\sin\\alpha & \\cos\\alpha \n" 5808 "\\end{bmatrix} $$ \n" 5809 "\n" 5810 "The same rotation matrix as obtained before.\n" 5811 "\n" 5812 "Note that the order of the direction cosines is because in our convention, the first row is for the $\\mathbf{X}$ coordinate and the second row for the $\\mathbf{Y}$ coordinate (the outputs). For the inputs, we followed the same order, first column for the $\\mathbf{x}$ coordinate, second column for the $\\mathbf{y}$ coordinate." 5813 )); 5814 5815 testMarkdown(entry, QString::fromUtf8( 5816 "### 3. Using a basis\n" 5817 "\n" 5818 "Another way to deduce the rotation matrix is to view the axes of the rotated coordinate system as unit vectors, versors, of a <a href=\"http://en.wikipedia.org/wiki/Basis_(linear_algebra)\">basis</a> as illustrated in the figure below.\n" 5819 "\n" 5820 "> A basis is a set of linearly independent vectors that can represent every vector in a given vector space, i.e., a basis defines a coordinate system.\n" 5821 "\n" 5822 "<figure><img src='./../images/basis2D2.png' alt='basis 2D'/> <figcaption><center><i>Figure. Definition of the rotation matrix using a basis at the two-dimensional space.</i></center></figcaption> </figure>\n" 5823 "\n" 5824 "The coordinates of these two versors at the local coordinate system in terms of the Global coordinate system are:\n" 5825 "\n" 5826 "$$ \\begin{array}{l l}\n" 5827 "\\mathbf{e}_x = \\cos\\alpha\\:\\mathbf{e_X} + \\sin\\alpha\\:\\mathbf{e_Y} \\\\\n" 5828 "\\mathbf{e}_y = -\\sin\\alpha\\:\\mathbf{e_X} + \\cos\\alpha\\:\\mathbf{e_Y}\n" 5829 "\\end{array}$$\n" 5830 "\n" 5831 "Note that as unit vectors, each of the versors above should have norm (length) equals to one, which indeed is the case.\n" 5832 "\n" 5833 "If we express each versor above as different columns of a matrix, we obtain the rotation matrix again: \n" 5834 "\n" 5835 "$$ \\mathbf{R_{Gl}} = \\begin{bmatrix}\n" 5836 "\\cos\\alpha & -\\sin\\alpha \\\\\\\n" 5837 "\\sin\\alpha & \\cos\\alpha \n" 5838 "\\end{bmatrix} $$\n" 5839 "\n" 5840 "This means that the rotation matrix can be viewed as the basis of the rotated coordinate system defined by its versors. \n" 5841 "\n" 5842 "This third way to derive the rotation matrix is in fact the method most commonly used in motion analysis because the coordinates of markers (in the Global/laboratory coordinate system) are what we measure with cameras. " 5843 )); 5844 5845 testMarkdown(entry, QString::fromUtf8( 5846 "### 4. Using the inner (dot or scalar) product between versors\n" 5847 "\n" 5848 "Yet another way to deduce the rotation matrix is to define it as the dot product between the versors of the bases related to the two coordinate systems:\n" 5849 "\n" 5850 "$$\n" 5851 "\\mathbf{R_{Gl}} = \\begin{bmatrix}\n" 5852 "\\mathbf{\\hat{e}_X}\\! \\cdot \\mathbf{\\hat{e}_x} & \\mathbf{\\hat{e}_X}\\! \\cdot \\mathbf{\\hat{e}_y} \\\\\n" 5853 "\\mathbf{\\hat{e}_Y}\\! \\cdot \\mathbf{\\hat{e}_x} & \\mathbf{\\hat{e}_Y}\\! \\cdot \\mathbf{\\hat{e}_y} \n" 5854 "\\end{bmatrix}\n" 5855 "$$ \n" 5856 "\n" 5857 "By definition:\n" 5858 "\n" 5859 "$$ \\hat{\\mathbf{e}}_1\\! \\cdot \\hat{\\mathbf{e}}_2 = ||\\hat{\\mathbf{e}}_1|| \\times ||\\hat{\\mathbf{e}}_2||\\cos(e_1,e_2)=\\cos(e_1,e_2)$$\n" 5860 "\n" 5861 "And the rotation matrix will be equal to the matrix deduced based on the direction cosines." 5862 )); 5863 5864 testMarkdown(entry, QString::fromUtf8( 5865 "### Local-to-Global and Global-to-local coordinate systems' rotations" 5866 )); 5867 5868 testMarkdown(entry, QString::fromUtf8( 5869 "If we want the inverse operation, to express the position of point $\\mathbf{P}$ in the local coordinate system in terms of the Global coordinate system, the figure below illustrates that using trigonometry. \n" 5870 "<br>\n" 5871 "<figure><img src='./../images/rotation2Dc.png' alt='rotation 2D'/> <figcaption><center><i>Figure. The coordinates of a point at the local coordinate system in terms of the coordinates at the Global coordinate system.</i></center></figcaption> </figure>\n" 5872 "\n" 5873 "Then:\n" 5874 "\n" 5875 "$$ \\mathbf{P}_x = \\;\\;\\mathbf{P_X} \\cos \\alpha + \\mathbf{P_Y} \\sin \\alpha $$\n" 5876 "\n" 5877 "$$ \\mathbf{P}_y = -\\mathbf{P_X} \\sin \\alpha + \\mathbf{P_Y} \\cos \\alpha $$\n" 5878 "\n" 5879 "And in matrix form:\n" 5880 "\n" 5881 "$$\n" 5882 "\\begin{bmatrix} \n" 5883 "\\mathbf{P}_x \\\\\n" 5884 "\\mathbf{P}_y \n" 5885 "\\end{bmatrix} =\n" 5886 "\\begin{bmatrix}\n" 5887 "\\cos\\alpha & \\sin\\alpha \\\\\n" 5888 "-\\sin\\alpha & \\cos\\alpha \n" 5889 "\\end{bmatrix} \\begin{bmatrix}\n" 5890 "\\mathbf{P_X} \\\\\n" 5891 "\\mathbf{P_Y} \n" 5892 "\\end{bmatrix} $$\n" 5893 "\n" 5894 "$$ \\mathbf{P_l} = \\mathbf{R_{lG}}\\mathbf{P_G} $$\n" 5895 "\n" 5896 "Where $\\mathbf{R_{lG}}$ is the rotation matrix that rotates the coordinates from the Global to the local coordinate system (note the inverse order of the subscripts):\n" 5897 "\n" 5898 "$$ \\mathbf{R_{lG}} = \\begin{bmatrix}\n" 5899 "\\cos\\alpha & \\sin\\alpha \\\\\n" 5900 "-\\sin\\alpha & \\cos\\alpha \n" 5901 "\\end{bmatrix} $$\n" 5902 "\n" 5903 "If we use the direction cosines to calculate the rotation matrix, because the axes didn't change, the cosines are the same, only the order changes, now $\\mathbf{x, y}$ are the rows (outputs) and $\\mathbf{X, Y}$ are the columns (inputs):\n" 5904 "\n" 5905 "$$ \\mathbf{R_{lG}} = \\begin{bmatrix}\n" 5906 "\\cos\\mathbf{X}x & \\cos\\mathbf{Y}x \\\\\n" 5907 "\\cos\\mathbf{X}y & \\cos\\mathbf{Y}y \n" 5908 "\\end{bmatrix} = \n" 5909 "\\begin{bmatrix}\n" 5910 "\\cos(\\alpha) & \\cos(90^o-\\alpha) \\\\\n" 5911 "\\cos(90^o+\\alpha) & \\cos(\\alpha)\n" 5912 "\\end{bmatrix} = \n" 5913 "\\begin{bmatrix}\n" 5914 "\\cos\\alpha & \\sin\\alpha \\\\\n" 5915 "-\\sin\\alpha & \\cos\\alpha \n" 5916 "\\end{bmatrix} $$\n" 5917 "\n" 5918 "And defining the versors of the axes in the Global coordinate system for a basis in terms of the local coordinate system would also produce this latter rotation matrix.\n" 5919 "\n" 5920 "The two sets of equations and matrices for the rotations from Global-to-local and local-to-Global coordinate systems are very similar, this is no coincidence. Each of the rotation matrices we deduced, $\\mathbf{R_{Gl}}$ and $\\mathbf{R_{lG}}$, perform the inverse operation in relation to the other. Each matrix is the inverse of the other. \n" 5921 "\n" 5922 "In other words, the relation between the two rotation matrices means it is equivalent to instead of rotating the local coordinate system by $\\alpha$ in relation to the Global coordinate system, to rotate the Global coordinate system by $-\\alpha$ in relation to the local coordinate system; remember that $\\cos(-\\alpha)=\\cos(\\alpha)$ and $\\sin(-\\alpha)=-\\sin(\\alpha)$." 5923 )); 5924 5925 testMarkdown(entry, QString::fromUtf8( 5926 "### Rotation of a Vector\n" 5927 "\n" 5928 "We can also use the rotation matrix to rotate a vector by a given angle around an axis of the coordinate system as shown in the figure below. \n" 5929 "<br>\n" 5930 "<figure><img src='./../images/rotation2Dvector.png' alt='rotation 2D of a vector'/> <figcaption><center><i>Figure. Rotation of a position vector $\\mathbf{P}$ by an angle $\\alpha$ in the two-dimensional space.</i></center></figcaption> </figure>\n" 5931 "\n" 5932 "We will not prove that we use the same rotation matrix, but think that in this case the vector position rotates by the same angle instead of the coordinate system. The new coordinates of the vector position $\\mathbf{P'}$ rotated by an angle $\\alpha$ is simply the rotation matrix (for the angle $\\alpha$) multiplied by the coordinates of the vector position $\\mathbf{P}$:\n" 5933 "\n" 5934 "$$ \\mathbf{P'} = \\mathbf{R}_\\alpha\\mathbf{P} $$\n" 5935 "\n" 5936 "Consider for example that $\\mathbf{P}=[2,1]$ and $\\alpha=30^o$; the coordinates of $\\mathbf{P'}$ are:" 5937 )); 5938 5939 qDebug() << "command entry 7"; 5940 testCommandEntry(entry, 7, 1, QString::fromUtf8( 5941 "a = np.pi/6\n" 5942 "R = np.array([[np.cos(a), -np.sin(a)], [np.sin(a), np.cos(a)]])\n" 5943 "P = np.array([[2, 1]]).T\n" 5944 "Pl = np.dot(R, P)\n" 5945 "print(\"P':\\n\", Pl)" 5946 )); 5947 testTextResult(entry, 0, QString::fromUtf8( 5948 "P':\n" 5949 " [[1.23205081]\n" 5950 " [1.8660254 ]]" 5951 )); 5952 entry = entry->next(); 5953 5954 testMarkdown(entry, QString::fromUtf8( 5955 "### The rotation matrix\n" 5956 "\n" 5957 "**[See here for a review about matrix and its main properties](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/Matrix.ipynb)**.\n" 5958 "\n" 5959 "A nice property of the rotation matrix is that its inverse is the transpose of the matrix (because the columns/rows are mutually orthogonal and have norm equal to one). \n" 5960 "This property can be shown with the rotation matrices we deduced:\n" 5961 "\n" 5962 "$$ \\begin{array}{l l}\n" 5963 "\\mathbf{R}\\:\\mathbf{R^T} & = \n" 5964 "\\begin{bmatrix}\n" 5965 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 5966 "\\sin\\alpha & \\cos\\alpha \n" 5967 "\\end{bmatrix} \n" 5968 "\\begin{bmatrix}\n" 5969 "\\cos\\alpha & \\sin\\alpha \\\\\n" 5970 "-\\sin\\alpha & \\cos\\alpha \n" 5971 "\\end{bmatrix} \\\\\n" 5972 "& = \\begin{bmatrix}\n" 5973 "\\cos^2\\alpha+\\sin^2\\alpha & \\cos\\alpha \\sin\\alpha-\\sin\\alpha \\cos\\alpha\\;\\; \\\\\n" 5974 "\\sin\\alpha \\cos\\alpha-\\cos\\alpha \\sin\\alpha & \\sin^2\\alpha+\\cos^2\\alpha\\;\\;\n" 5975 "\\end{bmatrix} \\\\\n" 5976 "& = \\begin{bmatrix}\n" 5977 "1 & 0 \\\\\n" 5978 "0 & 1 \n" 5979 "\\end{bmatrix} \\\\\n" 5980 "& = \\mathbf{I} \\\\\n" 5981 "\\mathbf{R^{-1}} = \\mathbf{R^T}\n" 5982 "\\end{array} $$\n" 5983 "\n" 5984 "This means that if we have a rotation matrix, we know its inverse. \n" 5985 "\n" 5986 "The transpose and inverse operators in NumPy are methods of the array:" 5987 )); 5988 5989 qDebug() << "command entry 8"; 5990 testCommandEntry(entry, 8, 1, QString::fromUtf8( 5991 "RGl = np.mat([[np.cos(np.pi/4), -np.sin(np.pi/4)], [np.sin(np.pi/4), np.cos(np.pi/4)]])\n" 5992 "\n" 5993 "print('Orthogonal matrix (RGl):\\n', np.around(RGl, 4))\n" 5994 "print('Transpose (RGl.T):\\n', np.around(RGl.T, 4))\n" 5995 "print('Inverse (RGl.I):\\n', np.around(RGl.I, 4))" 5996 )); 5997 testTextResult(entry, 0, QString::fromUtf8( 5998 "Orthogonal matrix (RGl):\n" 5999 " [[ 0.7071 -0.7071]\n" 6000 " [ 0.7071 0.7071]]\n" 6001 "Transpose (RGl.T):\n" 6002 " [[ 0.7071 0.7071]\n" 6003 " [-0.7071 0.7071]]\n" 6004 "Inverse (RGl.I):\n" 6005 " [[ 0.7071 0.7071]\n" 6006 " [-0.7071 0.7071]]" 6007 )); 6008 entry = entry->next(); 6009 6010 testMarkdown(entry, QString::fromUtf8( 6011 "Using the inverse and the transpose mathematical operations, the coordinates at the local coordinate system given the coordinates at the Global coordinate system and the rotation matrix can be obtained by: \n" 6012 "\n" 6013 "$$ \\begin{array}{l l}\n" 6014 "\\mathbf{P_G} = \\mathbf{R_{Gl}}\\mathbf{P_l} \\implies \\\\\n" 6015 "\\\\\n" 6016 "\\mathbf{R_{Gl}^{-1}}\\mathbf{P_G} = \\mathbf{R_{Gl}^{-1}}\\mathbf{R_{Gl}}\\mathbf{P_l} \\implies \\\\\n" 6017 "\\\\\n" 6018 "\\mathbf{R_{Gl}^{-1}}\\mathbf{P_G} = \\mathbf{I}\\:\\mathbf{P_l} \\implies \\\\\n" 6019 "\\\\\n" 6020 "\\mathbf{P_l} = \\mathbf{R_{Gl}^{-1}}\\mathbf{P_G} = \\mathbf{R_{Gl}^T}\\mathbf{P_G} \\quad \\text{or}\n" 6021 "\\quad \\mathbf{P_l} = \\mathbf{R_{lG}}\\mathbf{P_G}\n" 6022 "\\end{array} $$\n" 6023 "\n" 6024 "Where we referred the inverse of $\\mathbf{R_{Gl}}\\;(\\:\\mathbf{R_{Gl}^{-1}})$ as $\\mathbf{R_{lG}}$ (note the different order of the subscripts). \n" 6025 "\n" 6026 "Let's show this calculation in NumPy:" 6027 )); 6028 6029 qDebug() << "command entry 9"; 6030 testCommandEntry(entry, 9, 1, QString::fromUtf8( 6031 "RGl = np.array([[np.cos(np.pi/4), -np.sin(np.pi/4)], [np.sin(np.pi/4), np.cos(np.pi/4)]])\n" 6032 "print('Rotation matrix (RGl):\\n', np.around(RGl, 4))\n" 6033 "\n" 6034 "Pl = np.array([[1, 1]]).T # transpose the array for correct matrix multiplication\n" 6035 "print('Position at the local coordinate system (Pl):\\n', Pl)\n" 6036 "\n" 6037 "PG = np.dot(RGl, Pl) # the function dot() is used for matrix multiplication with arrays\n" 6038 "print('Position at the Global coordinate system (PG=RGl*Pl):\\n', np.around(PG,2))\n" 6039 "\n" 6040 "Pl = np.dot(RGl.T, PG)\n" 6041 "print('Position at the local coordinate system using the inverse of RGl (Pl=RlG*PG):\\n', Pl)" 6042 )); 6043 testTextResult(entry, 0, QString::fromUtf8( 6044 "Rotation matrix (RGl):\n" 6045 " [[ 0.7071 -0.7071]\n" 6046 " [ 0.7071 0.7071]]\n" 6047 "Position at the local coordinate system (Pl):\n" 6048 " [[1]\n" 6049 " [1]]\n" 6050 "Position at the Global coordinate system (PG=RGl*Pl):\n" 6051 " [[0. ]\n" 6052 " [1.41]]\n" 6053 "Position at the local coordinate system using the inverse of RGl (Pl=RlG*PG):\n" 6054 " [[1.]\n" 6055 " [1.]]" 6056 )); 6057 entry = entry->next(); 6058 6059 testMarkdown(entry, QString::fromUtf8( 6060 "**In summary, some of the properties of the rotation matrix are:** \n" 6061 "1. The columns of the rotation matrix form a basis of (independent) unit vectors (versors) and the rows are also independent versors since the transpose of the rotation matrix is another rotation matrix. \n" 6062 "2. The rotation matrix is orthogonal. There is no linear combination of one of the lines or columns of the matrix that would lead to the other row or column, i.e., the lines and columns of the rotation matrix are independent, orthogonal, to each other (this is property 1 rewritten). Because each row and column have norm equal to one, this matrix is also sometimes said to be orthonormal. \n" 6063 "3. The determinant of the rotation matrix is equal to one (or equal to -1 if a left-hand coordinate system was used, but you should rarely use that). For instance, the determinant of the rotation matrix we deduced is $cos\\alpha cos\\alpha - sin\\alpha(-sin\\alpha)=1$.\n" 6064 "4. The inverse of the rotation matrix is equals to its transpose.\n" 6065 "\n" 6066 "**On the different meanings of the rotation matrix:** \n" 6067 "- It represents the coordinate transformation between the coordinates of a point expressed in two different coordinate systems. \n" 6068 "- It describes the rotation between two coordinate systems. The columns are the direction cosines (versors) of the axes of the rotated coordinate system in relation to the other coordinate system and the rows are also direction cosines (versors) for the inverse rotation. \n" 6069 "- It is an operator for the calculation of the rotation of a vector in a coordinate system.\n" 6070 "- Rotation matrices provide a means of numerically representing rotations without appealing to angular specification.\n" 6071 "\n" 6072 "**Which matrix to use, from local to Global or Global to local?** \n" 6073 "- A typical use of the transformation is in movement analysis, where there are the fixed Global (laboratory) coordinate system and the local (moving, e.g. anatomical) coordinate system attached to each body segment. Because the movement of the body segment is measured in the Global coordinate system, using cameras for example, and we want to reconstruct the coordinates of the markers at the anatomical coordinate system, we want the transformation leading from the Global coordinate system to the local coordinate system.\n" 6074 "- Of course, if you have one matrix, it is simple to get the other; you just have to pay attention to use the right one." 6075 )); 6076 6077 testMarkdown(entry, QString::fromUtf8( 6078 "## Translation and rotation\n" 6079 "\n" 6080 "Consider now the case where the local coordinate system is translated and rotated in relation to the Global coordinate system and a point is described in both coordinate systems as illustrated in the figure below (once again, remember that this is equivalent to describing a translation and a rotation between two rigid bodies). \n" 6081 "<br>\n" 6082 "<figure><img src='./../images/transrot2D.png' alt='translation and rotation 2D'/> <figcaption><center><i>Figure. A point in two-dimensional space represented in two coordinate systems, with one system translated and rotated.</i></center></figcaption> </figure>\n" 6083 "\n" 6084 "The position of point $\\mathbf{P}$ originally described in the local coordinate system, but now described in the Global coordinate system in vector form is:\n" 6085 "\n" 6086 "$$ \\mathbf{P_G} = \\mathbf{L_G} + \\mathbf{R_{Gl}}\\mathbf{P_l} $$\n" 6087 "\n" 6088 "And in matrix form:\n" 6089 "\n" 6090 "$$ \\begin{bmatrix}\n" 6091 "\\mathbf{P_X} \\\\\n" 6092 "\\mathbf{P_Y} \n" 6093 "\\end{bmatrix} =\n" 6094 "\\begin{bmatrix} \\mathbf{L_{X}} \\\\\\ \\mathbf{L_{Y}} \\end{bmatrix} + \n" 6095 "\\begin{bmatrix}\n" 6096 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 6097 "\\sin\\alpha & \\cos\\alpha \n" 6098 "\\end{bmatrix} \\begin{bmatrix}\n" 6099 "\\mathbf{P}_x \\\\\n" 6100 "\\mathbf{P}_y \n" 6101 "\\end{bmatrix} $$\n" 6102 "\n" 6103 "This means that we first *disrotate* the local coordinate system and then correct for the translation between the two coordinate systems. Note that we can't invert this order: the point position is expressed in the local coordinate system and we can't add this vector to another vector expressed in the Global coordinate system, first we have to convert the vectors to the same coordinate system.\n" 6104 "\n" 6105 "If now we want to find the position of a point at the local coordinate system given its position in the Global coordinate system, the rotation matrix and the translation vector, we have to invert the expression above:\n" 6106 "\n" 6107 "$$ \\begin{array}{l l}\n" 6108 "\\mathbf{P_G} = \\mathbf{L_G} + \\mathbf{R_{Gl}}\\mathbf{P_l} \\implies \\\\\n" 6109 "\\\\\n" 6110 "\\mathbf{R_{Gl}^{-1}}(\\mathbf{P_G} - \\mathbf{L_G}) = \\mathbf{R_{Gl}^{-1}}\\mathbf{R_{Gl}}\\mathbf{P_l} \\implies \\\\\n" 6111 "\\\\\n" 6112 "\\mathbf{P_l} = \\mathbf{R_{Gl}^{-1}}\\left(\\mathbf{P_G}-\\mathbf{L_G}\\right) = \\mathbf{R_{Gl}^T}\\left(\\mathbf{P_G}-\\mathbf{L_G}\\right) \\quad \\text{or} \\quad \\mathbf{P_l} = \\mathbf{R_{lG}}\\left(\\mathbf{P_G}-\\mathbf{L_G}\\right) \n" 6113 "\\end{array} $$\n" 6114 "\n" 6115 "The expression above indicates that to perform the inverse operation, to go from the Global to the local coordinate system, we first translate and then rotate the coordinate system." 6116 )); 6117 6118 testMarkdown(entry, QString::fromUtf8( 6119 "### Transformation matrix\n" 6120 "\n" 6121 "It is possible to combine the translation and rotation operations in only one matrix, called the transformation matrix (also referred as homogeneous transformation matrix):\n" 6122 "\n" 6123 "$$ \\begin{bmatrix}\n" 6124 "\\mathbf{P_X} \\\\\n" 6125 "\\mathbf{P_Y} \\\\\n" 6126 "1\n" 6127 "\\end{bmatrix} =\n" 6128 "\\begin{bmatrix}\n" 6129 "\\cos\\alpha & -\\sin\\alpha & \\mathbf{L_{X}} \\\\\n" 6130 "\\sin\\alpha & \\cos\\alpha & \\mathbf{L_{Y}} \\\\\n" 6131 "0 & 0 & 1\n" 6132 "\\end{bmatrix} \\begin{bmatrix}\n" 6133 "\\mathbf{P}_x \\\\\n" 6134 "\\mathbf{P}_y \\\\\n" 6135 "1\n" 6136 "\\end{bmatrix} $$\n" 6137 "\n" 6138 "Or simply:\n" 6139 "\n" 6140 "$$ \\mathbf{P_G} = \\mathbf{T_{Gl}}\\mathbf{P_l} $$\n" 6141 "\n" 6142 "The inverse operation, to express the position at the local coordinate system in terms of the Global coordinate system, is:\n" 6143 "\n" 6144 "$$ \\mathbf{P_l} = \\mathbf{T_{Gl}^{-1}}\\mathbf{P_G} $$\n" 6145 "\n" 6146 "However, because $\\mathbf{T_{Gl}}$ is not orthonormal when there is a translation, its inverse is not its transpose. Its inverse in matrix form is given by:\n" 6147 "\n" 6148 "$$ \\begin{bmatrix}\n" 6149 "\\mathbf{P}_x \\\\\n" 6150 "\\mathbf{P}_y \\\\\n" 6151 "1\n" 6152 "\\end{bmatrix} =\n" 6153 "\\begin{bmatrix}\n" 6154 "\\mathbf{R^{-1}_{Gl}} & \\cdot & - \\mathbf{R^{-1}_{Gl}}\\mathbf{L_{G}} \\\\\n" 6155 "\\cdot & \\cdot & \\cdot \\\\\n" 6156 "0 & 0 & 1\n" 6157 "\\end{bmatrix} \\begin{bmatrix}\n" 6158 "\\mathbf{P_X} \\\\\n" 6159 "\\mathbf{P_Y} \\\\\n" 6160 "1\n" 6161 "\\end{bmatrix} $$" 6162 )); 6163 6164 testMarkdown(entry, QString::fromUtf8( 6165 "### Calculation of a basis\n" 6166 "\n" 6167 "A typical scenario in motion analysis is to calculate the rotation matrix using the position of markers placed on the moving rigid body. With the markers' positions, we create a local basis, which by definition is the rotation matrix for the rigid body with respect to the Global (laboratory) coordinate system. To define a coordinate system using a basÃs, we also will need to define an origin. \n" 6168 "\n" 6169 "Let's see how to calculate a basis given the markers' positions. \n" 6170 "Consider the markers at m1=[1,1]`, m2=[1,2] and m3=[-1,1] measured in the Global coordinate system as illustrated in the figure below: \n" 6171 "<br>\n" 6172 "<figure><img src='./../images/transrot2Db.png' alt='translation and rotation 2D'/> <figcaption><center><i>Figure. Three points in the two-dimensional space, two possible vectors given these points, and the corresponding basis.</i></center></figcaption> </figure>\n" 6173 "\n" 6174 "A possible local coordinate system with origin at the position of m1 is also illustrated in the figure above. Intentionally, the three markers were chosen to form orthogonal vectors. \n" 6175 "The translation vector between the two coordinate system is:\n" 6176 "\n" 6177 "$$\\mathbf{L_{Gl}} = m_1 - [0,0] = [1,1]$$\n" 6178 "\n" 6179 "The vectors expressing the axes of the local coordinate system are:\n" 6180 "\n" 6181 "$$ x = m_2 - m_1 = [1,2] - [1,1] = [0,1] $$\n" 6182 "\n" 6183 "$$ y = m_3 - m_1 = [-1,1] - [1,1] = [-2,0] $$\n" 6184 "\n" 6185 "Note that these two vectors do not form a basis yet because they are not unit vectors (in fact, only *y* is not a unit vector). Let's normalize these vectors:\n" 6186 "\n" 6187 "$$ \\begin{array}{}\n" 6188 "e_x = \\frac{x}{||x||} = \\frac{[0,1]}{\\sqrt{0^2+1^2}} = [0,1] \\\\\n" 6189 "\\\\\n" 6190 "e_y = \\frac{y}{||y||} = \\frac{[-2,0]}{\\sqrt{2^2+0^2}} = [-1,0] \n" 6191 "\\end{array} $$\n" 6192 "\n" 6193 "Beware that the versors above are not exactly the same as the ones shown in the right plot of the last figure, the versors above if plotted will start at the origin of the coordinate system, not at [1,1] as shown in the figure.\n" 6194 "\n" 6195 "We could have done this calculation in NumPy (we will need to do that when dealing with real data later):" 6196 )); 6197 6198 qDebug() << "command entry 10"; 6199 testCommandEntry(entry, 10, 1, QString::fromUtf8( 6200 "m1 = np.array([1.,1.]) # marker 1\n" 6201 "m2 = np.array([1.,2.]) # marker 2\n" 6202 "m3 = np.array([-1.,1.]) # marker 3\n" 6203 "\n" 6204 "x = m2 - m1 # vector x\n" 6205 "y = m3 - m1 # vector y\n" 6206 "\n" 6207 "vx = x/np.linalg.norm(x) # versor x\n" 6208 "vy = y/np.linalg.norm(y) # verson y\n" 6209 "\n" 6210 "print(\"x =\", x, \", y =\", y, \"\\nex=\", vx, \", ey=\", vy)" 6211 )); 6212 testTextResult(entry, 0, QString::fromUtf8( 6213 "x = [0. 1.] , y = [-2. 0.] \n" 6214 "ex= [0. 1.] , ey= [-1. 0.]" 6215 )); 6216 entry = entry->next(); 6217 6218 testMarkdown(entry, QString::fromUtf8( 6219 "Now, both $\\mathbf{e}_x$ and $\\mathbf{e}_y$ are unit vectors (versors) and they are orthogonal, a basis can be formed with these two versors, and we can represent the rotation matrix using this basis (just place the versors of this basis as columns of the rotation matrix):\n" 6220 "\n" 6221 "$$ \\mathbf{R_{Gl}} = \\begin{bmatrix}\n" 6222 "0 & -1 \\\\\n" 6223 "1 & 0 \n" 6224 "\\end{bmatrix} $$\n" 6225 "\n" 6226 "This rotation matrix makes sense because from the figure above we see that the local coordinate system we defined is rotated by 90$^o$ in relation to the Global coordinate system and if we use the general form for the rotation matrix:\n" 6227 "\n" 6228 "$$ \\mathbf{R} = \\begin{bmatrix}\n" 6229 "\\cos\\alpha & -\\sin\\alpha \\\\\n" 6230 "\\sin\\alpha & \\cos\\alpha \n" 6231 "\\end{bmatrix} = \n" 6232 "\\begin{bmatrix}\n" 6233 "\\cos90^o & -\\sin90^o \\\\\n" 6234 "\\sin90^o & \\cos90^o \n" 6235 "\\end{bmatrix} =\n" 6236 "\\begin{bmatrix}\n" 6237 "0 & -1 \\\\\n" 6238 "1 & 0 \n" 6239 "\\end{bmatrix} $$\n" 6240 "\n" 6241 "So, the position of any point in the local coordinate system can be represented in the Global coordinate system by:\n" 6242 "\n" 6243 "$$ \\begin{array}{l l}\n" 6244 "\\mathbf{P_G} =& \\mathbf{L_{Gl}} + \\mathbf{R_{Gl}}\\mathbf{P_l} \\\\\n" 6245 "\\\\\n" 6246 "\\mathbf{P_G} =& \\begin{bmatrix} 1 \\\\ 1 \\end{bmatrix} + \\begin{bmatrix} 0 & -1 \\\\ 1 & 0 \\end{bmatrix} \\mathbf{P_l} \n" 6247 "\\end{array} $$\n" 6248 "\n" 6249 "For example, the point $\\mathbf{P_l}=[1,1]$ has the following position at the Global coordinate system:" 6250 )); 6251 6252 qDebug() << "command entry 11"; 6253 testCommandEntry(entry, 11, 1, QString::fromUtf8( 6254 "LGl = np.array([[1, 1]]).T\n" 6255 "print('Translation vector:\\n', LGl)\n" 6256 "\n" 6257 "RGl = np.array([[0, -1], [1, 0]])\n" 6258 "print('Rotation matrix:\\n', RGl)\n" 6259 "\n" 6260 "Pl = np.array([[1, 1]]).T\n" 6261 "print('Position at the local coordinate system:\\n', Pl)\n" 6262 "\n" 6263 "PG = LGl + np.dot(RGl, Pl)\n" 6264 "print('Position at the Global coordinate system, PG = LGl + RGl*Pl:\\n', PG)" 6265 )); 6266 testTextResult(entry, 0, QString::fromUtf8( 6267 "Translation vector:\n" 6268 " [[1]\n" 6269 " [1]]\n" 6270 "Rotation matrix:\n" 6271 " [[ 0 -1]\n" 6272 " [ 1 0]]\n" 6273 "Position at the local coordinate system:\n" 6274 " [[1]\n" 6275 " [1]]\n" 6276 "Position at the Global coordinate system, PG = LGl + RGl*Pl:\n" 6277 " [[0]\n" 6278 " [2]]" 6279 )); 6280 entry = entry->next(); 6281 6282 testMarkdown(entry, QString::fromUtf8( 6283 "### Determination of the unknown angle of rotation\n" 6284 "\n" 6285 "If we didn't know the angle of rotation between the two coordinate systems, which is the typical situation in motion analysis, we simply would equate one of the terms of the two-dimensional rotation matrix in its algebraic form to its correspondent value in the numerical rotation matrix we calculated.\n" 6286 "\n" 6287 "For instance, taking the first term of the rotation matrices above: $\\cos\\alpha = 0$ implies that $\\theta$ is 90$^o$ or 270$^o$, but combining with another matrix term, $\\sin\\alpha = 1$, implies that $\\alpha=90^o$. We can solve this problem in one step using the tangent $(\\sin\\alpha/\\cos\\alpha)$ function with two terms of the rotation matrix and calculating the angle with the `arctan2(y, x)` function:" 6288 )); 6289 6290 qDebug() << "command entry 12"; 6291 testCommandEntry(entry, 12, 1, QString::fromUtf8( 6292 "ang = np.arctan2(RGl[1, 0], RGl[0, 0])*180/np.pi\n" 6293 "print('The angle is:', ang)" 6294 )); 6295 testTextResult(entry, 0, QString::fromUtf8( 6296 "The angle is: 90.0" 6297 )); 6298 entry = entry->next(); 6299 6300 testMarkdown(entry, QString::fromUtf8( 6301 "And this procedure would be repeated for each segment and for each instant of the analyzed movement to find the rotation of each segment." 6302 )); 6303 6304 testMarkdown(entry, QString::fromUtf8( 6305 "#### Joint angle as a sequence of rotations of adjacent segments\n" 6306 "\n" 6307 "In the notebook about [two-dimensional angular kinematics](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/AngularKinematics2D.ipynb), we calculated segment and joint angles using simple trigonometric relations. We can also calculate these two-dimensional angles using what we learned here about the rotation matrix.\n" 6308 "\n" 6309 "The segment angle will be given by the matrix representing the rotation from the laboratory coordinate system (G) to a coordinate system attached to the segment and the joint angle will be given by the matrix representing the rotation from one segment coordinate system (l1) to the other segment coordinate system (l2). So, we have to calculate two basis now, one for each segment and the joint angle will be given by the product between the two rotation matrices. \n" 6310 "\n" 6311 "To define a two-dimensional basis, we need to calculate vectors perpendicular to each of these lines. Here is a way of doing that. First, let's find three non-collinear points for each basis:" 6312 )); 6313 6314 qDebug() << "command entry 13"; 6315 testCommandEntry(entry, 13, QString::fromUtf8( 6316 "x1, y1, x2, y2 = 0, 0, 1, 1 # points at segment 1\n" 6317 "x3, y3, x4, y4 = 1.1, 1, 2.1, 0 # points at segment 2\n" 6318 "\n" 6319 "#The slope of the perpendicular line is minus the inverse of the slope of the line\n" 6320 "xl1 = x1 - (y2-y1); yl1 = y1 + (x2-x1) # point at the perpendicular line 1\n" 6321 "xl2 = x4 - (y3-y4); yl2 = y4 + (x3-x4) # point at the perpendicular line 2" 6322 )); 6323 6324 testMarkdown(entry, QString::fromUtf8( 6325 "With these three points, we can create a basis and the corresponding rotation matrix:" 6326 )); 6327 6328 qDebug() << "command entry 14"; 6329 testCommandEntry(entry, 14, QString::fromUtf8( 6330 "b1x = np.array([x2-x1, y2-y1])\n" 6331 "b1x = b1x/np.linalg.norm(b1x) # versor x of basis 1\n" 6332 "b1y = np.array([xl1-x1, yl1-y1])\n" 6333 "b1y = b1y/np.linalg.norm(b1y) # versor y of basis 1\n" 6334 "b2x = np.array([x3-x4, y3-y4])\n" 6335 "b2x = b2x/np.linalg.norm(b2x) # versor x of basis 2\n" 6336 "b2y = np.array([xl2-x4, yl2-y4])\n" 6337 "b2y = b2y/np.linalg.norm(b2y) # versor y of basis 2\n" 6338 "\n" 6339 "RGl1 = np.array([b1x, b1y]).T # rotation matrix from segment 1 to the laboratory\n" 6340 "RGl2 = np.array([b2x, b2y]).T # rotation matrix from segment 2 to the laboratory" 6341 )); 6342 6343 testMarkdown(entry, QString::fromUtf8( 6344 "Now, the segment and joint angles are simply matrix operations:" 6345 )); 6346 6347 qDebug() << "command entry 15"; 6348 testCommandEntry(entry, 15, 1, QString::fromUtf8( 6349 "print('Rotation matrix for segment 1:\\n', np.around(RGl1, 4))\n" 6350 "print('\\nRotation angle of segment 1:', np.arctan2(RGl1[1,0], RGl1[0,0])*180/np.pi)\n" 6351 "print('\\nRotation matrix for segment 2:\\n', np.around(RGl2, 4))\n" 6352 "print('\\nRotation angle of segment 2:', np.arctan2(RGl1[1,0], RGl2[0,0])*180/np.pi)\n" 6353 "\n" 6354 "Rl1l2 = np.dot(RGl1.T, RGl2) # Rl1l2 = Rl1G*RGl2\n" 6355 "\n" 6356 "print('\\nJoint rotation matrix (Rl1l2 = Rl1G*RGl2):\\n', np.around(Rl1l2, 4))\n" 6357 "print('\\nJoint angle:', np.arctan2(Rl1l2[1,0], Rl1l2[0,0])*180/np.pi)" 6358 )); 6359 testTextResult(entry, 0, QString::fromUtf8( 6360 "Rotation matrix for segment 1:\n" 6361 " [[ 0.7071 -0.7071]\n" 6362 " [ 0.7071 0.7071]]\n" 6363 "\n" 6364 "Rotation angle of segment 1: 45.0\n" 6365 "\n" 6366 "Rotation matrix for segment 2:\n" 6367 " [[-0.7071 -0.7071]\n" 6368 " [ 0.7071 -0.7071]]\n" 6369 "\n" 6370 "Rotation angle of segment 2: 135.0\n" 6371 "\n" 6372 "Joint rotation matrix (Rl1l2 = Rl1G*RGl2):\n" 6373 " [[ 0. -1.]\n" 6374 " [ 1. -0.]]\n" 6375 "\n" 6376 "Joint angle: 90.0" 6377 )); 6378 entry = entry->next(); 6379 6380 testMarkdown(entry, QString::fromUtf8( 6381 "Same result as obtained in [Angular kinematics in a plane (2D)](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/AngularKinematics2D.ipynb). " 6382 )); 6383 6384 testMarkdown(entry, QString::fromUtf8( 6385 "### Kinematic chain in a plain (2D)\n" 6386 "\n" 6387 "The fact that we simply multiplied the rotation matrices to calculate the rotation matrix of one segment in relation to the other is powerful and can be generalized for any number of segments: given a serial kinematic chain with links 1, 2, ..., n and 0 is the base/laboratory, the rotation matrix between the base and last link is: $\\mathbf{R_{n,n-1}R_{n-1,n-2} \\dots R_{2,1}R_{1,0}}$, where each matrix in this product (calculated from right to left) is the rotation of one link with respect to the next one. \n" 6388 "\n" 6389 "For instance, consider a kinematic chain with two links, the link 1 is rotated by $\\alpha_1$ with respect to the base (0) and the link 2 is rotated by $\\alpha_2$ with respect to the link 1. \n" 6390 "Using Sympy, the rotation matrices for link 2 w.r.t. link 1 $(R_{12})$ and for link 1 w.r.t. base 0 $(R_{01})$ are: " 6391 )); 6392 6393 qDebug() << "command entry 16"; 6394 testCommandEntry(entry, 16, QString::fromUtf8( 6395 "from IPython.display import display, Math\n" 6396 "from sympy import sin, cos, Matrix, simplify, latex, symbols\n" 6397 "from sympy.interactive import printing\n" 6398 "printing.init_printing()" 6399 )); 6400 6401 qDebug() << "command entry 17"; 6402 testCommandEntry(entry, 17, 2, QString::fromUtf8( 6403 "a1, a2 = symbols('alpha1 alpha2')\n" 6404 "\n" 6405 "R12 = Matrix([[cos(a2), -sin(a2)], [sin(a2), cos(a2)]])\n" 6406 "display(Math(latex(r'\\mathbf{R_{12}}=') + latex(R12)))\n" 6407 "R01 = Matrix([[cos(a1), -sin(a1)], [sin(a1), cos(a1)]])\n" 6408 "display(Math(latex(r'\\mathbf{R_{01}}=') + latex(R01)))" 6409 )); 6410 { 6411 QCOMPARE(expression(entry)->results()[0]->type(), (int)Cantor::LatexResult::Type); 6412 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->results()[0]); 6413 QCOMPARE(result->code(), QLatin1String( 6414 "$$\\mathbf{R_{12}}=\\left[\\begin{matrix}\\cos{\\left(\\alpha_{2} \\right)} & - \\sin{\\left(\\alpha_{2} \\right)}\\\\\\sin{\\left(\\alpha_{2} \\right)} & \\cos{\\left(\\alpha_{2} \\right)}\\end{matrix}\\right]$$" 6415 )); 6416 QCOMPARE(result->plain(), QLatin1String( 6417 "<IPython.core.display.Math object>" 6418 )); 6419 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 6420 } 6421 { 6422 QCOMPARE(expression(entry)->results()[1]->type(), (int)Cantor::LatexResult::Type); 6423 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->results()[1]); 6424 QCOMPARE(result->code(), QLatin1String( 6425 "$$\\mathbf{R_{01}}=\\left[\\begin{matrix}\\cos{\\left(\\alpha_{1} \\right)} & - \\sin{\\left(\\alpha_{1} \\right)}\\\\\\sin{\\left(\\alpha_{1} \\right)} & \\cos{\\left(\\alpha_{1} \\right)}\\end{matrix}\\right]$$" 6426 )); 6427 QCOMPARE(result->plain(), QLatin1String( 6428 "<IPython.core.display.Math object>" 6429 )); 6430 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 6431 } 6432 entry = entry->next(); 6433 6434 testMarkdown(entry, QString::fromUtf8( 6435 "The rotation matrix of link 2 w.r.t. the base $(R_{02})$ is given simply by $R_{01}*R_{12}$:" 6436 )); 6437 6438 qDebug() << "command entry 18"; 6439 testCommandEntry(entry, 18, 1, QString::fromUtf8( 6440 "R02 = R01*R12\n" 6441 "display(Math(latex(r'\\mathbf{R_{02}}=') + latex(R02)))" 6442 )); 6443 { 6444 QCOMPARE(expression(entry)->results()[0]->type(), (int)Cantor::LatexResult::Type); 6445 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->results()[0]); 6446 QCOMPARE(result->code(), QLatin1String( 6447 "$$\\mathbf{R_{02}}=\\left[\\begin{matrix}- \\sin{\\left(\\alpha_{1} \\right)} \\sin{\\left(\\alpha_{2} \\right)} + \\cos{\\left(\\alpha_{1} \\right)} \\cos{\\left(\\alpha_{2} \\right)} & - \\sin{\\left(\\alpha_{1} \\right)} \\cos{\\left(\\alpha_{2} \\right)} - \\sin{\\left(\\alpha_{2} \\right)} \\cos{\\left(\\alpha_{1} \\right)}\\\\\\sin{\\left(\\alpha_{1} \\right)} \\cos{\\left(\\alpha_{2} \\right)} + \\sin{\\left(\\alpha_{2} \\right)} \\cos{\\left(\\alpha_{1} \\right)} & - \\sin{\\left(\\alpha_{1} \\right)} \\sin{\\left(\\alpha_{2} \\right)} + \\cos{\\left(\\alpha_{1} \\right)} \\cos{\\left(\\alpha_{2} \\right)}\\end{matrix}\\right]$$" 6448 )); 6449 QCOMPARE(result->plain(), QLatin1String( 6450 "<IPython.core.display.Math object>" 6451 )); 6452 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 6453 } 6454 entry = entry->next(); 6455 6456 testMarkdown(entry, QString::fromUtf8( 6457 "Which simplifies to:" 6458 )); 6459 6460 qDebug() << "command entry 19"; 6461 testCommandEntry(entry, 19, 1, QString::fromUtf8( 6462 "display(Math(latex(r'\\mathbf{R_{02}}=') + latex(simplify(R02))))" 6463 )); 6464 { 6465 QCOMPARE(expression(entry)->results()[0]->type(), (int)Cantor::LatexResult::Type); 6466 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->results()[0]); 6467 QCOMPARE(result->code(), QLatin1String( 6468 "$$\\mathbf{R_{02}}=\\left[\\begin{matrix}\\cos{\\left(\\alpha_{1} + \\alpha_{2} \\right)} & - \\sin{\\left(\\alpha_{1} + \\alpha_{2} \\right)}\\\\\\sin{\\left(\\alpha_{1} + \\alpha_{2} \\right)} & \\cos{\\left(\\alpha_{1} + \\alpha_{2} \\right)}\\end{matrix}\\right]$$" 6469 )); 6470 QCOMPARE(result->plain(), QLatin1String( 6471 "<IPython.core.display.Math object>" 6472 )); 6473 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 6474 } 6475 entry = entry->next(); 6476 6477 testMarkdown(entry, QString::fromUtf8( 6478 "As expected.\n" 6479 "\n" 6480 "The typical use of all these concepts is in the three-dimensional motion analysis where we will have to deal with angles in different planes, which needs a special manipulation as we will see next." 6481 )); 6482 6483 testMarkdown(entry, QString::fromUtf8( 6484 "## Problems\n" 6485 "\n" 6486 "1. A local coordinate system is rotated 30$^o$ clockwise in relation to the Global reference system. \n" 6487 " A. Determine the matrices for rotating one coordinate system to another (two-dimensional). \n" 6488 " B. What are the coordinates of the point [1, 1] (local coordinate system) at the global coordinate system? \n" 6489 " C. And if this point is at the Global coordinate system and we want the coordinates at the local coordinate system? \n" 6490 " D. Consider that the local coordinate system, besides the rotation is also translated by [2, 2]. What are the matrices for rotation, translation, and transformation from one coordinate system to another (two-dimensional)? \n" 6491 " E. Repeat B and C considering this translation.\n" 6492 " \n" 6493 "2. Consider a local coordinate system U rotated 45$^o$ clockwise in relation to the Global reference system and another local coordinate system V rotated 45$^o$ clockwise in relation to the local reference system U. \n" 6494 " A. Determine the rotation matrices of all possible transformations between the coordinate systems. \n" 6495 " B. For the point [1, 1] in the coordinate system U, what are its coordinates in coordinate system V and in the Global coordinate system? \n" 6496 " \n" 6497 "3. Using the rotation matrix, deduce the new coordinates of a square figure with coordinates [0, 0], [1, 0], [1, 1], and [0, 1] when rotated by 0$^o$, 45$^o$, 90$^o$, 135$^o$, and 180$^o$ (always clockwise).\n" 6498 " \n" 6499 "4. Solve the problem 2 of [Angular kinematics in a plane (2D)](http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/AngularKinematics2D.ipynb) but now using the concept of two-dimensional transformations. " 6500 )); 6501 6502 testMarkdown(entry, QString::fromUtf8( 6503 "## References\n" 6504 "\n" 6505 "- Robertson G, Caldwell G, Hamill J, Kamen G (2013) [Research Methods in Biomechanics](http://books.google.com.br/books?id=gRn8AAAAQBAJ). 2nd Edition. Human Kinetics. \n" 6506 "- Ruina A, Rudra P (2013) [Introduction to Statics and Dynamics](http://ruina.tam.cornell.edu/Book/index.html). Oxford University Press. \n" 6507 "- Winter DA (2009) [Biomechanics and motor control of human movement](http://books.google.com.br/books?id=_bFHL08IWfwC). 4 ed. Hoboken, EUA: Wiley. \n" 6508 "- Zatsiorsky VM (1997) [Kinematics of Human Motion](http://books.google.com.br/books/about/Kinematics_of_Human_Motion.html?id=Pql_xXdbrMcC&redir_esc=y). Champaign, Human Kinetics." 6509 )); 6510 6511 QCOMPARE(entry, nullptr); 6512 } 6513 6514 void WorksheetTest::testMarkdownAttachment() 6515 { 6516 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6517 if (backend && backend->isEnabled() == false) 6518 QSKIP("Skip, because python backend don't available", SkipSingle); 6519 6520 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestMarkdownAttachment.ipynb"))); 6521 6522 QCOMPARE(w->isReadOnly(), false); 6523 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6524 6525 WorksheetEntry* entry = w->firstEntry(); 6526 6527 testCommandEntry(entry, 1, 1, QString::fromUtf8( 6528 "2+2" 6529 )); 6530 testTextResult(entry, 0, QString::fromLatin1( 6531 "4" 6532 )); 6533 entry = entry->next(); 6534 6535 // Tests attachments via toJupyterJson: ugly, but works 6536 QVERIFY(entry); 6537 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 6538 QCOMPARE(plainMarkdown(entry), QString::fromLatin1( 6539 "![CantorLogo.png](attachment:CantorLogo.png)\n" 6540 "![CantorLogo.png](attachment:CantorLogo.png)" 6541 )); 6542 QJsonValue value = entry->toJupyterJson(); 6543 QVERIFY(value.isObject()); 6544 QVERIFY(value.toObject().contains(QLatin1String("attachments"))); 6545 entry = entry->next(); 6546 6547 QVERIFY(entry); 6548 QCOMPARE(entry->type(), (int)MarkdownEntry::Type); 6549 QCOMPARE(plainMarkdown(entry), QString::fromLatin1( 6550 "![CantorLogo.png](attachment:CantorLogo.png)" 6551 )); 6552 value = entry->toJupyterJson(); 6553 QVERIFY(value.isObject()); 6554 QVERIFY(value.toObject().contains(QLatin1String("attachments"))); 6555 entry = entry->next(); 6556 6557 testCommandEntry(entry, -1, QString::fromUtf8( 6558 "" 6559 )); 6560 6561 QCOMPARE(entry, nullptr); 6562 } 6563 6564 void WorksheetTest::testEntryLoad1() 6565 { 6566 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6567 if (backend && backend->isEnabled() == false) 6568 QSKIP("Skip, because python backend don't available", SkipSingle); 6569 6570 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestEntryLoad1.ipynb"))); 6571 6572 QCOMPARE(w->isReadOnly(), false); 6573 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6574 6575 WorksheetEntry* entry = w->firstEntry(); 6576 6577 testCommandEntry(entry, 2, 1, QString::fromUtf8( 6578 "2+2" 6579 )); 6580 testTextResult(entry, 0, QString::fromLatin1( 6581 "4" 6582 )); 6583 entry = entry->next(); 6584 6585 testLatexEntry(entry, QString::fromLatin1( 6586 "$$\\Gamma$$" 6587 )); 6588 6589 testMarkdown(entry, QString::fromLatin1( 6590 "### Test Entry" 6591 )); 6592 6593 testMarkdown(entry, QString::fromLatin1( 6594 "Text" 6595 )); 6596 6597 QCOMPARE(entry, nullptr); 6598 } 6599 6600 void WorksheetTest::testEntryLoad2() 6601 { 6602 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6603 if (backend && backend->isEnabled() == false) 6604 QSKIP("Skip, because python backend don't available", SkipSingle); 6605 6606 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestEntryLoad2.ipynb"))); 6607 6608 QCOMPARE(w->isReadOnly(), false); 6609 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6610 6611 WorksheetEntry* entry = w->firstEntry(); 6612 6613 testCommandEntry(entry, 0, 1, QString::fromUtf8( 6614 "2+2" 6615 )); 6616 testTextResult(entry, 0, QString::fromLatin1( 6617 "ans = 4" 6618 )); 6619 entry = entry->next(); 6620 6621 testTextEntry(entry, QString::fromLatin1( 6622 "Text entry" 6623 )); 6624 6625 testMarkdown(entry, QString::fromLatin1( 6626 "#### Markdown entry" 6627 )); 6628 6629 testLatexEntry(entry, QString::fromLatin1( 6630 "\\LaTeX\\ entry" 6631 )); 6632 6633 QCOMPARE(entry, nullptr); 6634 } 6635 6636 void WorksheetTest::testResultsLoad() 6637 { 6638 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("sage")); 6639 if (backend && backend->isEnabled() == false) 6640 QSKIP("Skip, because sage backend don't available", SkipSingle); 6641 6642 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestResultsLoad.ipynb"))); 6643 6644 QCOMPARE(w->isReadOnly(), false); 6645 QCOMPARE(w->session()->backend()->id(), QLatin1String("sage")); 6646 6647 WorksheetEntry* entry = w->firstEntry(); 6648 6649 testCommandEntry(entry, 9, QString::fromUtf8( 6650 "from IPython.display import Latex" 6651 )); 6652 6653 testCommandEntry(entry, 16, 1, QString::fromUtf8( 6654 "print(\"Hello world\")" 6655 )); 6656 testTextResult(entry, 0, QString::fromUtf8( 6657 "Hello world" 6658 )); 6659 entry = entry->next(); 6660 6661 testCommandEntry(entry, 17, 1, QString::fromUtf8( 6662 "plot(x^2, (x,0,5))" 6663 )); 6664 testImageResult(entry, 0); 6665 entry = entry->next(); 6666 6667 testCommandEntry(entry, 6, 1, QString::fromUtf8( 6668 "sines = [plot(c*sin(x), (-2*pi,2*pi), color=Color(c,0,0), ymin=-1, ymax=1) for c in sxrange(0,1,.05)]\n" 6669 "a = animate(sines)\n" 6670 "a.show()" 6671 )); 6672 QVERIFY(expression(entry)); 6673 QCOMPARE(expression(entry)->results().at(0)->type(), (int)Cantor::AnimationResult::Type); 6674 QVERIFY(static_cast<Cantor::AnimationResult*>(expression(entry)->results().at(0))->url().isValid()); 6675 entry = entry->next(); 6676 6677 testCommandEntry(entry, 15, 1, QString::fromUtf8( 6678 "Latex(\"$$\\Gamma$$\")" 6679 )); 6680 QCOMPARE(expression(entry)->result()->type(), (int)Cantor::LatexResult::Type); 6681 { 6682 Cantor::LatexResult* result = static_cast<Cantor::LatexResult*>(expression(entry)->result()); 6683 QCOMPARE(result->code(), QLatin1String( 6684 "$$\\Gamma$$" 6685 )); 6686 QCOMPARE(result->plain(), QLatin1String( 6687 "<IPython.core.display.Latex object>" 6688 )); 6689 QCOMPARE(result->mimeType(), QStringLiteral("image/x-eps")); 6690 } 6691 entry = entry->next(); 6692 6693 QCOMPARE(entry, nullptr); 6694 } 6695 6696 void WorksheetTest::testMimeResult() 6697 { 6698 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6699 if (backend && backend->isEnabled() == false) 6700 QSKIP("Skip, because python backend don't available", SkipSingle); 6701 6702 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestNotebookWithJson.ipynb"))); 6703 6704 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6705 6706 WorksheetEntry* entry = w->firstEntry(); 6707 6708 testCommandEntry(entry, 6, QString::fromUtf8( 6709 "import json\n" 6710 "import uuid\n" 6711 "from IPython.display import display_javascript, display_html, display\n" 6712 "\n" 6713 "class RenderJSON(object):\n" 6714 " def __init__(self, json_data):\n" 6715 " if isinstance(json_data, dict) or isinstance(json_data, list):\n" 6716 " self.json_str = json.dumps(json_data)\n" 6717 " else:\n" 6718 " self.json_str = json_data\n" 6719 " self.uuid = str(uuid.uuid4())\n" 6720 "\n" 6721 " def _ipython_display_(self):\n" 6722 " display_html('<div id=\"{}\" style=\"height: 600px; width:100%;font: 12px/18px monospace !important;\"></div>'.format(self.uuid), raw=True)\n" 6723 " display_javascript(\"\"\"\n" 6724 " require([\"https://rawgit.com/caldwell/renderjson/master/renderjson.js\"], function() {\n" 6725 " renderjson.set_show_to_level(2);\n" 6726 " document.getElementById('%s').appendChild(renderjson(%s))\n" 6727 " });\n" 6728 " \"\"\" % (self.uuid, self.json_str), raw=True)" 6729 )); 6730 6731 testCommandEntry(entry, 7, 2, QString::fromUtf8( 6732 "RenderJSON([\n" 6733 " {\n" 6734 " \"a\": 1\n" 6735 " }, \n" 6736 " {\n" 6737 " \"b\": 2,\n" 6738 " \"in1\": {\n" 6739 " \"key\": \"value\"\n" 6740 " }\n" 6741 " }\n" 6742 "])" 6743 )); 6744 testHtmlResult(entry, 0, QString::fromLatin1( 6745 "" 6746 ), QString::fromLatin1( 6747 "<div id=\"bb6d9031-c990-4aee-849e-6d697430777c\" style=\"height: 600px; width:100%;font: 12px/18px monospace !important;\"></div>" 6748 )); 6749 { 6750 QVERIFY(expression(entry)->results().size() > 1); 6751 QCOMPARE(expression(entry)->results().at(1)->type(), (int)Cantor::MimeResult::Type); 6752 Cantor::MimeResult* result = static_cast<Cantor::MimeResult*>(expression(entry)->results().at(1)); 6753 QJsonObject mimeData = result->data().value<QJsonObject>(); 6754 QStringList mimeKeys = mimeData.keys(); 6755 QCOMPARE(mimeKeys.size(), 1); 6756 QVERIFY(mimeKeys.contains(QLatin1String("application/javascript"))); 6757 QJsonArray value = QJsonArray::fromStringList(QStringList{ 6758 QLatin1String("\n"), 6759 QLatin1String(" require([\"https://rawgit.com/caldwell/renderjson/master/renderjson.js\"], function() {\n"), 6760 QLatin1String(" renderjson.set_show_to_level(2);\n"), 6761 QLatin1String(" document.getElementById('bb6d9031-c990-4aee-849e-6d697430777c').appendChild(renderjson([{\"a\": 1}, {\"b\": 2, \"in1\": {\"key\": \"value\"}}]))\n"), 6762 QLatin1String(" });\n"), 6763 QLatin1String(" ") 6764 }); 6765 QCOMPARE(mimeData[QLatin1String("application/javascript")].toArray(), value); 6766 } 6767 entry = entry->next(); 6768 6769 testCommandEntry(entry, -1, QString::fromUtf8( 6770 "" 6771 )); 6772 6773 QCOMPARE(entry, nullptr); 6774 } 6775 6776 void WorksheetTest::testMimeResultWithPlain() 6777 { 6778 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6779 if (backend && backend->isEnabled() == false) 6780 QSKIP("Skip, because python backend don't available", SkipSingle); 6781 6782 QScopedPointer<Worksheet> w(loadWorksheet(QLatin1String("TestNotebookWithModJson.ipynb"))); 6783 6784 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6785 6786 WorksheetEntry* entry = w->firstEntry(); 6787 6788 testCommandEntry(entry, 6, QString::fromUtf8( 6789 "import json\n" 6790 "import uuid\n" 6791 "from IPython.display import display_javascript, display_html, display\n" 6792 "\n" 6793 "class RenderJSON(object):\n" 6794 " def __init__(self, json_data):\n" 6795 " if isinstance(json_data, dict) or isinstance(json_data, list):\n" 6796 " self.json_str = json.dumps(json_data)\n" 6797 " else:\n" 6798 " self.json_str = json_data\n" 6799 " self.uuid = str(uuid.uuid4())\n" 6800 "\n" 6801 " def _ipython_display_(self):\n" 6802 " display_html('<div id=\"{}\" style=\"height: 600px; width:100%;font: 12px/18px monospace !important;\"></div>'.format(self.uuid), raw=True)\n" 6803 " display_javascript(\"\"\"\n" 6804 " require([\"https://rawgit.com/caldwell/renderjson/master/renderjson.js\"], function() {\n" 6805 " renderjson.set_show_to_level(2);\n" 6806 " document.getElementById('%s').appendChild(renderjson(%s))\n" 6807 " });\n" 6808 " \"\"\" % (self.uuid, self.json_str), raw=True)" 6809 )); 6810 6811 testCommandEntry(entry, 7, 2, QString::fromUtf8( 6812 "RenderJSON([\n" 6813 " {\n" 6814 " \"a\": 1\n" 6815 " }, \n" 6816 " {\n" 6817 " \"b\": 2,\n" 6818 " \"in1\": {\n" 6819 " \"key\": \"value\"\n" 6820 " }\n" 6821 " }\n" 6822 "])" 6823 )); 6824 testHtmlResult(entry, 0, QString::fromLatin1( 6825 "" 6826 ), QString::fromLatin1( 6827 "<div id=\"bb6d9031-c990-4aee-849e-6d697430777c\" style=\"height: 600px; width:100%;font: 12px/18px monospace !important;\"></div>" 6828 )); 6829 { 6830 QVERIFY(expression(entry)->results().size() > 1); 6831 QCOMPARE(expression(entry)->results().at(1)->type(), (int)Cantor::MimeResult::Type); 6832 Cantor::MimeResult* result = static_cast<Cantor::MimeResult*>(expression(entry)->results().at(1)); 6833 QJsonObject mimeData = result->data().value<QJsonObject>(); 6834 QStringList mimeKeys = mimeData.keys(); 6835 QCOMPARE(mimeKeys.size(), 2); 6836 QVERIFY(mimeKeys.contains(QLatin1String("application/javascript"))); 6837 QVERIFY(mimeKeys.contains(QLatin1String("text/plain"))); 6838 QCOMPARE(mimeData[QLatin1String("text/plain")].toString(), QLatin1String("")); 6839 QJsonArray value = QJsonArray::fromStringList(QStringList{ 6840 QLatin1String("\n"), 6841 QLatin1String(" require([\"https://rawgit.com/caldwell/renderjson/master/renderjson.js\"], function() {\n"), 6842 QLatin1String(" renderjson.set_show_to_level(2);\n"), 6843 QLatin1String(" document.getElementById('bb6d9031-c990-4aee-849e-6d697430777c').appendChild(renderjson([{\"a\": 1}, {\"b\": 2, \"in1\": {\"key\": \"value\"}}]))\n"), 6844 QLatin1String(" });\n"), 6845 QLatin1String(" ") 6846 }); 6847 QCOMPARE(mimeData[QLatin1String("application/javascript")].toArray(), value); 6848 } 6849 entry = entry->next(); 6850 6851 testCommandEntry(entry, -1, QString::fromUtf8( 6852 "" 6853 )); 6854 6855 QCOMPARE(entry, nullptr); 6856 } 6857 6858 void WorksheetTest::testCommandEntryExecutionAction1() 6859 { 6860 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6861 if (backend && backend->isEnabled() == false) 6862 QSKIP("Skip, because python backend don't available", SkipSingle); 6863 6864 QScopedPointer<Worksheet> w(std::move(loadWorksheet(QLatin1String("EmptyPythonWorksheet.cws")))); 6865 6866 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6867 6868 CommandEntry* entry = static_cast<CommandEntry*>(WorksheetEntry::create(CommandEntry::Type, w.data())); 6869 entry->setContent(QLatin1String("2+2")); 6870 entry->evaluate(); 6871 waitForSignal(entry->expression(), SIGNAL(gotResult())); 6872 QCOMPARE(entry->isExcludedFromExecution(), false); 6873 6874 testCommandEntry(entry, 0, 1, QLatin1String("2+2")); 6875 testTextResult(entry, 0, QLatin1String("4")); 6876 6877 entry->setContent(QLatin1String("8**2")); 6878 entry->excludeFromExecution(); 6879 entry->evaluate(); 6880 6881 testCommandEntry(entry, 0, 1, QLatin1String("8**2")); 6882 testTextResult(entry, 0, QLatin1String("4")); 6883 QCOMPARE(entry->isExcludedFromExecution(), true); 6884 6885 entry->addToExecution(); 6886 entry->evaluate(); 6887 waitForSignal(entry->expression(), SIGNAL(gotResult())); 6888 6889 testCommandEntry(entry, 1, 1, QLatin1String("8**2")); 6890 testTextResult(entry, 0, QLatin1String("64")); 6891 QCOMPARE(entry->isExcludedFromExecution(), false); 6892 } 6893 6894 void WorksheetTest::testCommandEntryExecutionAction2() 6895 { 6896 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6897 if (backend && backend->isEnabled() == false) 6898 QSKIP("Skip, because python backend don't available", SkipSingle); 6899 6900 QScopedPointer<Worksheet> w(std::move(loadWorksheet(QLatin1String("TestCommandEntryExecutionAction.cws")))); 6901 6902 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6903 6904 testCommandEntry(w->firstEntry(), -1, 1, QLatin1String("2+2")); 6905 testTextResult(w->firstEntry(), 0, QLatin1String("4")); 6906 6907 CommandEntry* entry = static_cast<CommandEntry*>(w->firstEntry()); 6908 QCOMPARE(entry->isExcludedFromExecution(), true); 6909 } 6910 6911 void WorksheetTest::testCollapsingAllResultsAction() 6912 { 6913 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6914 if (backend && backend->isEnabled() == false) 6915 QSKIP("Skip, because python backend don't available", SkipSingle); 6916 6917 QScopedPointer<Worksheet> w(std::move(loadWorksheet(QLatin1String("TwoCommandEntryWithResults.cws")))); 6918 6919 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6920 6921 WorksheetEntry* entry = w->firstEntry(); 6922 6923 testCommandEntry(entry, -1, 1, QLatin1String("2**2")); 6924 testTextResult(entry, 0, QLatin1String("4")); 6925 entry = entry->next(); 6926 6927 testCommandEntry(entry, -1, 1, QLatin1String("2**3")); 6928 testTextResult(entry, 0, QLatin1String("8")); 6929 entry = entry->next(); 6930 6931 QCOMPARE(entry, nullptr); 6932 6933 w->collapseAllResults(); 6934 6935 CommandEntry* comEntry = static_cast<CommandEntry*>(w->firstEntry()); 6936 QCOMPARE(comEntry->isResultCollapsed(), true); 6937 comEntry = static_cast<CommandEntry*>(comEntry->next()); 6938 QCOMPARE(comEntry->isResultCollapsed(), true); 6939 comEntry = static_cast<CommandEntry*>(comEntry->next()); 6940 QCOMPARE(entry, nullptr); 6941 6942 w->uncollapseAllResults(); 6943 comEntry = static_cast<CommandEntry*>(w->firstEntry()); 6944 QCOMPARE(comEntry->isResultCollapsed(), false); 6945 comEntry = static_cast<CommandEntry*>(comEntry->next()); 6946 QCOMPARE(comEntry->isResultCollapsed(), false); 6947 comEntry = static_cast<CommandEntry*>(comEntry->next()); 6948 QCOMPARE(entry, nullptr); 6949 } 6950 6951 void WorksheetTest::testRemovingAllResultsAction() 6952 { 6953 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6954 if (backend && backend->isEnabled() == false) 6955 QSKIP("Skip, because python backend don't available", SkipSingle); 6956 6957 QScopedPointer<Worksheet> w(std::move(loadWorksheet(QLatin1String("TwoCommandEntryWithResults.cws")))); 6958 6959 QCOMPARE(w->session()->backend()->id(), QLatin1String("python")); 6960 6961 WorksheetEntry* entry = w->firstEntry(); 6962 6963 testCommandEntry(entry, -1, 1, QLatin1String("2**2")); 6964 testTextResult(entry, 0, QLatin1String("4")); 6965 entry = entry->next(); 6966 6967 testCommandEntry(entry, -1, 1, QLatin1String("2**3")); 6968 testTextResult(entry, 0, QLatin1String("8")); 6969 entry = entry->next(); 6970 6971 QCOMPARE(entry, nullptr); 6972 6973 w->removeAllResults(); 6974 6975 entry = w->firstEntry(); 6976 6977 testCommandEntry(entry, -1, 0, QLatin1String("2**2")); 6978 entry = entry->next(); 6979 testCommandEntry(entry, -1, 0, QLatin1String("2**3")); 6980 entry = entry->next(); 6981 6982 QCOMPARE(entry, nullptr); 6983 } 6984 6985 void WorksheetTest::testMathRender() 6986 { 6987 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 6988 if (backend && backend->isEnabled() == false) 6989 QSKIP("Skip, because python backend don't available", SkipSingle); 6990 6991 Worksheet* w = new Worksheet(Cantor::Backend::getBackend(QLatin1String("python")), nullptr); 6992 WorksheetView v(w, nullptr); 6993 v.setEnabled(false); 6994 w->enableEmbeddedMath(true); 6995 6996 if (!w->mathRenderer()->mathRenderAvailable()) 6997 QSKIP("This test needs workable embedded math (pdflatex)", SkipSingle); 6998 6999 MarkdownEntry* entry = static_cast<MarkdownEntry*>(WorksheetEntry::create(MarkdownEntry::Type, w)); 7000 entry->setContent(QLatin1String("$$12$$")); 7001 entry->evaluate(WorksheetEntry::InternalEvaluation); 7002 7003 // Give 1 second to math renderer 7004 QTest::qWait(1000); 7005 7006 QDomDocument doc; 7007 QBuffer buffer; 7008 KZip archive(&buffer); 7009 QDomElement elem = entry->toXml(doc, &archive); 7010 7011 QDomNodeList list = elem.elementsByTagName(QLatin1String("EmbeddedMath")); 7012 QCOMPARE(list.count(), 1); 7013 QDomElement mathNode = list.at(0).toElement(); 7014 bool rendered = mathNode.attribute(QStringLiteral("rendered")).toInt(); 7015 QCOMPARE(rendered, true); 7016 } 7017 7018 void WorksheetTest::testMathRender2() 7019 { 7020 Cantor::Backend* backend = Cantor::Backend::getBackend(QLatin1String("python")); 7021 if (backend && backend->isEnabled() == false) 7022 QSKIP("Skip, because python backend don't available", SkipSingle); 7023 7024 Worksheet* w = new Worksheet(Cantor::Backend::getBackend(QLatin1String("python")), nullptr); 7025 WorksheetView v(w, nullptr); 7026 v.setEnabled(false); 7027 w->enableEmbeddedMath(true); 7028 7029 if (!w->mathRenderer()->mathRenderAvailable()) 7030 QSKIP("This test needs workable embedded math (pdflatex)", SkipSingle); 7031 7032 MarkdownEntry* entry = static_cast<MarkdownEntry*>(WorksheetEntry::create(MarkdownEntry::Type, w)); 7033 entry->setContent(QLatin1String("2 $12$ 4")); 7034 entry->evaluate(WorksheetEntry::InternalEvaluation); 7035 7036 // Give 1 second to math renderer 7037 QTest::qWait(1000); 7038 7039 QDomDocument doc; 7040 QBuffer buffer; 7041 KZip archive(&buffer); 7042 QDomElement elem = entry->toXml(doc, &archive); 7043 7044 QDomNodeList list = elem.elementsByTagName(QLatin1String("EmbeddedMath")); 7045 QCOMPARE(list.count(), 1); 7046 QDomElement mathNode = list.at(0).toElement(); 7047 bool rendered = mathNode.attribute(QStringLiteral("rendered")).toInt(); 7048 QCOMPARE(rendered, true); 7049 QCOMPARE(mathNode.text(), QLatin1String("$12$")); 7050 } 7051 7052 QTEST_MAIN( WorksheetTest )