mycroft-gui/import/mycroftcontroller.cpp

0001 /*
0002  * Copyright 2018 by Marco Martin <mart@kde.org>
0003  * Copyright 2018 David Edmundson <davidedmundson@kde.org>
0004  *
0005  * Licensed under the Apache License, Version 2.0 (the "License");
0006  * you may not use this file except in compliance with the License.
0007  * You may obtain a copy of the License at
0008  *
0009  *    http://www.apache.org/licenses/LICENSE-2.0
0010  *
0011  * Unless required by applicable law or agreed to in writing, software
0012  * distributed under the License is distributed on an "AS IS" BASIS,
0013  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
0014  * See the License for the specific language governing permissions and
0015  * limitations under the License.
0016  *
0017  */
0018
0019 #include "mycroftcontroller.h"
0020 #include "globalsettings.h"
0021 #include "abstractdelegate.h"
0022 #include "activeskillsmodel.h"
0023 #include "abstractskillview.h"
0024 #include "controllerconfig.h"
0025
0026 #include <QtGlobal>
0027 #include <QFile>
0028 #include <QJsonObject>
0029 #include <QJsonArray>
0030 #include <QJsonDocument>
0031 #include <QDebug>
0032 #include <QProcess>
0033 #include <QQmlPropertyMap>
0034 #include <QStandardItemModel>
0035 #include <QQmlEngine>
0036 #include <QQmlContext>
0037 #include <QUuid>
0038 #include <QWebSocket>
0039 #include <QMediaPlayer>
0040
0041 MycroftController *MycroftController::instance()
0042 {
0043     static MycroftController* s_self = nullptr;
0044     if (!s_self) {
0045         s_self = new MycroftController;
0046     }
0047     return s_self;
0048 }
0049
0050
0051 MycroftController::MycroftController(QObject *parent)
0052     : QObject(parent),
0053       m_appSettingObj(new GlobalSettings)
0054 {
0055     m_qt_version_context = QStringLiteral("6");
0056     connect(&m_mainWebSocket, &QWebSocket::connected, this,
0057             [this] () {
0058                 m_reconnectTimer.stop();
0059                 emit socketStatusChanged();
0060             });
0061     connect(&m_mainWebSocket, &QWebSocket::disconnected, this, &MycroftController::closed);
0062     connect(&m_mainWebSocket, &QWebSocket::stateChanged, this,
0063             [this] (QAbstractSocket::SocketState state) {
0064                 emit socketStatusChanged();
0065                 if (state == QAbstractSocket::ConnectedState) {
0066                     qWarning() << "Main Socket connected, trying to connect gui";
0067                     #if QT_VERSION >= 0x060000
0068                         m_qt_version_context = QStringLiteral("6");
0069                     #else
0070                         m_qt_version_context = QStringLiteral("5");
0071                     #endif
0072                     for (const auto &guiId : m_views.keys()) {
0073                         sendRequest(QStringLiteral("mycroft.gui.connected"),
0074                                     QVariantMap({{QStringLiteral("gui_id"), guiId}}),
0075                                     QVariantMap({{QStringLiteral("qt_version"), m_qt_version_context}}));
0076                     }
0077                     m_reannounceGuiTimer.start();
0078
0079                     sendRequest(QStringLiteral("mycroft.skills.all_loaded"), QVariantMap());
0080                 } else {
0081                     if (m_serverReady) {
0082                         m_serverReady = false;
0083                         emit serverReadyChanged();
0084                     }
0085                 }
0086             });
0087
0088     connect(&m_mainWebSocket, &QWebSocket::textMessageReceived, this, &MycroftController::onMainSocketMessageReceived);
0089
0090     m_reconnectTimer.setInterval(1000);
0091     connect(&m_reconnectTimer, &QTimer::timeout, this, [this]() {
0092         QString socket = m_appSettingObj->webSocketAddress() + QStringLiteral(":8181/core");
0093         m_mainWebSocket.open(QUrl(socket));
0094     });
0095
0096     m_reannounceGuiTimer.setInterval(10000);
0097     connect(&m_reannounceGuiTimer, &QTimer::timeout, this, [this]() {
0098         if (m_mainWebSocket.state() != QAbstractSocket::ConnectedState) {
0099             return;
0100         }
0101         for (const auto &guiId : m_views.keys()) {
0102             if (m_views[guiId]->status() != Open) {
0103                 qWarning()<<"Retrying to announce gui";
0104                 sendRequest(QStringLiteral("mycroft.gui.connected"),
0105                             QVariantMap({{QStringLiteral("gui_id"), guiId}}),
0106                             QVariantMap({{QStringLiteral("qt_version"), m_qt_version_context}}));
0107             }
0108         }
0109     });
0110
0111 #ifdef Q_OS_ANDROID
0112     m_speech = new QTextToSpeech(this);
0113     connect(m_speech, &QTextToSpeech::stateChanged, this, [this] () {
0114         if (!ttsqueue.isEmpty() && m_speech->state() != QTextToSpeech::Speaking) {
0115             m_speech->say(ttsqueue.dequeue());
0116         }
0117
0118         if (ttsqueue.isEmpty() && m_speech->state() != QTextToSpeech::Speaking && m_isExpectingSpeechResponse) {
0119             emit speechRequestedChanged(m_isExpectingSpeechResponse);
0120             m_isExpectingSpeechResponse = false;
0121         }
0122     });
0123 #endif
0124
0125 }
0126
0127
0128 void MycroftController::start()
0129 {
0130     //auto appSettingObj = new GlobalSettings;
0131     QString socket = m_appSettingObj->webSocketAddress() + QStringLiteral(":8181/core");
0132     m_mainWebSocket.open(QUrl(socket));
0133     connect(&m_mainWebSocket, QOverload<QAbstractSocket::SocketError>::of(&QWebSocket::error),
0134             this, [this] (const QAbstractSocket::SocketError &error) {
0135         //qDebug() << error;
0136
0137         if (error != QAbstractSocket::HostNotFoundError && error != QAbstractSocket::ConnectionRefusedError) {
0138             qWarning() << "Mycroft is running but the connection failed for some reason. Kill Mycroft manually.";
0139
0140             return;
0141         }
0142
0143         //don't try to launch mycroft more than once
0144         if (!m_mycroftLaunched) {
0145             QProcess::startDetached(QStringLiteral("mycroft-gui-core-loader"), QStringList());
0146             m_mycroftLaunched = true;
0147             if(m_appSettingObj->usePTTClient()){
0148                 QProcess::startDetached(QStringLiteral("mycroft-gui-ptt-loader"), QStringList());
0149             }
0150         }
0151         m_reconnectTimer.start();
0152         emit socketStatusChanged();
0153     });
0154
0155     emit socketStatusChanged();
0156 }
0157
0158 void MycroftController::disconnectSocket()
0159 {
0160     qDebug() << "in reconnect";
0161     m_mainWebSocket.close();
0162     m_reconnectTimer.stop();
0163     if (m_mycroftLaunched) {
0164         QProcess::startDetached(QStringLiteral("mycroft-gui-core-stop"), QStringList());
0165         m_mycroftLaunched = false;
0166     }
0167     emit socketStatusChanged();
0168 }
0169
0170 void MycroftController::reconnect()
0171 {
0172     qDebug() << "in reconnect";
0173     m_mainWebSocket.close();
0174     m_reconnectTimer.start();
0175     emit socketStatusChanged();
0176 }
0177
0178 void MycroftController::startPTTClient()
0179 {
0180     QProcess::startDetached(QStringLiteral("mycroft-gui-ptt-loader"), QStringList());
0181 }
0182
0183 void MycroftController::onMainSocketMessageReceived(const QString &message)
0184 {
0185     auto doc = QJsonDocument::fromJson(message.toUtf8());
0186
0187     if (doc.isEmpty()) {
0188         qWarning() << "Empty or invalid JSON message arrived on the main socket:" << message;
0189         return;
0190     }
0191
0192     auto type = doc[QStringLiteral("type")].toString();
0193
0194     if (type.isEmpty()) {
0195         qWarning() << "Empty type in the JSON message on the main socket";
0196         return;
0197     }
0198
0199     //filter out the noise so we can print debug stuff later without drowning in noise
0200     if (type.startsWith(QStringLiteral("enclosure")) || type.startsWith(QStringLiteral("mycroft-date"))) {
0201         return;
0202     }
0203
0204 #ifdef DEBUG_MYCROFT_MESSAGEBUS
0205     qDebug() << "type" << type;
0206 #endif
0207
0208     emit intentRecevied(type, doc[QStringLiteral("data")].toVariant().toMap());
0209
0210 #ifdef Q_OS_ANDROID
0211     if (type == QLatin1String("speak") && m_speech->state() != QTextToSpeech::Speaking) {
0212         m_speech->say(doc[QStringLiteral("data")][QStringLiteral("utterance")].toString());
0213     } else if (type == QLatin1String("speak") && m_speech->state() == QTextToSpeech::Speaking) {
0214         ttsqueue.enqueue(doc[QStringLiteral("data")][QStringLiteral("utterance")].toString());
0215     }
0216
0217     if (type == QLatin1String("mycroft.mic.listen")) {
0218         m_isExpectingSpeechResponse = true;
0219     }
0220 #endif
0221
0222     if (type == QLatin1String("remote.tts.audio") && m_appSettingObj->usesRemoteTTS()) {
0223         QString aud = doc[QStringLiteral("data")][QStringLiteral("wave")].toString();
0224         auto innerdoc = QJsonDocument::fromJson(aud.toUtf8());
0225         QJsonValue qjv = innerdoc[QStringLiteral("py/b64")];
0226         QString aud_values = qjv.toString();
0227         QByteArray audioopt;
0228         audioopt.append(qUtf8Printable(aud_values));
0229         QByteArray aud_array = QByteArray::fromBase64(audioopt, QByteArray::Base64UrlEncoding);
0230         QByteArray ret_aud = QByteArray::fromBase64(aud_array);
0231         //error: variable has incomplete type 'QFile'
0232         // fix: #include <QFile>
0233
0234         QFile file(QStringLiteral("/tmp/incoming.wav"));
0235         file.open(QIODevice::WriteOnly);
0236         file.write(ret_aud);
0237         file.close();
0238         QMediaPlayer *player = new QMediaPlayer;
0239         player->setSource(QUrl::fromLocalFile(QStringLiteral("/tmp/incoming.wav")));
0240         player->play();
0241     }
0242
0243     // Try catching intent_failure from another method because of issue: https://github.com/MycroftAI/mycroft-core/issues/2490
0244     if (type == QLatin1String("active_skill_request")) {
0245         QString skill_id = doc[QStringLiteral("data")][QStringLiteral("skill_id")].toString();
0246         if (skill_id == QStringLiteral("fallback-unknown.mycroftai")) {
0247             m_isListening = false;
0248             emit isListeningChanged();
0249             emit notUnderstood();
0250         }
0251         return;
0252     }
0253     // Instead of intent_failure which is handled by fallback skills, use complete_intent_failure where all skills failed to parse intent
0254     if (type == QLatin1String("complete_intent_failure")) {
0255         m_isListening = false;
0256         emit isListeningChanged();
0257         emit notUnderstood();
0258     }
0259     if (type == QLatin1String("recognizer_loop:audio_output_start")) {
0260         m_isSpeaking = true;
0261         emit isSpeakingChanged();
0262         return;
0263     }
0264     if (type == QLatin1String("recognizer_loop:audio_output_end")) {
0265         m_isSpeaking = false;
0266         emit isSpeakingChanged();
0267         return;
0268     }
0269     if (type == QLatin1String("recognizer_loop:wakeword")) {
0270         m_isListening = true;
0271         emit isListeningChanged();
0272         return;
0273     }
0274     if (type == QLatin1String("recognizer_loop:record_begin") && !m_isListening) {
0275         m_isListening = true;
0276         emit isListeningChanged();
0277         return;
0278     }
0279     if (type == QLatin1String("recognizer_loop:record_end")) {
0280         m_isListening = false;
0281         emit isListeningChanged();
0282         return;
0283     }
0284     if (type == QLatin1String("mycroft.speech.recognition.unknown")) {
0285         emit notUnderstood();
0286         return;
0287     }
0288
0289     if (type == QLatin1String("mycroft.skill.handler.start")) {
0290         m_currentSkill = doc[QStringLiteral("data")][QStringLiteral("name")].toString();
0291         qDebug() << "Current intent:" << m_currentIntent;
0292         emit currentIntentChanged();
0293     } else if (type == QLatin1String("mycroft.skill.handler.complete")) {
0294         m_currentSkill = QString();
0295         emit currentSkillChanged();
0296     } else if (type == QLatin1String("speak")) {
0297         emit fallbackTextRecieved(m_currentSkill, doc[QStringLiteral("data")].toVariant().toMap());
0298     } else if (type == QLatin1String("mycroft.stop.handled") || type == QLatin1String("mycroft.stop")) {
0299         emit stopped();
0300
0301     } else if (type == QLatin1String("mycroft.gui.port")) {
0302         const int port = doc[QStringLiteral("data")][QStringLiteral("port")].toInt();
0303         const QString guiId = doc[QStringLiteral("data")][QStringLiteral("gui_id")].toString();
0304         if (port < 0 || port > 65535) {
0305             qWarning() << "Invalid port from mycroft.gui.port";
0306             return;
0307         }
0308
0309         qWarning() << "Received port" << port << "for gui" << guiId;
0310         if (!m_views.contains(guiId)) {
0311             qWarning() << "Unknown guiId from mycroft.gui.port";
0312             return;
0313         }
0314
0315         QUrl url(QStringLiteral("%1:%2/gui").arg(m_appSettingObj->webSocketAddress()).arg(port));
0316         m_views[guiId]->setUrl(url);
0317         m_reannounceGuiTimer.stop();
0318     } else if (type == QLatin1String("mycroft.skills.all_loaded.response")) {
0319         if (doc[QStringLiteral("data")][QStringLiteral("status")].toBool() == true) {
0320             m_serverReady = true;
0321             emit serverReadyChanged();
0322         }
0323     } else if (type == QLatin1String("mycroft.ready")) {
0324         m_serverReady = true;
0325         emit serverReadyChanged();
0326     }
0327
0328     if (type == QLatin1String("screen.close.idle.event")) {
0329         QString skill_idle_event_id = doc[QStringLiteral("data")][QStringLiteral("skill_idle_event_id")].toString();
0330         emit skillTimeoutReceived(skill_idle_event_id);
0331     }
0332
0333     // Check if it's an utterance recognized as an intent
0334     if (type.contains(QLatin1Char(':')) && !doc[QStringLiteral("data")][QStringLiteral("utterance")].toString().isEmpty()) {
0335         const QString skill = type.split(QLatin1Char(':')).first();
0336         if (skill.contains(QLatin1Char('.'))) {
0337             m_currentSkill = skill;
0338             qDebug() << "Current skill:" << m_currentSkill;
0339             emit utteranceManagedBySkill(m_currentSkill);
0340             emit currentSkillChanged();
0341         }
0342     }
0343 }
0344
0345 void MycroftController::sendRequest(const QString &type, const QVariantMap &data, const QVariantMap &context)
0346 {
0347     if (m_mainWebSocket.state() != QAbstractSocket::ConnectedState) {
0348         qWarning() << "mycroft connection not open!";
0349         return;
0350     }
0351     QJsonObject root;
0352
0353     root[QStringLiteral("type")] = type;
0354     root[QStringLiteral("data")] = QJsonObject::fromVariantMap(data);
0355     root[QStringLiteral("context")] = QJsonObject::fromVariantMap(context);
0356
0357     QJsonDocument doc(root);
0358     m_mainWebSocket.sendTextMessage(QString::fromUtf8(doc.toJson()));
0359 }
0360
0361 void MycroftController::sendBinary(const QString &type, const QJsonObject &data, const QVariantMap &context)
0362 {
0363     if (m_mainWebSocket.state() != QAbstractSocket::ConnectedState) {
0364         qWarning() << "mycroft connection not open!";
0365         return;
0366     }
0367     QJsonObject socketObject;
0368     socketObject[QStringLiteral("type")] = type;
0369     socketObject[QStringLiteral("data")] = data;
0370
0371     if(m_appSettingObj->useHivemindProtocol()){
0372         socketObject[QStringLiteral("context")] = QJsonObject::fromVariantMap(context);
0373     }
0374
0375     QJsonDocument doc;
0376     doc.setObject(socketObject);
0377     QByteArray docbin = doc.toJson(QJsonDocument::Compact);
0378     m_mainWebSocket.sendBinaryMessage(docbin);
0379 }
0380
0381 void MycroftController::sendText(const QString &message)
0382 {
0383     if(!m_appSettingObj->useHivemindProtocol()){
0384         sendRequest(QStringLiteral("recognizer_loop:utterance"), QVariantMap({{QStringLiteral("utterances"), QStringList({message})}}), QVariantMap({{QStringLiteral("source"), QStringLiteral("debug_cli")}, {QStringLiteral("destination"), QStringLiteral("skills")}, {QStringLiteral("qt_version"), m_qt_version_context}}));
0385     } else {
0386         sendRequest(QStringLiteral("recognizer_loop:utterance"), QVariantMap({{QStringLiteral("utterances"), QStringList({message})}}), QVariantMap({{QStringLiteral("source"), QStringLiteral("mycroft-gui")}, {QStringLiteral("destination"), QStringLiteral("skills")}, {QStringLiteral("qt_version"), m_qt_version_context}}));
0387     }
0388 }
0389
0390 void MycroftController::registerView(AbstractSkillView *view)
0391 {
0392     Q_ASSERT(!view->id().isEmpty());
0393     Q_ASSERT(!m_views.contains(view->id()));
0394     m_views[view->id()] = view;
0395 //TODO: manage view destruction
0396     if (m_mainWebSocket.state() == QAbstractSocket::ConnectedState) {
0397         sendRequest(QStringLiteral("mycroft.gui.connected"),
0398                     QVariantMap({{QStringLiteral("gui_id"), view->id()}}),
0399                     QVariantMap({{QStringLiteral("qt_version"), m_qt_version_context}}));
0400     }
0401 }
0402
0403 MycroftController::Status MycroftController::status() const
0404 {
0405     if (m_reconnectTimer.isActive()) {
0406         return Connecting;
0407     }
0408
0409     switch(m_mainWebSocket.state())
0410     {
0411     case QAbstractSocket::ConnectingState:
0412     case QAbstractSocket::BoundState:
0413     case QAbstractSocket::HostLookupState:
0414         return Connecting;
0415     case QAbstractSocket::UnconnectedState:
0416         return Closed;
0417     case QAbstractSocket::ConnectedState:
0418         return Open;
0419     case QAbstractSocket::ClosingState:
0420         return Closing;
0421     default:
0422         return Connecting;
0423     }
0424 }
0425
0426 //FIXME: remove
0427 QString MycroftController::currentSkill() const
0428 {
0429     return m_currentSkill;
0430 }
0431
0432 QString MycroftController::currentIntent() const
0433 {
0434     return m_currentIntent;
0435 }
0436
0437 bool MycroftController::isSpeaking() const
0438 {
0439     return m_isSpeaking;
0440 }
0441
0442 bool MycroftController::isListening() const
0443 {
0444     return m_isListening;
0445 }
0446
0447 bool MycroftController::serverReady() const
0448 {
0449     return m_serverReady;
0450 }
0451
0452 #include "moc_mycroftcontroller.cpp"