libdiscover/appstream/OdrsReviewsBackend.cpp

0001 /*
0002  *   SPDX-FileCopyrightText: 2013 Aleix Pol Gonzalez <aleixpol@blue-systems.com>
0003  *   SPDX-FileCopyrightText: 2017 Jan Grulich <jgrulich@redhat.com>
0004  *
0005  *   SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
0006  */
0007
0008 #include "OdrsReviewsBackend.h"
0009 #include "AppStreamIntegration.h"
0010 #include "CachedNetworkAccessManager.h"
0011
0012 #include <ReviewsBackend/Rating.h>
0013 #include <ReviewsBackend/Review.h>
0014
0015 #include <qnumeric.h>
0016 #include <resources/AbstractResource.h>
0017 #include <resources/AbstractResourcesBackend.h>
0018
0019 #include <KIO/FileCopyJob>
0020 #include <KLocalizedString>
0021 #include <KUser>
0022
0023 #include "libdiscover_debug.h"
0024 #include <QCryptographicHash>
0025 #include <QDir>
0026 #include <QFile>
0027 #include <QFileInfo>
0028 #include <QJsonArray>
0029 #include <QJsonObject>
0030 #include <QNetworkAccessManager>
0031 #include <QNetworkRequest>
0032 #include <QStandardPaths>
0033
0034 #include <QFutureWatcher>
0035 #include <QtConcurrentRun>
0036
0037 // #define APIURL "http://127.0.0.1:5000/1.0/reviews/api"
0038 #define APIURL "https://odrs.gnome.org/1.0/reviews/api"
0039
0040 QSharedPointer<OdrsReviewsBackend> OdrsReviewsBackend::global()
0041 {
0042     static QSharedPointer<OdrsReviewsBackend> var = nullptr;
0043     if (!var) {
0044         var = QSharedPointer<OdrsReviewsBackend>(new OdrsReviewsBackend());
0045     }
0046
0047     return var;
0048 }
0049
0050 OdrsReviewsBackend::OdrsReviewsBackend()
0051     : AbstractReviewsBackend(nullptr)
0052 {
0053     fetchRatings();
0054 }
0055
0056 OdrsReviewsBackend::~OdrsReviewsBackend() noexcept
0057 {
0058     qDeleteAll(m_ratings);
0059 }
0060
0061 void OdrsReviewsBackend::fetchRatings()
0062 {
0063     bool fetchRatings = false;
0064     const QUrl ratingsUrl(QStringLiteral(APIURL "/ratings"));
0065     const QUrl fileUrl = QUrl::fromLocalFile(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/ratings/ratings"));
0066     const QDir cacheDir(QStandardPaths::writableLocation(QStandardPaths::CacheLocation));
0067
0068     // Create $HOME/.cache/discover/ratings folder
0069     cacheDir.mkpath(QStringLiteral("ratings"));
0070
0071     if (QFileInfo::exists(fileUrl.toLocalFile())) {
0072         QFileInfo file(fileUrl.toLocalFile());
0073         // Refresh the cached ratings if they are older than one day
0074         if (file.lastModified().msecsTo(QDateTime::currentDateTime()) > 1000 * 60 * 60 * 24) {
0075             fetchRatings = true;
0076         }
0077     } else {
0078         fetchRatings = true;
0079     }
0080
0081     qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Fetch ratings:" << fetchRatings;
0082     if (fetchRatings) {
0083         setFetching(true);
0084         KIO::FileCopyJob *getJob = KIO::file_copy(ratingsUrl, fileUrl, -1, KIO::Overwrite | KIO::HideProgressInfo);
0085         connect(getJob, &KIO::FileCopyJob::result, this, &OdrsReviewsBackend::ratingsFetched);
0086     } else {
0087         parseRatings();
0088     }
0089 }
0090
0091 void OdrsReviewsBackend::setFetching(bool fetching)
0092 {
0093     if (fetching == m_isFetching) {
0094         return;
0095     }
0096     m_isFetching = fetching;
0097     Q_EMIT fetchingChanged(fetching);
0098 }
0099
0100 void OdrsReviewsBackend::ratingsFetched(KJob *job)
0101 {
0102     setFetching(false);
0103     if (job->error()) {
0104         qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Failed to fetch ratings:" << job->errorString();
0105     } else {
0106         parseRatings();
0107     }
0108 }
0109
0110 static QString osName()
0111 {
0112     return AppStreamIntegration::global()->osRelease()->name();
0113 }
0114
0115 static QString userHash()
0116 {
0117     QString machineId;
0118     QFile file(QStringLiteral("/etc/machine-id"));
0119     if (file.open(QIODevice::ReadOnly)) {
0120         machineId = QString::fromUtf8(file.readAll());
0121         file.close();
0122     }
0123
0124     if (machineId.isEmpty()) {
0125         return QString();
0126     }
0127
0128     QString salted = QStringLiteral("gnome-software[%1:%2]").arg(KUser().loginName(), machineId);
0129     return QString::fromUtf8(QCryptographicHash::hash(salted.toUtf8(), QCryptographicHash::Sha1).toHex());
0130 }
0131
0132 void OdrsReviewsBackend::fetchReviews(AbstractResource *resource, int page)
0133 {
0134     if (resource->appstreamId().isEmpty()) {
0135         return;
0136     }
0137     Q_UNUSED(page)
0138     QString version = resource->isInstalled() ? resource->installedVersion() : resource->availableVersion();
0139     if (version.isEmpty()) {
0140         version = QStringLiteral("unknown");
0141     }
0142     setFetching(true);
0143
0144     const QJsonDocument document(QJsonObject{
0145         {QStringLiteral("app_id"), resource->appstreamId()},
0146         {QStringLiteral("distro"), osName()},
0147         {QStringLiteral("user_hash"), userHash()},
0148         {QStringLiteral("version"), version},
0149         {QStringLiteral("locale"), QLocale::system().name()},
0150         {QStringLiteral("limit"), -1},
0151     });
0152
0153     const auto json = document.toJson(QJsonDocument::Compact);
0154     QNetworkRequest request(QUrl(QStringLiteral(APIURL "/fetch")));
0155     request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json; charset=utf-8"));
0156     request.setHeader(QNetworkRequest::ContentLengthHeader, json.size());
0157     // Store reference to the resource for which we request reviews
0158     request.setOriginatingObject(resource);
0159
0160     auto reply = nam()->post(request, json);
0161     connect(reply, &QNetworkReply::finished, this, &OdrsReviewsBackend::reviewsFetched);
0162 }
0163
0164 void OdrsReviewsBackend::reviewsFetched()
0165 {
0166     const auto reply = qobject_cast<QNetworkReply *>(sender());
0167     QScopedPointer<QNetworkReply, QScopedPointerDeleteLater> replyPtr(reply);
0168     const QByteArray data = reply->readAll();
0169     const auto networkError = reply->error();
0170     if (networkError != QNetworkReply::NoError) {
0171         qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Error fetching reviews:" << reply->errorString() << data;
0172         m_errorMessage = i18n("Technical error message: %1", reply->errorString());
0173         Q_EMIT errorMessageChanged();
0174         setFetching(false);
0175         return;
0176     }
0177
0178     QJsonParseError error;
0179     const QJsonDocument document = QJsonDocument::fromJson(data, &error);
0180     if (error.error) {
0181         qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Error parsing reviews:" << reply->url() << error.errorString();
0182     }
0183
0184     const auto resource = qobject_cast<AbstractResource *>(reply->request().originatingObject());
0185     Q_ASSERT(resource);
0186     parseReviews(document, resource);
0187 }
0188
0189 Rating *OdrsReviewsBackend::ratingForApplication(AbstractResource *resource) const
0190 {
0191     if (resource->appstreamId().isEmpty()) {
0192         return nullptr;
0193     }
0194
0195     return m_ratings[resource->appstreamId()];
0196 }
0197
0198 void OdrsReviewsBackend::submitUsefulness(Review *review, bool useful)
0199 {
0200     const QJsonDocument document(QJsonObject{
0201         {QStringLiteral("app_id"), review->applicationName()},
0202         {QStringLiteral("user_skey"), review->getMetadata(QStringLiteral("ODRS::user_skey")).toString()},
0203         {QStringLiteral("user_hash"), userHash()},
0204         {QStringLiteral("distro"), osName()},
0205         {QStringLiteral("review_id"), QJsonValue(double(review->id()))}, // if we really need uint64 we should get it in QJsonValue
0206     });
0207
0208     QNetworkRequest request(QUrl(QStringLiteral(APIURL) + (useful ? QLatin1String("/upvote") : QLatin1String("/downvote"))));
0209     request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json; charset=utf-8"));
0210     request.setHeader(QNetworkRequest::ContentLengthHeader, document.toJson().size());
0211
0212     auto reply = nam()->post(request, document.toJson());
0213     connect(reply, &QNetworkReply::finished, this, &OdrsReviewsBackend::usefulnessSubmitted);
0214 }
0215
0216 void OdrsReviewsBackend::usefulnessSubmitted()
0217 {
0218     const auto reply = qobject_cast<QNetworkReply *>(sender());
0219     const auto networkError = reply->error();
0220     if (networkError == QNetworkReply::NoError) {
0221         qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Usefulness submitted";
0222     } else {
0223         qCWarning(LIBDISCOVER_LOG).noquote() << "OdrsReviewsBackend: Failed to submit usefulness:" << reply->errorString();
0224         Q_EMIT error(i18n("Error while submitting usefulness: %1", reply->errorString()));
0225     }
0226     reply->deleteLater();
0227 }
0228
0229 QString OdrsReviewsBackend::userName() const
0230 {
0231     return KUser().property(KUser::FullName).toString();
0232 }
0233
0234 void OdrsReviewsBackend::sendReview(AbstractResource *resource,
0235                                     const QString &summary,
0236                                     const QString &reviewText,
0237                                     const QString &rating,
0238                                     const QString &userName)
0239 {
0240     Q_ASSERT(resource);
0241     QJsonObject map = {
0242         {QStringLiteral("app_id"), resource->appstreamId()},
0243         {QStringLiteral("user_skey"), resource->getMetadata(QStringLiteral("ODRS::user_skey")).toString()},
0244         {QStringLiteral("user_hash"), userHash()},
0245         {QStringLiteral("version"), resource->isInstalled() ? resource->installedVersion() : resource->availableVersion()},
0246         {QStringLiteral("locale"), QLocale::system().name()},
0247         {QStringLiteral("distro"), osName()},
0248         {QStringLiteral("user_display"), QJsonValue::fromVariant(userName)},
0249         {QStringLiteral("summary"), summary},
0250         {QStringLiteral("description"), reviewText},
0251         {QStringLiteral("rating"), rating.toInt() * 10},
0252     };
0253
0254     const QJsonDocument document(map);
0255
0256     const auto accessManager = nam();
0257     QNetworkRequest request(QUrl(QStringLiteral(APIURL "/submit")));
0258     request.setHeader(QNetworkRequest::ContentTypeHeader, QStringLiteral("application/json; charset=utf-8"));
0259     request.setHeader(QNetworkRequest::ContentLengthHeader, document.toJson().size());
0260
0261     // Store what we need so we can immediately show our review once it is submitted
0262     // Use review_id 0 for now as odrs starts numbering from 1 and once reviews are re-downloaded we get correct id
0263     map.insert(QStringLiteral("review_id"), 0);
0264     resource->addMetadata(QStringLiteral("ODRS::review_map"), map);
0265     request.setOriginatingObject(resource);
0266
0267     accessManager->post(request, document.toJson());
0268     connect(accessManager, &QNetworkAccessManager::finished, this, &OdrsReviewsBackend::reviewSubmitted);
0269 }
0270
0271 void OdrsReviewsBackend::reviewSubmitted(QNetworkReply *reply)
0272 {
0273     const auto networkError = reply->error();
0274     if (networkError == QNetworkReply::NoError) {
0275         const auto resource = qobject_cast<AbstractResource *>(reply->request().originatingObject());
0276         Q_ASSERT(resource);
0277         qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Review submitted for" << resource;
0278         if (resource) {
0279             const QJsonDocument document({resource->getMetadata(QStringLiteral("ODRS::review_map")).toObject()});
0280             parseReviews(document, resource);
0281         } else {
0282             qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Failed to submit review: missing object";
0283         }
0284     } else {
0285         qCWarning(LIBDISCOVER_LOG).noquote() << "OdrsReviewsBackend: Failed to submit review:" << reply->errorString();
0286         Q_EMIT error(i18n("Error while submitting review: %1", reply->errorString()));
0287     }
0288     reply->deleteLater();
0289 }
0290
0291 void OdrsReviewsBackend::parseRatings()
0292 {
0293     auto fw = new QFutureWatcher<QJsonDocument>(this);
0294     connect(fw, &QFutureWatcher<QJsonDocument>::finished, this, [this, fw] {
0295         const auto jsonDocument = fw->result();
0296         fw->deleteLater();
0297         const auto jsonObject = jsonDocument.object();
0298         m_ratings.reserve(jsonObject.size());
0299         for (auto it = jsonObject.begin(); it != jsonObject.end(); it++) {
0300             const auto appJsonObject = it.value().toObject();
0301
0302             const int ratingCount = appJsonObject.value(QLatin1String("total")).toInt();
0303             int ratingMap[] = {
0304                 appJsonObject.value(QLatin1String("star0")).toInt(),
0305                 appJsonObject.value(QLatin1String("star1")).toInt(),
0306                 appJsonObject.value(QLatin1String("star2")).toInt(),
0307                 appJsonObject.value(QLatin1String("star3")).toInt(),
0308                 appJsonObject.value(QLatin1String("star4")).toInt(),
0309                 appJsonObject.value(QLatin1String("star5")).toInt(),
0310             };
0311
0312             const auto rating = new Rating(it.key(), ratingCount, ratingMap);
0313             m_ratings.insert(it.key(), rating);
0314
0315             const auto finder = [rating](Rating *review) {
0316                 return review->ratingPoints() < rating->ratingPoints();
0317             };
0318             const auto topIt = std::find_if(m_top.begin(), m_top.end(), finder);
0319             if (topIt == m_top.end()) {
0320                 if (m_top.size() < 25) {
0321                     m_top.append(rating);
0322                 }
0323             } else {
0324                 m_top.insert(topIt, rating);
0325             }
0326             if (m_top.size() > 25) {
0327                 m_top.resize(25);
0328             }
0329         }
0330         Q_EMIT ratingsReady();
0331     });
0332     fw->setFuture(QtConcurrent::run([] {
0333         QFile ratingsDocument(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QStringLiteral("/ratings/ratings"));
0334         if (!ratingsDocument.open(QIODevice::ReadOnly)) {
0335             qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Could not open file" << ratingsDocument.fileName();
0336             return QJsonDocument::fromJson({});
0337         }
0338
0339         QJsonParseError error;
0340         const auto ret = QJsonDocument::fromJson(ratingsDocument.readAll(), &error);
0341         if (error.error) {
0342             qCWarning(LIBDISCOVER_LOG) << "OdrsReviewsBackend: Error parsing ratings:" << ratingsDocument.errorString() << error.errorString();
0343         }
0344         return ret;
0345     }));
0346 }
0347
0348 void OdrsReviewsBackend::parseReviews(const QJsonDocument &document, AbstractResource *resource)
0349 {
0350     setFetching(false);
0351     Q_ASSERT(resource);
0352     if (!resource) {
0353         return;
0354     }
0355
0356     const auto reviews = document.array();
0357     if (!reviews.isEmpty()) {
0358         QList<ReviewPtr> reviewsList;
0359         for (auto it = reviews.begin(); it != reviews.end(); it++) {
0360             const QJsonObject review = it->toObject();
0361             if (!review.isEmpty()) {
0362                 // Same ranking algorythm Gnome Software uses
0363                 const int usefulFavorable = review.value(QStringLiteral("karma_up")).toInt();
0364                 const int usefulNegative = review.value(QStringLiteral("karma_down")).toInt();
0365                 const int usefulTotal = usefulFavorable + usefulNegative;
0366
0367                 qreal usefulWilson = 0.f;
0368
0369                 /* from http://www.evanmiller.org/how-not-to-sort-by-average-rating.html */
0370                 if (usefulFavorable > 0 || usefulNegative > 0) {
0371                     usefulWilson = ((usefulFavorable + 1.9208) / (usefulFavorable + usefulNegative)
0372                                     - 1.96 * sqrt((usefulFavorable * usefulNegative) / qreal(usefulFavorable + usefulNegative) + 0.9604)
0373                                         / (usefulFavorable + usefulNegative))
0374                         / (1 + 3.8416 / (usefulFavorable + usefulNegative));
0375                     usefulWilson *= 100.f;
0376                 }
0377
0378                 QDateTime dateTime;
0379                 dateTime.setSecsSinceEpoch(review.value(QStringLiteral("date_created")).toInt());
0380
0381                 // If there is no score or the score is the same, base on the age
0382                 const auto currentDateTime = QDateTime::currentDateTime();
0383                 const auto totalDays = static_cast<qreal>(dateTime.daysTo(currentDateTime));
0384
0385                 // use also the longest common subsequence of the version string to compute relevance
0386                 const auto reviewVersion = review.value(QStringLiteral("version")).toString();
0387                 const auto availableVersion = resource->availableVersion();
0388                 qreal versionScore = 0;
0389                 const int minLength = std::min(reviewVersion.length(), availableVersion.length());
0390                 if (minLength > 0) {
0391                     for (int i = 0; i < minLength; ++i) {
0392                         if (reviewVersion[i] != availableVersion[i] || i == minLength - 1) {
0393                             versionScore = i;
0394                             break;
0395                         }
0396                     }
0397                     // Normalize
0398                     versionScore = versionScore / qreal(std::max(reviewVersion.length(), availableVersion.length()) - 1);
0399                 }
0400
0401                 // Very random heuristic which weights usefulness with age and versioin similarity. Don't penalise usefulness more than 6 months
0402                 usefulWilson = versionScore + 1.0 / std::max(1.0, totalDays) + usefulWilson / std::clamp(totalDays, 1.0, 93.0);
0403
0404                 ReviewPtr r(new Review(review.value(QStringLiteral("app_id")).toString(),
0405                                        resource->packageName(),
0406                                        review.value(QStringLiteral("locale")).toString(),
0407                                        review.value(QStringLiteral("summary")).toString(),
0408                                        review.value(QStringLiteral("description")).toString(),
0409                                        review.value(QStringLiteral("user_display")).toString(),
0410                                        dateTime,
0411                                        usefulFavorable >= usefulNegative * 2 || review.value(QStringLiteral("reported")).toInt() > 4,
0412                                        review.value(QStringLiteral("review_id")).toInt(),
0413                                        review.value(QStringLiteral("rating")).toInt() / 10,
0414                                        usefulTotal,
0415                                        usefulFavorable,
0416                                        usefulWilson,
0417                                        reviewVersion));
0418                 // We can also receive just a json with app name and user info so filter these out as there is no review
0419                 if (!r->summary().isEmpty() && !r->reviewText().isEmpty()) {
0420                     reviewsList.append(r);
0421                     // Needed for submitting usefulness
0422                     r->addMetadata(QStringLiteral("ODRS::user_skey"), review.value(QStringLiteral("user_skey")).toString());
0423                 }
0424
0425                 // We should get at least user_skey needed for posting reviews
0426                 resource->addMetadata(QStringLiteral("ODRS::user_skey"), review.value(QStringLiteral("user_skey")).toString());
0427             }
0428         }
0429
0430         Q_EMIT reviewsReady(resource, reviewsList, false);
0431     }
0432 }
0433
0434 bool OdrsReviewsBackend::isResourceSupported(AbstractResource *resource) const
0435 {
0436     return !resource->appstreamId().isEmpty();
0437 }
0438
0439 void OdrsReviewsBackend::emitRatingFetched(AbstractResourcesBackend *backend, const QList<AbstractResource *> &resources) const
0440 {
0441     backend->emitRatingsReady();
0442     for (const auto resource : resources) {
0443         if (m_ratings.contains(resource->appstreamId())) {
0444             Q_EMIT resource->ratingFetched();
0445         }
0446     }
0447 }
0448
0449 QNetworkAccessManager *OdrsReviewsBackend::nam()
0450 {
0451     if (!m_delayedNam) {
0452         m_delayedNam = new CachedNetworkAccessManager(QStringLiteral("odrs"), this);
0453     }
0454     return m_delayedNam;
0455 }