src/utils/updatefeedjob.cpp

0001
0002 /**
0003  * SPDX-FileCopyrightText: 2021-2022 Bart De Vries <bart@mogwai.be>
0004  *
0005  * SPDX-License-Identifier: GPL-2.0-only OR GPL-3.0-only OR LicenseRef-KDE-Accepted-GPL
0006  */
0007
0008 #include "updatefeedjob.h"
0009
0010 #include <QDir>
0011 #include <QDomElement>
0012 #include <QMultiMap>
0013 #include <QNetworkReply>
0014 #include <QSqlQuery>
0015 #include <QTextDocumentFragment>
0016 #include <QTimer>
0017
0018 #include <KLocalizedString>
0019 #include <ThreadWeaver/Thread>
0020
0021 #include "database.h"
0022 #include "enclosure.h"
0023 #include "fetcher.h"
0024 #include "fetcherlogging.h"
0025 #include "kasts-version.h"
0026 #include "settingsmanager.h"
0027 #include "storagemanager.h"
0028
0029 using namespace ThreadWeaver;
0030
0031 UpdateFeedJob::UpdateFeedJob(const QString &url, const QByteArray &data, QObject *parent)
0032     : QObject(parent)
0033     , m_url(url)
0034     , m_data(data)
0035 {
0036     // connect to signals in Fetcher such that GUI can pick up the changes
0037     connect(this, &UpdateFeedJob::feedDetailsUpdated, &Fetcher::instance(), &Fetcher::feedDetailsUpdated);
0038     connect(this, &UpdateFeedJob::feedUpdated, &Fetcher::instance(), &Fetcher::feedUpdated);
0039     connect(this, &UpdateFeedJob::entryAdded, &Fetcher::instance(), &Fetcher::entryAdded);
0040     connect(this, &UpdateFeedJob::entryUpdated, &Fetcher::instance(), &Fetcher::entryUpdated);
0041     connect(this, &UpdateFeedJob::feedUpdateStatusChanged, &Fetcher::instance(), &Fetcher::feedUpdateStatusChanged);
0042 }
0043
0044 void UpdateFeedJob::run(JobPointer, Thread *)
0045 {
0046     if (m_abort) {
0047         Q_EMIT finished();
0048         return;
0049     }
0050
0051     Database::openDatabase(m_url);
0052
0053     // First check if the RSS file has changed since last update; we do this by
0054     // comparing to the SHA1 hash we saved last time
0055     bool skipUpdate = false;
0056
0057     m_newHash = QString::fromLatin1(QCryptographicHash::hash(m_data, QCryptographicHash::Sha256).toHex());
0058
0059     QSqlQuery query(QSqlDatabase::database(m_url));
0060     query.prepare(QStringLiteral("SELECT lastHash FROM Feeds WHERE url=:url;"));
0061     query.bindValue(QStringLiteral(":url"), m_url);
0062     Database::execute(query);
0063     if (query.next()) {
0064         m_oldHash = query.value(QStringLiteral("lastHash")).toString();
0065         qCDebug(kastsFetcher) << "RSS hashes (old and new)" << m_url << m_oldHash << m_newHash;
0066         if (m_newHash == m_oldHash) {
0067             skipUpdate = true;
0068             qCDebug(kastsFetcher) << "same RSS feed hash as last time; skipping feed update for" << m_url;
0069         }
0070     }
0071     query.clear(); // release lock on database
0072
0073     if (!skipUpdate) {
0074         Syndication::DocumentSource document(m_data, m_url);
0075         Syndication::FeedPtr feed = Syndication::parserCollection()->parse(document, QStringLiteral("Atom"));
0076         processFeed(feed);
0077     }
0078
0079     Database::closeDatabase(m_url);
0080
0081     Q_EMIT finished();
0082 }
0083
0084 void UpdateFeedJob::processFeed(Syndication::FeedPtr feed)
0085 {
0086     qCDebug(kastsFetcher) << "start process feed" << feed;
0087
0088     if (feed.isNull())
0089         return;
0090
0091     // First check if this is a newly added feed and get current name and dirname
0092     m_isNewFeed = false;
0093     QString oldName, oldDirname;
0094     QSqlQuery query(QSqlDatabase::database(m_url));
0095     query.prepare(QStringLiteral("SELECT new, name, dirname FROM Feeds WHERE url=:url;"));
0096     query.bindValue(QStringLiteral(":url"), m_url);
0097     Database::execute(query);
0098     if (query.next()) {
0099         m_isNewFeed = query.value(QStringLiteral("new")).toBool();
0100         oldName = query.value(QStringLiteral("name")).toString();
0101         oldDirname = query.value(QStringLiteral("dirname")).toString();
0102     } else {
0103         qCDebug(kastsFetcher) << "Feed not found in database" << m_url;
0104         return;
0105     }
0106     if (m_isNewFeed) {
0107         qCDebug(kastsFetcher) << "New feed" << feed->title();
0108     }
0109
0110     m_markUnreadOnNewFeed = !(SettingsManager::self()->markUnreadOnNewFeed() == 2);
0111
0112     // Retrieve "other" fields; this will include the "itunes" tags
0113     QMultiMap<QString, QDomElement> otherItems = feed->additionalProperties();
0114
0115     query.prepare(QStringLiteral(
0116         "UPDATE Feeds SET name=:name, image=:image, link=:link, description=:description, lastUpdated=:lastUpdated, dirname=:dirname WHERE url=:url;"));
0117     query.bindValue(QStringLiteral(":name"), feed->title());
0118     query.bindValue(QStringLiteral(":url"), m_url);
0119     query.bindValue(QStringLiteral(":link"), feed->link());
0120     query.bindValue(QStringLiteral(":description"), feed->description());
0121
0122     QDateTime current = QDateTime::currentDateTime();
0123     query.bindValue(QStringLiteral(":lastUpdated"), current.toSecsSinceEpoch());
0124
0125     QString image;
0126     // First try the itunes tags, if not, fall back to regular image tag
0127     if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
0128         image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
0129     } else {
0130         image = feed->image()->url();
0131     }
0132
0133     if (image.startsWith(QStringLiteral("/")))
0134         image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + image;
0135     query.bindValue(QStringLiteral(":image"), image);
0136
0137     // if the title has changed, we need to rename the corresponding enclosure
0138     // download directory name and move the files
0139     m_dirname = oldDirname;
0140     if (oldName != feed->title() || oldDirname.isEmpty() || m_isNewFeed) {
0141         QString generatedDirname = generateFeedDirname(feed->title());
0142         if (generatedDirname != oldDirname) {
0143             m_dirname = generatedDirname;
0144             QString enclosurePath = StorageManager::instance().enclosureDirPath();
0145             if (QDir(enclosurePath + oldDirname).exists()) {
0146                 QDir().rename(enclosurePath + oldDirname, enclosurePath + m_dirname);
0147             } else {
0148                 QDir().mkpath(enclosurePath + m_dirname);
0149             }
0150         }
0151     }
0152     query.bindValue(QStringLiteral(":dirname"), m_dirname);
0153
0154     // Do the actual database UPDATE of this feed
0155     Database::execute(query);
0156
0157     // Now that we have the feed details, we make vectors of the data that's
0158     // already in the database relating to this feed
0159     // NOTE: We will do the feed authors after this step, because otherwise
0160     // we can't check for duplicates and we'll keep adding more of the same!
0161     query.prepare(QStringLiteral("SELECT * FROM Entries WHERE feed=:feed;"));
0162     query.bindValue(QStringLiteral(":feed"), m_url);
0163     Database::execute(query);
0164     while (query.next()) {
0165         EntryDetails entryDetails;
0166         entryDetails.feed = m_url;
0167         entryDetails.id = query.value(QStringLiteral("id")).toString();
0168         entryDetails.title = query.value(QStringLiteral("title")).toString();
0169         entryDetails.content = query.value(QStringLiteral("content")).toString();
0170         entryDetails.created = query.value(QStringLiteral("created")).toInt();
0171         entryDetails.updated = query.value(QStringLiteral("updated")).toInt();
0172         entryDetails.read = query.value(QStringLiteral("read")).toBool();
0173         entryDetails.isNew = query.value(QStringLiteral("new")).toBool();
0174         entryDetails.link = query.value(QStringLiteral("link")).toString();
0175         entryDetails.hasEnclosure = query.value(QStringLiteral("hasEnclosure")).toBool();
0176         entryDetails.image = query.value(QStringLiteral("image")).toString();
0177         m_entries += entryDetails;
0178     }
0179
0180     query.prepare(QStringLiteral("SELECT * FROM Enclosures WHERE feed=:feed;"));
0181     query.bindValue(QStringLiteral(":feed"), m_url);
0182     Database::execute(query);
0183     while (query.next()) {
0184         EnclosureDetails enclosureDetails;
0185         enclosureDetails.feed = m_url;
0186         enclosureDetails.id = query.value(QStringLiteral("id")).toString();
0187         enclosureDetails.duration = query.value(QStringLiteral("duration")).toInt();
0188         enclosureDetails.size = query.value(QStringLiteral("size")).toInt();
0189         enclosureDetails.title = query.value(QStringLiteral("title")).toString();
0190         enclosureDetails.type = query.value(QStringLiteral("type")).toString();
0191         enclosureDetails.url = query.value(QStringLiteral("url")).toString();
0192         enclosureDetails.playPosition = query.value(QStringLiteral("id")).toInt();
0193         enclosureDetails.downloaded = Enclosure::dbToStatus(query.value(QStringLiteral("downloaded")).toInt());
0194         m_enclosures += enclosureDetails;
0195     }
0196
0197     query.prepare(QStringLiteral("SELECT * FROM Authors WHERE feed=:feed;"));
0198     query.bindValue(QStringLiteral(":feed"), m_url);
0199     Database::execute(query);
0200     while (query.next()) {
0201         AuthorDetails authorDetails;
0202         authorDetails.feed = m_url;
0203         authorDetails.id = query.value(QStringLiteral("id")).toString();
0204         authorDetails.name = query.value(QStringLiteral("name")).toString();
0205         authorDetails.uri = query.value(QStringLiteral("uri")).toString();
0206         authorDetails.email = query.value(QStringLiteral("email")).toString();
0207         m_authors += authorDetails;
0208     }
0209
0210     query.prepare(QStringLiteral("SELECT * FROM Chapters WHERE feed=:feed;"));
0211     query.bindValue(QStringLiteral(":feed"), m_url);
0212     Database::execute(query);
0213     while (query.next()) {
0214         ChapterDetails chapterDetails;
0215         chapterDetails.feed = m_url;
0216         chapterDetails.id = query.value(QStringLiteral("id")).toString();
0217         chapterDetails.start = query.value(QStringLiteral("start")).toInt();
0218         chapterDetails.title = query.value(QStringLiteral("title")).toString();
0219         chapterDetails.link = query.value(QStringLiteral("link")).toString();
0220         chapterDetails.image = query.value(QStringLiteral("image")).toString();
0221         m_chapters += chapterDetails;
0222     }
0223
0224     query.clear(); // make sure this query is not blocking anything anymore
0225
0226     // Process feed authors
0227     if (feed->authors().count() > 0) {
0228         for (auto &author : feed->authors()) {
0229             processAuthor(QLatin1String(""), author->name(), QLatin1String(""), QLatin1String(""));
0230         }
0231     } else {
0232         // Try to find itunes fields if plain author doesn't exist
0233         QString authorname, authoremail;
0234         // First try the "itunes:owner" tag, if that doesn't succeed, then try the "itunes:author" tag
0235         if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).hasChildNodes()) {
0236             QDomNodeList nodelist = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdowner")).childNodes();
0237             for (int i = 0; i < nodelist.length(); i++) {
0238                 if (nodelist.item(i).nodeName() == QStringLiteral("itunes:name")) {
0239                     authorname = nodelist.item(i).toElement().text();
0240                 } else if (nodelist.item(i).nodeName() == QStringLiteral("itunes:email")) {
0241                     authoremail = nodelist.item(i).toElement().text();
0242                 }
0243             }
0244         } else {
0245             authorname = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
0246             qCDebug(kastsFetcher) << "authorname" << authorname;
0247         }
0248         if (!authorname.isEmpty()) {
0249             processAuthor(QLatin1String(""), authorname, QLatin1String(""), authoremail);
0250         }
0251     }
0252
0253     qCDebug(kastsFetcher) << "Updated feed details:" << feed->title();
0254
0255     // TODO: Only emit signal if the details have really changed
0256     Q_EMIT feedDetailsUpdated(m_url, feed->title(), image, feed->link(), feed->description(), current, m_dirname);
0257
0258     if (m_abort)
0259         return;
0260
0261     // Now deal with the entries, enclosures, entry authors and chapter marks
0262     bool updatedEntries = false;
0263     for (const auto &entry : feed->items()) {
0264         if (m_abort)
0265             return;
0266
0267         bool isNewEntry = processEntry(entry);
0268         updatedEntries = updatedEntries || isNewEntry;
0269     }
0270
0271     writeToDatabase();
0272
0273     if (m_isNewFeed) {
0274         // Finally, reset the new flag to false now that the new feed has been
0275         // fully processed.  If we would reset the flag sooner, then too many
0276         // episodes will get flagged as new if the initial import gets
0277         // interrupted somehow.
0278         query.prepare(QStringLiteral("UPDATE Feeds SET new=:new WHERE url=:url;"));
0279         query.bindValue(QStringLiteral(":url"), m_url);
0280         query.bindValue(QStringLiteral(":new"), false);
0281         Database::execute(query);
0282     }
0283
0284     if (m_oldHash != m_newHash) {
0285         query.prepare(QStringLiteral("UPDATE Feeds SET lastHash=:lastHash WHERE url=:url;"));
0286         query.bindValue(QStringLiteral(":url"), m_url);
0287         query.bindValue(QStringLiteral(":lastHash"), m_newHash);
0288         Database::execute(query);
0289     }
0290
0291     if (updatedEntries || m_isNewFeed)
0292         Q_EMIT feedUpdated(m_url);
0293     qCDebug(kastsFetcher) << "done processing feed" << feed;
0294 }
0295
0296 bool UpdateFeedJob::processEntry(Syndication::ItemPtr entry)
0297 {
0298     qCDebug(kastsFetcher) << "Processing" << entry->title();
0299     bool isNewEntry = true;
0300     bool isUpdateEntry = false;
0301     bool isUpdateDependencies = false;
0302     EntryDetails currentEntry;
0303
0304     // check against existing entries and the list of new entries
0305     for (const EntryDetails &entryDetails : (m_entries + m_newEntries)) {
0306         if (entryDetails.id == entry->id()) {
0307             isNewEntry = false;
0308             currentEntry = entryDetails;
0309         }
0310     }
0311
0312     // stop here if doFullUpdate is set to false and this is an existing entry
0313     if (!isNewEntry && !SettingsManager::self()->doFullUpdate()) {
0314         return false;
0315     }
0316
0317     // Retrieve "other" fields; this will include the "itunes" tags
0318     QMultiMap<QString, QDomElement> otherItems = entry->additionalProperties();
0319
0320     for (const QString &key : otherItems.uniqueKeys()) {
0321         qCDebug(kastsFetcher) << "other elements";
0322         qCDebug(kastsFetcher) << key << otherItems.value(key).tagName();
0323     }
0324
0325     EntryDetails entryDetails;
0326     entryDetails.feed = m_url;
0327     entryDetails.id = entry->id();
0328     entryDetails.title = QTextDocumentFragment::fromHtml(entry->title()).toPlainText();
0329     entryDetails.created = static_cast<int>(entry->datePublished());
0330     entryDetails.updated = static_cast<int>(entry->dateUpdated());
0331     entryDetails.link = entry->link();
0332     entryDetails.hasEnclosure = (entry->enclosures().length() > 0);
0333     entryDetails.read = m_isNewFeed ? m_markUnreadOnNewFeed : false; // if new feed, then check settings
0334     entryDetails.isNew = !m_isNewFeed; // if new feed, then mark none as new
0335
0336     // Take the longest text, either content or description
0337     if (entry->content().length() > entry->description().length()) {
0338         entryDetails.content = entry->content();
0339     } else {
0340         entryDetails.content = entry->description();
0341     }
0342
0343     // Look for image in itunes tags
0344     if (otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).hasAttribute(QStringLiteral("href"))) {
0345         entryDetails.image = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdimage")).attribute(QStringLiteral("href"));
0346     } else if (otherItems.contains(QStringLiteral("http://search.yahoo.com/mrss/thumbnail"))) {
0347         entryDetails.image = otherItems.value(QStringLiteral("http://search.yahoo.com/mrss/thumbnail")).attribute(QStringLiteral("url"));
0348     }
0349     if (entryDetails.image.startsWith(QStringLiteral("/"))) {
0350         entryDetails.image = QUrl(m_url).adjusted(QUrl::RemovePath).toString() + entryDetails.image;
0351     }
0352     qCDebug(kastsFetcher) << "Entry image found" << entryDetails.image;
0353
0354     // if this is an existing episode, check if it needs updating
0355     if (!isNewEntry) {
0356         if ((currentEntry.title != entryDetails.title) || (currentEntry.content != entryDetails.content) || (currentEntry.created != entryDetails.created)
0357             || (currentEntry.updated != entryDetails.updated) || (currentEntry.link != entryDetails.link)
0358             || (currentEntry.hasEnclosure != entryDetails.hasEnclosure) || (currentEntry.image != entryDetails.image)) {
0359             qCDebug(kastsFetcher) << "episode details have been updated:" << entry->id();
0360             isUpdateEntry = true;
0361             m_updateEntries += entryDetails;
0362         } else {
0363             qCDebug(kastsFetcher) << "episode details are unchanged:" << entry->id();
0364         }
0365     } else {
0366         qCDebug(kastsFetcher) << "this is a new episode:" << entry->id();
0367         m_newEntries += entryDetails;
0368     }
0369
0370     // Process authors
0371     if (entry->authors().count() > 0) {
0372         for (const auto &author : entry->authors()) {
0373             isUpdateDependencies = isUpdateDependencies | processAuthor(entry->id(), author->name(), author->uri(), author->email());
0374         }
0375     } else {
0376         // As fallback, check if there is itunes "author" information
0377         QString authorName = otherItems.value(QStringLiteral("http://www.itunes.com/dtds/podcast-1.0.dtdauthor")).text();
0378         if (!authorName.isEmpty())
0379             isUpdateDependencies = isUpdateDependencies | processAuthor(entry->id(), authorName, QLatin1String(""), QLatin1String(""));
0380     }
0381
0382     // Process chapters
0383     if (otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).hasChildNodes()) {
0384         QDomNodeList nodelist = otherItems.value(QStringLiteral("http://podlove.org/simple-chapterschapters")).childNodes();
0385         for (int i = 0; i < nodelist.length(); i++) {
0386             if (nodelist.item(i).nodeName() == QStringLiteral("psc:chapter")) {
0387                 QDomElement element = nodelist.at(i).toElement();
0388                 QString title = element.attribute(QStringLiteral("title"));
0389                 QString start = element.attribute(QStringLiteral("start"));
0390                 QStringList startParts = start.split(QStringLiteral(":"));
0391                 // Some podcasts use colon for milliseconds as well
0392                 while (startParts.count() > 3) {
0393                     startParts.removeLast();
0394                 }
0395                 int startInt = 0;
0396                 for (QString part : startParts) {
0397                     // strip off decimal point if it's present
0398                     startInt = part.split(QStringLiteral("."))[0].toInt() + startInt * 60;
0399                 }
0400                 qCDebug(kastsFetcher) << "Found chapter mark:" << start << "; in seconds:" << startInt;
0401                 QString images = element.attribute(QStringLiteral("image"));
0402                 isUpdateDependencies = isUpdateDependencies | processChapter(entry->id(), startInt, title, entry->link(), images);
0403             }
0404         }
0405     }
0406
0407     // Process enclosures
0408     // only process first enclosure if there are multiple (e.g. mp3 and ogg);
0409     // the first one is probably the podcast author's preferred version
0410     // TODO: handle more than one enclosure?
0411     if (entry->enclosures().count() > 0) {
0412         isUpdateDependencies = isUpdateDependencies | processEnclosure(entry->enclosures()[0], entryDetails, currentEntry);
0413     }
0414
0415     return isNewEntry | isUpdateEntry | isUpdateDependencies; // this is a new or updated entry, or an enclosure, chapter or author has been changed/added
0416 }
0417
0418 bool UpdateFeedJob::processAuthor(const QString &entryId, const QString &authorName, const QString &authorUri, const QString &authorEmail)
0419 {
0420     bool isNewAuthor = true;
0421     bool isUpdateAuthor = false;
0422     AuthorDetails currentAuthor;
0423
0424     // check against existing authors already in database
0425     for (const AuthorDetails &authorDetails : (m_authors + m_newAuthors)) {
0426         if ((authorDetails.id == entryId) && (authorDetails.name == authorName)) {
0427             isNewAuthor = false;
0428             currentAuthor = authorDetails;
0429         }
0430     }
0431
0432     AuthorDetails authorDetails;
0433     authorDetails.feed = m_url;
0434     authorDetails.id = entryId;
0435     authorDetails.name = authorName;
0436     authorDetails.uri = authorUri;
0437     authorDetails.email = authorEmail;
0438
0439     if (!isNewAuthor) {
0440         if ((currentAuthor.uri != authorDetails.uri) || (currentAuthor.email != authorDetails.email)) {
0441             qCDebug(kastsFetcher) << "author details have been updated for:" << entryId << authorName;
0442             isUpdateAuthor = true;
0443             m_updateAuthors += authorDetails;
0444         } else {
0445             qCDebug(kastsFetcher) << "author details are unchanged:" << entryId << authorName;
0446         }
0447     } else {
0448         qCDebug(kastsFetcher) << "this is a new author:" << entryId << authorName;
0449         m_newAuthors += authorDetails;
0450     }
0451
0452     return isNewAuthor | isUpdateAuthor;
0453 }
0454
0455 bool UpdateFeedJob::processEnclosure(Syndication::EnclosurePtr enclosure, const EntryDetails &newEntry, const EntryDetails &oldEntry)
0456 {
0457     bool isNewEnclosure = true;
0458     bool isUpdateEnclosure = false;
0459     EnclosureDetails currentEnclosure;
0460
0461     // check against existing enclosures already in database
0462     for (const EnclosureDetails &enclosureDetails : (m_enclosures + m_newEnclosures)) {
0463         if (enclosureDetails.id == newEntry.id) {
0464             isNewEnclosure = false;
0465             currentEnclosure = enclosureDetails;
0466         }
0467     }
0468
0469     EnclosureDetails enclosureDetails;
0470     enclosureDetails.feed = m_url;
0471     enclosureDetails.id = newEntry.id;
0472     enclosureDetails.duration = enclosure->duration();
0473     enclosureDetails.size = enclosure->length();
0474     enclosureDetails.title = enclosure->title();
0475     enclosureDetails.type = enclosure->type();
0476     enclosureDetails.url = enclosure->url();
0477     enclosureDetails.playPosition = 0;
0478     enclosureDetails.downloaded = Enclosure::Downloadable;
0479
0480     if (!isNewEnclosure) {
0481         if ((currentEnclosure.url != enclosureDetails.url) || (currentEnclosure.title != enclosureDetails.title)
0482             || (currentEnclosure.type != enclosureDetails.type)) {
0483             qCDebug(kastsFetcher) << "enclosure details have been updated for:" << newEntry.id;
0484             isUpdateEnclosure = true;
0485             m_updateEnclosures += enclosureDetails;
0486         } else {
0487             qCDebug(kastsFetcher) << "enclosure details are unchanged:" << newEntry.id;
0488         }
0489
0490         // Check if entry title or enclosure URL has changed
0491         if (newEntry.title != oldEntry.title) {
0492             QString oldFilename = StorageManager::instance().enclosurePath(oldEntry.title, currentEnclosure.url, m_dirname);
0493             QString newFilename = StorageManager::instance().enclosurePath(newEntry.title, enclosureDetails.url, m_dirname);
0494
0495             if (oldFilename != newFilename) {
0496                 if (currentEnclosure.url == enclosureDetails.url) {
0497                     // If entry title has changed but URL is still the same, the existing enclosure needs to be renamed
0498                     QFile::rename(oldFilename, newFilename);
0499                 } else {
0500                     // If enclosure URL has changed, the old enclosure needs to be deleted
0501                     if (QFile(oldFilename).exists()) {
0502                         QFile(oldFilename).remove();
0503                     }
0504                 }
0505             }
0506         }
0507     } else {
0508         qCDebug(kastsFetcher) << "this is a new enclosure:" << newEntry.id;
0509         m_newEnclosures += enclosureDetails;
0510     }
0511
0512     return isNewEnclosure | isUpdateEnclosure;
0513 }
0514
0515 bool UpdateFeedJob::processChapter(const QString &entryId, const int &start, const QString &chapterTitle, const QString &link, const QString &image)
0516 {
0517     bool isNewChapter = true;
0518     bool isUpdateChapter = false;
0519     ChapterDetails currentChapter;
0520
0521     // check against existing enclosures already in database
0522     for (const ChapterDetails &chapterDetails : (m_chapters + m_newChapters)) {
0523         if ((chapterDetails.id == entryId) && (chapterDetails.start == start)) {
0524             isNewChapter = false;
0525             currentChapter = chapterDetails;
0526         }
0527     }
0528
0529     ChapterDetails chapterDetails;
0530     chapterDetails.feed = m_url;
0531     chapterDetails.id = entryId;
0532     chapterDetails.start = start;
0533     chapterDetails.title = chapterTitle;
0534     chapterDetails.link = link;
0535     chapterDetails.image = image;
0536
0537     if (!isNewChapter) {
0538         if ((currentChapter.title != chapterDetails.title) || (currentChapter.link != chapterDetails.link) || (currentChapter.image != chapterDetails.image)) {
0539             qCDebug(kastsFetcher) << "chapter details have been updated for:" << entryId << start;
0540             isUpdateChapter = true;
0541             m_updateChapters += chapterDetails;
0542         } else {
0543             qCDebug(kastsFetcher) << "chapter details are unchanged:" << entryId << start;
0544         }
0545     } else {
0546         qCDebug(kastsFetcher) << "this is a new chapter:" << entryId << start;
0547         m_newChapters += chapterDetails;
0548     }
0549
0550     return isNewChapter | isUpdateChapter;
0551 }
0552
0553 void UpdateFeedJob::writeToDatabase()
0554 {
0555     QSqlQuery writeQuery(QSqlDatabase::database(m_url));
0556
0557     Database::transaction(m_url);
0558
0559     // new entries
0560     writeQuery.prepare(
0561         QStringLiteral("INSERT INTO Entries VALUES (:feed, :id, :title, :content, :created, :updated, :link, :read, :new, :hasEnclosure, :image, :favorite);"));
0562     for (const EntryDetails &entryDetails : m_newEntries) {
0563         writeQuery.bindValue(QStringLiteral(":feed"), entryDetails.feed);
0564         writeQuery.bindValue(QStringLiteral(":id"), entryDetails.id);
0565         writeQuery.bindValue(QStringLiteral(":title"), entryDetails.title);
0566         writeQuery.bindValue(QStringLiteral(":content"), entryDetails.content);
0567         writeQuery.bindValue(QStringLiteral(":created"), entryDetails.created);
0568         writeQuery.bindValue(QStringLiteral(":updated"), entryDetails.updated);
0569         writeQuery.bindValue(QStringLiteral(":link"), entryDetails.link);
0570         writeQuery.bindValue(QStringLiteral(":hasEnclosure"), entryDetails.hasEnclosure);
0571         writeQuery.bindValue(QStringLiteral(":read"), entryDetails.read);
0572         writeQuery.bindValue(QStringLiteral(":new"), entryDetails.isNew);
0573         writeQuery.bindValue(QStringLiteral(":image"), entryDetails.image);
0574         writeQuery.bindValue(QStringLiteral(":favorite"), false);
0575         Database::execute(writeQuery);
0576     }
0577
0578     // update entries
0579     writeQuery.prepare(
0580         QStringLiteral("UPDATE Entries SET title=:title, content=:content, created=:created, updated=:updated, link=:link, hasEnclosure=:hasEnclosure, "
0581                        "image=:image WHERE id=:id AND feed=:feed;"));
0582     for (const EntryDetails &entryDetails : m_updateEntries) {
0583         writeQuery.bindValue(QStringLiteral(":feed"), entryDetails.feed);
0584         writeQuery.bindValue(QStringLiteral(":id"), entryDetails.id);
0585         writeQuery.bindValue(QStringLiteral(":title"), entryDetails.title);
0586         writeQuery.bindValue(QStringLiteral(":content"), entryDetails.content);
0587         writeQuery.bindValue(QStringLiteral(":created"), entryDetails.created);
0588         writeQuery.bindValue(QStringLiteral(":updated"), entryDetails.updated);
0589         writeQuery.bindValue(QStringLiteral(":link"), entryDetails.link);
0590         writeQuery.bindValue(QStringLiteral(":hasEnclosure"), entryDetails.hasEnclosure);
0591         writeQuery.bindValue(QStringLiteral(":image"), entryDetails.image);
0592         Database::execute(writeQuery);
0593     }
0594
0595     // new authors
0596     writeQuery.prepare(QStringLiteral("INSERT INTO Authors VALUES(:feed, :id, :name, :uri, :email);"));
0597     for (const AuthorDetails &authorDetails : m_newAuthors) {
0598         writeQuery.bindValue(QStringLiteral(":feed"), authorDetails.feed);
0599         writeQuery.bindValue(QStringLiteral(":id"), authorDetails.id);
0600         writeQuery.bindValue(QStringLiteral(":name"), authorDetails.name);
0601         writeQuery.bindValue(QStringLiteral(":uri"), authorDetails.uri);
0602         writeQuery.bindValue(QStringLiteral(":email"), authorDetails.email);
0603         Database::execute(writeQuery);
0604     }
0605
0606     // update authors
0607     writeQuery.prepare(QStringLiteral("UPDATE Authors SET uri=:uri, email=:email WHERE feed=:feed AND id=:id AND name=:name;"));
0608     for (const AuthorDetails &authorDetails : m_updateAuthors) {
0609         writeQuery.bindValue(QStringLiteral(":feed"), authorDetails.feed);
0610         writeQuery.bindValue(QStringLiteral(":id"), authorDetails.id);
0611         writeQuery.bindValue(QStringLiteral(":name"), authorDetails.name);
0612         writeQuery.bindValue(QStringLiteral(":uri"), authorDetails.uri);
0613         writeQuery.bindValue(QStringLiteral(":email"), authorDetails.email);
0614         Database::execute(writeQuery);
0615     }
0616
0617     // new enclosures
0618     writeQuery.prepare(QStringLiteral("INSERT INTO Enclosures VALUES (:feed, :id, :duration, :size, :title, :type, :url, :playposition, :downloaded);"));
0619     for (const EnclosureDetails &enclosureDetails : m_newEnclosures) {
0620         writeQuery.bindValue(QStringLiteral(":feed"), enclosureDetails.feed);
0621         writeQuery.bindValue(QStringLiteral(":id"), enclosureDetails.id);
0622         writeQuery.bindValue(QStringLiteral(":duration"), enclosureDetails.duration);
0623         writeQuery.bindValue(QStringLiteral(":size"), enclosureDetails.size);
0624         writeQuery.bindValue(QStringLiteral(":title"), enclosureDetails.title);
0625         writeQuery.bindValue(QStringLiteral(":type"), enclosureDetails.type);
0626         writeQuery.bindValue(QStringLiteral(":url"), enclosureDetails.url);
0627         writeQuery.bindValue(QStringLiteral(":playposition"), enclosureDetails.playPosition);
0628         writeQuery.bindValue(QStringLiteral(":downloaded"), Enclosure::statusToDb(enclosureDetails.downloaded));
0629         Database::execute(writeQuery);
0630     }
0631
0632     // update enclosures
0633     writeQuery.prepare(QStringLiteral("UPDATE Enclosures SET duration=:duration, size=:size, title=:title, type=:type, url=:url WHERE feed=:feed AND id=:id;"));
0634     for (const EnclosureDetails &enclosureDetails : m_updateEnclosures) {
0635         writeQuery.bindValue(QStringLiteral(":feed"), enclosureDetails.feed);
0636         writeQuery.bindValue(QStringLiteral(":id"), enclosureDetails.id);
0637         writeQuery.bindValue(QStringLiteral(":duration"), enclosureDetails.duration);
0638         writeQuery.bindValue(QStringLiteral(":size"), enclosureDetails.size);
0639         writeQuery.bindValue(QStringLiteral(":title"), enclosureDetails.title);
0640         writeQuery.bindValue(QStringLiteral(":type"), enclosureDetails.type);
0641         writeQuery.bindValue(QStringLiteral(":url"), enclosureDetails.url);
0642         Database::execute(writeQuery);
0643     }
0644
0645     // new chapters
0646     writeQuery.prepare(QStringLiteral("INSERT INTO Chapters VALUES(:feed, :id, :start, :title, :link, :image);"));
0647     for (const ChapterDetails &chapterDetails : m_newChapters) {
0648         writeQuery.bindValue(QStringLiteral(":feed"), chapterDetails.feed);
0649         writeQuery.bindValue(QStringLiteral(":id"), chapterDetails.id);
0650         writeQuery.bindValue(QStringLiteral(":start"), chapterDetails.start);
0651         writeQuery.bindValue(QStringLiteral(":title"), chapterDetails.title);
0652         writeQuery.bindValue(QStringLiteral(":link"), chapterDetails.link);
0653         writeQuery.bindValue(QStringLiteral(":image"), chapterDetails.image);
0654         Database::execute(writeQuery);
0655     }
0656
0657     // update chapters
0658     writeQuery.prepare(QStringLiteral("UPDATE Chapters SET title=:title, link=:link, image=:image WHERE feed=:feed AND id=:id AND start=:start;"));
0659     for (const ChapterDetails &chapterDetails : m_updateChapters) {
0660         writeQuery.bindValue(QStringLiteral(":feed"), chapterDetails.feed);
0661         writeQuery.bindValue(QStringLiteral(":id"), chapterDetails.id);
0662         writeQuery.bindValue(QStringLiteral(":start"), chapterDetails.start);
0663         writeQuery.bindValue(QStringLiteral(":title"), chapterDetails.title);
0664         writeQuery.bindValue(QStringLiteral(":link"), chapterDetails.link);
0665         writeQuery.bindValue(QStringLiteral(":image"), chapterDetails.image);
0666         Database::execute(writeQuery);
0667     }
0668
0669     // set custom amount of episodes to unread/new if required
0670     if (m_isNewFeed && (SettingsManager::self()->markUnreadOnNewFeed() == 1) && (SettingsManager::self()->markUnreadOnNewFeedCustomAmount() > 0)) {
0671         writeQuery.prepare(QStringLiteral(
0672             "UPDATE Entries SET read=:read, new=:new WHERE id in (SELECT id FROM Entries WHERE feed =:feed ORDER BY updated DESC LIMIT :recentUnread);"));
0673         writeQuery.bindValue(QStringLiteral(":feed"), m_url);
0674         writeQuery.bindValue(QStringLiteral(":read"), false);
0675         writeQuery.bindValue(QStringLiteral(":new"), true);
0676         writeQuery.bindValue(QStringLiteral(":recentUnread"), SettingsManager::self()->markUnreadOnNewFeedCustomAmount());
0677         Database::execute(writeQuery);
0678     }
0679
0680     if (Database::commit(m_url)) {
0681         QStringList newIds, updateIds;
0682
0683         // emit signals for new entries
0684         for (const EntryDetails &entryDetails : m_newEntries) {
0685             if (!newIds.contains(entryDetails.id)) {
0686                 newIds += entryDetails.id;
0687             }
0688         }
0689
0690         for (const QString &id : newIds) {
0691             Q_EMIT entryAdded(m_url, id);
0692         }
0693
0694         // emit signals for updated entries or entries with new/updated authors,
0695         // enclosures or chapters
0696         for (const EntryDetails &entryDetails : m_updateEntries) {
0697             if (!updateIds.contains(entryDetails.id) && !newIds.contains(entryDetails.id)) {
0698                 updateIds += entryDetails.id;
0699             }
0700         }
0701         for (const EnclosureDetails &enclosureDetails : (m_newEnclosures + m_updateEnclosures)) {
0702             if (!updateIds.contains(enclosureDetails.id) && !newIds.contains(enclosureDetails.id)) {
0703                 updateIds += enclosureDetails.id;
0704             }
0705         }
0706         for (const AuthorDetails &authorDetails : (m_newAuthors + m_updateAuthors)) {
0707             if (!updateIds.contains(authorDetails.id) && !newIds.contains(authorDetails.id)) {
0708                 updateIds += authorDetails.id;
0709             }
0710         }
0711         for (const ChapterDetails &chapterDetails : (m_newChapters + m_updateChapters)) {
0712             if (!updateIds.contains(chapterDetails.id) && !newIds.contains(chapterDetails.id)) {
0713                 updateIds += chapterDetails.id;
0714             }
0715         }
0716
0717         for (const QString &id : updateIds) {
0718             qCDebug(kastsFetcher) << "updated episode" << id;
0719             Q_EMIT entryUpdated(m_url, id);
0720         }
0721     }
0722 }
0723
0724 QString UpdateFeedJob::generateFeedDirname(const QString &name) const
0725 {
0726     // Generate directory name for enclosures based on feed name
0727     // NOTE: Any changes here require a database migration!
0728     QString dirBaseName = StorageManager::instance().sanitizedFilePath(name);
0729     QString dirName = dirBaseName;
0730
0731     QStringList dirNameList;
0732     QSqlQuery query(QSqlDatabase::database(m_url));
0733     query.prepare(QStringLiteral("SELECT name FROM Feeds;"));
0734     while (query.next()) {
0735         dirNameList << query.value(QStringLiteral("name")).toString();
0736     }
0737
0738     // Check for duplicate names in database and on filesystem
0739     int numDups = 1; // Minimum to append is " (1)" if file already exists
0740     while (dirNameList.contains(dirName) || QDir(StorageManager::instance().enclosureDirPath() + dirName).exists()) {
0741         dirName = QStringLiteral("%1 (%2)").arg(dirBaseName, QString::number(numDups));
0742         numDups++;
0743     }
0744     return dirName;
0745 }
0746
0747 void UpdateFeedJob::abort()
0748 {
0749     m_abort = true;
0750     Q_EMIT aborting();
0751 }