File indexing completed on 2024-05-12 15:55:23
0001 // SPDX-FileCopyrightText: 2006-2008 Tuomas Suutari <tuomas@nepnep.net> 0002 // SPDX-FileCopyrightText: 2006-2014 Jesper K. Pedersen <jesper.pedersen@kdab.com> 0003 // SPDX-FileCopyrightText: 2007 Dirk Mueller <mueller@kde.org> 0004 // SPDX-FileCopyrightText: 2007 Laurent Montel <montel@kde.org> 0005 // SPDX-FileCopyrightText: 2008-2011 Jan Kundrát <jkt@flaska.net> 0006 // SPDX-FileCopyrightText: 2008-2009 Henner Zeller <h.zeller@acm.org> 0007 // SPDX-FileCopyrightText: 2012 Yuri Chornoivan <yurchor@ukr.net> 0008 // SPDX-FileCopyrightText: 2012-2013 Miika Turkia <miika.turkia@gmail.com> 0009 // SPDX-FileCopyrightText: 2014-2020 Tobias Leupold <tl@stonemx.de> 0010 // SPDX-FileCopyrightText: 2018-2020 Robert Krawitz <rlk@alum.mit.edu> 0011 // SPDX-FileCopyrightText: 2012-2023 Johannes Zarl-Zierl <johannes@zarl-zierl.at> 0012 // 0013 // SPDX-License-Identifier: GPL-2.0-or-later 0014 0015 #include "FileWriter.h" 0016 0017 #include "CompressFileInfo.h" 0018 #include "ElementWriter.h" 0019 #include "NumberedBackup.h" 0020 0021 #include <DB/Category.h> 0022 #include <DB/ImageDB.h> 0023 #include <DB/TagInfo.h> 0024 #include <Utilities/List.h> 0025 #include <kpabase/Logging.h> 0026 #include <kpabase/SettingsData.h> 0027 #include <kpabase/UIDelegate.h> 0028 0029 #include <KLocalizedString> 0030 #include <QElapsedTimer> 0031 #include <QFile> 0032 #include <QFileInfo> 0033 #include <QMutexLocker> 0034 #include <QXmlStreamWriter> 0035 0036 // 0037 // 0038 // 0039 // +++++++++++++++++++++++++++++++ REMEMBER ++++++++++++++++++++++++++++++++ 0040 // 0041 // 0042 // 0043 // 0044 // Update DB::ImageDB::fileVersion every time you update the file format! 0045 // 0046 // 0047 // 0048 // 0049 // 0050 // 0051 // 0052 // 0053 // (sorry for the noise, but it is really important :-) 0054 0055 using Utilities::StringSet; 0056 0057 namespace 0058 { 0059 constexpr QFileDevice::Permissions FILE_PERMISSIONS { QFile::ReadOwner | QFile::WriteOwner | QFile::ReadGroup | QFile::WriteGroup | QFile::ReadOther }; 0060 } 0061 0062 void DB::FileWriter::save(const QString &fileName, bool isAutoSave) 0063 { 0064 setUseCompressedFileFormat(Settings::SettingsData::instance()->useCompressedIndexXML()); 0065 0066 if (!isAutoSave) 0067 NumberedBackup(m_db->uiDelegate()).makeNumberedBackup(); 0068 0069 // prepare XML document for saving: 0070 m_db->m_categoryCollection.initIdMap(); 0071 QFile out(fileName + QStringLiteral(".tmp")); 0072 if (!out.open(QIODevice::WriteOnly | QIODevice::Text)) { 0073 m_db->uiDelegate().error( 0074 DB::LogMessage { DBLog(), QStringLiteral("Error saving to file '%1': %2").arg(out.fileName(), out.errorString()) }, i18n("<p>Could not save the image database to XML.</p>" 0075 "File %1 could not be opened because of the following error: %2", 0076 out.fileName(), out.errorString()), 0077 i18n("Error while saving...")); 0078 return; 0079 } 0080 if (!out.setPermissions(FILE_PERMISSIONS)) { 0081 qCWarning(DBLog, "Could not set permissions on file %s!", qPrintable(out.fileName())); 0082 } 0083 QElapsedTimer timer; 0084 if (TimingLog().isDebugEnabled()) 0085 timer.start(); 0086 QXmlStreamWriter writer(&out); 0087 writer.setAutoFormatting(true); 0088 writer.writeStartDocument(); 0089 0090 { 0091 ElementWriter dummy(writer, QStringLiteral("KPhotoAlbum")); 0092 writer.writeAttribute(QStringLiteral("version"), QString::number(DB::ImageDB::fileVersion())); 0093 writer.writeAttribute(QStringLiteral("compressed"), QString::number(useCompressedFileFormat())); 0094 0095 saveCategories(writer); 0096 saveImages(writer); 0097 saveBlockList(writer); 0098 saveMemberGroups(writer); 0099 // saveSettings(writer); 0100 saveGlobalSortOrder(writer); 0101 } 0102 writer.writeEndDocument(); 0103 qCDebug(TimingLog) << "DB::FileWriter::save(): Saving took" << timer.elapsed() << "ms"; 0104 0105 // State: index.xml has previous DB version, index.xml.tmp has the current version. 0106 0107 // original file can be safely deleted 0108 if ((!QFile::remove(fileName)) && QFile::exists(fileName)) { 0109 m_db->uiDelegate().error( 0110 DB::LogMessage { DBLog(), QStringLiteral("Removal of file '%1' failed.").arg(fileName) }, i18n("<p>Failed to remove old version of image database.</p>" 0111 "<p>Please try again or replace the file %1 with file %2 manually!</p>", 0112 fileName, out.fileName()), 0113 i18n("Error while saving...")); 0114 return; 0115 } 0116 // State: index.xml doesn't exist, index.xml.tmp has the current version. 0117 if (!out.rename(fileName)) { 0118 m_db->uiDelegate().error( 0119 DB::LogMessage { DBLog(), QStringLiteral("Renaming index.xml to '%1' failed.").arg(out.fileName()) }, i18n("<p>Failed to move temporary XML file to permanent location.</p>" 0120 "<p>Please try again or rename file %1 to %2 manually!</p>", 0121 out.fileName(), fileName), 0122 i18n("Error while saving...")); 0123 // State: index.xml.tmp has the current version. 0124 return; 0125 } 0126 // State: index.xml has the current version. 0127 } 0128 0129 void DB::FileWriter::saveCategories(QXmlStreamWriter &writer) 0130 { 0131 const QStringList categories = DB::ImageDB::instance()->categoryCollection()->categoryNames(); 0132 ElementWriter dummy(writer, QStringLiteral("Categories")); 0133 0134 DB::CategoryPtr tokensCategory = DB::ImageDB::instance()->categoryCollection()->categoryForSpecial(DB::Category::TokensCategory); 0135 const DB::TagInfo *untaggedTag = DB::ImageDB::instance()->untaggedTag(); 0136 for (const QString &name : categories) { 0137 DB::CategoryPtr category = DB::ImageDB::instance()->categoryCollection()->categoryForName(name); 0138 0139 if (!shouldSaveCategory(name)) { 0140 continue; 0141 } 0142 0143 ElementWriter dummy(writer, QStringLiteral("Category")); 0144 writer.writeAttribute(QStringLiteral("name"), name); 0145 writer.writeAttribute(QStringLiteral("icon"), category->iconName()); 0146 writer.writeAttribute(QStringLiteral("show"), QString::number(category->doShow())); 0147 writer.writeAttribute(QStringLiteral("viewtype"), QString::number(category->viewType())); 0148 writer.writeAttribute(QStringLiteral("thumbnailsize"), QString::number(category->thumbnailSize())); 0149 writer.writeAttribute(QStringLiteral("positionable"), QString::number(category->positionable())); 0150 if (category == tokensCategory) { 0151 writer.writeAttribute(QStringLiteral("meta"), QStringLiteral("tokens")); 0152 } 0153 0154 // As bug 423334 shows, it is easy to forget to add a group to the respective category 0155 // when it's created. We can not enforce correct creation of member groups in our API, 0156 // but we can prevent incorrect data from entering index.xml. 0157 const auto categoryItems = Utilities::mergeListsUniqly(category->items(), m_db->memberMap().groups(name)); 0158 for (const QString &tagName : categoryItems) { 0159 ElementWriter dummy(writer, QStringLiteral("value")); 0160 writer.writeAttribute(QStringLiteral("value"), tagName); 0161 writer.writeAttribute(QStringLiteral("id"), QString::number(category->idForName(tagName))); 0162 QDate birthDate = category->birthDate(tagName); 0163 if (!birthDate.isNull()) 0164 writer.writeAttribute(QStringLiteral("birthDate"), birthDate.toString(Qt::ISODate)); 0165 if (untaggedTag && untaggedTag->category() == category.data() && untaggedTag->tagName() == tagName) { 0166 writer.writeAttribute(QStringLiteral("meta"), QStringLiteral("mark-untagged")); 0167 } 0168 } 0169 } 0170 } 0171 0172 void DB::FileWriter::saveImages(QXmlStreamWriter &writer) 0173 { 0174 DB::ImageInfoList list = m_db->m_images; 0175 0176 // Copy files from clipboard to end of overview, so we don't loose them 0177 const auto clipBoardImages = m_db->m_clipboard; 0178 for (const DB::ImageInfoPtr &infoPtr : clipBoardImages) { 0179 list.append(infoPtr); 0180 } 0181 0182 { 0183 ElementWriter dummy(writer, QStringLiteral("images")); 0184 0185 for (const DB::ImageInfoPtr &infoPtr : qAsConst(list)) { 0186 save(writer, infoPtr); 0187 } 0188 } 0189 } 0190 0191 void DB::FileWriter::saveBlockList(QXmlStreamWriter &writer) 0192 { 0193 ElementWriter dummy(writer, QStringLiteral("blocklist")); 0194 QList<DB::FileName> blockList(m_db->m_blockList.begin(), m_db->m_blockList.end()); 0195 // sort blocklist to get diffable files 0196 std::sort(blockList.begin(), blockList.end()); 0197 for (const DB::FileName &block : qAsConst(blockList)) { 0198 ElementWriter dummy(writer, QStringLiteral("block")); 0199 writer.writeAttribute(QStringLiteral("file"), block.relative()); 0200 } 0201 } 0202 0203 void DB::FileWriter::saveMemberGroups(QXmlStreamWriter &writer) 0204 { 0205 if (m_db->m_members.isEmpty()) 0206 return; 0207 0208 ElementWriter dummy(writer, QStringLiteral("member-groups")); 0209 for (QMap<QString, QMap<QString, StringSet>>::ConstIterator memberMapIt = m_db->m_members.memberMap().constBegin(); 0210 memberMapIt != m_db->m_members.memberMap().constEnd(); ++memberMapIt) { 0211 const QString categoryName = memberMapIt.key(); 0212 0213 // FIXME (l3u): This can happen when an empty sub-category (group) is present. 0214 // Would be fine to fix the reason why this happens in the first place. 0215 if (categoryName.isEmpty()) { 0216 continue; 0217 } 0218 0219 if (!shouldSaveCategory(categoryName)) 0220 continue; 0221 0222 QMap<QString, StringSet> groupMap = memberMapIt.value(); 0223 for (QMap<QString, StringSet>::ConstIterator groupMapIt = groupMap.constBegin(); groupMapIt != groupMap.constEnd(); ++groupMapIt) { 0224 0225 // FIXME (l3u): This can happen when an empty sub-category (group) is present. 0226 // Would be fine to fix the reason why this happens in the first place. 0227 if (groupMapIt.key().isEmpty()) { 0228 continue; 0229 } 0230 0231 if (useCompressedFileFormat()) { 0232 const StringSet members = groupMapIt.value(); 0233 ElementWriter dummy(writer, QStringLiteral("member")); 0234 writer.writeAttribute(QStringLiteral("category"), categoryName); 0235 writer.writeAttribute(QStringLiteral("group-name"), groupMapIt.key()); 0236 QStringList idList; 0237 for (const QString &member : members) { 0238 DB::CategoryPtr catPtr = m_db->m_categoryCollection.categoryForName(categoryName); 0239 if (catPtr->idForName(member) == 0) 0240 qCWarning(DBLog) << "Member" << member << "in group" << categoryName << "->" << groupMapIt.key() << "has no id!"; 0241 idList.append(QString::number(catPtr->idForName(member))); 0242 } 0243 std::sort(idList.begin(), idList.end()); 0244 writer.writeAttribute(QStringLiteral("members"), idList.join(QStringLiteral(","))); 0245 } else { 0246 const auto groupMapItValue = groupMapIt.value(); 0247 QStringList members(groupMapItValue.begin(), groupMapItValue.end()); 0248 std::sort(members.begin(), members.end()); 0249 for (const QString &member : qAsConst(members)) { 0250 ElementWriter dummy(writer, QStringLiteral("member")); 0251 writer.writeAttribute(QStringLiteral("category"), memberMapIt.key()); 0252 writer.writeAttribute(QStringLiteral("group-name"), groupMapIt.key()); 0253 writer.writeAttribute(QStringLiteral("member"), member); 0254 } 0255 0256 // Add an entry even if the group is empty 0257 // (this is not necessary for the compressed format) 0258 if (members.size() == 0) { 0259 ElementWriter dummy(writer, QStringLiteral("member")); 0260 writer.writeAttribute(QStringLiteral("category"), memberMapIt.key()); 0261 writer.writeAttribute(QStringLiteral("group-name"), groupMapIt.key()); 0262 } 0263 } 0264 } 0265 } 0266 } 0267 0268 void DB::FileWriter::saveGlobalSortOrder(QXmlStreamWriter &writer) 0269 { 0270 ElementWriter dummy(writer, QStringLiteral("global-sort-order")); 0271 for (const auto &item : m_db->categoryCollection()->globalSortOrder()->modifiedSortOrder()) { 0272 ElementWriter dummy(writer, QStringLiteral("item")); 0273 writer.writeAttribute(QStringLiteral("category"), item.category); 0274 writer.writeAttribute(QStringLiteral("item"), item.item); 0275 } 0276 } 0277 0278 /* 0279 Perhaps, we may need this later ;-) 0280 0281 void DB::FileWriter::saveSettings(QXmlStreamWriter& writer) 0282 { 0283 ElementWriter dummy(writer, settingsString); 0284 0285 QMap<QString, QString> settings; 0286 // For testing 0287 settings.insert(QStringLiteral("tokensCategory"), QStringLiteral("Tokens")); 0288 settings.insert(QStringLiteral("untaggedCategory"), QStringLiteral("Events")); 0289 settings.insert(QStringLiteral("untaggedTag"), QStringLiteral("untagged")); 0290 0291 QMapIterator<QString, QString> settingsIterator(settings); 0292 while (settingsIterator.hasNext()) { 0293 ElementWriter dummy(writer, settingString); 0294 settingsIterator.next(); 0295 writer.writeAttribute(QStringLiteral("key"), escape(settingsIterator.key())); 0296 writer.writeAttribute(QStringLiteral("value"), escape(settingsIterator.value())); 0297 } 0298 } 0299 */ 0300 0301 static const QString &stdDateTimeToString(const Utilities::FastDateTime &date) 0302 { 0303 static QString s_lastDateTimeString; 0304 static Utilities::FastDateTime s_lastDateTime; 0305 static QMutex s_lastDateTimeLocker; 0306 QMutexLocker dummy(&s_lastDateTimeLocker); 0307 if (date.isValid() && date != s_lastDateTime) { 0308 s_lastDateTime = date; 0309 s_lastDateTimeString = date.toString(Qt::ISODate); 0310 } 0311 return s_lastDateTimeString; 0312 } 0313 0314 void DB::FileWriter::save(QXmlStreamWriter &writer, const DB::ImageInfoPtr &info) 0315 { 0316 ElementWriter dummy(writer, QStringLiteral("image")); 0317 writer.writeAttribute(QStringLiteral("file"), info->fileName().relative()); 0318 if (info->label() != QFileInfo(info->fileName().relative()).completeBaseName()) 0319 writer.writeAttribute(QStringLiteral("label"), info->label()); 0320 if (!info->description().isEmpty()) 0321 writer.writeAttribute(QStringLiteral("description"), info->description()); 0322 0323 DB::ImageDate date = info->date(); 0324 Utilities::FastDateTime start = date.start(); 0325 Utilities::FastDateTime end = date.end(); 0326 0327 writer.writeAttribute(QStringLiteral("startDate"), stdDateTimeToString(start)); 0328 if (start != end) 0329 writer.writeAttribute(QStringLiteral("endDate"), stdDateTimeToString(end)); 0330 0331 if (info->angle() != 0) 0332 writer.writeAttribute(QStringLiteral("angle"), QString::number(info->angle())); 0333 writer.writeAttribute(QStringLiteral("md5sum"), info->MD5Sum().toHexString()); 0334 writer.writeAttribute(QStringLiteral("width"), QString::number(info->size().width())); 0335 writer.writeAttribute(QStringLiteral("height"), QString::number(info->size().height())); 0336 0337 if (info->rating() != -1) { 0338 writer.writeAttribute(QStringLiteral("rating"), QString::number(info->rating())); 0339 } 0340 0341 if (info->stackId()) { 0342 writer.writeAttribute(QStringLiteral("stackId"), QString::number(info->stackId())); 0343 writer.writeAttribute(QStringLiteral("stackOrder"), QString::number(info->stackOrder())); 0344 } 0345 0346 if (info->isVideo()) 0347 writer.writeAttribute(QStringLiteral("videoLength"), QString::number(info->videoLength())); 0348 0349 if (useCompressedFileFormat()) 0350 writeCategoriesCompressed(writer, info); 0351 else 0352 writeCategories(writer, info); 0353 } 0354 0355 QString DB::FileWriter::areaToString(QRect area) const 0356 { 0357 QStringList areaString; 0358 areaString.append(QString::number(area.x())); 0359 areaString.append(QString::number(area.y())); 0360 areaString.append(QString::number(area.width())); 0361 areaString.append(QString::number(area.height())); 0362 return areaString.join(QStringLiteral(" ")); 0363 } 0364 0365 void DB::FileWriter::writeCategories(QXmlStreamWriter &writer, const DB::ImageInfoPtr &info) 0366 { 0367 ElementWriter topElm(writer, QStringLiteral("options"), false); 0368 0369 QStringList grps = info->availableCategories(); 0370 // in contrast to CategoryCollection::categories, availableCategories is randomly sorted (since it is now a QHash) 0371 grps.sort(); 0372 for (const QString &name : grps) { 0373 if (!shouldSaveCategory(name)) 0374 continue; 0375 0376 ElementWriter categoryElm(writer, QStringLiteral("option"), false); 0377 0378 const auto itemsOfCategory = info->itemsOfCategory(name); 0379 QStringList items(itemsOfCategory.begin(), itemsOfCategory.end()); 0380 std::sort(items.begin(), items.end()); 0381 if (!items.isEmpty()) { 0382 topElm.writeStartElement(); 0383 categoryElm.writeStartElement(); 0384 writer.writeAttribute(QStringLiteral("name"), name); 0385 } 0386 0387 for (const QString &itemValue : qAsConst(items)) { 0388 ElementWriter dummy(writer, QStringLiteral("value")); 0389 writer.writeAttribute(QStringLiteral("value"), itemValue); 0390 0391 QRect area = info->areaForTag(name, itemValue); 0392 if (!area.isNull()) { 0393 writer.writeAttribute(QStringLiteral("area"), areaToString(area)); 0394 } 0395 } 0396 } 0397 } 0398 0399 void DB::FileWriter::writeCategoriesCompressed(QXmlStreamWriter &writer, const DB::ImageInfoPtr &info) 0400 { 0401 QMap<QString, QList<QPair<QString, QRect>>> positionedTags; 0402 0403 const QList<DB::CategoryPtr> categoryList = DB::ImageDB::instance()->categoryCollection()->categories(); 0404 for (const DB::CategoryPtr &category : categoryList) { 0405 QString categoryName = category->name(); 0406 0407 if (!shouldSaveCategory(categoryName)) 0408 continue; 0409 0410 const StringSet items = info->itemsOfCategory(categoryName); 0411 if (!items.empty()) { 0412 QStringList idList; 0413 0414 for (const QString &itemValue : items) { 0415 QRect area = info->areaForTag(categoryName, itemValue); 0416 0417 if (area.isValid()) { 0418 // Positioned tags can't be stored in the "fast" format 0419 // so we have to handle them separately 0420 positionedTags[categoryName] << QPair<QString, QRect>(itemValue, area); 0421 } else { 0422 int id = category->idForName(itemValue); 0423 idList.append(QString::number(id)); 0424 } 0425 } 0426 0427 // Possibly all ids of a category have area information, so only 0428 // write the category attribute if there are actually ids to write 0429 if (!idList.isEmpty()) { 0430 std::sort(idList.begin(), idList.end()); 0431 writer.writeAttribute(escape(categoryName), idList.join(QStringLiteral(","))); 0432 } 0433 } 0434 } 0435 0436 // Add a "readable" sub-element for the positioned tags 0437 // FIXME: can this be merged with the code in writeCategories()? 0438 if (!positionedTags.isEmpty()) { 0439 ElementWriter topElm(writer, QStringLiteral("options"), false); 0440 topElm.writeStartElement(); 0441 0442 QMapIterator<QString, QList<QPair<QString, QRect>>> categoryWithAreas(positionedTags); 0443 while (categoryWithAreas.hasNext()) { 0444 categoryWithAreas.next(); 0445 0446 ElementWriter categoryElm(writer, QStringLiteral("option"), false); 0447 categoryElm.writeStartElement(); 0448 writer.writeAttribute(QStringLiteral("name"), categoryWithAreas.key()); 0449 0450 QList<QPair<QString, QRect>> areas = categoryWithAreas.value(); 0451 std::sort(areas.begin(), areas.end(), 0452 [](QPair<QString, QRect> a, QPair<QString, QRect> b) { return a.first < b.first; }); 0453 for (const auto &positionedTag : qAsConst(areas)) { 0454 ElementWriter dummy(writer, QStringLiteral("value")); 0455 writer.writeAttribute(QStringLiteral("value"), positionedTag.first); 0456 writer.writeAttribute(QStringLiteral("area"), areaToString(positionedTag.second)); 0457 } 0458 } 0459 } 0460 } 0461 0462 bool DB::FileWriter::shouldSaveCategory(const QString &categoryName) const 0463 { 0464 // Profiling indicated that this function was a hotspot, so this cache improved saving speed with 25% 0465 static QHash<QString, bool> cache; 0466 if (cache.contains(categoryName)) 0467 return cache[categoryName]; 0468 0469 // A few bugs has shown up, where an invalid category name has crashed KPA. It therefore checks for such invalid names here. 0470 if (!m_db->m_categoryCollection.categoryForName(categoryName)) { 0471 qCWarning(DBLog, "Invalid category name: %s", qPrintable(categoryName)); 0472 cache.insert(categoryName, false); 0473 return false; 0474 } 0475 0476 const auto category = m_db->m_categoryCollection.categoryForName(categoryName).data(); 0477 Q_ASSERT(category); 0478 const bool shouldSave = category->shouldSave(); 0479 cache.insert(categoryName, shouldSave); 0480 return shouldSave; 0481 } 0482 0483 /** 0484 * @brief Escape problematic characters in a string that forms an XML attribute name. 0485 * 0486 * N.B.: Attribute values do not need to be escaped! 0487 * @see DB::FileReader::unescape 0488 * 0489 * @param str the string to be escaped 0490 * @return the escaped string 0491 */ 0492 QString DB::FileWriter::escape(const QString &str) 0493 { 0494 static bool hashUsesCompressedFormat = useCompressedFileFormat(); 0495 static QHash<QString, QString> s_cache; 0496 if (hashUsesCompressedFormat != useCompressedFileFormat()) 0497 s_cache.clear(); 0498 0499 if (s_cache.contains(str)) 0500 return s_cache[str]; 0501 0502 QString tmp(str); 0503 // Regex to match characters that are not allowed to start XML attribute names 0504 static const QRegExp rx(QStringLiteral("([^a-zA-Z0-9:_])")); 0505 int pos = 0; 0506 0507 // Encoding special characters if compressed XML is selected 0508 if (useCompressedFileFormat()) { 0509 while ((pos = rx.indexIn(tmp, pos)) != -1) { 0510 QString before = rx.cap(1); 0511 QString after = QString::asprintf("_.%0X", rx.cap(1).data()->toLatin1()); 0512 tmp.replace(pos, before.length(), after); 0513 pos += after.length(); 0514 } 0515 } else 0516 tmp.replace(QStringLiteral(" "), QStringLiteral("_")); 0517 s_cache.insert(str, tmp); 0518 return tmp; 0519 } 0520 0521 // vi:expandtab:tabstop=4 shiftwidth=4: