Warning, file /frameworks/ki18n/src/localedata/isocodescache.cpp was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).

0001 /*
0002     SPDX-FileCopyrightText: 2021 Volker Krause <vkrause@kde.org>
0003 
0004     SPDX-License-Identifier: LGPL-2.0-or-later
0005 */
0006 
0007 #include "config-localedata.h"
0008 
0009 #include "isocodes_p.h"
0010 #include "isocodescache_p.h"
0011 #include "logging.h"
0012 
0013 #include <QDir>
0014 #include <QFile>
0015 #include <QFileInfo>
0016 #include <QJsonArray>
0017 #include <QJsonDocument>
0018 #include <QJsonObject>
0019 #include <QStandardPaths>
0020 
0021 // increment those when changing the format
0022 enum : uint32_t {
0023     Iso3166_1CacheHeader = 0x4B493101,
0024     Iso3166_2CacheHeader = 0x4B493201,
0025 };
0026 
0027 static QString isoCodesPath(QStringView file)
0028 {
0029 #ifndef Q_OS_ANDROID
0030     auto path = QStandardPaths::locate(QStandardPaths::GenericDataLocation, QLatin1String("iso-codes/json/") + file, QStandardPaths::LocateFile);
0031     if (!path.isEmpty()) {
0032         return path;
0033     }
0034 
0035     // search manually in the compile-time determined prefix
0036     // needed for example for non-installed Windows binaries to work, such as unit tests
0037     for (const char *installLocation : {"/share", "/bin/data"}) {
0038         path = QLatin1String(ISO_CODES_PREFIX) + QLatin1String(installLocation) + QLatin1String("/iso-codes/json/") + file;
0039         if (QFileInfo::exists(path)) {
0040             return path;
0041         }
0042     }
0043 
0044     return {};
0045 #else
0046     return QLatin1String("assets:/share/iso-codes/json/") + file;
0047 #endif
0048 }
0049 
0050 static QString cachePath()
0051 {
0052     return QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.ki18n/iso-codes/");
0053 }
0054 
0055 static QString cacheFilePath(QStringView file)
0056 {
0057     return cachePath() + file;
0058 }
0059 
0060 IsoCodesCache::~IsoCodesCache() = default;
0061 
0062 IsoCodesCache *IsoCodesCache::instance()
0063 {
0064     static IsoCodesCache s_cache;
0065     return &s_cache;
0066 }
0067 
0068 void IsoCodesCache::loadIso3166_1()
0069 {
0070     if (!m_iso3166_1CacheData && !loadIso3166_1Cache()) {
0071         createIso3166_1Cache();
0072         loadIso3166_1Cache();
0073     }
0074 }
0075 
0076 bool IsoCodesCache::loadIso3166_1Cache()
0077 {
0078     QFileInfo jsonFi(isoCodesPath(u"iso_3166-1.json"));
0079     auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-1"));
0080     if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) {
0081         return false;
0082     }
0083     m_iso3166_1CacheSize = f->size();
0084 
0085     // validate cache file is usable
0086     // header matches
0087     const auto data = f->map(0, m_iso3166_1CacheSize);
0088     if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_1CacheHeader) {
0089         return false;
0090     }
0091     // lookup tables fit into the available size
0092     const auto size = *(reinterpret_cast<const uint32_t *>(data) + 1);
0093     if (sizeof(Iso3166_1CacheHeader) + sizeof(size) + size * sizeof(MapEntry<uint16_t>) * 2 >= m_iso3166_1CacheSize) {
0094         return false;
0095     }
0096     // string table is 0 terminated
0097     if (data[m_iso3166_1CacheSize - 1] != '\0') {
0098         return false;
0099     }
0100 
0101     m_iso3166_1CacheFile = std::move(f);
0102     m_iso3166_1CacheData = data;
0103     return true;
0104 }
0105 
0106 uint32_t IsoCodesCache::countryCount() const
0107 {
0108     return m_iso3166_1CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_1CacheData) + 1) : 0;
0109 }
0110 
0111 const MapEntry<uint16_t> *IsoCodesCache::countryNameMapBegin() const
0112 {
0113     return m_iso3166_1CacheData ? reinterpret_cast<const MapEntry<uint16_t> *>(m_iso3166_1CacheData + sizeof(uint32_t) * 2) : nullptr;
0114 }
0115 
0116 const MapEntry<uint16_t> *IsoCodesCache::countryAlpha3MapBegin() const
0117 {
0118     return m_iso3166_1CacheData ? countryNameMapBegin() + countryCount() : nullptr;
0119 }
0120 
0121 const char *IsoCodesCache::countryStringTableLookup(uint16_t offset) const
0122 {
0123     if (m_iso3166_1CacheData) {
0124         const auto pos = offset + 2 * sizeof(uint32_t) + 2 * countryCount() * sizeof(MapEntry<uint16_t>);
0125         return m_iso3166_1CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_1CacheData + pos) : nullptr;
0126     }
0127     return nullptr;
0128 }
0129 
0130 void IsoCodesCache::createIso3166_1Cache()
0131 {
0132     qCDebug(KI18NLD) << "Rebuilding ISO 3166-1 cache";
0133     const auto path = isoCodesPath(u"iso_3166-1.json");
0134 
0135     QFile file(path);
0136     if (!file.open(QFile::ReadOnly)) {
0137         qCWarning(KI18NLD) << "Unable to open iso_3166-1.json" << path << file.errorString();
0138         return;
0139     }
0140 
0141     std::vector<MapEntry<uint16_t>> alpha2NameMap;
0142     std::vector<MapEntry<uint16_t>> alpha3alpha2Map;
0143     QByteArray iso3166_1stringTable;
0144 
0145     const auto doc = QJsonDocument::fromJson(file.readAll());
0146     const auto array = doc.object().value(QLatin1String("3166-1")).toArray();
0147     for (const auto &entryVal : array) {
0148         const auto entry = entryVal.toObject();
0149         const auto alpha2 = entry.value(QLatin1String("alpha_2")).toString();
0150         if (alpha2.size() != 2) {
0151             continue;
0152         }
0153         const auto alpha2Key = IsoCodes::alpha2CodeToKey(alpha2);
0154 
0155         assert(std::numeric_limits<uint16_t>::max() > iso3166_1stringTable.size());
0156         alpha2NameMap.push_back({alpha2Key, (uint16_t)iso3166_1stringTable.size()});
0157         iso3166_1stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8());
0158         iso3166_1stringTable.append('\0');
0159 
0160         const auto alpha3Key = IsoCodes::alpha3CodeToKey(entry.value(QLatin1String("alpha_3")).toString());
0161         alpha3alpha2Map.push_back({alpha3Key, alpha2Key});
0162     }
0163 
0164     std::sort(alpha2NameMap.begin(), alpha2NameMap.end());
0165     std::sort(alpha3alpha2Map.begin(), alpha3alpha2Map.end());
0166 
0167     // write out binary cache file
0168     QDir().mkpath(cachePath());
0169     QFile cache(cacheFilePath(u"iso_3166-1"));
0170     if (!cache.open(QFile::WriteOnly)) {
0171         qCWarning(KI18NLD) << "Failed to write ISO 3166-1 cache:" << cache.errorString() << cache.fileName();
0172         return;
0173     }
0174 
0175     uint32_t n = Iso3166_1CacheHeader;
0176     cache.write(reinterpret_cast<const char *>(&n), 4); // header
0177     n = alpha2NameMap.size();
0178     cache.write(reinterpret_cast<const char *>(&n), 4); // size
0179     for (auto entry : alpha2NameMap) {
0180         cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
0181     }
0182     for (auto entry : alpha3alpha2Map) {
0183         cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
0184     }
0185     cache.write(iso3166_1stringTable);
0186 }
0187 
0188 void IsoCodesCache::loadIso3166_2()
0189 {
0190     if (!m_iso3166_2CacheData && !loadIso3166_2Cache()) {
0191         createIso3166_2Cache();
0192         loadIso3166_2Cache();
0193     }
0194 }
0195 
0196 bool IsoCodesCache::loadIso3166_2Cache()
0197 {
0198     QFileInfo jsonFi(isoCodesPath(u"iso_3166-2.json"));
0199     auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-2"));
0200     if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) {
0201         return false;
0202     }
0203     m_iso3166_2CacheSize = f->size();
0204 
0205     // validate cache file is usable
0206     // header matches
0207     const auto data = f->map(0, m_iso3166_2CacheSize);
0208     if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_2CacheHeader) {
0209         return false;
0210     }
0211     // name lookup table fits into the available size
0212     auto size = *(reinterpret_cast<const uint32_t *>(data) + 1);
0213     auto offset = 3 * sizeof(uint32_t) + size * sizeof(MapEntry<uint32_t>);
0214     if (offset >= m_iso3166_2CacheSize) {
0215         return false;
0216     }
0217     // hierarchy map boundary check
0218     size = *(reinterpret_cast<const uint32_t *>(data + offset) - 1);
0219     offset += size * sizeof(MapEntry<uint32_t>);
0220     if (offset >= m_iso3166_2CacheSize) {
0221         return false;
0222     }
0223     // string table is 0 terminated
0224     if (data[m_iso3166_2CacheSize - 1] != '\0') {
0225         return false;
0226     }
0227 
0228     m_iso3166_2CacheFile = std::move(f);
0229     m_iso3166_2CacheData = data;
0230     return true;
0231 }
0232 
0233 uint32_t IsoCodesCache::subdivisionCount() const
0234 {
0235     return m_iso3166_2CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData) + 1) : 0;
0236 }
0237 
0238 const MapEntry<uint32_t> *IsoCodesCache::subdivisionNameMapBegin() const
0239 {
0240     return m_iso3166_2CacheData ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t)) : nullptr;
0241 }
0242 
0243 uint32_t IsoCodesCache::subdivisionHierachyMapSize() const
0244 {
0245     return m_iso3166_2CacheData
0246         ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>)))
0247         : 0;
0248 }
0249 
0250 const MapEntry<uint32_t> *IsoCodesCache::subdivisionParentMapBegin() const
0251 {
0252     return m_iso3166_2CacheData
0253         ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 3 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>))
0254         : nullptr;
0255 }
0256 
0257 const char *IsoCodesCache::subdivisionStringTableLookup(uint16_t offset) const
0258 {
0259     if (m_iso3166_2CacheData) {
0260         const auto pos = offset + 3 * sizeof(uint32_t) + (subdivisionCount() + subdivisionHierachyMapSize()) * sizeof(MapEntry<uint32_t>);
0261         return m_iso3166_2CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_2CacheData + pos) : nullptr;
0262     }
0263     return nullptr;
0264 }
0265 
0266 void IsoCodesCache::createIso3166_2Cache()
0267 {
0268     qCDebug(KI18NLD) << "Rebuilding ISO 3166-2 cache";
0269     const auto path = isoCodesPath(u"iso_3166-2.json");
0270 
0271     QFile file(path);
0272     if (!file.open(QFile::ReadOnly)) {
0273         qCWarning(KI18NLD) << "Unable to open iso_3166-2.json" << path << file.errorString();
0274         return;
0275     }
0276 
0277     std::vector<MapEntry<uint32_t>> subdivNameMap;
0278     std::vector<MapEntry<uint32_t>> subdivParentMap;
0279     QByteArray iso3166_2stringTable;
0280 
0281     const auto doc = QJsonDocument::fromJson(file.readAll());
0282     const auto array = doc.object().value(QLatin1String("3166-2")).toArray();
0283     for (const auto &entryVal : array) {
0284         const auto entry = entryVal.toObject();
0285         const auto key = IsoCodes::subdivisionCodeToKey(entry.value(QLatin1String("code")).toString());
0286 
0287         assert(std::numeric_limits<uint16_t>::max() > iso3166_2stringTable.size());
0288         subdivNameMap.push_back({key, (uint16_t)iso3166_2stringTable.size()});
0289         iso3166_2stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8());
0290         iso3166_2stringTable.append('\0');
0291 
0292         const auto parentKey = IsoCodes::parentCodeToKey(entry.value(QLatin1String("parent")).toString());
0293         if (parentKey) {
0294             subdivParentMap.push_back({key, parentKey});
0295         }
0296     }
0297 
0298     std::sort(subdivNameMap.begin(), subdivNameMap.end());
0299     std::sort(subdivParentMap.begin(), subdivParentMap.end());
0300 
0301     // write out binary cache file
0302     QDir().mkpath(cachePath());
0303     QFile cache(cacheFilePath(u"iso_3166-2"));
0304     if (!cache.open(QFile::WriteOnly)) {
0305         qCWarning(KI18NLD) << "Failed to write ISO 3166-2 cache:" << cache.errorString() << cache.fileName();
0306         return;
0307     }
0308 
0309     uint32_t n = Iso3166_2CacheHeader;
0310     cache.write(reinterpret_cast<const char *>(&n), 4); // header
0311     n = subdivNameMap.size();
0312     cache.write(reinterpret_cast<const char *>(&n), 4); // size of the name map
0313     for (auto entry : subdivNameMap) {
0314         cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
0315     }
0316     n = subdivParentMap.size();
0317     cache.write(reinterpret_cast<const char *>(&n), 4); // size of the hierarchy map
0318     for (auto entry : subdivParentMap) {
0319         cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry));
0320     }
0321     cache.write(iso3166_2stringTable);
0322 }