Warning, file /frameworks/ki18n/src/localedata/isocodescache.cpp was not indexed or was modified since last indexation (in which case cross-reference links may be missing, inaccurate or erroneous).
0001 /* 0002 SPDX-FileCopyrightText: 2021 Volker Krause <vkrause@kde.org> 0003 0004 SPDX-License-Identifier: LGPL-2.0-or-later 0005 */ 0006 0007 #include "config-localedata.h" 0008 0009 #include "isocodes_p.h" 0010 #include "isocodescache_p.h" 0011 #include "logging.h" 0012 0013 #include <QDir> 0014 #include <QFile> 0015 #include <QFileInfo> 0016 #include <QJsonArray> 0017 #include <QJsonDocument> 0018 #include <QJsonObject> 0019 #include <QStandardPaths> 0020 0021 // increment those when changing the format 0022 enum : uint32_t { 0023 Iso3166_1CacheHeader = 0x4B493101, 0024 Iso3166_2CacheHeader = 0x4B493201, 0025 }; 0026 0027 static QString isoCodesPath(QStringView file) 0028 { 0029 #ifndef Q_OS_ANDROID 0030 auto path = QStandardPaths::locate(QStandardPaths::GenericDataLocation, QLatin1String("iso-codes/json/") + file, QStandardPaths::LocateFile); 0031 if (!path.isEmpty()) { 0032 return path; 0033 } 0034 0035 // search manually in the compile-time determined prefix 0036 // needed for example for non-installed Windows binaries to work, such as unit tests 0037 for (const char *installLocation : {"/share", "/bin/data"}) { 0038 path = QLatin1String(ISO_CODES_PREFIX) + QLatin1String(installLocation) + QLatin1String("/iso-codes/json/") + file; 0039 if (QFileInfo::exists(path)) { 0040 return path; 0041 } 0042 } 0043 0044 return {}; 0045 #else 0046 return QLatin1String("assets:/share/iso-codes/json/") + file; 0047 #endif 0048 } 0049 0050 static QString cachePath() 0051 { 0052 return QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation) + QLatin1String("/org.kde.ki18n/iso-codes/"); 0053 } 0054 0055 static QString cacheFilePath(QStringView file) 0056 { 0057 return cachePath() + file; 0058 } 0059 0060 IsoCodesCache::~IsoCodesCache() = default; 0061 0062 IsoCodesCache *IsoCodesCache::instance() 0063 { 0064 static IsoCodesCache s_cache; 0065 return &s_cache; 0066 } 0067 0068 void IsoCodesCache::loadIso3166_1() 0069 { 0070 if (!m_iso3166_1CacheData && !loadIso3166_1Cache()) { 0071 createIso3166_1Cache(); 0072 loadIso3166_1Cache(); 0073 } 0074 } 0075 0076 bool IsoCodesCache::loadIso3166_1Cache() 0077 { 0078 QFileInfo jsonFi(isoCodesPath(u"iso_3166-1.json")); 0079 auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-1")); 0080 if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) { 0081 return false; 0082 } 0083 m_iso3166_1CacheSize = f->size(); 0084 0085 // validate cache file is usable 0086 // header matches 0087 const auto data = f->map(0, m_iso3166_1CacheSize); 0088 if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_1CacheHeader) { 0089 return false; 0090 } 0091 // lookup tables fit into the available size 0092 const auto size = *(reinterpret_cast<const uint32_t *>(data) + 1); 0093 if (sizeof(Iso3166_1CacheHeader) + sizeof(size) + size * sizeof(MapEntry<uint16_t>) * 2 >= m_iso3166_1CacheSize) { 0094 return false; 0095 } 0096 // string table is 0 terminated 0097 if (data[m_iso3166_1CacheSize - 1] != '\0') { 0098 return false; 0099 } 0100 0101 m_iso3166_1CacheFile = std::move(f); 0102 m_iso3166_1CacheData = data; 0103 return true; 0104 } 0105 0106 uint32_t IsoCodesCache::countryCount() const 0107 { 0108 return m_iso3166_1CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_1CacheData) + 1) : 0; 0109 } 0110 0111 const MapEntry<uint16_t> *IsoCodesCache::countryNameMapBegin() const 0112 { 0113 return m_iso3166_1CacheData ? reinterpret_cast<const MapEntry<uint16_t> *>(m_iso3166_1CacheData + sizeof(uint32_t) * 2) : nullptr; 0114 } 0115 0116 const MapEntry<uint16_t> *IsoCodesCache::countryAlpha3MapBegin() const 0117 { 0118 return m_iso3166_1CacheData ? countryNameMapBegin() + countryCount() : nullptr; 0119 } 0120 0121 const char *IsoCodesCache::countryStringTableLookup(uint16_t offset) const 0122 { 0123 if (m_iso3166_1CacheData) { 0124 const auto pos = offset + 2 * sizeof(uint32_t) + 2 * countryCount() * sizeof(MapEntry<uint16_t>); 0125 return m_iso3166_1CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_1CacheData + pos) : nullptr; 0126 } 0127 return nullptr; 0128 } 0129 0130 void IsoCodesCache::createIso3166_1Cache() 0131 { 0132 qCDebug(KI18NLD) << "Rebuilding ISO 3166-1 cache"; 0133 const auto path = isoCodesPath(u"iso_3166-1.json"); 0134 0135 QFile file(path); 0136 if (!file.open(QFile::ReadOnly)) { 0137 qCWarning(KI18NLD) << "Unable to open iso_3166-1.json" << path << file.errorString(); 0138 return; 0139 } 0140 0141 std::vector<MapEntry<uint16_t>> alpha2NameMap; 0142 std::vector<MapEntry<uint16_t>> alpha3alpha2Map; 0143 QByteArray iso3166_1stringTable; 0144 0145 const auto doc = QJsonDocument::fromJson(file.readAll()); 0146 const auto array = doc.object().value(QLatin1String("3166-1")).toArray(); 0147 for (const auto &entryVal : array) { 0148 const auto entry = entryVal.toObject(); 0149 const auto alpha2 = entry.value(QLatin1String("alpha_2")).toString(); 0150 if (alpha2.size() != 2) { 0151 continue; 0152 } 0153 const auto alpha2Key = IsoCodes::alpha2CodeToKey(alpha2); 0154 0155 assert(std::numeric_limits<uint16_t>::max() > iso3166_1stringTable.size()); 0156 alpha2NameMap.push_back({alpha2Key, (uint16_t)iso3166_1stringTable.size()}); 0157 iso3166_1stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8()); 0158 iso3166_1stringTable.append('\0'); 0159 0160 const auto alpha3Key = IsoCodes::alpha3CodeToKey(entry.value(QLatin1String("alpha_3")).toString()); 0161 alpha3alpha2Map.push_back({alpha3Key, alpha2Key}); 0162 } 0163 0164 std::sort(alpha2NameMap.begin(), alpha2NameMap.end()); 0165 std::sort(alpha3alpha2Map.begin(), alpha3alpha2Map.end()); 0166 0167 // write out binary cache file 0168 QDir().mkpath(cachePath()); 0169 QFile cache(cacheFilePath(u"iso_3166-1")); 0170 if (!cache.open(QFile::WriteOnly)) { 0171 qCWarning(KI18NLD) << "Failed to write ISO 3166-1 cache:" << cache.errorString() << cache.fileName(); 0172 return; 0173 } 0174 0175 uint32_t n = Iso3166_1CacheHeader; 0176 cache.write(reinterpret_cast<const char *>(&n), 4); // header 0177 n = alpha2NameMap.size(); 0178 cache.write(reinterpret_cast<const char *>(&n), 4); // size 0179 for (auto entry : alpha2NameMap) { 0180 cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry)); 0181 } 0182 for (auto entry : alpha3alpha2Map) { 0183 cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry)); 0184 } 0185 cache.write(iso3166_1stringTable); 0186 } 0187 0188 void IsoCodesCache::loadIso3166_2() 0189 { 0190 if (!m_iso3166_2CacheData && !loadIso3166_2Cache()) { 0191 createIso3166_2Cache(); 0192 loadIso3166_2Cache(); 0193 } 0194 } 0195 0196 bool IsoCodesCache::loadIso3166_2Cache() 0197 { 0198 QFileInfo jsonFi(isoCodesPath(u"iso_3166-2.json")); 0199 auto f = std::make_unique<QFile>(cacheFilePath(u"iso_3166-2")); 0200 if (!f->open(QFile::ReadOnly) || f->fileTime(QFile::FileModificationTime) < jsonFi.lastModified() || f->size() < 8) { 0201 return false; 0202 } 0203 m_iso3166_2CacheSize = f->size(); 0204 0205 // validate cache file is usable 0206 // header matches 0207 const auto data = f->map(0, m_iso3166_2CacheSize); 0208 if (*reinterpret_cast<const uint32_t *>(data) != Iso3166_2CacheHeader) { 0209 return false; 0210 } 0211 // name lookup table fits into the available size 0212 auto size = *(reinterpret_cast<const uint32_t *>(data) + 1); 0213 auto offset = 3 * sizeof(uint32_t) + size * sizeof(MapEntry<uint32_t>); 0214 if (offset >= m_iso3166_2CacheSize) { 0215 return false; 0216 } 0217 // hierarchy map boundary check 0218 size = *(reinterpret_cast<const uint32_t *>(data + offset) - 1); 0219 offset += size * sizeof(MapEntry<uint32_t>); 0220 if (offset >= m_iso3166_2CacheSize) { 0221 return false; 0222 } 0223 // string table is 0 terminated 0224 if (data[m_iso3166_2CacheSize - 1] != '\0') { 0225 return false; 0226 } 0227 0228 m_iso3166_2CacheFile = std::move(f); 0229 m_iso3166_2CacheData = data; 0230 return true; 0231 } 0232 0233 uint32_t IsoCodesCache::subdivisionCount() const 0234 { 0235 return m_iso3166_2CacheData ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData) + 1) : 0; 0236 } 0237 0238 const MapEntry<uint32_t> *IsoCodesCache::subdivisionNameMapBegin() const 0239 { 0240 return m_iso3166_2CacheData ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t)) : nullptr; 0241 } 0242 0243 uint32_t IsoCodesCache::subdivisionHierachyMapSize() const 0244 { 0245 return m_iso3166_2CacheData 0246 ? *(reinterpret_cast<const uint32_t *>(m_iso3166_2CacheData + 2 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>))) 0247 : 0; 0248 } 0249 0250 const MapEntry<uint32_t> *IsoCodesCache::subdivisionParentMapBegin() const 0251 { 0252 return m_iso3166_2CacheData 0253 ? reinterpret_cast<const MapEntry<uint32_t> *>(m_iso3166_2CacheData + 3 * sizeof(uint32_t) + subdivisionCount() * sizeof(MapEntry<uint32_t>)) 0254 : nullptr; 0255 } 0256 0257 const char *IsoCodesCache::subdivisionStringTableLookup(uint16_t offset) const 0258 { 0259 if (m_iso3166_2CacheData) { 0260 const auto pos = offset + 3 * sizeof(uint32_t) + (subdivisionCount() + subdivisionHierachyMapSize()) * sizeof(MapEntry<uint32_t>); 0261 return m_iso3166_2CacheSize > pos ? reinterpret_cast<const char *>(m_iso3166_2CacheData + pos) : nullptr; 0262 } 0263 return nullptr; 0264 } 0265 0266 void IsoCodesCache::createIso3166_2Cache() 0267 { 0268 qCDebug(KI18NLD) << "Rebuilding ISO 3166-2 cache"; 0269 const auto path = isoCodesPath(u"iso_3166-2.json"); 0270 0271 QFile file(path); 0272 if (!file.open(QFile::ReadOnly)) { 0273 qCWarning(KI18NLD) << "Unable to open iso_3166-2.json" << path << file.errorString(); 0274 return; 0275 } 0276 0277 std::vector<MapEntry<uint32_t>> subdivNameMap; 0278 std::vector<MapEntry<uint32_t>> subdivParentMap; 0279 QByteArray iso3166_2stringTable; 0280 0281 const auto doc = QJsonDocument::fromJson(file.readAll()); 0282 const auto array = doc.object().value(QLatin1String("3166-2")).toArray(); 0283 for (const auto &entryVal : array) { 0284 const auto entry = entryVal.toObject(); 0285 const auto key = IsoCodes::subdivisionCodeToKey(entry.value(QLatin1String("code")).toString()); 0286 0287 assert(std::numeric_limits<uint16_t>::max() > iso3166_2stringTable.size()); 0288 subdivNameMap.push_back({key, (uint16_t)iso3166_2stringTable.size()}); 0289 iso3166_2stringTable.append(entry.value(QLatin1String("name")).toString().toUtf8()); 0290 iso3166_2stringTable.append('\0'); 0291 0292 const auto parentKey = IsoCodes::parentCodeToKey(entry.value(QLatin1String("parent")).toString()); 0293 if (parentKey) { 0294 subdivParentMap.push_back({key, parentKey}); 0295 } 0296 } 0297 0298 std::sort(subdivNameMap.begin(), subdivNameMap.end()); 0299 std::sort(subdivParentMap.begin(), subdivParentMap.end()); 0300 0301 // write out binary cache file 0302 QDir().mkpath(cachePath()); 0303 QFile cache(cacheFilePath(u"iso_3166-2")); 0304 if (!cache.open(QFile::WriteOnly)) { 0305 qCWarning(KI18NLD) << "Failed to write ISO 3166-2 cache:" << cache.errorString() << cache.fileName(); 0306 return; 0307 } 0308 0309 uint32_t n = Iso3166_2CacheHeader; 0310 cache.write(reinterpret_cast<const char *>(&n), 4); // header 0311 n = subdivNameMap.size(); 0312 cache.write(reinterpret_cast<const char *>(&n), 4); // size of the name map 0313 for (auto entry : subdivNameMap) { 0314 cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry)); 0315 } 0316 n = subdivParentMap.size(); 0317 cache.write(reinterpret_cast<const char *>(&n), 4); // size of the hierarchy map 0318 for (auto entry : subdivParentMap) { 0319 cache.write(reinterpret_cast<const char *>(&entry), sizeof(entry)); 0320 } 0321 cache.write(iso3166_2stringTable); 0322 }