File indexing completed on 2025-02-16 04:48:29
0001 /* 0002 SPDX-FileCopyrightText: 2018 Volker Krause <vkrause@kde.org> 0003 0004 SPDX-License-Identifier: LGPL-2.0-or-later 0005 */ 0006 0007 #include "weatherforecastmanager.h" 0008 #include "weatherforecast.h" 0009 0010 #include <QCoreApplication> 0011 #include <QDateTime> 0012 #include <QDebug> 0013 #include <QDir> 0014 #include <QDirIterator> 0015 #include <QNetworkAccessManager> 0016 #include <QNetworkReply> 0017 #include <QStandardPaths> 0018 #include <QUrlQuery> 0019 #include <QVariant> 0020 #include <QXmlStreamReader> 0021 0022 #include <zlib.h> 0023 0024 #include <cmath> 0025 0026 static void alignToHour(QDateTime &dt) 0027 { 0028 dt.setTime(QTime(dt.time().hour(), 0, 0, 0)); 0029 } 0030 0031 static void roundToHour(QDateTime &dt) 0032 { 0033 if (dt.time().minute() >= 30) { 0034 alignToHour(dt); 0035 dt = dt.addSecs(3600); 0036 } else { 0037 alignToHour(dt); 0038 } 0039 } 0040 0041 /* 0042 * ATTENTION! 0043 * Before touching anything in here, especially regarding the network operations 0044 * make sure to read and understand https://api.met.no/conditions_service.html! 0045 */ 0046 0047 WeatherForecastManager::WeatherForecastManager(QObject *parent) 0048 : QObject(parent) 0049 { 0050 connect(&m_updateTimer, &QTimer::timeout, this, &WeatherForecastManager::updateAll); 0051 m_updateTimer.setSingleShot(true); 0052 } 0053 0054 WeatherForecastManager::~WeatherForecastManager() = default; 0055 0056 bool WeatherForecastManager::allowNetworkAccess() const 0057 { 0058 return m_allowNetwork || m_testMode; 0059 } 0060 0061 void WeatherForecastManager::setAllowNetworkAccess(bool enabled) 0062 { 0063 m_allowNetwork = enabled; 0064 Q_EMIT forecastUpdated(); 0065 if (enabled) { 0066 scheduleUpdate(); 0067 } else { 0068 m_updateTimer.stop(); 0069 } 0070 fetchNext(); 0071 } 0072 0073 void WeatherForecastManager::monitorLocation(float latitude, float longitude) 0074 { 0075 WeatherTile t{latitude, longitude}; 0076 qDebug() << latitude << longitude << t.lat << t.lon; 0077 0078 auto it = std::lower_bound(m_monitoredTiles.begin(), m_monitoredTiles.end(), t); 0079 if (it != m_monitoredTiles.end() && (*it) == t) { 0080 return; 0081 } 0082 0083 m_monitoredTiles.insert(it, t); 0084 fetchTile(t); 0085 } 0086 0087 WeatherForecast WeatherForecastManager::forecast(float latitude, float longitude, const QDateTime &dt) const 0088 { 0089 return forecast(latitude, longitude, dt, dt.addSecs(3600)); 0090 } 0091 0092 WeatherForecast WeatherForecastManager::forecast(float latitude, float longitude, const QDateTime &begin, const QDateTime &end) const 0093 { 0094 if (Q_UNLIKELY(m_testMode)) { 0095 WeatherForecast fc; 0096 auto beginDt = begin; 0097 roundToHour(beginDt); 0098 fc.setDateTime(beginDt); 0099 auto endDt = end; 0100 roundToHour(endDt); 0101 if (beginDt == endDt) { 0102 endDt = endDt.addSecs(3600); 0103 } 0104 const auto range = beginDt.secsTo(endDt) / 3600; 0105 fc.setRange(range); 0106 fc.setTile({latitude, longitude}); 0107 fc.setMinimumTemperature(std::min(latitude, longitude)); 0108 fc.setMaximumTemperature(std::max(latitude, longitude)); 0109 fc.setPrecipitation(23.0f); 0110 fc.setSymbolType(WeatherForecast::LightClouds); 0111 return fc; 0112 } 0113 0114 const auto now = QDateTime::currentDateTimeUtc(); 0115 if (end < now) { 0116 return {}; 0117 } 0118 0119 auto beginDt = std::max(begin, now); 0120 roundToHour(beginDt); 0121 auto endDt = std::max(end, now); 0122 roundToHour(endDt); 0123 if (!beginDt.isValid() || !endDt.isValid() || beginDt > endDt) { 0124 return {}; 0125 } 0126 if (beginDt == endDt) { 0127 endDt = endDt.addSecs(3600); 0128 } 0129 0130 WeatherTile tile{latitude, longitude}; 0131 if (!loadForecastData(tile)) { 0132 return {}; 0133 } 0134 0135 const auto &forecasts = m_forecastData[tile]; 0136 const auto beginIt = std::lower_bound(forecasts.begin(), forecasts.end(), beginDt, [](const WeatherForecast &lhs, const QDateTime &rhs) { 0137 return lhs.dateTime() < rhs; 0138 }); 0139 const auto endIt = std::lower_bound(forecasts.begin(), forecasts.end(), endDt, [](const WeatherForecast &lhs, const QDateTime &rhs) { 0140 return lhs.dateTime() < rhs; 0141 }); 0142 if (beginIt == forecasts.end() || beginIt == endIt) { 0143 return {}; 0144 } 0145 const auto range = beginDt.secsTo(std::min((*std::prev(endIt)).dateTime().addSecs(3600), endDt)) / 3600; 0146 0147 WeatherForecast fc(*beginIt); 0148 fc.setRange(range); 0149 for (auto it = beginIt; it != endIt; ++it) { 0150 fc.merge(*it); 0151 } 0152 return fc; 0153 } 0154 0155 void WeatherForecastManager::fetchTile(WeatherTile tile) 0156 { 0157 QFileInfo fi(cachePath(tile) + QLatin1StringView("forecast.xml")); 0158 if (fi.exists() && fi.lastModified().toUTC().addSecs(3600 * 2) >= QDateTime::currentDateTimeUtc()) { // cache is already new enough 0159 return; 0160 } 0161 0162 m_pendingTiles.push_back(tile); 0163 fetchNext(); 0164 } 0165 0166 void WeatherForecastManager::fetchNext() 0167 { 0168 if (!m_allowNetwork || m_pendingReply || m_pendingTiles.empty()) { 0169 return; 0170 } 0171 0172 const auto tile = m_pendingTiles.front(); 0173 m_pendingTiles.pop_front(); 0174 0175 if (!m_nam) { 0176 m_nam = new QNetworkAccessManager(this); 0177 } 0178 0179 QUrl url; 0180 url.setScheme(QStringLiteral("https")); 0181 url.setHost(QStringLiteral("api.met.no")); 0182 url.setPath(QStringLiteral("/weatherapi/locationforecast/2.0/classic")); 0183 QUrlQuery query; 0184 query.addQueryItem(QStringLiteral("lat"), QString::number(tile.latitude())); 0185 query.addQueryItem(QStringLiteral("lon"), QString::number(tile.longitude())); 0186 url.setQuery(query); 0187 0188 qDebug() << url; 0189 QNetworkRequest req(url); 0190 req.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy); 0191 req.setAttribute(QNetworkRequest::User, QVariant::fromValue(tile)); 0192 0193 // see §Identification on https://api.met.no/conditions_service.html 0194 req.setHeader(QNetworkRequest::UserAgentHeader, QString(QCoreApplication::applicationName() + 0195 QLatin1Char(' ') + QCoreApplication::applicationVersion() + QLatin1StringView(" (kde-pim@kde.org)"))); 0196 // TODO see §Cache on https://api.met.no/conditions_service.html 0197 // see §Compression on https://api.met.no/conditions_service.html 0198 req.setRawHeader("Accept-Encoding", "gzip"); 0199 0200 m_pendingReply = m_nam->get(req); 0201 connect(m_pendingReply, &QNetworkReply::finished, this, &WeatherForecastManager::tileDownloaded); 0202 } 0203 0204 void WeatherForecastManager::tileDownloaded() 0205 { 0206 // TODO handle 304 Not Modified 0207 // TODO handle 429 Too Many Requests 0208 if (m_pendingReply->error() != QNetworkReply::NoError) { 0209 qWarning() << m_pendingReply->errorString(); 0210 } else { 0211 writeToCacheFile(m_pendingReply); 0212 } 0213 0214 m_pendingReply->deleteLater(); 0215 m_pendingReply = nullptr; 0216 if (m_pendingTiles.empty()) { 0217 Q_EMIT forecastUpdated(); 0218 } 0219 fetchNext(); 0220 } 0221 0222 QString WeatherForecastManager::cachePath(WeatherTile tile) const 0223 { 0224 const auto path = QString(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) 0225 + QLatin1StringView("/weather/") 0226 + QString::number(tile.lat) + QLatin1Char('/') 0227 + QString::number(tile.lon) + QLatin1Char('/')); 0228 QDir().mkpath(path); 0229 return path; 0230 } 0231 0232 void WeatherForecastManager::writeToCacheFile(QNetworkReply* reply) const 0233 { 0234 const auto tile = reply->request().attribute(QNetworkRequest::User).value<WeatherTile>(); 0235 qDebug() << tile.lat << tile.lon; 0236 qDebug() << reply->rawHeaderPairs(); 0237 QFile f(cachePath(tile) + QLatin1StringView("forecast.xml")); 0238 if (!f.open(QFile::WriteOnly)) { 0239 qWarning() << "Failed to open weather cache location:" << f.errorString(); 0240 return; 0241 } 0242 0243 const auto contentEncoding = reply->rawHeader("Content-Encoding"); 0244 if (contentEncoding == "gzip") { 0245 const auto data = reply->readAll(); 0246 if (data.size() < 4 || data.at(0) != 0x1f || data.at(1) != char(0x8b)) { 0247 qWarning() << "Invalid gzip format"; 0248 return; 0249 } 0250 0251 z_stream stream; 0252 unsigned char buffer[1024]; 0253 0254 stream.zalloc = nullptr; 0255 stream.zfree = nullptr; 0256 stream.opaque = nullptr; 0257 stream.avail_in = data.size(); 0258 stream.next_in = reinterpret_cast<unsigned char*>(const_cast<char*>(data.data())); 0259 0260 auto ret = inflateInit2(&stream, 15 + 32); // see docs, the magic numbers enable gzip decoding 0261 if (ret != Z_OK) { 0262 qWarning() << "Failed to initialize zlib stream."; 0263 return; 0264 } 0265 0266 do { 0267 stream.avail_out = sizeof(buffer); 0268 stream.next_out = buffer; 0269 0270 ret = inflate(&stream, Z_NO_FLUSH); 0271 if (ret != Z_OK && ret != Z_STREAM_END) { 0272 qWarning() << "Zlib decoding failed!" << ret; 0273 break; 0274 } 0275 0276 f.write(reinterpret_cast<char*>(buffer), sizeof(buffer) - stream.avail_out); 0277 } while (stream.avail_out == 0); 0278 inflateEnd(&stream); 0279 } else { 0280 f.write(reply->readAll()); 0281 } 0282 0283 m_forecastData.erase(tile); 0284 } 0285 0286 bool WeatherForecastManager::loadForecastData(WeatherTile tile) const 0287 { 0288 const auto it = m_forecastData.find(tile); 0289 if (it != m_forecastData.end()) { 0290 return true; 0291 } 0292 0293 QFile f(cachePath(tile) + QLatin1StringView("forecast.xml")); 0294 if (!f.exists() || !f.open(QFile::ReadOnly)) { 0295 return false; 0296 } 0297 0298 QXmlStreamReader reader(&f); 0299 auto forecasts = parseForecast(reader, tile); 0300 mergeForecasts(forecasts); 0301 if (forecasts.empty()) { 0302 return false; 0303 } 0304 0305 m_forecastData.insert(it, {tile, std::move(forecasts)}); 0306 return true; 0307 } 0308 0309 void WeatherForecastManager::mergeForecasts(std::vector<WeatherForecast>& forecasts) const 0310 { 0311 std::stable_sort(forecasts.begin(), forecasts.end(), [](const WeatherForecast &lhs, const WeatherForecast &rhs) { 0312 if (lhs.dateTime() == rhs.dateTime()) 0313 return lhs.range() < rhs.range(); 0314 return lhs.dateTime() < rhs.dateTime(); 0315 }); 0316 0317 // merge duplicated time slices 0318 auto storeIt = forecasts.begin(); 0319 for (auto it = forecasts.begin(); it != forecasts.end();) { 0320 (*storeIt) = (*it); 0321 auto mergeIt = it; 0322 for (; mergeIt != forecasts.end(); ++mergeIt) { 0323 if ((*it).dateTime() == (*mergeIt).dateTime()) { 0324 (*storeIt).merge(*mergeIt); 0325 } else { 0326 (*mergeIt).setRange(1); 0327 break; 0328 } 0329 } 0330 ++storeIt; 0331 it = mergeIt; 0332 } 0333 forecasts.erase(storeIt, forecasts.end()); 0334 } 0335 0336 std::vector<WeatherForecast> WeatherForecastManager::parseForecast(QXmlStreamReader &reader, WeatherTile tile) const 0337 { 0338 std::vector<WeatherForecast> result; 0339 0340 auto beginDt = QDateTime::currentDateTimeUtc(); 0341 alignToHour(beginDt); 0342 0343 while (!reader.atEnd()) { 0344 if (reader.tokenType() == QXmlStreamReader::StartElement) { 0345 if (reader.name() == QLatin1StringView("weatherdata") || reader.name() == QLatin1StringView("product")) { 0346 reader.readNext(); // enter these elements 0347 continue; 0348 } 0349 if (reader.name() == QLatin1StringView("time") && reader.attributes().value(QLatin1StringView("datatype")) == QLatin1StringView("forecast")) { 0350 // normalize time ranges to 1 hour 0351 auto from = QDateTime::fromString(reader.attributes().value(QLatin1StringView("from")).toString(), Qt::ISODate); 0352 from = std::max(from, beginDt); 0353 alignToHour(from); 0354 auto to = QDateTime::fromString(reader.attributes().value(QLatin1StringView("to")).toString(), Qt::ISODate); 0355 alignToHour(to); 0356 const auto range = from.secsTo(to) / 3600; 0357 if (to == from) { 0358 to = to.addSecs(3600); 0359 } 0360 if (to < beginDt || to <= from || !to.isValid() || !from.isValid()) { 0361 reader.skipCurrentElement(); 0362 continue; 0363 } 0364 auto fc = parseForecastElement(reader); 0365 for (int i = 0; i < from.secsTo(to); i += 3600) { 0366 fc.setTile(tile); 0367 fc.setDateTime(from.addSecs(i)); 0368 fc.setRange(range); 0369 result.push_back(fc); 0370 } 0371 continue; 0372 } 0373 // unknown element 0374 reader.skipCurrentElement(); 0375 } else { 0376 reader.readNext(); 0377 } 0378 } 0379 0380 return result; 0381 } 0382 0383 // Icon mapping: https://api.met.no/weatherapi/weathericon/1.1/documentation 0384 struct symbol_map_t { 0385 uint8_t id; 0386 WeatherForecast::SymbolType type; 0387 }; 0388 0389 static const symbol_map_t symbol_map[] = { 0390 { 1, WeatherForecast::Clear }, // 1 Sun 0391 { 2, WeatherForecast::Clear | WeatherForecast::LightClouds }, // 2 LightCloud 0392 { 3, WeatherForecast::Clear | WeatherForecast::Clouds }, // 3 PartlyCloud 0393 { 4, WeatherForecast::Clouds }, // 4 Cloud 0394 { 5, WeatherForecast::Clear | WeatherForecast::LightRain }, // 5 LightRainSun 0395 { 6, WeatherForecast::Clear | WeatherForecast::LightRain | WeatherForecast::ThunderStorm }, // 6 LightRainThunderSun 0396 { 7, WeatherForecast::Clear | WeatherForecast::Hail }, // 7 SleetSun 0397 { 8, WeatherForecast::Clear | WeatherForecast::Snow }, // 8 SnowSun 0398 { 9, WeatherForecast::LightRain }, // 9 LightRain 0399 { 10, WeatherForecast::Rain }, // 10 Rain 0400 { 11, WeatherForecast::Rain | WeatherForecast::ThunderStorm }, // 11 RainThunder 0401 { 12, WeatherForecast::Hail }, // 12 Sleet 0402 { 13, WeatherForecast::Snow }, // 13 Snow 0403 { 14, WeatherForecast::Snow | WeatherForecast::ThunderStorm }, // 14 SnowThunder 0404 { 15, WeatherForecast::Fog }, // 15 Fog 0405 { 20, WeatherForecast::Clear | WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 20 SleetSunThunder 0406 { 21, WeatherForecast::Clear | WeatherForecast::Snow | WeatherForecast::ThunderStorm }, // 21 SnowSunThunder 0407 { 22, WeatherForecast::LightRain | WeatherForecast::ThunderStorm }, // 22 LightRainThunder 0408 { 23, WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 23 SleetThunder 0409 { 24, WeatherForecast::Clear | WeatherForecast::LightRain | WeatherForecast::ThunderStorm }, // 24 DrizzleThunderSun 0410 { 25, WeatherForecast::Clear | WeatherForecast::Rain | WeatherForecast::ThunderStorm }, // 25 RainThunderSun 0411 { 26, WeatherForecast::Clear | WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 26 LightSleetThunderSun 0412 { 27, WeatherForecast::Clear | WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 27 HeavySleetThunderSun 0413 { 28, WeatherForecast::Clear | WeatherForecast::LightSnow | WeatherForecast::ThunderStorm }, // 28 LightSnowThunderSun 0414 { 29, WeatherForecast::Clear | WeatherForecast::Snow | WeatherForecast::ThunderStorm }, // 29 HeavySnowThunderSun 0415 { 30, WeatherForecast::LightRain | WeatherForecast::ThunderStorm }, // 30 DrizzleThunder 0416 { 31, WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 31 LightSleetThunder 0417 { 32, WeatherForecast::Hail | WeatherForecast::ThunderStorm }, // 32 HeavySleetThunder 0418 { 33, WeatherForecast::LightSnow | WeatherForecast::ThunderStorm }, // 33 LightSnowThunder 0419 { 34, WeatherForecast::Snow | WeatherForecast::ThunderStorm }, // 34 HeavySnowThunder 0420 { 40, WeatherForecast::Clear | WeatherForecast::LightRain }, // 40 DrizzleSun 0421 { 41, WeatherForecast::Clear | WeatherForecast::Rain}, // 41 RainSun 0422 { 42, WeatherForecast::Clear | WeatherForecast::Hail }, // 42 LightSleetSun 0423 { 43, WeatherForecast::Clear | WeatherForecast::Hail }, // 43 HeavySleetSun 0424 { 44, WeatherForecast::Clear | WeatherForecast::LightSnow }, // 44 LightSnowSun 0425 { 45, WeatherForecast::Clear | WeatherForecast::Snow }, // 45 HeavysnowSun 0426 { 46, WeatherForecast::LightRain }, // 46 Drizzle 0427 { 47, WeatherForecast::Hail }, // 47 LightSleet 0428 { 48, WeatherForecast::Hail }, // 48 HeavySleet 0429 { 49, WeatherForecast::LightSnow }, // 49 LightSnow 0430 { 50, WeatherForecast::Snow } // 50 HeavySnow 0431 }; 0432 0433 WeatherForecast WeatherForecastManager::parseForecastElement(QXmlStreamReader &reader) const 0434 { 0435 WeatherForecast fc; 0436 while (!reader.atEnd()) { 0437 switch (reader.tokenType()) { 0438 case QXmlStreamReader::StartElement: 0439 if (reader.name() == QLatin1StringView("temperature")) { 0440 const auto t = reader.attributes().value(QLatin1StringView("value")).toFloat(); 0441 fc.setMinimumTemperature(t); 0442 fc.setMaximumTemperature(t); 0443 } else if (reader.name() == QLatin1StringView("minTemperature")) { 0444 fc.setMinimumTemperature(reader.attributes().value(QLatin1StringView("value")).toFloat()); 0445 } else if (reader.name() == QLatin1StringView("maxTemperature")) { 0446 fc.setMaximumTemperature(reader.attributes().value(QLatin1StringView("value")).toFloat()); 0447 } else if (reader.name() == QLatin1StringView("windSpeed")) { 0448 fc.setWindSpeed(reader.attributes().value(QLatin1StringView("mps")).toFloat()); 0449 } else if (reader.name() == QLatin1StringView("symbol")) { 0450 auto symId = reader.attributes().value(QLatin1StringView("number")).toInt(); 0451 if (symId > 100) { 0452 symId -= 100; // map polar night symbols 0453 } 0454 const auto it = std::lower_bound(std::begin(symbol_map), std::end(symbol_map), symId, [](symbol_map_t lhs, uint8_t rhs) { 0455 return lhs.id < rhs; 0456 }); 0457 if (it != std::end(symbol_map) && (*it).id == symId) { 0458 fc.setSymbolType((*it).type); 0459 } 0460 } else if (reader.name() == QLatin1StringView("precipitation")) { 0461 fc.setPrecipitation(reader.attributes().value(QLatin1StringView("value")).toFloat()); 0462 } 0463 break; 0464 case QXmlStreamReader::EndElement: 0465 if (reader.name() == QLatin1StringView("time")) { 0466 return fc; 0467 } 0468 break; 0469 default: 0470 break; 0471 } 0472 reader.readNext(); 0473 } 0474 0475 return fc; 0476 } 0477 0478 QDateTime WeatherForecastManager::maximumForecastTime(const QDate &today) const 0479 { 0480 return QDateTime(today.addDays(9), QTime(0, 0)); 0481 } 0482 0483 void WeatherForecastManager::setTestModeEnabled(bool testMode) 0484 { 0485 m_testMode = testMode; 0486 } 0487 0488 void WeatherForecastManager::scheduleUpdate() 0489 { 0490 if (m_updateTimer.isActive()) { 0491 return; 0492 } 0493 0494 // see §Updates on https://api.met.no/conditions_service.html 0495 m_updateTimer.setInterval(std::chrono::hours(2) + std::chrono::minutes(QTime::currentTime().msec() % 30)); 0496 qDebug() << "Next weather update:" << m_updateTimer.interval(); 0497 m_updateTimer.start(); 0498 } 0499 0500 void WeatherForecastManager::updateAll() 0501 { 0502 for (const auto tile : m_monitoredTiles) { 0503 fetchTile(tile); 0504 } 0505 purgeCache(); 0506 scheduleUpdate(); 0507 } 0508 0509 void WeatherForecastManager::purgeCache() 0510 { 0511 const auto basePath = QString(QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + QLatin1StringView("/weather/")); 0512 const auto cutoffDate = QDateTime::currentDateTimeUtc().addDays(-9); 0513 0514 QDirIterator it(basePath, QDir::Files | QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks | QDir::Writable, QDirIterator::Subdirectories); 0515 while (it.hasNext()) { 0516 it.next(); 0517 if (it.fileInfo().isFile() && it.fileInfo().lastModified() < cutoffDate) { 0518 qDebug() << "Purging old weather data:" << it.filePath(); 0519 QFile::remove(it.filePath()); 0520 } else if (it.fileInfo().isDir() && QDir(it.filePath()).isEmpty()) { 0521 qDebug() << "Purging old weather cache folder:" << it.filePath(); 0522 QDir().rmdir(it.filePath()); 0523 } 0524 } 0525 } 0526 0527 #include "moc_weatherforecastmanager.cpp"