diff --git a/src/QGCApplication.cc b/src/QGCApplication.cc index 2d819399802d..00b95e24ec49 100644 --- a/src/QGCApplication.cc +++ b/src/QGCApplication.cc @@ -35,6 +35,7 @@ #include "QGCImageProvider.h" #include "QGCLoggingCategory.h" #include "QGCLoggingCategoryManager.h" +#include "QGCMapEngineManager.h" #include "SettingsManager.h" #include "MavlinkSettings.h" #include "AppSettings.h" @@ -242,6 +243,10 @@ void QGCApplication::init() qCWarning(QGCApplicationLog) << "Could not load /fonts/opensans-demibold font"; } + // Register here, not in _initQmlRootWindow, so the unit-test harness (which builds + // its own engine) also resolves the QGroundControl.QGCMapEngineManager QML import. + QGCMapEngineManager::registerQmlTypes(); + if (_simpleBootTest) { // Since GStream builds are so problematic we initialize video during the simple boot test // to make sure it works and verfies plugin availability. diff --git a/src/QtLocationPlugin/CMakeLists.txt b/src/QtLocationPlugin/CMakeLists.txt index dc85dd4bd3a1..e00350f649eb 100644 --- a/src/QtLocationPlugin/CMakeLists.txt +++ b/src/QtLocationPlugin/CMakeLists.txt @@ -39,9 +39,17 @@ qt_add_plugin(QGCLocation QGCTileCacheDatabase.cpp QGCTileCacheDatabase.h QGCTileCacheTypes.h + QGCTileDatabaseSchema.cpp + QGCTileDatabaseSchema.h + QGCTileFallback.cpp + QGCTileFallback.h QGCTileCacheWorker.cpp QGCTileCacheWorker.h + QGCTileSetImporter.cpp + QGCTileSetImporter.h QGCTileSet.h + QGCTileCache.cpp + QGCTileCache.h QGeoFileTileCacheQGC.cpp QGeoFileTileCacheQGC.h QGeoMapReplyQGC.cpp @@ -54,6 +62,8 @@ qt_add_plugin(QGCLocation QGeoTiledMapQGC.h QGeoTileFetcherQGC.cpp QGeoTileFetcherQGC.h + TileFetchMetrics.cpp + TileFetchMetrics.h ) if(IOS) @@ -71,6 +81,7 @@ target_link_libraries(QGCLocation QGCNetwork PUBLIC Qt6::Core + Qt6::Gui Qt6::Location Qt6::LocationPrivate Qt6::Network diff --git a/src/QtLocationPlugin/Providers/BingMapProvider.cpp b/src/QtLocationPlugin/Providers/BingMapProvider.cpp index d0803da14b41..71919e352892 100644 --- a/src/QtLocationPlugin/Providers/BingMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/BingMapProvider.cpp @@ -3,5 +3,5 @@ QString BingMapProvider::_getURL(int x, int y, int zoom) const { const QString key = _tileXYToQuadKey(x, y, zoom); - return _mapUrl.arg(_getServerNum(x, y, 4)).arg(_mapTypeId, key, _imageFormat, _versionBingMaps, _language); + return _mapUrl.arg(_getServerNum(x, y, kServerCount)).arg(_mapTypeId, key, _imageFormat, _versionBingMaps, _language); } diff --git a/src/QtLocationPlugin/Providers/BingMapProvider.h b/src/QtLocationPlugin/Providers/BingMapProvider.h index 720d524b9a8a..d1626b01003d 100644 --- a/src/QtLocationPlugin/Providers/BingMapProvider.h +++ b/src/QtLocationPlugin/Providers/BingMapProvider.h @@ -3,25 +3,24 @@ #include "MapProvider.h" static constexpr const quint32 AVERAGE_BING_STREET_MAP = 1297; -static constexpr const quint32 AVERAGE_BING_SAT_MAP = 19597; +static constexpr const quint32 AVERAGE_BING_SAT_MAP = 19597; class BingMapProvider : public MapProvider { -protected: - BingMapProvider(const QString &mapName, const QString &mapTypeCode, const QString &imageFormat, quint32 averageSize, - MapProvider::MapStyle mapType) - : MapProvider(mapName, QStringLiteral("https://www.bing.com/maps/"), imageFormat, averageSize, mapType) - , _mapTypeId(mapTypeCode) {} - public: - bool isBingProvider() const final { return true; } + BingMapProvider(const QString& mapName, const QString& mapTypeCode, const QString& imageFormat, quint32 averageSize, + MapProvider::MapStyle mapType) + : MapProvider(mapName, QStringLiteral("https://www.bing.com/maps/"), imageFormat, averageSize, mapType), + _mapTypeId(mapTypeCode) + {} private: QString _getURL(int x, int y, int zoom) const final; const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("http://ecn.t%1.tiles.virtualearth.net/tiles/%2%3.%4?g=%5&mkt=%6"); + const QString _mapUrl = QStringLiteral("https://ecn.t%1.tiles.virtualearth.net/tiles/%2%3.%4?g=%5&mkt=%6"); const QString _versionBingMaps = QStringLiteral("2981"); + static constexpr int kServerCount = 4; /*QUrl m_url; const QString m_scheme = QStringLiteral("http"); @@ -29,39 +28,3 @@ class BingMapProvider : public MapProvider const QString m_path = QStringLiteral("tiles/%1%2.%3"); const QUrlQuery m_query = QStringLiteral("g=%1&mkt=%2");*/ }; - -class BingRoadMapProvider : public BingMapProvider -{ -public: - BingRoadMapProvider() - : BingMapProvider( - QStringLiteral("Bing Road"), - QStringLiteral("r"), - QStringLiteral("png"), - AVERAGE_BING_STREET_MAP, - MapProvider::StreetMap) {} -}; - -class BingSatelliteMapProvider : public BingMapProvider -{ -public: - BingSatelliteMapProvider() - : BingMapProvider( - QStringLiteral("Bing Satellite"), - QStringLiteral("a"), - QStringLiteral("jpg"), - AVERAGE_BING_SAT_MAP, - MapProvider::SatelliteMapDay) {} -}; - -class BingHybridMapProvider : public BingMapProvider -{ -public: - BingHybridMapProvider() - : BingMapProvider( - QStringLiteral("Bing Hybrid"), - QStringLiteral("h"), - QStringLiteral("jpg"), - AVERAGE_BING_SAT_MAP, - MapProvider::HybridMap) {} -}; diff --git a/src/QtLocationPlugin/Providers/ElevationMapProvider.cpp b/src/QtLocationPlugin/Providers/ElevationMapProvider.cpp index 3a24b9e738cf..659dbbcecd93 100644 --- a/src/QtLocationPlugin/Providers/ElevationMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/ElevationMapProvider.cpp @@ -17,6 +17,15 @@ int CopernicusElevationProvider::lat2tileY(double lat, int z) const return static_cast(floor((lat + 90.0) / TerrainTileCopernicus::kTileSizeDegrees)); } +bool CopernicusElevationProvider::isValidTileCoordinate(int x, int y, int zoom) const +{ + // Copernicus tiles a global 0.01° grid, not a 2^zoom web-mercator pyramid, so the + // base bound rejects every real tile. Bound by the provider's own grid extents. + const int maxX = long2tileX(180.0, zoom); + const int maxY = lat2tileY(90.0, zoom); + return (x >= 0) && (x <= maxX) && (y >= 0) && (y <= maxY); +} + QString CopernicusElevationProvider::_getURL(int x, int y, int zoom) const { Q_UNUSED(zoom) @@ -38,6 +47,14 @@ QGCTileSet CopernicusElevationProvider::getTileCount(int zoom, double topleftLon set.tileX1 = long2tileX(bottomRightLon, zoom); set.tileY1 = lat2tileY(topleftLat, zoom); + // Guard against an inverted bbox: an unsigned subtraction with tileX1 < tileX0 + // (or tileY1 < tileY0) would underflow to ~2^64 and report a bogus tile count. + if ((set.tileX1 < set.tileX0) || (set.tileY1 < set.tileY0)) { + set.tileCount = 0; + set.tileSize = 0; + return set; + } + set.tileCount = (static_cast(set.tileX1) - static_cast(set.tileX0) + 1) * (static_cast(set.tileY1) - diff --git a/src/QtLocationPlugin/Providers/ElevationMapProvider.h b/src/QtLocationPlugin/Providers/ElevationMapProvider.h index 873eee116e30..557029fd2c7d 100644 --- a/src/QtLocationPlugin/Providers/ElevationMapProvider.h +++ b/src/QtLocationPlugin/Providers/ElevationMapProvider.h @@ -5,18 +5,13 @@ class ElevationProvider : public MapProvider { protected: - ElevationProvider(const QString &mapName, const QString &referrer, const QString &imageFormat, quint32 averageSize, + ElevationProvider(const QString& mapName, const QString& referrer, const QString& imageFormat, quint32 averageSize, MapProvider::MapStyle mapType) - : MapProvider( - mapName, - referrer, - imageFormat, - averageSize, - mapType) {} + : MapProvider(mapName, referrer, imageFormat, averageSize, mapType) + {} public: - bool isElevationProvider() const final { return true; } - virtual QByteArray serialize(const QByteArray &image) const = 0; + virtual QByteArray serialize(const QByteArray& image) const = 0; }; /// \brief https://spacedata.copernicus.eu/collections/copernicus-digital-elevation-model @@ -25,25 +20,22 @@ class CopernicusElevationProvider : public ElevationProvider { public: CopernicusElevationProvider() - : ElevationProvider( - kProviderKey, - kProviderURL, - QStringLiteral("bin"), - kAvgElevSize, - MapProvider::TerrainMap) {} + : ElevationProvider(kProviderKey, kProviderURL, QStringLiteral("bin"), kAvgElevSize, MapProvider::TerrainMap) + {} int long2tileX(double lon, int z) const final; int lat2tileY(double lat, int z) const final; - QGCTileSet getTileCount(int zoom, double topleftLon, - double topleftLat, double bottomRightLon, + bool isValidTileCoordinate(int x, int y, int zoom) const final; + + QGCTileSet getTileCount(int zoom, double topleftLon, double topleftLat, double bottomRightLon, double bottomRightLat) const final; - QByteArray serialize(const QByteArray &image) const final; + QByteArray serialize(const QByteArray& image) const final; - static constexpr const char *kProviderKey = "Copernicus"; - static constexpr const char *kProviderNotice = "© Airbus Defence and Space GmbH"; - static constexpr const char *kProviderURL = "https://terrain-ce.suite.auterion.com"; + static constexpr const char* kProviderKey = "Copernicus"; + static constexpr const char* kProviderNotice = "© Airbus Defence and Space GmbH"; + static constexpr const char* kProviderURL = "https://terrain-ce.suite.auterion.com"; static constexpr quint32 kAvgElevSize = 2786; private: diff --git a/src/QtLocationPlugin/Providers/EsriMapProvider.cpp b/src/QtLocationPlugin/Providers/EsriMapProvider.cpp index 33555bad2784..9f013792fa9f 100644 --- a/src/QtLocationPlugin/Providers/EsriMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/EsriMapProvider.cpp @@ -4,7 +4,7 @@ QByteArray EsriMapProvider::getToken() const { - return SettingsManager::instance()->appSettings()->esriToken()->rawValue().toString().toUtf8(); + return factString(appSettings()->esriToken()).toUtf8(); } QString EsriMapProvider::_getURL(int x, int y, int zoom) const diff --git a/src/QtLocationPlugin/Providers/EsriMapProvider.h b/src/QtLocationPlugin/Providers/EsriMapProvider.h index 842fc05538ad..3d3b2036fa97 100644 --- a/src/QtLocationPlugin/Providers/EsriMapProvider.h +++ b/src/QtLocationPlugin/Providers/EsriMapProvider.h @@ -4,55 +4,18 @@ class EsriMapProvider : public MapProvider { -protected: - EsriMapProvider(const QString &mapName, const QString &mapTypeId, quint32 averageSize, MapProvider::MapStyle mapType) - : MapProvider( - mapName, - QStringLiteral(""), - QStringLiteral(""), - averageSize, - mapType) - , _mapTypeId(mapTypeId) {} - public: + EsriMapProvider(const QString& mapName, const QString& mapTypeId, quint32 averageSize, + MapProvider::MapStyle mapType) + : MapProvider(mapName, QStringLiteral(""), QStringLiteral(""), averageSize, mapType), _mapTypeId(mapTypeId) + {} + QByteArray getToken() const final; private: QString _getURL(int x, int y, int zoom) const final; const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("http://services.arcgisonline.com/ArcGIS/rest/services/%1/MapServer/tile/%2/%3/%4"); -}; - -class EsriWorldStreetMapProvider : public EsriMapProvider -{ -public: - EsriWorldStreetMapProvider() - : EsriMapProvider( - QStringLiteral("Esri World Street"), - QStringLiteral("World_Street_Map"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} -}; - -class EsriWorldSatelliteMapProvider : public EsriMapProvider -{ -public: - EsriWorldSatelliteMapProvider() - : EsriMapProvider( - QStringLiteral("Esri World Satellite"), - QStringLiteral("World_Imagery"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::SatelliteMapDay) {} -}; - -class EsriTerrainMapProvider : public EsriMapProvider -{ -public: - EsriTerrainMapProvider() - : EsriMapProvider( - QStringLiteral("Esri Terrain"), - QStringLiteral("World_Terrain_Base"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::TerrainMap) {} + const QString _mapUrl = + QStringLiteral("https://services.arcgisonline.com/ArcGIS/rest/services/%1/MapServer/tile/%2/%3/%4"); }; diff --git a/src/QtLocationPlugin/Providers/GenericMapProvider.cpp b/src/QtLocationPlugin/Providers/GenericMapProvider.cpp index 0993f12ae9b4..8e7a5afd3979 100644 --- a/src/QtLocationPlugin/Providers/GenericMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/GenericMapProvider.cpp @@ -2,62 +2,87 @@ #include "SettingsManager.h" #include "AppSettings.h" +#include + +// Percent-encode a user-supplied token before substituting it into a URL so it +// cannot inject query/path separators. +static QString encodeToken(const QString& token) +{ + return QString::fromUtf8(QUrl::toPercentEncoding(token)); +} + QString CustomURLMapProvider::_getURL(int x, int y, int zoom) const { - QString url = SettingsManager::instance()->appSettings()->customURL()->rawValue().toString(); + QString url = factString(appSettings()->customURL()); (void) url.replace("{x}", QString::number(x)); (void) url.replace("{y}", QString::number(y)); static const QRegularExpression zoomRegExp("\\{(z|zoom)\\}"); (void) url.replace(zoomRegExp, QString::number(zoom)); + + // SSRF guard: a user-settings URL must be http(s) and must not target the + // loopback or link-local ranges. Callers treat an empty URL as "no tile". + const QUrl parsed(url, QUrl::StrictMode); + const QString scheme = parsed.scheme().toLower(); + if (!parsed.isValid() || ((scheme != QStringLiteral("http")) && (scheme != QStringLiteral("https")))) { + return QString(); + } + const QString host = parsed.host().toLower(); + if (host.isEmpty() || (host == QStringLiteral("localhost")) || host.startsWith(QStringLiteral("127.")) || + host.startsWith(QStringLiteral("169.254.")) || (host == QStringLiteral("::1"))) { + return QString(); + } + return url; } -QString CyberJapanMapProvider::_getURL(int x, int y, int zoom) const +QString TemplateMapProvider::_getURL(int x, int y, int zoom) const { - return _mapUrl.arg(_mapName).arg(zoom).arg(x).arg(y).arg(_imageFormat); -} + const int tileY = _config.flipY ? ((1 << zoom) - 1 - y) : y; -QString LINZBasemapMapProvider::_getURL(int x, int y, int zoom) const -{ - return _mapUrl.arg(zoom).arg(x).arg(y).arg(_imageFormat); + QString url = _config.urlTemplate; + if (_config.serverCount > 0) { + url = url.arg(_getServerNum(x, y, _config.serverCount)); + } + if (!_config.mapTypeId.isEmpty()) { + url = url.arg(_config.mapTypeId); + } + url = url.arg(zoom); + if (_config.axisOrder == MapProviderConfig::ZYX) { + url = url.arg(tileY).arg(x); + } else { + url = url.arg(x).arg(tileY); + } + if (_config.appendImageFormat) { + url = url.arg(_imageFormat); + } + + return url; } -QString OpenAIPMapProvider::_getURL(int x, int y, int zoom) const +QString LINZBasemapMapProvider::_getURL(int x, int y, int zoom) const { - const QString apiKey = SettingsManager::instance()->appSettings()->openaipToken()->rawValue().toString(); + const QString apiKey = factString(appSettings()->linzToken()); - QString url = _mapUrl.arg(zoom).arg(x).arg(y); + QString url = _mapUrl.arg(zoom).arg(x).arg(y).arg(_imageFormat); if (!apiKey.isEmpty()) { - url += QStringLiteral("?apiKey=%1").arg(apiKey); + url += QStringLiteral("?api=%1").arg(encodeToken(apiKey)); } return url; } -QString OpenStreetMapProvider::_getURL(int x, int y, int zoom) const -{ - return _mapUrl.arg(zoom).arg(x).arg(y); -} - -QString StatkartMapProvider::_getURL(int x, int y, int zoom) const +QString OpenAIPMapProvider::_getURL(int x, int y, int zoom) const { - return _mapUrl.arg(zoom).arg(y).arg(x); -} + const QString apiKey = factString(appSettings()->openaipToken()); -QString EniroMapProvider::_getURL(int x, int y, int zoom) const -{ - return _mapUrl.arg(zoom).arg(x).arg((1 << zoom) - 1 - y).arg(_imageFormat); -} + QString url = _mapUrl.arg(zoom).arg(x).arg(y); -QString SvalbardMapProvider::_getURL(int x, int y, int zoom) const -{ - return _mapUrl.arg(zoom).arg(y).arg(x); -} + if (!apiKey.isEmpty()) { + url += QStringLiteral("?apiKey=%1").arg(encodeToken(apiKey)); + } -QString MapQuestMapProvider::_getURL(int x, int y, int zoom) const -{ - return _mapUrl.arg(_getServerNum(x, y, 4)).arg(_mapName).arg(zoom).arg(x).arg(y).arg(_imageFormat); + return url; } QString VWorldMapProvider::_getURL(int x, int y, int zoom) const @@ -67,19 +92,20 @@ QString VWorldMapProvider::_getURL(int x, int y, int zoom) const } const int gap = zoom - 6; + const qint64 span = static_cast(1) << gap; - const int x_min = 53 * pow(2, gap); - const int x_max = (55 * pow(2, gap)) + (2 * gap - 1); + const qint64 x_min = 53 * span; + const qint64 x_max = (55 * span) + (2 * gap - 1); if ((x < x_min) || (x > x_max)) { return QString(); } - const int y_min = 22 * pow(2, gap); - const int y_max = (26 * pow(2, gap)) + (2 * gap - 1); + const qint64 y_min = 22 * span; + const qint64 y_max = (26 * span) + (2 * gap - 1); if ((y < y_min) || (y > y_max)) { return QString(); } - const QString VWorldMapToken = SettingsManager::instance()->appSettings()->vworldToken()->rawValue().toString(); - return _mapUrl.arg(VWorldMapToken, _mapName).arg(zoom).arg(y).arg(x).arg(_imageFormat); + const QString VWorldMapToken = encodeToken(factString(appSettings()->vworldToken())); + return _mapUrl.arg(VWorldMapToken, _mapTypeId).arg(zoom).arg(y).arg(x).arg(_imageFormat); } diff --git a/src/QtLocationPlugin/Providers/GenericMapProvider.h b/src/QtLocationPlugin/Providers/GenericMapProvider.h index 86004c17e7d6..0759e97cd432 100644 --- a/src/QtLocationPlugin/Providers/GenericMapProvider.h +++ b/src/QtLocationPlugin/Providers/GenericMapProvider.h @@ -6,113 +6,75 @@ class CustomURLMapProvider : public MapProvider { public: CustomURLMapProvider() - : MapProvider( - QStringLiteral("CustomURL Custom"), - QStringLiteral(""), - QStringLiteral(""), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} + : MapProvider(QStringLiteral("CustomURL Custom"), QStringLiteral(""), QStringLiteral(""), QGC_AVERAGE_TILE_SIZE, + MapProvider::CustomMap) + {} private: QString _getURL(int x, int y, int zoom) const final; }; -class CyberJapanMapProvider : public MapProvider +// Data-driven provider for the common XYZ/TMS tile schemes; _getURL applies the +// config (server-rotation arg, layer id, axis order, TMS y-flip, image-format token). +struct MapProviderConfig { -protected: - CyberJapanMapProvider(const QString &mapName, const QString &mapTypeId, const QString &imageFormat) - : MapProvider( - mapName, - QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), - imageFormat, - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) - , _mapTypeId(mapName) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"); -}; + enum AxisOrder + { + ZXY, + ZYX + }; -class JapanStdMapProvider : public CyberJapanMapProvider -{ -public: - JapanStdMapProvider() - : CyberJapanMapProvider( - QStringLiteral("Japan-GSI Contour"), - QStringLiteral("std"), - QStringLiteral("png")) {} + QString name; + QString referrer; + QString urlTemplate; + QString imageFormat; + quint32 averageSize = QGC_AVERAGE_TILE_SIZE; + MapProvider::MapStyle mapStyle = MapProvider::CustomMap; + QString mapTypeId; + AxisOrder axisOrder = ZXY; + bool flipY = false; + bool appendImageFormat = false; + int serverCount = 0; + bool isOSM = false; }; -class JapanSeamlessMapProvider : public CyberJapanMapProvider +class TemplateMapProvider : public MapProvider { public: - JapanSeamlessMapProvider() - : CyberJapanMapProvider( - QStringLiteral("Japan-GSI Seamless"), - QStringLiteral("seamlessphoto"), - QStringLiteral("jpg")) {} -}; + explicit TemplateMapProvider(const MapProviderConfig &config) + : MapProvider(config.name, config.referrer, config.imageFormat, config.averageSize, config.mapStyle), + _config(config) + {} -class JapanAnaglyphMapProvider : public CyberJapanMapProvider -{ -public: - JapanAnaglyphMapProvider() - : CyberJapanMapProvider( - QStringLiteral("Japan-GSI Anaglyph"), - QStringLiteral("anaglyphmap_color"), - QStringLiteral("png")) {} -}; + bool isOSMProvider() const final { return _config.isOSM; } -class JapanSlopeMapProvider : public CyberJapanMapProvider -{ -public: - JapanSlopeMapProvider() - : CyberJapanMapProvider( - QStringLiteral("Japan-GSI Slope"), - QStringLiteral("slopemap"), - QStringLiteral("png")) {} -}; +private: + QString _getURL(int x, int y, int zoom) const final; -class JapanReliefMapProvider : public CyberJapanMapProvider -{ -public: - JapanReliefMapProvider() - : CyberJapanMapProvider( - QStringLiteral("Japan-GSI Relief"), - QStringLiteral("relief"), - QStringLiteral("png")) {} + const MapProviderConfig _config; }; class LINZBasemapMapProvider : public MapProvider { public: LINZBasemapMapProvider() - : MapProvider( - QStringLiteral("LINZ Basemap"), - QStringLiteral("https://basemaps.linz.govt.nz/v1/tiles/aerial"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::SatelliteMapDay) {} + : MapProvider(QStringLiteral("LINZ Basemap"), QStringLiteral("https://basemaps.linz.govt.nz/v1/tiles/aerial"), + QStringLiteral("png"), QGC_AVERAGE_TILE_SIZE, MapProvider::SatelliteMapDay) + {} private: QString _getURL(int x, int y, int zoom) const final; - const QString _mapUrl = QStringLiteral("https://basemaps.linz.govt.nz/v1/tiles/aerial/EPSG:3857/%1/%2/%3.%4?api=d01ev80nqcjxddfvc6amyvkk1ka"); + const QString _mapUrl = QStringLiteral("https://basemaps.linz.govt.nz/v1/tiles/aerial/EPSG:3857/%1/%2/%3.%4"); }; class OpenAIPMapProvider : public MapProvider { public: OpenAIPMapProvider() - : MapProvider( - QStringLiteral("OpenAIP"), - QStringLiteral("https://www.openaip.net"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} + : MapProvider(QStringLiteral("OpenAIP"), QStringLiteral("https://www.openaip.net"), QStringLiteral("png"), + QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap) + {} private: QString _getURL(int x, int y, int zoom) const final; @@ -120,173 +82,36 @@ class OpenAIPMapProvider : public MapProvider const QString _mapUrl = QStringLiteral("https://api.tiles.openaip.net/api/data/openaip/%1/%2/%3.png"); }; -class OpenStreetMapProvider : public MapProvider -{ -public: - OpenStreetMapProvider() - : MapProvider( - QStringLiteral("Street Map"), - QStringLiteral("https://www.openstreetmap.org"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapUrl = QStringLiteral("http://tile.openstreetmap.org/%1/%2/%3.png"); -}; - -class StatkartMapProvider : public MapProvider -{ -protected: - StatkartMapProvider(const QString &mapName, const QString &mapTypeId) - : MapProvider( - mapName, - QStringLiteral("https://norgeskart.no/"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) - , _mapTypeId(mapName) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("https://cache.kartverket.no/v1/wmts/1.0.0/topo/default/webmercator/%1/%2/%3.png"); -}; - -class StatkartTopoMapProvider : public StatkartMapProvider -{ -public: - StatkartTopoMapProvider() - : StatkartMapProvider( - QStringLiteral("Statkart Topo"), - QStringLiteral("topo4")) {} -}; - -class StatkartBaseMapProvider : public StatkartMapProvider -{ -public: - StatkartBaseMapProvider() - : StatkartMapProvider( - QStringLiteral("Statkart Basemap"), - QStringLiteral("norgeskart_bakgrunn")) {} -}; - -class SvalbardMapProvider : public MapProvider -{ -public: - SvalbardMapProvider() - : MapProvider( - QStringLiteral("Svalbard Topo"), - QStringLiteral("https://www.npolar.no/"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapUrl = QStringLiteral("https://geodata.npolar.no/arcgis/rest/services/Basisdata/NP_Basiskart_Svalbard_WMTS_3857/MapServer/WMTS/tile/1.0.0/Basisdata_NP_Basiskart_Svalbard_WMTS_3857/default/default028mm/%1/%2/%3"); -}; - - -class EniroMapProvider : public MapProvider -{ -public: - EniroMapProvider() - : MapProvider( - QStringLiteral("Eniro Topo"), - QStringLiteral("https://www.eniro.se/"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapUrl = QStringLiteral("http://map.eniro.com/geowebcache/service/tms1.0.0/map/%1/%2/%3.%4"); -}; - -class MapQuestMapProvider : public MapProvider -{ -protected: - MapQuestMapProvider(const QString &mapName, const QString &mapTypeId, MapProvider::MapStyle mapType) - : MapProvider( - mapName, - QStringLiteral("https://mapquest.com"), - QStringLiteral("jpg"), - QGC_AVERAGE_TILE_SIZE, - mapType) - , _mapTypeId(mapName) {} - -private: - QString _getURL(int x, int y, int zoom) const final; - - const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("http://otile%1.mqcdn.com/tiles/1.0.0/%2/%3/%4/%5.%6"); -}; - -class MapQuestMapMapProvider : public MapQuestMapProvider -{ -public: - MapQuestMapMapProvider() - : MapQuestMapProvider( - QStringLiteral("MapQuest Map"), - QStringLiteral("map"), - MapProvider::StreetMap) {} -}; - -class MapQuestSatMapProvider : public MapQuestMapProvider -{ -public: - MapQuestSatMapProvider() - : MapQuestMapProvider( - QStringLiteral("MapQuest Sat"), - QStringLiteral("sat"), - MapProvider::SatelliteMapDay) {} -}; - class VWorldMapProvider : public MapProvider { protected: - VWorldMapProvider(const QString &mapName, const QString &mapTypeId, const QString &imageFormat, quint32 averageSize, MapProvider::MapStyle mapStyle) - : MapProvider( - mapName, - QStringLiteral("www.vworld.kr"), - imageFormat, - averageSize, - mapStyle) - , _mapTypeId(mapName) {} + VWorldMapProvider(const QString& mapName, const QString& mapTypeId, const QString& imageFormat, quint32 averageSize, + MapProvider::MapStyle mapStyle) + : MapProvider(mapName, QStringLiteral("www.vworld.kr"), imageFormat, averageSize, mapStyle), + _mapTypeId(mapTypeId) + {} private: QString _getURL(int x, int y, int zoom) const final; const QString _mapTypeId; - const QString _mapUrl = QStringLiteral("http://api.vworld.kr/req/wmts/1.0.0/%1/%2/%3/%4/%5.%6"); + const QString _mapUrl = QStringLiteral("https://api.vworld.kr/req/wmts/1.0.0/%1/%2/%3/%4/%5.%6"); }; class VWorldStreetMapProvider : public VWorldMapProvider { public: VWorldStreetMapProvider() - : VWorldMapProvider( - QStringLiteral("VWorld Street Map"), - QStringLiteral("Base"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} + : VWorldMapProvider(QStringLiteral("VWorld Street Map"), QStringLiteral("Base"), QStringLiteral("png"), + QGC_AVERAGE_TILE_SIZE, MapProvider::StreetMap) + {} }; class VWorldSatMapProvider : public VWorldMapProvider { public: VWorldSatMapProvider() - : VWorldMapProvider( - QStringLiteral("VWorld Satellite Map"), - QStringLiteral("Satellite"), - QStringLiteral("jpeg"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::SatelliteMapDay) {} + : VWorldMapProvider(QStringLiteral("VWorld Satellite Map"), QStringLiteral("Satellite"), QStringLiteral("jpeg"), + QGC_AVERAGE_TILE_SIZE, MapProvider::SatelliteMapDay) + {} }; diff --git a/src/QtLocationPlugin/Providers/GoogleMapProvider.cpp b/src/QtLocationPlugin/Providers/GoogleMapProvider.cpp index dc8a60aa7033..a1136eb118da 100644 --- a/src/QtLocationPlugin/Providers/GoogleMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/GoogleMapProvider.cpp @@ -1,5 +1,7 @@ #include "GoogleMapProvider.h" +#include "QGCMapUrlEngine.h" + void GoogleMapProvider::_getSecGoogleWords(int x, int y, QString& sec1, QString& sec2) const { sec1 = QStringLiteral(""); // after &x=... @@ -17,11 +19,11 @@ QString GoogleMapProvider::_getURL(int x, int y, int zoom) const QString sec2; _getSecGoogleWords(x, y, sec1, sec2); return _mapUrl - .arg(_getServerNum(x, y, 4)) + .arg(_getServerNum(x, y, kServerCount)) .arg(_versionRequest, _version, _language) .arg(x) .arg(sec1) .arg(y) .arg(zoom) - .arg(sec2, _scale); + .arg(sec2, QString::number(UrlFactory::tilePixelScale())); } diff --git a/src/QtLocationPlugin/Providers/GoogleMapProvider.h b/src/QtLocationPlugin/Providers/GoogleMapProvider.h index 5136a0527952..f2b713139a49 100644 --- a/src/QtLocationPlugin/Providers/GoogleMapProvider.h +++ b/src/QtLocationPlugin/Providers/GoogleMapProvider.h @@ -16,23 +16,20 @@ // size: 256x256 // maxZoom: 20 -static constexpr const quint32 AVERAGE_GOOGLE_STREET_MAP = 4913; -static constexpr const quint32 AVERAGE_GOOGLE_SAT_MAP = 56887; +static constexpr const quint32 AVERAGE_GOOGLE_STREET_MAP = 4913; +static constexpr const quint32 AVERAGE_GOOGLE_SAT_MAP = 56887; static constexpr const quint32 AVERAGE_GOOGLE_TERRAIN_MAP = 19391; class GoogleMapProvider : public MapProvider { -protected: - GoogleMapProvider(const QString &mapName, const QString &versionRequest, const QString &version, const QString &imageFormat, quint32 averageSize, - MapProvider::MapStyle mapType) - : MapProvider( - mapName, - QStringLiteral("https://www.google.com/maps/preview"), - imageFormat, - averageSize, - mapType) - , _versionRequest(versionRequest) - , _version(version) {} +public: + GoogleMapProvider(const QString& mapName, const QString& versionRequest, const QString& version, + const QString& imageFormat, quint32 averageSize, MapProvider::MapStyle mapType) + : MapProvider(mapName, QStringLiteral("https://www.google.com/maps/preview"), imageFormat, averageSize, + mapType), + _versionRequest(versionRequest), + _version(version) + {} private: void _getSecGoogleWords(int x, int y, QString& sec1, QString& sec2) const; @@ -40,72 +37,7 @@ class GoogleMapProvider : public MapProvider const QString _versionRequest; const QString _version; - const QString _mapUrl = QStringLiteral("http://mt%1.google.com/vt/%2=%3&hl=%4&x=%5%6&y=%7&z=%8&s=%9&scale=%10"); + const QString _mapUrl = QStringLiteral("https://mt%1.google.com/vt/%2=%3&hl=%4&x=%5%6&y=%7&z=%8&s=%9&scale=%10"); const QString _secGoogleWord = QStringLiteral("Galileo"); - const QString _scale = QStringLiteral("1"); -}; - -class GoogleStreetMapProvider : public GoogleMapProvider -{ -public: - GoogleStreetMapProvider() - : GoogleMapProvider( - QStringLiteral("Google Street Map"), - QStringLiteral("lyrs"), - QStringLiteral("m"), - QStringLiteral("png"), - AVERAGE_GOOGLE_STREET_MAP, - MapProvider::StreetMap) {} -}; - -class GoogleSatelliteMapProvider : public GoogleMapProvider -{ -public: - GoogleSatelliteMapProvider() - : GoogleMapProvider( - QStringLiteral("Google Satellite"), - QStringLiteral("lyrs"), - QStringLiteral("s"), - QStringLiteral("jpg"), - AVERAGE_GOOGLE_SAT_MAP, - MapProvider::SatelliteMapDay) {} -}; - -class GoogleLabelsMapProvider : public GoogleMapProvider -{ -public: - GoogleLabelsMapProvider() - : GoogleMapProvider( - QStringLiteral("Google Labels"), - QStringLiteral("lyrs"), - QStringLiteral("h"), - QStringLiteral("png"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; - -class GoogleTerrainMapProvider : public GoogleMapProvider -{ -public: - GoogleTerrainMapProvider() - : GoogleMapProvider( - QStringLiteral("Google Terrain"), - QStringLiteral("v"), - QStringLiteral("t,r"), - QStringLiteral("png"), - AVERAGE_GOOGLE_TERRAIN_MAP, - MapProvider::TerrainMap) {} -}; - -class GoogleHybridMapProvider : public GoogleMapProvider -{ -public: - GoogleHybridMapProvider() - : GoogleMapProvider( - QStringLiteral("Google Hybrid"), - QStringLiteral("lyrs"), - QStringLiteral("y"), - QStringLiteral("png"), - AVERAGE_GOOGLE_SAT_MAP, - MapProvider::HybridMap) {} + static constexpr int kServerCount = 4; }; diff --git a/src/QtLocationPlugin/Providers/MapProvider.cpp b/src/QtLocationPlugin/Providers/MapProvider.cpp index 020fb73a78df..042ed613227b 100644 --- a/src/QtLocationPlugin/Providers/MapProvider.cpp +++ b/src/QtLocationPlugin/Providers/MapProvider.cpp @@ -1,46 +1,81 @@ #include "MapProvider.h" -#include - -#include "QGCTileSet.h" +#include #include #include +#include #include +#include + +#include "AppSettings.h" +#include "Fact.h" +#include "QGCTileSet.h" +#include "SettingsManager.h" QGC_LOGGING_CATEGORY(MapProviderLog, "QtLocationPlugin.MapProvider") -// MapProvider::MapStyle mirrors QGeoMapType::MapStyle to keep the public -// header free of . Catch drift at compile -// time so the two enums never disagree. -static_assert(static_cast(MapProvider::NoMap) == static_cast(QGeoMapType::NoMap)); -static_assert(static_cast(MapProvider::StreetMap) == static_cast(QGeoMapType::StreetMap)); -static_assert(static_cast(MapProvider::SatelliteMapDay) == static_cast(QGeoMapType::SatelliteMapDay)); -static_assert(static_cast(MapProvider::SatelliteMapNight)== static_cast(QGeoMapType::SatelliteMapNight)); -static_assert(static_cast(MapProvider::TerrainMap) == static_cast(QGeoMapType::TerrainMap)); -static_assert(static_cast(MapProvider::HybridMap) == static_cast(QGeoMapType::HybridMap)); -static_assert(static_cast(MapProvider::TransitMap) == static_cast(QGeoMapType::TransitMap)); -static_assert(static_cast(MapProvider::GrayStreetMap) == static_cast(QGeoMapType::GrayStreetMap)); -static_assert(static_cast(MapProvider::PedestrianMap) == static_cast(QGeoMapType::PedestrianMap)); -static_assert(static_cast(MapProvider::CarNavigationMap) == static_cast(QGeoMapType::CarNavigationMap)); -static_assert(static_cast(MapProvider::CycleMap) == static_cast(QGeoMapType::CycleMap)); -static_assert(static_cast(MapProvider::CustomMap) == static_cast(QGeoMapType::CustomMap)); +// MapProvider::MapStyle mirrors QGeoMapType::MapStyle so the public header stays +// free of . Catch any enumerator drift here +// at compile time. +static_assert(static_cast(MapProvider::NoMap) == static_cast(QGeoMapType::NoMap)); +static_assert(static_cast(MapProvider::StreetMap) == static_cast(QGeoMapType::StreetMap)); +static_assert(static_cast(MapProvider::SatelliteMapDay) == static_cast(QGeoMapType::SatelliteMapDay)); +static_assert(static_cast(MapProvider::SatelliteMapNight) == static_cast(QGeoMapType::SatelliteMapNight)); +static_assert(static_cast(MapProvider::TerrainMap) == static_cast(QGeoMapType::TerrainMap)); +static_assert(static_cast(MapProvider::HybridMap) == static_cast(QGeoMapType::HybridMap)); +static_assert(static_cast(MapProvider::TransitMap) == static_cast(QGeoMapType::TransitMap)); +static_assert(static_cast(MapProvider::GrayStreetMap) == static_cast(QGeoMapType::GrayStreetMap)); +static_assert(static_cast(MapProvider::PedestrianMap) == static_cast(QGeoMapType::PedestrianMap)); +static_assert(static_cast(MapProvider::CarNavigationMap) == static_cast(QGeoMapType::CarNavigationMap)); +static_assert(static_cast(MapProvider::CycleMap) == static_cast(QGeoMapType::CycleMap)); +static_assert(static_cast(MapProvider::CustomMap) == static_cast(QGeoMapType::CustomMap)); // QtLocation expects MapIds to start at 1 and be sequential. int MapProvider::_mapIdIndex = 1; -MapProvider::MapProvider( - const QString &mapName, - const QString &referrer, - const QString &imageFormat, - quint32 averageSize, - MapStyle mapStyle) - : _mapName(mapName) - , _referrer(referrer) - , _imageFormat(imageFormat) - , _averageSize(averageSize) - , _mapStyle(mapStyle) - , _language(!QLocale::system().uiLanguages().isEmpty() ? QLocale::system().uiLanguages().constFirst() : "en") - , _mapId(_mapIdIndex++) +AppSettings* MapProvider::appSettings() +{ + return SettingsManager::instance()->appSettings(); +} + +QString MapProvider::factString(const Fact* fact) +{ + return fact ? fact->rawValue().toString() : QString(); +} + +bool MapProvider::isValidLongitude(double lon) +{ + return (lon >= -180.0) && (lon <= 180.0); +} + +bool MapProvider::isValidLatitude(double lat) +{ + return (lat >= -90.0) && (lat <= 90.0); +} + +bool MapProvider::isValidZoom(int zoom) +{ + return (zoom >= 0) && (zoom <= QGC_MAX_MAP_ZOOM); +} + +bool MapProvider::isValidTileCoordinate(int x, int y, int zoom) const +{ + if (!isValidZoom(zoom)) { + return false; + } + const int maxTile = (1 << zoom) - 1; + return (x >= 0) && (x <= maxTile) && (y >= 0) && (y <= maxTile); +} + +MapProvider::MapProvider(const QString& mapName, const QString& referrer, const QString& imageFormat, + quint32 averageSize, MapStyle mapStyle) + : _mapName(mapName), + _referrer(referrer), + _imageFormat(imageFormat), + _averageSize(averageSize), + _mapStyle(mapStyle), + _language(!QLocale::system().uiLanguages().isEmpty() ? QLocale::system().uiLanguages().constFirst() : "en"), + _mapId(_mapIdIndex++) { // qCDebug(MapProviderLog) << Q_FUNC_INFO << this << _mapId; } @@ -71,17 +106,39 @@ QString MapProvider::getImageFormat(QByteArrayView image) const return QStringLiteral("jpg"); } + if (image.size() >= 12) { + static constexpr QByteArrayView riffSignature("RIFF"); + static constexpr QByteArrayView webpSignature("WEBP"); + if (image.startsWith(riffSignature) && QByteArrayView(image.data() + 8, 4).startsWith(webpSignature)) { + return QStringLiteral("webp"); + } + } + static constexpr QByteArrayView gifSignature("\x47\x49\x46\x38"); if (image.startsWith(gifSignature)) { return QStringLiteral("gif"); } + if (image.size() >= 12) { + const char* d = image.data(); + if (d[4] == 'f' && d[5] == 't' && d[6] == 'y' && d[7] == 'p' && d[8] == 'a' && d[9] == 'v' && d[10] == 'i' && + (d[11] == 'f' || d[11] == 's')) { + return QStringLiteral("avif"); + } + } + + static constexpr QByteArrayView bmpSignature("BM"); + if (image.startsWith(bmpSignature)) { + return QStringLiteral("bmp"); + } + return _imageFormat; } QString MapProvider::_tileXYToQuadKey(int tileX, int tileY, int levelOfDetail) const { QString quadKey; + quadKey.reserve(levelOfDetail); for (int i = levelOfDetail; i > 0; i--) { char digit = '0'; const int mask = 1 << (i - 1); @@ -106,12 +163,17 @@ int MapProvider::_getServerNum(int x, int y, int max) const int MapProvider::long2tileX(double lon, int z) const { + lon = qBound(-180.0, std::isnan(lon) ? 0.0 : lon, 180.0); + z = qBound(0, z, QGC_MAX_MAP_ZOOM); return static_cast(floor((lon + 180.0) / 360.0 * pow(2.0, z))); } int MapProvider::lat2tileY(double lat, int z) const { - return static_cast(floor((1.0 - log(tan(lat * M_PI / 180.0) + 1.0 / cos(lat * M_PI / 180.0)) / M_PI) / 2.0 * pow(2.0, z))); + lat = qBound(-QGC_MAX_MERCATOR_LATITUDE, std::isnan(lat) ? 0.0 : lat, QGC_MAX_MERCATOR_LATITUDE); + z = qBound(0, z, QGC_MAX_MAP_ZOOM); + return static_cast( + floor((1.0 - log(tan(lat * M_PI / 180.0) + 1.0 / cos(lat * M_PI / 180.0)) / M_PI) / 2.0 * pow(2.0, z))); } double MapProvider::tileX2long(int x, int z) const @@ -125,8 +187,7 @@ double MapProvider::tileY2lat(int y, int z) const return qRadiansToDegrees(std::atan(std::sinh(n))); } -QGCTileSet MapProvider::getTileCount(int zoom, double topleftLon, - double topleftLat, double bottomRightLon, +QGCTileSet MapProvider::getTileCount(int zoom, double topleftLon, double topleftLat, double bottomRightLon, double bottomRightLat) const { QGCTileSet set; @@ -135,10 +196,16 @@ QGCTileSet MapProvider::getTileCount(int zoom, double topleftLon, set.tileX1 = long2tileX(bottomRightLon, zoom); set.tileY1 = lat2tileY(bottomRightLat, zoom); - set.tileCount = (static_cast(set.tileX1) - - static_cast(set.tileX0) + 1) * - (static_cast(set.tileY1) - - static_cast(set.tileY0) + 1); + // Guard against an inverted bbox: an unsigned subtraction with tileX1 < tileX0 + // (or tileY1 < tileY0) would underflow to ~2^64 and report a bogus tile count. + if ((set.tileX1 < set.tileX0) || (set.tileY1 < set.tileY0)) { + set.tileCount = 0; + set.tileSize = 0; + return set; + } + + set.tileCount = (static_cast(set.tileX1) - static_cast(set.tileX0) + 1) * + (static_cast(set.tileY1) - static_cast(set.tileY0) + 1); set.tileSize = getAverageSize() * set.tileCount; return set; diff --git a/src/QtLocationPlugin/Providers/MapProvider.h b/src/QtLocationPlugin/Providers/MapProvider.h index ac22acb23308..2be463806004 100644 --- a/src/QtLocationPlugin/Providers/MapProvider.h +++ b/src/QtLocationPlugin/Providers/MapProvider.h @@ -6,17 +6,24 @@ #include "QGCTileSet.h" -#define QGC_MAX_MAP_ZOOM 23 -static constexpr const quint32 QGC_AVERAGE_TILE_SIZE = 13652; +class AppSettings; +class Fact; + +static constexpr int QGC_MAX_MAP_ZOOM = 23; +// Web Mercator is only defined to ±85.05112878°; tan/cos blow up at the poles. +static constexpr double QGC_MAX_MERCATOR_LATITUDE = 85.05112878; +static constexpr quint32 QGC_AVERAGE_TILE_SIZE = 13652; // TODO: Inherit from QGeoMapType class MapProvider { public: - // Mirror of QGeoMapType::MapStyle (kept in sync manually so this header - // doesn't need to pull QtLocation/private/qgeomaptype_p.h). Converted at - // the boundary in QGeoTiledMappingManagerEngineQGC.cpp. - enum MapStyle { + // Mirror of QGeoMapType::MapStyle (kept in sync manually so this header does + // not pull QtLocation/private/qgeomaptype_p.h). Drift is caught at compile + // time via static_assert in MapProvider.cpp; values are converted at the + // QGeoMapType boundary in QGeoTiledMappingManagerEngineQGC.cpp. + enum MapStyle + { NoMap = 0, StreetMap, SatelliteMapDay, @@ -31,8 +38,8 @@ class MapProvider CustomMap = 100 }; - MapProvider(const QString &mapName, const QString &referrer, const QString &imageFormat, quint32 averageSize = QGC_AVERAGE_TILE_SIZE, - MapStyle mapStyle = CustomMap); + MapProvider(const QString& mapName, const QString& referrer, const QString& imageFormat, + quint32 averageSize = QGC_AVERAGE_TILE_SIZE, MapStyle mapStyle = CustomMap); virtual ~MapProvider(); QUrl getTileURL(int x, int y, int zoom) const; @@ -43,9 +50,13 @@ class MapProvider quint32 getAverageSize() const { return _averageSize; } MapStyle getMapStyle() const { return _mapStyle; } + const QString& getMapName() const { return _mapName; } + int getMapId() const { return _mapId; } + const QString& getReferrer() const { return _referrer; } + virtual QByteArray getToken() const { return QByteArray(); } virtual int long2tileX(double lon, int z) const; @@ -53,14 +64,26 @@ class MapProvider virtual double tileX2long(int x, int z) const; virtual double tileY2lat(int y, int z) const; - virtual bool isElevationProvider() const { return false; } - virtual bool isBingProvider() const { return false; } + // OSM tile-usage policy requires an app-identifiable User-Agent. + virtual bool isOSMProvider() const { return false; } + + static bool isValidLongitude(double lon); + static bool isValidLatitude(double lat); + static bool isValidZoom(int zoom); - virtual QGCTileSet getTileCount(int zoom, double topleftLon, - double topleftLat, double bottomRightLon, + // Virtual so providers with non-web-mercator tiling (e.g. the Copernicus + // degree grid) can replace the default 2^zoom bound. + virtual bool isValidTileCoordinate(int x, int y, int zoom) const; + + virtual QGCTileSet getTileCount(int zoom, double topleftLon, double topleftLat, double bottomRightLon, double bottomRightLat) const; protected: + // Centralizes the SettingsManager -> AppSettings -> Fact -> rawValue chain; + // factString() returns QString() when the Fact is null. + static AppSettings* appSettings(); + static QString factString(const Fact* fact); + QString _tileXYToQuadKey(int tileX, int tileY, int levelOfDetail) const; int _getServerNum(int x, int y, int max) const; diff --git a/src/QtLocationPlugin/Providers/MapboxMapProvider.cpp b/src/QtLocationPlugin/Providers/MapboxMapProvider.cpp index c61864dbcdc9..b8d63da5575d 100644 --- a/src/QtLocationPlugin/Providers/MapboxMapProvider.cpp +++ b/src/QtLocationPlugin/Providers/MapboxMapProvider.cpp @@ -1,20 +1,33 @@ #include "MapboxMapProvider.h" +#include "QGCMapUrlEngine.h" #include "SettingsManager.h" #include "AppSettings.h" +#include + QString MapboxMapProvider::_getURL(int x, int y, int zoom) const { - const QString mapBoxToken = SettingsManager::instance()->appSettings()->mapboxToken()->rawValue().toString(); + const QString mapBoxToken = QString::fromUtf8(QUrl::toPercentEncoding(factString(appSettings()->mapboxToken()))); if (!mapBoxToken.isEmpty()) { if (_mapTypeId == QStringLiteral("mapbox.custom")) { - static const QString MapBoxUrlCustom = QStringLiteral("https://api.mapbox.com/styles/v1/%1/%2/tiles/256/%3/%4/%5?access_token=%6"); - const QString mapBoxAccount = SettingsManager::instance()->appSettings()->mapboxAccount()->rawValue().toString(); - const QString mapBoxStyle = SettingsManager::instance()->appSettings()->mapboxStyle()->rawValue().toString(); - return MapBoxUrlCustom.arg(mapBoxAccount).arg(mapBoxStyle).arg(zoom).arg(x).arg(y).arg(mapBoxToken); + const bool retina = UrlFactory::useRetinaTiles(); + const QString tileSize = retina ? QStringLiteral("512") : QStringLiteral("256"); + const QString retinaSuffix = retina ? QStringLiteral("@2x") : QString(); + const QString MapBoxUrlCustom = + QStringLiteral("https://api.mapbox.com/styles/v1/%1/%2/tiles/%3/%4/%5/%6%7?access_token=%8"); + const QString mapBoxAccount = factString(appSettings()->mapboxAccount()); + const QString mapBoxStyle = factString(appSettings()->mapboxStyle()); + return MapBoxUrlCustom.arg(mapBoxAccount, mapBoxStyle, tileSize) + .arg(zoom) + .arg(x) + .arg(y) + .arg(retinaSuffix, mapBoxToken); } - static const QString MapBoxUrl = QStringLiteral("https://api.mapbox.com/styles/v1/mapbox/%1/tiles/%2/%3/%4?access_token=%5"); - return MapBoxUrl.arg(_mapTypeId).arg(zoom).arg(x).arg(y).arg(mapBoxToken); + const QString retinaSuffix = UrlFactory::useRetinaTiles() ? QStringLiteral("@2x") : QString(); + const QString MapBoxUrl = + QStringLiteral("https://api.mapbox.com/styles/v1/mapbox/%1/tiles/%2/%3/%4%5?access_token=%6"); + return MapBoxUrl.arg(_mapTypeId).arg(zoom).arg(x).arg(y).arg(retinaSuffix, mapBoxToken); } return QString(); } diff --git a/src/QtLocationPlugin/Providers/MapboxMapProvider.h b/src/QtLocationPlugin/Providers/MapboxMapProvider.h index d1953000f7a9..e7e9e93a2033 100644 --- a/src/QtLocationPlugin/Providers/MapboxMapProvider.h +++ b/src/QtLocationPlugin/Providers/MapboxMapProvider.h @@ -2,122 +2,20 @@ #include "MapProvider.h" -static constexpr const quint32 AVERAGE_MAPBOX_SAT_MAP = 15739; -static constexpr const quint32 AVERAGE_MAPBOX_STREET_MAP = 5648; +static constexpr const quint32 AVERAGE_MAPBOX_SAT_MAP = 15739; +static constexpr const quint32 AVERAGE_MAPBOX_STREET_MAP = 5648; class MapboxMapProvider : public MapProvider { -protected: - MapboxMapProvider(const QString &mapName, const QString &mapTypeId, quint32 averageSize, MapProvider::MapStyle mapType) - : MapProvider( - mapName, - QStringLiteral("https://www.mapbox.com/"), - QStringLiteral("jpg"), - averageSize, - mapType) - , _mapTypeId(mapTypeId) {} +public: + MapboxMapProvider(const QString& mapName, const QString& mapTypeId, quint32 averageSize, + MapProvider::MapStyle mapType) + : MapProvider(mapName, QStringLiteral("https://www.mapbox.com/"), QStringLiteral("jpg"), averageSize, mapType), + _mapTypeId(mapTypeId) + {} private: QString _getURL(int x, int y, int zoom) const final; const QString _mapTypeId; }; - -class MapboxStreetMapProvider : public MapboxMapProvider -{ -public: - MapboxStreetMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Streets"), - QStringLiteral("streets-v10"), - AVERAGE_MAPBOX_STREET_MAP, - MapProvider::StreetMap) {} -}; - -class MapboxLightMapProvider : public MapboxMapProvider -{ -public: - MapboxLightMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Light"), - QStringLiteral("light-v9"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; - -class MapboxDarkMapProvider : public MapboxMapProvider -{ -public: - MapboxDarkMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Dark"), - QStringLiteral("dark-v9"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; - -class MapboxSatelliteMapProvider : public MapboxMapProvider -{ -public: - MapboxSatelliteMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Satellite"), - QStringLiteral("satellite-v9"), - AVERAGE_MAPBOX_SAT_MAP, - MapProvider::SatelliteMapDay) {} -}; - -class MapboxHybridMapProvider : public MapboxMapProvider -{ -public: - MapboxHybridMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Hybrid"), - QStringLiteral("satellite-streets-v10"), - AVERAGE_MAPBOX_SAT_MAP, - MapProvider::HybridMap) {} -}; - -class MapboxBrightMapProvider : public MapboxMapProvider -{ -public: - MapboxBrightMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Bright"), - QStringLiteral("bright-v9"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; - -class MapboxStreetsBasicMapProvider : public MapboxMapProvider -{ -public: - MapboxStreetsBasicMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox StreetsBasic"), - QStringLiteral("basic-v9"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::StreetMap) {} -}; - -class MapboxOutdoorsMapProvider : public MapboxMapProvider -{ -public: - MapboxOutdoorsMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Outdoors"), - QStringLiteral("outdoors-v10"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; - -class MapboxCustomMapProvider : public MapboxMapProvider -{ -public: - MapboxCustomMapProvider() - : MapboxMapProvider( - QStringLiteral("Mapbox Custom"), - QStringLiteral("mapbox.custom"), - QGC_AVERAGE_TILE_SIZE, - MapProvider::CustomMap) {} -}; diff --git a/src/QtLocationPlugin/Providers/TianDiTuProvider.cpp b/src/QtLocationPlugin/Providers/TianDiTuProvider.cpp index b1fe6cbc922b..5559300d0feb 100644 --- a/src/QtLocationPlugin/Providers/TianDiTuProvider.cpp +++ b/src/QtLocationPlugin/Providers/TianDiTuProvider.cpp @@ -2,13 +2,16 @@ #include "SettingsManager.h" #include "AppSettings.h" +#include + QString TianDiTuProvider::_getURL(int x, int y, int zoom) const { //"https://t%1.tianditu.gov.cn/DataServer?tk=%2&T=%3&x=%4&y=%5&l=%6" - const QString tiandituToken = SettingsManager::instance()->appSettings()->tiandituToken()->rawValue().toString(); + const QString tiandituToken = + QString::fromUtf8(QUrl::toPercentEncoding(factString(appSettings()->tiandituToken()))); if (!tiandituToken.isEmpty()) { return _mapUrl - .arg(_getServerNum(x, y, 8)) + .arg(_getServerNum(x, y, kServerCount)) .arg(tiandituToken) .arg(_mapType) .arg(x) diff --git a/src/QtLocationPlugin/Providers/TianDiTuProvider.h b/src/QtLocationPlugin/Providers/TianDiTuProvider.h index 8e1b9abd3668..170b0f2da870 100644 --- a/src/QtLocationPlugin/Providers/TianDiTuProvider.h +++ b/src/QtLocationPlugin/Providers/TianDiTuProvider.h @@ -5,43 +5,21 @@ #include "MapProvider.h" static constexpr const quint32 AVERAGE_TIANDITU_STREET_MAP = 1297; -static constexpr const quint32 AVERAGE_TIANDITU_SAT_MAP = 19597; +static constexpr const quint32 AVERAGE_TIANDITU_SAT_MAP = 19597; class TianDiTuProvider : public MapProvider { -protected: - TianDiTuProvider(const QString &mapName, const QString &mapTypeCode, const QString &imageFormat, quint32 averageSize, - MapProvider::MapStyle mapType) - : MapProvider(mapName, QStringLiteral("https://map.tianditu.gov.cn/"), imageFormat, averageSize, mapType) - , _mapType(mapTypeCode) {} +public: + TianDiTuProvider(const QString& mapName, const QString& mapTypeCode, const QString& imageFormat, + quint32 averageSize, MapProvider::MapStyle mapType) + : MapProvider(mapName, QStringLiteral("https://map.tianditu.gov.cn/"), imageFormat, averageSize, mapType), + _mapType(mapTypeCode) + {} private: QString _getURL(int x, int y, int zoom) const final; const QString _mapType; const QString _mapUrl = QStringLiteral("https://t%1.tianditu.gov.cn/DataServer?tk=%2&T=%3&x=%4&y=%5&l=%6"); -}; - -class TianDiTuRoadProvider : public TianDiTuProvider -{ -public: - TianDiTuRoadProvider() - : TianDiTuProvider( - QObject::tr("TianDiTu Road"), - QStringLiteral("cia_w"), - QStringLiteral("png"), - AVERAGE_TIANDITU_STREET_MAP, - MapProvider::StreetMap) {} -}; - -class TianDiTuSatelliteProvider : public TianDiTuProvider -{ -public: - TianDiTuSatelliteProvider() - : TianDiTuProvider( - QObject::tr("TianDiTu Satellite"), - QStringLiteral("img_w"), - QStringLiteral("jpg"), - AVERAGE_TIANDITU_SAT_MAP, - MapProvider::SatelliteMapDay) {} + static constexpr int kServerCount = 8; }; diff --git a/src/QtLocationPlugin/QGCCacheTile.h b/src/QtLocationPlugin/QGCCacheTile.h index 27783424741e..c92164160ad9 100644 --- a/src/QtLocationPlugin/QGCCacheTile.h +++ b/src/QtLocationPlugin/QGCCacheTile.h @@ -2,27 +2,35 @@ #include #include +#include #include struct QGCCacheTile { - QGCCacheTile(const QString &hash_, const QByteArray &img_, const QString &format_, const QString &type_, quint64 tileSet_ = UINT64_MAX) - : tileSet(tileSet_) - , hash(hash_) - , img(img_) - , format(format_) - , type(type_) - {} - QGCCacheTile(const QString &hash_, quint64 tileSet_) - : tileSet(tileSet_) - , hash(hash_) + QGCCacheTile(const QString& hash_, const QByteArray& img_, const QString& format_, const QString& type_, + quint64 tileSet_ = UINT64_MAX) + : tileSet(tileSet_), hash(hash_), img(img_), format(format_), type(type_) {} + QGCCacheTile(const QString& hash_, quint64 tileSet_) : tileSet(tileSet_), hash(hash_) {} + quint64 tileSet; QString hash; QByteArray img; QString format; QString type; + + // HTTP cache-validation metadata. etag/lastModified drive conditional GETs + // (If-None-Match / If-Modified-Since); expiresAt is a Unix epoch (seconds, + // 0 = unknown) past which the tile must be revalidated before reuse. + QByteArray etag; + QByteArray lastModified; + qint64 expiresAt = 0; + + // Set from response Cache-Control no-cache/must-revalidate. When true (or once + // expiresAt has passed) the cache serves the tile stale to force a conditional GET. + bool mustRevalidate = false; }; Q_DECLARE_METATYPE(QGCCacheTile) Q_DECLARE_METATYPE(QGCCacheTile*) +Q_DECLARE_METATYPE(QSharedPointer) diff --git a/src/QtLocationPlugin/QGCCachedTileSet.cpp b/src/QtLocationPlugin/QGCCachedTileSet.cpp index 8859a6d3bec6..be3d0db27929 100644 --- a/src/QtLocationPlugin/QGCCachedTileSet.cpp +++ b/src/QtLocationPlugin/QGCCachedTileSet.cpp @@ -1,21 +1,55 @@ #include "QGCCachedTileSet.h" +#include +#include +#include + #include "ElevationMapProvider.h" #include "QGCFormat.h" +#include "QGCHostCircuitBreaker.h" #include "QGCLoggingCategory.h" #include "QGCMapEngine.h" #include "QGCMapEngineManager.h" -#include "QGCNetworkHelper.h" #include "QGCMapTasks.h" #include "QGCMapUrlEngine.h" -#include "QGeoFileTileCacheQGC.h" +#include "QGCNetworkHelper.h" +#include "QGCTile.h" +#include "QGCTileCache.h" +#include "QGCTileCacheDatabase.h" +#include "QGCTileCacheWorker.h" #include "QGeoTileFetcherQGC.h" QGC_LOGGING_CATEGORY(QGCCachedTileSetLog, "QtLocationPlugin.QGCCachedTileSet") -QGCCachedTileSet::QGCCachedTileSet(const QString &name, QObject *parent) - : QObject(parent) - , _name(name) +namespace { +// O1: builds the fire-and-forget tile-download-state update as a generic command +// task, keeping the "*" wildcard (all-tiles) vs single-hash branch in one place. +QGCCommandTask* makeUpdateStateTask(quint64 setID, QGCTile::TileState state, const QString& hash, + QGCTile::TileState fromState = QGCTile::StatePending, + bool filterByFromState = false) +{ + return new QGCCommandTask( + QGCMapTask::TaskType::taskUpdateTileDownloadState, + [setID, state, hash, fromState, filterByFromState](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + bool ok; + if (hash == QStringLiteral("*")) { + const int from = filterByFromState ? static_cast(fromState) : -1; + ok = worker.database()->updateAllTileDownloadStates(setID, static_cast(state), from); + } else { + ok = worker.database()->updateTileDownloadState(setID, static_cast(state), hash); + } + if (!ok) { + self.setError("Error updating tile download state"); + } + return true; + }); +} +} // namespace + +QGCCachedTileSet::QGCCachedTileSet(const QString& name, QObject* parent) : QObject(parent), _name(name) { qCDebug(QGCCachedTileSetLog) << this; } @@ -40,7 +74,7 @@ QString QGCCachedTileSet::downloadStatus() const void QGCCachedTileSet::createDownloadTask() { - if (_cancelPending) { + if (_cancelPending || _paused) { setDownloading(false); return; } @@ -49,10 +83,14 @@ void QGCCachedTileSet::createDownloadTask() setErrorCount(0); setDownloading(true); _noMoreTiles = false; + _maxConcurrentDownloads = QGeoTileFetcherQGC::concurrentDownloads(_type); } - QGCGetTileDownloadListTask *task = new QGCGetTileDownloadListTask(_id, kTileBatchSize); - (void) connect(task, &QGCGetTileDownloadListTask::tileListFetched, this, &QGCCachedTileSet::_tileListFetched); + QGCGetTileDownloadListTask* task = new QGCGetTileDownloadListTask(_id, kTileBatchSize); + // Queued: the task signal is emitted from the cache-worker thread; this slot + // must run on the main thread where all download state lives. + (void) connect(task, &QGCGetTileDownloadListTask::tileListFetched, this, &QGCCachedTileSet::_tileListFetched, + Qt::QueuedConnection); if (_manager) { (void) connect(task, &QGCMapTask::error, _manager, &QGCMapEngineManager::taskError); } @@ -69,8 +107,12 @@ void QGCCachedTileSet::createDownloadTask() void QGCCachedTileSet::resumeDownloadTask() { _cancelPending = false; + setPaused(false); - QGCUpdateTileDownloadStateTask *task = new QGCUpdateTileDownloadStateTask(_id, QGCTile::StatePending, "*"); + // Reactivate tiles parked by a pause (Paused -> Pending) without disturbing + // already-completed rows. + QGCCommandTask* task = + makeUpdateStateTask(_id, QGCTile::StatePending, QStringLiteral("*"), QGCTile::StatePaused, true); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); } @@ -78,12 +120,58 @@ void QGCCachedTileSet::resumeDownloadTask() createDownloadTask(); } +void QGCCachedTileSet::pauseDownloadTask() +{ + if (!_downloading && !_paused) { + return; + } + + setPaused(true); + ++_downloadEpoch; + + // Abort in-flight replies without letting their finished/error slots run, so + // they don't mutate state or re-enter _prepareDownload after the pause. Their + // gate slots would otherwise leak (the handlers that release are blocked), so + // release here. + for (const DownloadReply& pending : std::as_const(_replies)) { + pending.reply->blockSignals(true); + pending.reply->abort(); + pending.reply->deleteLater(); + HostConcurrencyGate::instance().release(pending.host); + } + _replies.clear(); + + qDeleteAll(_tilesToDownload); + _tilesToDownload.clear(); + + QGCMapEngine* engine = getQGCMapEngine(); + + // Park both in-flight (Downloading) and queued (Pending) tiles as Paused so + // getTileDownloadList (which only pulls Pending) stops handing them out. + QGCCommandTask* downloadingTask = + makeUpdateStateTask(_id, QGCTile::StatePaused, QStringLiteral("*"), QGCTile::StateDownloading, true); + if (!engine || !engine->addTask(downloadingTask)) { + downloadingTask->deleteLater(); + } + + QGCCommandTask* pendingTask = + makeUpdateStateTask(_id, QGCTile::StatePaused, QStringLiteral("*"), QGCTile::StatePending, true); + if (!engine || !engine->addTask(pendingTask)) { + pendingTask->deleteLater(); + } + + _noMoreTiles = false; + _batchRequested = false; + setDownloading(false); +} + void QGCCachedTileSet::cancelDownloadTask() { _cancelPending = true; + ++_downloadEpoch; } -void QGCCachedTileSet::_tileListFetched(const QQueue &tiles) +void QGCCachedTileSet::_tileListFetched(const QQueue& tiles) { _batchRequested = false; if (tiles.size() < kTileBatchSize) { @@ -95,6 +183,11 @@ void QGCCachedTileSet::_tileListFetched(const QQueue &tiles) return; } + if (_paused || _cancelPending) { + qDeleteAll(tiles); + return; + } + if (!_networkManager) { _networkManager = new QNetworkAccessManager(this); QGCNetworkHelper::configureProxy(_networkManager); @@ -136,12 +229,18 @@ void QGCCachedTileSet::_prepareDownload() return; } - for (qsizetype i = _replies.count(); i < QGeoTileFetcherQGC::concurrentDownloads(_type); i++) { + if (_paused || _cancelPending) { + return; + } + + const qsizetype maxConcurrent = _maxConcurrentDownloads; + + for (qsizetype i = _replies.count(); i < maxConcurrent; i++) { if (_tilesToDownload.isEmpty()) { break; } + QGCTile* tile = _tilesToDownload.dequeue(); - QGCTile* const tile = _tilesToDownload.dequeue(); QNetworkRequest request = QGeoTileFetcherQGC::getNetworkRequest(tile->type, tile->x, tile->y, tile->z); if (!request.url().isValid()) { qCWarning(QGCCachedTileSetLog) << "Invalid URL for tile" << tile->hash << "- skipping"; @@ -152,32 +251,64 @@ void QGCCachedTileSet::_prepareDownload() request.setOriginatingObject(this); request.setAttribute(QNetworkRequest::User, tile->hash); - QNetworkReply* const reply = _networkManager->get(request); - reply->setParent(this); - QGCNetworkHelper::ignoreSslErrorsIfNeeded(reply); - (void) connect(reply, &QNetworkReply::finished, this, &QGCCachedTileSet::_networkReplyFinished); - (void) connect(reply, &QNetworkReply::errorOccurred, this, &QGCCachedTileSet::_networkReplyError); - { - QMutexLocker lock(&_repliesMutex); - (void) _replies.insert(tile->hash, reply); + const QString host = request.url().host(); + + // Circuit breaker (R2): a host the live fetcher has tripped is skipped here + // too, so bulk downloads don't keep hammering a dead endpoint. + if (!HostCircuitBreaker::instance().allowRequest(host)) { + qCDebug(QGCCachedTileSetLog) << "Circuit breaker open; skipping tile" << tile->hash; + setErrorCount(_errorCount + 1); + QGCCommandTask* task = makeUpdateStateTask(_id, QGCTile::StateError, tile->hash); + if (!getQGCMapEngine()->addTask(task)) { + task->deleteLater(); + } + delete tile; + continue; + } + + // Shared cross-path gate (A2): the combined live+bulk in-flight count for + // this host must stay under the provider cap. If the live fetcher already + // holds the host's slots, re-queue this tile (front) and stop filling; a + // gate release (here or in the live path) plus the next completion's + // _prepareDownload drains the backlog. + const SharedMapProvider provider = UrlFactory::getMapProviderFromProviderType(tile->type); + const bool isOSM = provider && provider->isOSMProvider(); + if (!HostConcurrencyGate::instance().tryAcquire(host, static_cast(maxConcurrent), isOSM)) { + _tilesToDownload.prepend(tile); + break; } + _startTileGet(tile->hash, request, 0, host); delete tile; - if (!_batchRequested && !_noMoreTiles && (_tilesToDownload.count() < (QGeoTileFetcherQGC::concurrentDownloads(_type) * 10))) { + + if (!_batchRequested && !_noMoreTiles && (_tilesToDownload.count() < (maxConcurrent * 10))) { createDownloadTask(); } } } -void QGCCachedTileSet::_networkReplyFinished() +void QGCCachedTileSet::_startTileGet(const QString& hash, const QNetworkRequest& request, int attempt, + const QString& host) +{ + QNetworkReply* const reply = _networkManager->get(request); + reply->setParent(this); + QGCNetworkHelper::ignoreSslErrorsIfNeeded(reply); + (void) connect(reply, &QNetworkReply::finished, this, [this, hash, reply]() { _replyFinished(hash, reply); }); + (void) connect(reply, &QNetworkReply::errorOccurred, this, + [this, hash, reply](QNetworkReply::NetworkError error) { _replyError(hash, reply, error); }); + _replies.insert(hash, DownloadReply{reply, request, attempt, host}); +} + +void QGCCachedTileSet::_replyFinished(const QString& hash, QNetworkReply* reply) { - QNetworkReply* const reply = qobject_cast(QObject::sender()); if (!reply) { qCWarning(QGCCachedTileSetLog) << "NULL Reply"; return; } reply->deleteLater(); + // errorOccurred fires before finished and owns the failure/retry path; bail + // here so a retried reply isn't also treated as a (failed) completion. if (reply->error() != QNetworkReply::NoError) { return; } @@ -187,20 +318,18 @@ void QGCCachedTileSet::_networkReplyFinished() return; } - const QString hash = reply->request().attribute(QNetworkRequest::User).toString(); - if (hash.isEmpty()) { - qCWarning(QGCCachedTileSetLog) << "Empty Hash"; - return; + const auto replyIt = _replies.constFind(hash); + const QString host = (replyIt != _replies.constEnd()) ? replyIt->host : reply->request().url().host(); + if (replyIt != _replies.constEnd()) { + (void) _replies.erase(replyIt); + } else { + qCWarning(QGCCachedTileSetLog) << "Reply not in list: " << hash; } - { - QMutexLocker lock(&_repliesMutex); - if (_replies.contains(hash)) { - (void) _replies.remove(hash); - } else { - qCWarning(QGCCachedTileSetLog) << "Reply not in list: " << hash; - } - } + // Release the shared per-host slot claimed in _prepareDownload so a parked + // live/bulk request for this host can proceed. + HostConcurrencyGate::instance().release(host); + HostCircuitBreaker::instance().recordSuccess(host); qCDebug(QGCCachedTileSetLog) << "Tile fetched:" << hash; QByteArray image = reply->readAll(); @@ -216,8 +345,8 @@ void QGCCachedTileSet::_networkReplyFinished() return; } - if (mapProvider->isElevationProvider()) { - const SharedElevationProvider elevationProvider = std::dynamic_pointer_cast(mapProvider); + if (const SharedElevationProvider elevationProvider = + std::dynamic_pointer_cast(mapProvider)) { image = elevationProvider->serialize(image); if (image.isEmpty()) { qCWarning(QGCCachedTileSetLog) << "Failed to Serialize Terrain Tile"; @@ -231,9 +360,9 @@ void QGCCachedTileSet::_networkReplyFinished() return; } - QGeoFileTileCacheQGC::cacheTile(type, hash, image, format, _id); + QGCTileCache::cacheTile(type, hash, image, format, _id); - QGCUpdateTileDownloadStateTask *task = new QGCUpdateTileDownloadStateTask(_id, QGCTile::StateComplete, hash); + QGCCommandTask* task = makeUpdateStateTask(_id, QGCTile::StateComplete, hash); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); } @@ -250,36 +379,36 @@ void QGCCachedTileSet::_networkReplyFinished() _prepareDownload(); } -void QGCCachedTileSet::_networkReplyError(QNetworkReply::NetworkError error) +void QGCCachedTileSet::_replyError(const QString& hash, QNetworkReply* reply, QNetworkReply::NetworkError error) { - QNetworkReply* const reply = qobject_cast(QObject::sender()); if (!reply) { return; } qCDebug(QGCCachedTileSetLog) << "Error fetching tile" << reply->errorString(); - setErrorCount(_errorCount + 1); - - const QString hash = reply->request().attribute(QNetworkRequest::User).toString(); - if (hash.isEmpty()) { - qCWarning(QGCCachedTileSetLog) << "Empty Hash"; - return; + const auto it = _replies.constFind(hash); + const DownloadReply context = (it != _replies.constEnd()) ? *it : DownloadReply{}; + if (it != _replies.constEnd()) { + (void) _replies.remove(hash); + } else { + qCWarning(QGCCachedTileSetLog) << "Reply not in list:" << hash; } - { - QMutexLocker lock(&_repliesMutex); - if (_replies.contains(hash)) { - (void) _replies.remove(hash); - } else { - qCWarning(QGCCachedTileSetLog) << "Reply not in list:" << hash; - } + const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + if (_retryOrFail(hash, context, statusCode, error, reply)) { + // Retry scheduled: the per-host slot stays held across the backoff timer + // and is reused by the re-issued GET (no re-acquire, no double-count). + return; } + // Terminal failure: release the per-host slot claimed in _prepareDownload. + HostConcurrencyGate::instance().release(context.host); + setErrorCount(_errorCount + 1); if (error != QNetworkReply::OperationCanceledError) { qCWarning(QGCCachedTileSetLog) << "Error:" << reply->errorString(); } - QGCUpdateTileDownloadStateTask *task = new QGCUpdateTileDownloadStateTask(_id, QGCTile::StateError, hash); + QGCCommandTask* task = makeUpdateStateTask(_id, QGCTile::StateError, hash); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); } @@ -287,6 +416,44 @@ void QGCCachedTileSet::_networkReplyError(QNetworkReply::NetworkError error) _prepareDownload(); } +bool QGCCachedTileSet::_retryOrFail(const QString& hash, const DownloadReply& context, int statusCode, + QNetworkReply::NetworkError error, const QNetworkReply* reply) +{ + if (!QGCNetworkHelper::isTransientError(error, statusCode)) { + return false; + } + HostCircuitBreaker::instance().recordFailure(context.request.url().host()); + if ((context.attempt >= kMaxRetryAttempts) || _paused || _cancelPending) { + return false; + } + + std::chrono::milliseconds delay = QGCNetworkHelper::retryBackoff(context.attempt); + if (statusCode == QGCNetworkHelper::kHttpTooManyRequests) { + delay = QGCNetworkHelper::retryAfterFromReply(reply).value_or(QGCNetworkHelper::kDefaultRateLimitDelay); + } + + const QNetworkRequest request = context.request; + const int nextAttempt = context.attempt + 1; + const quint64 epoch = _downloadEpoch; + qCDebug(QGCCachedTileSetLog) << "Retrying tile" << hash << "attempt" << nextAttempt << "in" << delay.count() + << "ms"; + + const QString host = context.host; + QTimer::singleShot(delay, this, [this, hash, request, nextAttempt, epoch, host]() { + // A pause/cancel between scheduling and firing bumps _downloadEpoch; bail so + // a resumed session that already re-queued this tile doesn't get a duplicate + // GET. The held per-host slot must be released on the bail path; pause/cancel + // teardown released the rest of the in-flight slots but this retry was not yet + // back in _replies, so it owns its slot here. + if ((epoch != _downloadEpoch) || _paused || _cancelPending || !_networkManager) { + HostConcurrencyGate::instance().release(host); + return; + } + _startTileGet(hash, request, nextAttempt, host); + }); + return true; +} + void QGCCachedTileSet::setSelected(bool sel) { if (sel != _selected) { diff --git a/src/QtLocationPlugin/QGCCachedTileSet.h b/src/QtLocationPlugin/QGCCachedTileSet.h index c8625af0eb80..3f831a976ed6 100644 --- a/src/QtLocationPlugin/QGCCachedTileSet.h +++ b/src/QtLocationPlugin/QGCCachedTileSet.h @@ -2,11 +2,11 @@ #include #include -#include #include #include #include #include +#include struct QGCTile; class QGCMapEngineManager; @@ -17,110 +17,229 @@ class QGCCachedTileSet : public QObject Q_OBJECT Q_MOC_INCLUDE("QGCTile.h") - Q_PROPERTY(QString name READ name NOTIFY nameChanged) - Q_PROPERTY(QString mapTypeStr READ mapTypeStr CONSTANT) - Q_PROPERTY(double topleftLon READ topleftLon CONSTANT) - Q_PROPERTY(double topleftLat READ topleftLat CONSTANT) - Q_PROPERTY(double bottomRightLon READ bottomRightLon CONSTANT) - Q_PROPERTY(double bottomRightLat READ bottomRightLat CONSTANT) - Q_PROPERTY(int minZoom READ minZoom CONSTANT) - Q_PROPERTY(int maxZoom READ maxZoom CONSTANT) - Q_PROPERTY(quint32 totalTileCount READ totalTileCount NOTIFY totalTileCountChanged) - Q_PROPERTY(QString totalTileCountStr READ totalTileCountStr NOTIFY totalTileCountChanged) - Q_PROPERTY(quint64 totalTilesSize READ totalTilesSize NOTIFY totalTilesSizeChanged) - Q_PROPERTY(QString totalTilesSizeStr READ totalTilesSizeStr NOTIFY totalTilesSizeChanged) - Q_PROPERTY(quint32 uniqueTileCount READ uniqueTileCount NOTIFY uniqueTileCountChanged) - Q_PROPERTY(QString uniqueTileCountStr READ uniqueTileCountStr NOTIFY uniqueTileCountChanged) - Q_PROPERTY(quint64 uniqueTileSize READ uniqueTileSize NOTIFY uniqueTileSizeChanged) - Q_PROPERTY(QString uniqueTileSizeStr READ uniqueTileSizeStr NOTIFY uniqueTileSizeChanged) - Q_PROPERTY(quint32 savedTileCount READ savedTileCount NOTIFY savedTileCountChanged) - Q_PROPERTY(QString savedTileCountStr READ savedTileCountStr NOTIFY savedTileCountChanged) - Q_PROPERTY(quint64 savedTileSize READ savedTileSize NOTIFY savedTileSizeChanged) - Q_PROPERTY(QString savedTileSizeStr READ savedTileSizeStr NOTIFY savedTileSizeChanged) - Q_PROPERTY(QString downloadStatus READ downloadStatus NOTIFY savedTileSizeChanged) - Q_PROPERTY(QDateTime creationDate READ creationDate CONSTANT) - Q_PROPERTY(bool complete READ complete NOTIFY completeChanged) - Q_PROPERTY(bool defaultSet READ defaultSet CONSTANT) - Q_PROPERTY(quint64 id READ id CONSTANT) - Q_PROPERTY(bool deleting READ deleting NOTIFY deletingChanged) - Q_PROPERTY(bool downloading READ downloading NOTIFY downloadingChanged) - Q_PROPERTY(quint32 errorCount READ errorCount NOTIFY errorCountChanged) - Q_PROPERTY(QString errorCountStr READ errorCountStr NOTIFY errorCountChanged) - Q_PROPERTY(bool selected READ selected WRITE setSelected NOTIFY selectedChanged) + Q_PROPERTY(QString name READ name NOTIFY nameChanged) + Q_PROPERTY(QString mapTypeStr READ mapTypeStr CONSTANT) + Q_PROPERTY(double topleftLon READ topleftLon CONSTANT) + Q_PROPERTY(double topleftLat READ topleftLat CONSTANT) + Q_PROPERTY(double bottomRightLon READ bottomRightLon CONSTANT) + Q_PROPERTY(double bottomRightLat READ bottomRightLat CONSTANT) + Q_PROPERTY(int minZoom READ minZoom CONSTANT) + Q_PROPERTY(int maxZoom READ maxZoom CONSTANT) + Q_PROPERTY(quint32 totalTileCount READ totalTileCount NOTIFY totalTileCountChanged) + Q_PROPERTY(QString totalTileCountStr READ totalTileCountStr NOTIFY totalTileCountChanged) + Q_PROPERTY(quint64 totalTilesSize READ totalTilesSize NOTIFY totalTilesSizeChanged) + Q_PROPERTY(QString totalTilesSizeStr READ totalTilesSizeStr NOTIFY totalTilesSizeChanged) + Q_PROPERTY(quint32 uniqueTileCount READ uniqueTileCount NOTIFY uniqueTileCountChanged) + Q_PROPERTY(QString uniqueTileCountStr READ uniqueTileCountStr NOTIFY uniqueTileCountChanged) + Q_PROPERTY(quint64 uniqueTileSize READ uniqueTileSize NOTIFY uniqueTileSizeChanged) + Q_PROPERTY(QString uniqueTileSizeStr READ uniqueTileSizeStr NOTIFY uniqueTileSizeChanged) + Q_PROPERTY(quint32 savedTileCount READ savedTileCount NOTIFY savedTileCountChanged) + Q_PROPERTY(QString savedTileCountStr READ savedTileCountStr NOTIFY savedTileCountChanged) + Q_PROPERTY(quint64 savedTileSize READ savedTileSize NOTIFY savedTileSizeChanged) + Q_PROPERTY(QString savedTileSizeStr READ savedTileSizeStr NOTIFY savedTileSizeChanged) + Q_PROPERTY(QString downloadStatus READ downloadStatus NOTIFY savedTileSizeChanged) + Q_PROPERTY(QDateTime creationDate READ creationDate CONSTANT) + Q_PROPERTY(bool complete READ complete NOTIFY completeChanged) + Q_PROPERTY(bool defaultSet READ defaultSet CONSTANT) + Q_PROPERTY(quint64 id READ id CONSTANT) + Q_PROPERTY(bool deleting READ deleting NOTIFY deletingChanged) + Q_PROPERTY(bool downloading READ downloading NOTIFY downloadingChanged) + Q_PROPERTY(bool paused READ paused NOTIFY pausedChanged) + Q_PROPERTY(quint32 errorCount READ errorCount NOTIFY errorCountChanged) + Q_PROPERTY(QString errorCountStr READ errorCountStr NOTIFY errorCountChanged) + Q_PROPERTY(bool selected READ selected WRITE setSelected NOTIFY selectedChanged) public: - explicit QGCCachedTileSet(const QString &name, QObject *parent = nullptr); + explicit QGCCachedTileSet(const QString& name, QObject* parent = nullptr); ~QGCCachedTileSet(); Q_INVOKABLE void createDownloadTask(); Q_INVOKABLE void resumeDownloadTask(); + Q_INVOKABLE void pauseDownloadTask(); Q_INVOKABLE void cancelDownloadTask(); - const QString &name() const { return _name; } - const QString &mapTypeStr() const { return _mapTypeStr; } + const QString& name() const { return _name; } + + const QString& mapTypeStr() const { return _mapTypeStr; } double topleftLat() const { return _topleftLat; } + double topleftLon() const { return _topleftLon; } + double bottomRightLat() const { return _bottomRightLat; } + double bottomRightLon() const { return _bottomRightLon; } quint32 totalTileCount() const { return _totalTileCount; } + QString totalTileCountStr() const; + quint64 totalTilesSize() const { return _totalTileSize; } + QString totalTilesSizeStr() const; + quint32 uniqueTileCount() const { return _uniqueTileCount; } + QString uniqueTileCountStr() const; + quint64 uniqueTileSize() const { return _uniqueTileSize; } + QString uniqueTileSizeStr() const; + quint32 savedTileCount() const { return _savedTileCount; } + QString savedTileCountStr() const; + quint64 savedTileSize() const { return _savedTileSize; } + QString savedTileSizeStr() const; QString downloadStatus() const; + int minZoom() const { return _minZoom; } + int maxZoom() const { return _maxZoom; } - const QDateTime &creationDate() const { return _creationDate; } + + const QDateTime& creationDate() const { return _creationDate; } + quint64 id() const { return _id; } - const QString &type() const { return _type; } + + const QString& type() const { return _type; } + bool complete() const { return (_defaultSet || (_totalTileCount <= _savedTileCount)); } + bool defaultSet() const { return _defaultSet; } + bool deleting() const { return _deleting; } + bool downloading() const { return _downloading; } + + bool paused() const { return _paused; } + quint32 errorCount() const { return _errorCount; } + QString errorCountStr() const; + bool selected() const { return _selected; } - void setManager(QGCMapEngineManager *mgr) { _manager = mgr; } + void setManager(QGCMapEngineManager* mgr) { _manager = mgr; } + void setSelected(bool sel); - void setName(const QString &name) { if (name != _name) { _name = name; emit nameChanged(); } } - void setMapTypeStr(const QString &typeStr) { _mapTypeStr = typeStr; } + void setName(const QString& name) + { + if (name != _name) { + _name = name; + emit nameChanged(); + } + } + + void setMapTypeStr(const QString& typeStr) { _mapTypeStr = typeStr; } + void setTopleftLat(double lat) { _topleftLat = lat; } + void setTopleftLon(double lon) { _topleftLon = lon; } + void setBottomRightLat(double lat) { _bottomRightLat = lat; } + void setBottomRightLon(double lon) { _bottomRightLon = lon; } - void setUniqueTileCount(quint32 num) { if (num != _uniqueTileCount) { _uniqueTileCount = num; emit uniqueTileCountChanged(); } } - void setTotalTileCount(quint32 num) { if (num != _totalTileCount) { _totalTileCount = num; emit totalTileCountChanged(); } } - void setSavedTileCount(quint32 num) { if (num != _savedTileCount) { _savedTileCount = num; emit savedTileCountChanged(); } } - void setUniqueTileSize(quint64 size) { if (size != _uniqueTileSize) { _uniqueTileSize = size; emit uniqueTileSizeChanged(); } } - void setTotalTileSize(quint64 size) { if (size != _totalTileSize) { _totalTileSize = size; emit totalTilesSizeChanged(); } } - void setSavedTileSize(quint64 size) { if (size != _savedTileSize) { _savedTileSize = size; emit savedTileSizeChanged(); } } + void setUniqueTileCount(quint32 num) + { + if (num != _uniqueTileCount) { + _uniqueTileCount = num; + emit uniqueTileCountChanged(); + } + } + + void setTotalTileCount(quint32 num) + { + if (num != _totalTileCount) { + _totalTileCount = num; + emit totalTileCountChanged(); + } + } + + void setSavedTileCount(quint32 num) + { + if (num != _savedTileCount) { + _savedTileCount = num; + emit savedTileCountChanged(); + } + } + + void setUniqueTileSize(quint64 size) + { + if (size != _uniqueTileSize) { + _uniqueTileSize = size; + emit uniqueTileSizeChanged(); + } + } + + void setTotalTileSize(quint64 size) + { + if (size != _totalTileSize) { + _totalTileSize = size; + emit totalTilesSizeChanged(); + } + } + + void setSavedTileSize(quint64 size) + { + if (size != _savedTileSize) { + _savedTileSize = size; + emit savedTileSizeChanged(); + } + } void setMinZoom(int zoom) { _minZoom = zoom; } + void setMaxZoom(int zoom) { _maxZoom = zoom; } - void setCreationDate(const QDateTime &date) { _creationDate = date; } + + void setCreationDate(const QDateTime& date) { _creationDate = date; } + void setId(quint64 id) { _id = id; } - void setType(const QString &type) { _type = type; } + + void setType(const QString& type) { _type = type; } + void setDefaultSet(bool def) { _defaultSet = def; } - void setDeleting(bool del) { if (del != _deleting) { _deleting = del; emit deletingChanged(); } } - void setDownloading(bool down) { if (down != _downloading) { _downloading = down; emit downloadingChanged(); } } - void setErrorCount(quint32 count) { if (count != _errorCount) { _errorCount = count; emit errorCountChanged(); } } + + void setDeleting(bool del) + { + if (del != _deleting) { + _deleting = del; + emit deletingChanged(); + } + } + + void setDownloading(bool down) + { + if (down != _downloading) { + _downloading = down; + emit downloadingChanged(); + } + } + + void setPaused(bool paused) + { + if (paused != _paused) { + _paused = paused; + emit pausedChanged(); + } + } + + void setErrorCount(quint32 count) + { + if (count != _errorCount) { + _errorCount = count; + emit errorCountChanged(); + } + } signals: void deletingChanged(); void downloadingChanged(); + void pausedChanged(); void totalTileCountChanged(); void uniqueTileCountChanged(); void uniqueTileSizeChanged(); @@ -133,13 +252,28 @@ class QGCCachedTileSet : public QObject void nameChanged(); private slots: - void _tileListFetched(const QQueue &tiles); - void _networkReplyFinished(); - void _networkReplyError(QNetworkReply::NetworkError error); + void _tileListFetched(const QQueue& tiles); private: + // In-flight bulk GET. Retains the request + attempt count so a transient failure + // can be retried with backoff (the tile's x/y/z aren't recoverable from the hash). + struct DownloadReply + { + QNetworkReply* reply = nullptr; + QNetworkRequest request; + int attempt = 0; + // Host whose shared HostConcurrencyGate slot this download holds; released + // when the download terminates (success, terminal error, pause, retry bail). + QString host; + }; + void _prepareDownload(); void _doneWithDownload(); + void _startTileGet(const QString& hash, const QNetworkRequest& request, int attempt, const QString& host); + void _replyFinished(const QString& hash, QNetworkReply* reply); + void _replyError(const QString& hash, QNetworkReply* reply, QNetworkReply::NetworkError error); + bool _retryOrFail(const QString& hash, const DownloadReply& context, int statusCode, + QNetworkReply::NetworkError error, const QNetworkReply* reply); QString _name; QString _mapTypeStr; @@ -161,17 +295,29 @@ private slots: bool _defaultSet = false; bool _deleting = false; bool _downloading = false; + bool _paused = false; bool _noMoreTiles = false; bool _batchRequested = false; bool _selected = false; bool _cancelPending = false; + // Bumped on pause/cancel so a retry timer scheduled in a prior download session + // can't fire a duplicate GET after resume re-fetches the same tile. + quint64 _downloadEpoch = 0; QDateTime _creationDate; - QHash _replies; - QMutex _repliesMutex; + // All download state below is touched only on the main thread: task-result + // signals (tileListFetched) are delivered via a queued connection and the + // QNAM replies are parented to this (main-thread) object, so no locking is + // needed. + QHash _replies; QQueue _tilesToDownload; - QGCMapEngineManager *_manager = nullptr; - QNetworkAccessManager *_networkManager = nullptr; + QGCMapEngineManager* _manager = nullptr; + QNetworkAccessManager* _networkManager = nullptr; + + // Constant for a download session; recomputed when a fresh download starts so + // a per-tile-completion call into QNetworkInformation is avoided. + qsizetype _maxConcurrentDownloads = 0; static constexpr uint32_t kTileBatchSize = 256; + static constexpr int kMaxRetryAttempts = 3; }; diff --git a/src/QtLocationPlugin/QGCHostCircuitBreaker.h b/src/QtLocationPlugin/QGCHostCircuitBreaker.h new file mode 100644 index 000000000000..78925ed43bec --- /dev/null +++ b/src/QtLocationPlugin/QGCHostCircuitBreaker.h @@ -0,0 +1,235 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "QGCLoggingCategory.h" + +Q_DECLARE_LOGGING_CATEGORY(QGeoTiledMapReplyQGCLog) + +// --------------------------------------------------------------------------- +// Per-host circuit breaker (R2) +// +// Process-wide and mutex-guarded: QGeoTiledMapReplyQGC instances can be created +// on different threads, so the breaker state is a static singleton rather than +// per-reply. Keyed by URL host. After kFailureThreshold consecutive transient +// failures within kFailureWindow, the breaker OPENs for kOpenDuration and new +// requests to that host fast-fail. After kOpenDuration one HALF-OPEN probe is +// allowed through: success CLOSEs and resets, failure re-OPENs for another +// kOpenDuration. Any 2xx success CLOSEs the breaker and clears the counter. +// --------------------------------------------------------------------------- +class HostCircuitBreaker +{ +public: + static HostCircuitBreaker& instance() + { + static HostCircuitBreaker s_instance; + return s_instance; + } + + // Returns true if a request to host may proceed (CLOSED, or HALF-OPEN probe + // slot just claimed). Returns false when the breaker is OPEN and not yet due + // for a probe. + bool allowRequest(const QString& host) + { + if (host.isEmpty()) { + return true; + } + const qint64 now = QDateTime::currentMSecsSinceEpoch(); + QMutexLocker lock(&_mutex); + // Read path: never default-insert. A host with no entry is CLOSED, so + // healthy hosts don't accumulate map entries that recordSuccess must prune. + auto it = _hosts.find(host); + if (it == _hosts.end()) { + return true; // Closed + } + HostState& st = *it; + + if (st.state == State::Open) { + if (now >= st.openUntilMs) { + // Cooldown elapsed: allow exactly one half-open probe. + st.state = State::HalfOpen; + st.probeInFlight = true; + qCDebug(QGeoTiledMapReplyQGCLog) << "Circuit breaker HALF-OPEN (probe) for host" << host; + return true; + } + return false; + } + + if (st.state == State::HalfOpen) { + // Only one probe at a time while half-open. + return !st.probeInFlight ? (st.probeInFlight = true) : false; + } + + return true; // Closed + } + + void recordSuccess(const QString& host) + { + if (host.isEmpty()) { + return; + } + QMutexLocker lock(&_mutex); + auto it = _hosts.find(host); + if (it == _hosts.end()) { + return; + } + if (it->state != State::Closed || it->failures != 0) { + qCDebug(QGeoTiledMapReplyQGCLog) << "Circuit breaker CLOSED (success) for host" << host; + } + _hosts.erase(it); + } + + void recordFailure(const QString& host) + { + if (host.isEmpty()) { + return; + } + const qint64 now = QDateTime::currentMSecsSinceEpoch(); + QMutexLocker lock(&_mutex); + // operator[] must default-insert to count consecutive failures, so a host + // that fails once and is never retried would leak an entry — bound the map. + if (_hosts.size() >= kPruneThreshold) { + _pruneStale(now); + } + HostState& st = _hosts[host]; + + if (st.state == State::HalfOpen) { + // Probe failed: re-open for another cooldown. + st.state = State::Open; + st.openUntilMs = now + kOpenDuration.count(); + st.probeInFlight = false; + qCDebug(QGeoTiledMapReplyQGCLog) << "Circuit breaker re-OPEN (probe failed) for host" << host; + return; + } + + // Reset the consecutive counter if the failure window has lapsed. + if ((st.firstFailureMs == 0) || ((now - st.firstFailureMs) > kFailureWindow.count())) { + st.firstFailureMs = now; + st.failures = 0; + } + ++st.failures; + + if (st.failures >= kFailureThreshold) { + st.state = State::Open; + st.openUntilMs = now + kOpenDuration.count(); + qCDebug(QGeoTiledMapReplyQGCLog) + << "Circuit breaker OPEN for host" << host << "after" << st.failures << "failures"; + } + } + +private: + enum class State + { + Closed, + Open, + HalfOpen + }; + + struct HostState + { + State state = State::Closed; + int failures = 0; + qint64 firstFailureMs = 0; + qint64 openUntilMs = 0; + bool probeInFlight = false; + }; + + // Erase Closed entries whose failure window has lapsed; Open/HalfOpen entries + // are load-bearing and kept. Caller must hold _mutex. + void _pruneStale(qint64 now) + { + for (auto it = _hosts.begin(); it != _hosts.end();) { + const bool lapsed = (it->firstFailureMs == 0) || ((now - it->firstFailureMs) > kFailureWindow.count()); + it = (it->state == State::Closed && lapsed) ? _hosts.erase(it) : std::next(it); + } + } + + static constexpr int kFailureThreshold = 5; + static constexpr int kPruneThreshold = 64; + static constexpr std::chrono::milliseconds kFailureWindow = std::chrono::seconds(30); + static constexpr std::chrono::milliseconds kOpenDuration = std::chrono::seconds(60); + + QMutex _mutex; + QHash _hosts; +}; + +// --------------------------------------------------------------------------- +// Per-host concurrency gate (A2) +// +// The live-tile fetcher (QtLocation engine thread) and the bulk downloader +// (QGCCachedTileSet, main thread) own separate QNetworkAccessManagers and so +// cannot share one socket pool safely. Left uncoordinated each runs its own +// budget, letting up to 2N simultaneous requests reach a single provider host +// and doubling the rate that host sees. This process-wide, mutex-guarded +// singleton enforces a SINGLE combined in-flight cap per host across both +// paths: a slot is claimed before a GET is issued and released when it +// finishes. OSM hosts are clamped to kOSMHostLimit so the tile-usage-policy +// guarantee holds no matter which path (or how many) issues the request. +// --------------------------------------------------------------------------- +class HostConcurrencyGate +{ +public: + static HostConcurrencyGate& instance() + { + static HostConcurrencyGate s_instance; + return s_instance; + } + + // Claim an in-flight slot for host if under its cap. limit is the caller's + // transport-scaled budget; the effective cap is min(limit, kOSMHostLimit) + // for OSM so a larger live budget can never exceed the OSM policy. An empty + // host (unit tests with no real URL) is never gated. + bool tryAcquire(const QString& host, int limit, bool isOSM) + { + if (host.isEmpty()) { + return true; + } + const int cap = isOSM ? qMin(limit, kOSMHostLimit) : limit; + QMutexLocker lock(&_mutex); + int& active = _active[host]; + if (active >= cap) { + return false; + } + ++active; + return true; + } + + void release(const QString& host) + { + if (host.isEmpty()) { + return; + } + QMutexLocker lock(&_mutex); + const auto it = _active.find(host); + if (it == _active.end()) { + return; + } + if (--(*it) <= 0) { + _active.erase(it); + } + } + +#ifdef QGC_UNITTEST_BUILD + int activeForTest(const QString& host) + { + QMutexLocker lock(&_mutex); + return _active.value(host, 0); + } + + void resetForTest() + { + QMutexLocker lock(&_mutex); + _active.clear(); + } +#endif + + static constexpr int kOSMHostLimit = 2; + +private: + QMutex _mutex; + QHash _active; +}; diff --git a/src/QtLocationPlugin/QGCMapEngine.cpp b/src/QtLocationPlugin/QGCMapEngine.cpp index 414c617565f6..6104a3a31782 100644 --- a/src/QtLocationPlugin/QGCMapEngine.cpp +++ b/src/QtLocationPlugin/QGCMapEngine.cpp @@ -1,27 +1,28 @@ #include "QGCMapEngine.h" #include +#include -#include "QGCCachedTileSet.h" #include "QGCCacheTile.h" +#include "QGCCachedTileSet.h" #include "QGCLoggingCategory.h" #include "QGCMapTasks.h" #include "QGCTile.h" +#include "QGCTileCache.h" +#include "QGCTileCacheDatabase.h" #include "QGCTileCacheWorker.h" #include "QGCTileSet.h" -#include "QGeoFileTileCacheQGC.h" QGC_LOGGING_CATEGORY(QGCMapEngineLog, "QtLocationPlugin.QGCMapEngine") Q_APPLICATION_STATIC(QGCMapEngine, _mapEngine); -QGCMapEngine *getQGCMapEngine() +QGCMapEngine* getQGCMapEngine() { return QGCMapEngine::instance(); } -QGCMapEngine::QGCMapEngine(QObject *parent) - : QObject(parent) +QGCMapEngine::QGCMapEngine(QObject* parent) : QObject(parent) { qCDebug(QGCMapEngineLog) << this; @@ -33,25 +34,27 @@ QGCMapEngine::QGCMapEngine(QObject *parent) (void) qRegisterMetaType("QGCTileSet*"); (void) qRegisterMetaType("QGCCacheTile"); (void) qRegisterMetaType("QGCCacheTile*"); + (void) qRegisterMetaType>("QSharedPointer"); } QGCMapEngine::~QGCMapEngine() { - if (m_initialized && m_worker) { + if (m_worker) { (void) disconnect(m_worker); - m_worker->stop(); - (void) m_worker->wait(); + // ~QGCCacheWorker() runs stop() + _thread.wait(). + delete m_worker; + m_worker = nullptr; } qCDebug(QGCMapEngineLog) << this; } -QGCMapEngine *QGCMapEngine::instance() +QGCMapEngine* QGCMapEngine::instance() { return _mapEngine(); } -void QGCMapEngine::init(const QString &databasePath) +void QGCMapEngine::init(const QString& databasePath) { if (m_initialized) { return; @@ -59,24 +62,24 @@ void QGCMapEngine::init(const QString &databasePath) m_initialized = true; - m_worker = new QGCCacheWorker(this); + m_worker = new QGCCacheWorker(); m_worker->setDatabaseFile(databasePath); (void) connect(m_worker, &QGCCacheWorker::updateTotals, this, &QGCMapEngine::_updateTotals); - QGCMapTask *task = new QGCMapTask(QGCMapTask::TaskType::taskInit); + QGCMapTask* task = new QGCMapTask(QGCMapTask::TaskType::taskInit); if (!addTask(task)) { task->deleteLater(); m_initialized = false; } } -bool QGCMapEngine::addTask(QGCMapTask *task) +bool QGCMapEngine::addTask(QGCMapTask* task) { - // DirectConnection is intentional: the worker thread uses a custom loop (not - // an event loop), so queued connections would never be delivered. The queue - // is mutex-protected in enqueueTask, so calling from the main thread is safe. + // DirectConnection: enqueueTask must run synchronously to return its result and + // start the worker; its queue is mutex-protected, so main-thread calls are safe. bool result = false; - (void) QMetaObject::invokeMethod(m_worker, &QGCCacheWorker::enqueueTask, Qt::DirectConnection, qReturnArg(result), task); + (void) QMetaObject::invokeMethod(m_worker, &QGCCacheWorker::enqueueTask, Qt::DirectConnection, qReturnArg(result), + task); return result; } @@ -84,13 +87,23 @@ void QGCMapEngine::_updateTotals(quint32 totaltiles, quint64 totalsize, quint32 { emit updateTotals(totaltiles, totalsize, defaulttiles, defaultsize); - const quint64 maxSize = static_cast(QGeoFileTileCacheQGC::getMaxDiskCacheSetting()) * qPow(1024, 2); + const quint64 maxSize = static_cast(QGCTileCache::getMaxDiskCacheSetting()) * qPow(1024, 2); if (!m_pruning && (defaultsize > maxSize)) { m_pruning = true; const quint64 amountToPrune = defaultsize - maxSize; - QGCPruneCacheTask *task = new QGCPruneCacheTask(amountToPrune); - (void) connect(task, &QGCPruneCacheTask::pruned, this, &QGCMapEngine::_pruned); + QGCCommandTask* task = new QGCCommandTask(QGCMapTask::TaskType::taskPruneCache, + [amountToPrune](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + if (!worker.database()->pruneCache(amountToPrune)) { + self.setError("Error pruning cache"); + return false; + } + return true; + }); + (void) connect(task, &QGCCommandTask::completed, this, &QGCMapEngine::_pruned); if (!addTask(task)) { task->deleteLater(); m_pruning = false; diff --git a/src/QtLocationPlugin/QGCMapEngineManager.cc b/src/QtLocationPlugin/QGCMapEngineManager.cc index 8dae3b153f62..569bfc339c69 100644 --- a/src/QtLocationPlugin/QGCMapEngineManager.cc +++ b/src/QtLocationPlugin/QGCMapEngineManager.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -12,13 +11,15 @@ #include "ElevationMapProvider.h" #include "FlightMapSettings.h" #include "QGCCachedTileSet.h" -#include "QGCFormat.h" #include "QGCCompression.h" #include "QGCCompressionJob.h" +#include "QGCFormat.h" #include "QGCLoggingCategory.h" #include "QGCMapEngine.h" #include "QGCMapTasks.h" #include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" +#include "QGCTileCacheWorker.h" #include "QGeoFileTileCacheQGC.h" #include "QmlObjectListModel.h" #include "SettingsManager.h" @@ -29,20 +30,26 @@ QGC_LOGGING_CATEGORY(QGCMapEngineManagerLog, "QtLocationPlugin.QGCMapEngineManag Q_APPLICATION_STATIC(QGCMapEngineManager, _mapEngineManager); -QGCMapEngineManager *QGCMapEngineManager::instance() +QGCMapEngineManager* QGCMapEngineManager::instance() { return _mapEngineManager(); } -QGCMapEngineManager::QGCMapEngineManager(QObject *parent) - : QObject(parent) - , _tileSets(new QmlObjectListModel(this)) +void QGCMapEngineManager::registerQmlTypes() { - qCDebug(QGCMapEngineManagerLog) << this; + // Imperative registration (QML_ELEMENT needs a qt_add_qml_module; this is a geo + // service plugin). Must run after QCoreApplication exists and before QML type + // resolution; QGCApplication calls this from _initQmlRootWindow(). + (void) qmlRegisterUncreatableType("QGroundControl.QGCMapEngineManager", 1, 0, + "QGCMapEngineManager", "Reference only"); +} - (void) qmlRegisterUncreatableType("QGroundControl.QGCMapEngineManager", 1, 0, "QGCMapEngineManager", "Reference only"); +QGCMapEngineManager::QGCMapEngineManager(QObject* parent) : QObject(parent), _tileSets(new QmlObjectListModel(this)) +{ + qCDebug(QGCMapEngineManagerLog) << this; - (void) connect(getQGCMapEngine(), &QGCMapEngine::updateTotals, this, &QGCMapEngineManager::_updateTotals, Qt::UniqueConnection); + (void) connect(getQGCMapEngine(), &QGCMapEngine::updateTotals, this, &QGCMapEngineManager::_updateTotals, + Qt::UniqueConnection); } QGCMapEngineManager::~QGCMapEngineManager() @@ -52,7 +59,8 @@ QGCMapEngineManager::~QGCMapEngineManager() qCDebug(QGCMapEngineManagerLog) << this; } -void QGCMapEngineManager::updateForCurrentView(double lon0, double lat0, double lon1, double lat1, int minZoom, int maxZoom, const QString &mapName) +void QGCMapEngineManager::updateForCurrentView(double lon0, double lat0, double lon1, double lat1, int minZoom, + int maxZoom, const QString& mapName) { _topleftLat = lat0; _topleftLon = lon0; @@ -70,7 +78,8 @@ void QGCMapEngineManager::updateForCurrentView(double lon0, double lat0, double } if (_fetchElevation) { - const QString elevationProviderName = SettingsManager::instance()->flightMapSettings()->elevationMapProvider()->rawValue().toString(); + const QString elevationProviderName = + SettingsManager::instance()->flightMapSettings()->elevationMapProvider()->rawValue().toString(); const QGCTileSet set = UrlFactory::getTileCount(1, lon0, lat0, lon1, lat1, elevationProviderName); _elevationSet += set; } @@ -98,7 +107,7 @@ void QGCMapEngineManager::loadTileSets() emit tileSetsChanged(); } - QGCFetchTileSetTask *task = new QGCFetchTileSetTask(); + QGCFetchTileSetTask* task = new QGCFetchTileSetTask(); (void) connect(task, &QGCFetchTileSetTask::tileSetFetched, this, &QGCMapEngineManager::_tileSetFetched); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { @@ -106,7 +115,7 @@ void QGCMapEngineManager::loadTileSets() } } -void QGCMapEngineManager::_tileSetFetched(QGCCachedTileSet *tileSet) +void QGCMapEngineManager::_tileSetFetched(QGCCachedTileSet* tileSet) { if (tileSet->type() == QStringLiteral("Invalid")) { tileSet->setMapTypeStr(QStringLiteral("Various")); @@ -117,7 +126,7 @@ void QGCMapEngineManager::_tileSetFetched(QGCCachedTileSet *tileSet) emit tileSetsChanged(); } -void QGCMapEngineManager::startDownload(const QString &name, const QString &mapType) +void QGCMapEngineManager::startDownload(const QString& name, const QString& mapType) { if (_imageSet.tileSize > 0) { QGCCachedTileSet* const set = new QGCCachedTileSet(name); @@ -132,7 +141,7 @@ void QGCMapEngineManager::startDownload(const QString &name, const QString &mapT set->setTotalTileCount(static_cast(_imageSet.tileCount)); set->setType(mapType); - QGCCreateTileSetTask *task = new QGCCreateTileSetTask(set); + QGCCreateTileSetTask* task = new QGCCreateTileSetTask(set); (void) connect(task, &QGCCreateTileSetTask::tileSetSaved, this, &QGCMapEngineManager::_tileSetSaved); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { @@ -145,7 +154,8 @@ void QGCMapEngineManager::startDownload(const QString &name, const QString &mapT const int mapid = UrlFactory::getQtMapIdFromProviderType(mapType); if (_fetchElevation && !UrlFactory::isElevation(mapid)) { QGCCachedTileSet* const set = new QGCCachedTileSet(name + QStringLiteral(" Elevation")); - const QString elevationProviderName = SettingsManager::instance()->flightMapSettings()->elevationMapProvider()->rawValue().toString(); + const QString elevationProviderName = + SettingsManager::instance()->flightMapSettings()->elevationMapProvider()->rawValue().toString(); set->setMapTypeStr(elevationProviderName); set->setTopleftLat(_topleftLat); set->setTopleftLon(_topleftLon); @@ -157,7 +167,7 @@ void QGCMapEngineManager::startDownload(const QString &name, const QString &mapT set->setTotalTileCount(static_cast(_elevationSet.tileCount)); set->setType(elevationProviderName); - QGCCreateTileSetTask *task = new QGCCreateTileSetTask(set); + QGCCreateTileSetTask* task = new QGCCreateTileSetTask(set); (void) connect(task, &QGCCreateTileSetTask::tileSetSaved, this, &QGCMapEngineManager::_tileSetSaved); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { @@ -168,7 +178,7 @@ void QGCMapEngineManager::startDownload(const QString &name, const QString &mapT } } -void QGCMapEngineManager::_tileSetSaved(QGCCachedTileSet *set) +void QGCMapEngineManager::_tileSetSaved(QGCCachedTileSet* set) { qCDebug(QGCMapEngineManagerLog) << "New tile set saved (" << set->name() << "). Starting download..."; @@ -177,21 +187,7 @@ void QGCMapEngineManager::_tileSetSaved(QGCCachedTileSet *set) set->createDownloadTask(); } -void QGCMapEngineManager::saveSetting(const QString &key, const QString &value) -{ - QSettings settings; - settings.beginGroup(kQmlOfflineMapKeyName); - settings.setValue(key, value); -} - -QString QGCMapEngineManager::loadSetting(const QString &key, const QString &defaultValue) -{ - QSettings settings; - settings.beginGroup(kQmlOfflineMapKeyName); - return settings.value(key, defaultValue).toString(); -} - -QStringList QGCMapEngineManager::mapTypeList(const QString &provider) +QStringList QGCMapEngineManager::mapTypeList(const QString& provider) { QStringList mapStringList = mapList(); mapStringList = mapStringList.filter(QRegularExpression(provider)); @@ -203,20 +199,31 @@ QStringList QGCMapEngineManager::mapTypeList(const QString &provider) return mapStringList; } -void QGCMapEngineManager::deleteTileSet(QGCCachedTileSet *tileSet) +void QGCMapEngineManager::deleteTileSet(QGCCachedTileSet* tileSet) { qCDebug(QGCMapEngineManagerLog) << "Deleting tile set" << tileSet->name(); if (tileSet->defaultSet()) { - for (qsizetype i = 0; i < _tileSets->count(); i++ ) { + for (qsizetype i = 0; i < _tileSets->count(); i++) { QGCCachedTileSet* const set = qobject_cast(_tileSets->get(i)); if (set) { set->setDeleting(true); } } - QGCResetTask *task = new QGCResetTask(); - (void) connect(task, &QGCResetTask::resetCompleted, this, &QGCMapEngineManager::_resetCompleted); + QGCCommandTask* task = + new QGCCommandTask(QGCMapTask::TaskType::taskReset, [](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + if (!worker.database()->resetDatabase()) { + self.setError("Error resetting cache database"); + return false; + } + worker.setDatabaseValid(worker.database()->isValid()); + return true; + }); + (void) connect(task, &QGCCommandTask::completed, this, &QGCMapEngineManager::_resetCompleted); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); @@ -224,8 +231,20 @@ void QGCMapEngineManager::deleteTileSet(QGCCachedTileSet *tileSet) } else { tileSet->setDeleting(true); - QGCDeleteTileSetTask *task = new QGCDeleteTileSetTask(tileSet->id()); - (void) connect(task, &QGCDeleteTileSetTask::tileSetDeleted, this, &QGCMapEngineManager::_tileSetDeleted); + const quint64 setID = tileSet->id(); + QGCCommandTask* task = new QGCCommandTask(QGCMapTask::TaskType::taskDeleteTileSet, + [setID](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + if (!worker.database()->deleteTileSet(setID)) { + self.setError("Error deleting tile set"); + return false; + } + worker.emitTotals(); + return true; + }); + (void) connect(task, &QGCCommandTask::completed, this, [this, setID]() { _tileSetDeleted(setID); }); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); @@ -233,7 +252,7 @@ void QGCMapEngineManager::deleteTileSet(QGCCachedTileSet *tileSet) } } -void QGCMapEngineManager::renameTileSet(QGCCachedTileSet *tileSet, const QString &newName) +void QGCMapEngineManager::renameTileSet(QGCCachedTileSet* tileSet, const QString& newName) { int idx = 1; QString name = newName; @@ -245,7 +264,18 @@ void QGCMapEngineManager::renameTileSet(QGCCachedTileSet *tileSet, const QString tileSet->setName(name); emit tileSet->nameChanged(); - QGCRenameTileSetTask *task = new QGCRenameTileSetTask(tileSet->id(), name); + const quint64 renameSetID = tileSet->id(); + QGCCommandTask* task = new QGCCommandTask(QGCMapTask::TaskType::taskRenameTileSet, + [renameSetID, name](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + if (!worker.database()->renameTileSet(renameSetID, name)) { + self.setError("Error renaming tile set"); + return false; + } + return true; + }); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); @@ -254,8 +284,8 @@ void QGCMapEngineManager::renameTileSet(QGCCachedTileSet *tileSet, const QString void QGCMapEngineManager::_tileSetDeleted(quint64 setID) { - for (qsizetype i = 0; i < _tileSets->count(); i++ ) { - QGCCachedTileSet *set = qobject_cast(_tileSets->get(i)); + for (qsizetype i = 0; i < _tileSets->count(); i++) { + QGCCachedTileSet* set = qobject_cast(_tileSets->get(i)); if (set && (set->id() == setID)) { (void) _tileSets->removeAt(i); delete set; @@ -265,34 +295,34 @@ void QGCMapEngineManager::_tileSetDeleted(quint64 setID) } } -void QGCMapEngineManager::taskError(QGCMapTask::TaskType type, const QString &error) +void QGCMapEngineManager::taskError(QGCMapTask::TaskType type, const QString& error) { QString task; switch (type) { - case QGCMapTask::TaskType::taskFetchTileSets: - task = QStringLiteral("Fetch Tile Set"); - break; - case QGCMapTask::TaskType::taskCreateTileSet: - task = QStringLiteral("Create Tile Set"); - break; - case QGCMapTask::TaskType::taskGetTileDownloadList: - task = QStringLiteral("Get Tile Download List"); - break; - case QGCMapTask::TaskType::taskUpdateTileDownloadState: - task = QStringLiteral("Update Tile Download Status"); - break; - case QGCMapTask::TaskType::taskDeleteTileSet: - task = QStringLiteral("Delete Tile Set"); - break; - case QGCMapTask::TaskType::taskReset: - task = QStringLiteral("Reset Tile Sets"); - break; - case QGCMapTask::TaskType::taskExport: - task = QStringLiteral("Export Tile Sets"); - break; - default: - task = QStringLiteral("Database Error"); - break; + case QGCMapTask::TaskType::taskFetchTileSets: + task = QStringLiteral("Fetch Tile Set"); + break; + case QGCMapTask::TaskType::taskCreateTileSet: + task = QStringLiteral("Create Tile Set"); + break; + case QGCMapTask::TaskType::taskGetTileDownloadList: + task = QStringLiteral("Get Tile Download List"); + break; + case QGCMapTask::TaskType::taskUpdateTileDownloadState: + task = QStringLiteral("Update Tile Download Status"); + break; + case QGCMapTask::TaskType::taskDeleteTileSet: + task = QStringLiteral("Delete Tile Set"); + break; + case QGCMapTask::TaskType::taskReset: + task = QStringLiteral("Reset Tile Sets"); + break; + case QGCMapTask::TaskType::taskExport: + task = QStringLiteral("Export Tile Sets"); + break; + default: + task = QStringLiteral("Database Error"); + break; } QString serror = QStringLiteral("Error in task: ") + task; @@ -304,7 +334,8 @@ void QGCMapEngineManager::taskError(QGCMapTask::TaskType type, const QString &er qCWarning(QGCMapEngineManagerLog) << serror; } -void QGCMapEngineManager::_updateTotals(quint32 totaltiles, quint64 totalsize, quint32 defaulttiles, quint64 defaultsize) +void QGCMapEngineManager::_updateTotals(quint32 totaltiles, quint64 totalsize, quint32 defaulttiles, + quint64 defaultsize) { for (qsizetype i = 0; i < _tileSets->count(); i++) { QGCCachedTileSet* const set = qobject_cast(_tileSets->get(i)); @@ -318,7 +349,7 @@ void QGCMapEngineManager::_updateTotals(quint32 totaltiles, quint64 totalsize, q } } -bool QGCMapEngineManager::findName(const QString &name) const +bool QGCMapEngineManager::findName(const QString& name) const { for (qsizetype i = 0; i < _tileSets->count(); i++) { const QGCCachedTileSet* const set = qobject_cast(_tileSets->get(i)); @@ -364,7 +395,7 @@ int QGCMapEngineManager::selectedCount() const return count; } -bool QGCMapEngineManager::importSets(const QString &path) +bool QGCMapEngineManager::importSets(const QString& path) { setImportAction(ImportAction::ActionNone); @@ -374,7 +405,7 @@ bool QGCMapEngineManager::importSets(const QString &path) setImportAction(ImportAction::ActionImporting); - QGCImportTileTask *task = new QGCImportTileTask(path, _importReplace); + QGCImportTileTask* task = new QGCImportTileTask(path, _importReplace); (void) connect(task, &QGCImportTileTask::actionCompleted, this, &QGCMapEngineManager::_actionCompleted); (void) connect(task, &QGCImportTileTask::actionProgress, this, &QGCMapEngineManager::_actionProgressHandler); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); @@ -386,7 +417,7 @@ bool QGCMapEngineManager::importSets(const QString &path) return true; } -bool QGCMapEngineManager::exportSets(const QString &path) +bool QGCMapEngineManager::exportSets(const QString& path) { setImportAction(ImportAction::ActionNone); @@ -409,7 +440,6 @@ bool QGCMapEngineManager::exportSets(const QString &path) rec.bottomRightLon = set->bottomRightLon(); rec.minZoom = set->minZoom(); rec.maxZoom = set->maxZoom(); - rec.type = UrlFactory::getQtMapIdFromProviderType(set->type()); rec.numTiles = set->totalTileCount(); rec.defaultSet = set->defaultSet(); rec.date = set->creationDate().toSecsSinceEpoch(); @@ -423,7 +453,7 @@ bool QGCMapEngineManager::exportSets(const QString &path) setImportAction(ImportAction::ActionExporting); - QGCExportTileTask *task = new QGCExportTileTask(records, path); + QGCExportTileTask* task = new QGCExportTileTask(records, path); (void) connect(task, &QGCExportTileTask::actionCompleted, this, &QGCMapEngineManager::_actionCompleted); (void) connect(task, &QGCExportTileTask::actionProgress, this, &QGCMapEngineManager::_actionProgressHandler); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); @@ -465,7 +495,7 @@ QStringList QGCMapEngineManager::mapProviderList() { QStringList mapStringList = mapList(); const QStringList elevationStringList = elevationProviderList(); - for (const QString &elevationProviderName : elevationStringList) { + for (const QString& elevationProviderName : elevationStringList) { (void) mapStringList.removeAll(elevationProviderName); } @@ -481,7 +511,7 @@ QStringList QGCMapEngineManager::elevationProviderList() return UrlFactory::getElevationProviderTypes(); } -bool QGCMapEngineManager::importArchive(const QString &archivePath) +bool QGCMapEngineManager::importArchive(const QString& archivePath) { if (archivePath.isEmpty()) { setErrorMessage(tr("No archive path specified")); @@ -503,7 +533,8 @@ bool QGCMapEngineManager::importArchive(const QString &archivePath) return false; } - const QString tempPath = QDir::temp().filePath(QStringLiteral("qgc_tiles_") + QString::number(QDateTime::currentMSecsSinceEpoch())); + const QString tempPath = + QDir::temp().filePath(QStringLiteral("qgc_tiles_") + QString::number(QDateTime::currentMSecsSinceEpoch())); if (!QDir().mkpath(tempPath)) { setErrorMessage(tr("Could not create temporary directory")); return false; @@ -513,10 +544,9 @@ bool QGCMapEngineManager::importArchive(const QString &archivePath) if (_extractionJob == nullptr) { _extractionJob = new QGCCompressionJob(this); - connect(_extractionJob, &QGCCompressionJob::progressChanged, - this, &QGCMapEngineManager::_handleExtractionProgress); - connect(_extractionJob, &QGCCompressionJob::finished, - this, &QGCMapEngineManager::_handleExtractionFinished); + connect(_extractionJob, &QGCCompressionJob::progressChanged, this, + &QGCMapEngineManager::_handleExtractionProgress); + connect(_extractionJob, &QGCCompressionJob::finished, this, &QGCMapEngineManager::_handleExtractionFinished); } setImportAction(ImportAction::ActionImporting); @@ -543,8 +573,8 @@ void QGCMapEngineManager::_handleExtractionFinished(bool success) } QString dbPath; - QDirIterator it(_extractionOutputDir, {QStringLiteral("*.db"), QStringLiteral("*.sqlite")}, - QDir::Files, QDirIterator::Subdirectories); + QDirIterator it(_extractionOutputDir, {QStringLiteral("*.db"), QStringLiteral("*.sqlite")}, QDir::Files, + QDirIterator::Subdirectories); if (it.hasNext()) { dbPath = it.next(); } @@ -559,15 +589,14 @@ void QGCMapEngineManager::_handleExtractionFinished(bool success) qCDebug(QGCMapEngineManagerLog) << "Found tile database:" << dbPath; - QGCImportTileTask *task = new QGCImportTileTask(dbPath, _importReplace); + QGCImportTileTask* task = new QGCImportTileTask(dbPath, _importReplace); (void) connect(task, &QGCImportTileTask::actionCompleted, this, [this]() { _actionCompleted(); QDir(_extractionOutputDir).removeRecursively(); _extractionOutputDir.clear(); }); - (void) connect(task, &QGCImportTileTask::actionProgress, this, [this](int percentage) { - setActionProgress(50 + (percentage / 2)); - }); + (void) connect(task, &QGCImportTileTask::actionProgress, this, + [this](int percentage) { setActionProgress(50 + (percentage / 2)); }); (void) connect(task, &QGCMapTask::error, this, &QGCMapEngineManager::taskError); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); diff --git a/src/QtLocationPlugin/QGCMapEngineManager.h b/src/QtLocationPlugin/QGCMapEngineManager.h index 27ba88eaddb9..a74e4bfcfa6c 100644 --- a/src/QtLocationPlugin/QGCMapEngineManager.h +++ b/src/QtLocationPlugin/QGCMapEngineManager.h @@ -2,8 +2,8 @@ #include -#include "QGCTileSet.h" #include "QGCMapTaskBase.h" +#include "QGCTileSet.h" class QGCCachedTileSet; class QGCCompressionJob; @@ -12,31 +12,31 @@ class QmlObjectListModel; class QGCMapEngineManager : public QObject { Q_OBJECT - // QML_ELEMENT - // QML_UNCREATABLE("") Q_MOC_INCLUDE("QmlObjectListModel.h") Q_MOC_INCLUDE("QGCCachedTileSet.h") - Q_PROPERTY(bool fetchElevation MEMBER _fetchElevation NOTIFY fetchElevationChanged) - Q_PROPERTY(bool importReplace MEMBER _importReplace NOTIFY importReplaceChanged) - Q_PROPERTY(ImportAction importAction READ importAction WRITE setImportAction NOTIFY importActionChanged) - Q_PROPERTY(int actionProgress READ actionProgress NOTIFY actionProgressChanged) - Q_PROPERTY(int selectedCount READ selectedCount NOTIFY selectedCountChanged) - Q_PROPERTY(QmlObjectListModel *tileSets READ tileSets NOTIFY tileSetsChanged) - Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorMessageChanged) - Q_PROPERTY(QString tileCountStr READ tileCountStr NOTIFY tileCountChanged) - Q_PROPERTY(QString tileSizeStr READ tileSizeStr NOTIFY tileSizeChanged) - Q_PROPERTY(QStringList mapList READ mapList CONSTANT) - Q_PROPERTY(QStringList mapProviderList READ mapProviderList CONSTANT) - Q_PROPERTY(QStringList elevationProviderList READ elevationProviderList CONSTANT) - Q_PROPERTY(quint64 tileCount READ tileCount NOTIFY tileCountChanged) - Q_PROPERTY(quint64 tileSize READ tileSize NOTIFY tileSizeChanged) + Q_PROPERTY(bool fetchElevation MEMBER _fetchElevation NOTIFY fetchElevationChanged) + Q_PROPERTY(bool importReplace MEMBER _importReplace NOTIFY importReplaceChanged) + Q_PROPERTY(ImportAction importAction READ importAction WRITE setImportAction NOTIFY importActionChanged) + Q_PROPERTY(int actionProgress READ actionProgress NOTIFY actionProgressChanged) + Q_PROPERTY(int selectedCount READ selectedCount NOTIFY selectedCountChanged) + Q_PROPERTY(QmlObjectListModel* tileSets READ tileSets NOTIFY tileSetsChanged) + Q_PROPERTY(QString errorMessage READ errorMessage NOTIFY errorMessageChanged) + Q_PROPERTY(QString tileCountStr READ tileCountStr NOTIFY tileCountChanged) + Q_PROPERTY(QString tileSizeStr READ tileSizeStr NOTIFY tileSizeChanged) + Q_PROPERTY(QStringList mapList READ mapList CONSTANT) + Q_PROPERTY(QStringList mapProviderList READ mapProviderList CONSTANT) + Q_PROPERTY(QStringList elevationProviderList READ elevationProviderList CONSTANT) + Q_PROPERTY(quint64 tileCount READ tileCount NOTIFY tileCountChanged) + Q_PROPERTY(quint64 tileSize READ tileSize NOTIFY tileSizeChanged) public: - explicit QGCMapEngineManager(QObject *parent = nullptr); + explicit QGCMapEngineManager(QObject* parent = nullptr); ~QGCMapEngineManager(); - static QGCMapEngineManager *instance(); + static QGCMapEngineManager* instance(); + static void registerQmlTypes(); - enum class ImportAction { + enum class ImportAction + { ActionNone, ActionImporting, ActionExporting, @@ -44,42 +44,70 @@ class QGCMapEngineManager : public QObject }; Q_ENUM(ImportAction) - Q_INVOKABLE bool exportSets(const QString &path = QString()); - Q_INVOKABLE bool findName(const QString &name) const; - Q_INVOKABLE bool importSets(const QString &path = QString()); + Q_INVOKABLE bool exportSets(const QString& path = QString()); + Q_INVOKABLE bool findName(const QString& name) const; + Q_INVOKABLE bool importSets(const QString& path = QString()); /// Import tile sets from an archive file (.zip, .tar.gz, etc.) /// If the path is an archive, it will be extracted first, then imported. /// @param archivePath Path to the archive file /// @return true if import/extraction started successfully - Q_INVOKABLE bool importArchive(const QString &archivePath); + Q_INVOKABLE bool importArchive(const QString& archivePath); Q_INVOKABLE QString getUniqueName() const; - Q_INVOKABLE void deleteTileSet(QGCCachedTileSet *tileSet); + Q_INVOKABLE void deleteTileSet(QGCCachedTileSet* tileSet); Q_INVOKABLE void loadTileSets(); - Q_INVOKABLE void renameTileSet(QGCCachedTileSet *tileSet, const QString &newName); + Q_INVOKABLE void renameTileSet(QGCCachedTileSet* tileSet, const QString& newName); + Q_INVOKABLE void resetAction() { setImportAction(ImportAction::ActionNone); } + Q_INVOKABLE void selectAll(); Q_INVOKABLE void selectNone(); - Q_INVOKABLE void startDownload(const QString &name, const QString &mapType); - Q_INVOKABLE void updateForCurrentView(double lon0, double lat0, double lon1, double lat1, int minZoom, int maxZoom, const QString &mapName); + Q_INVOKABLE void startDownload(const QString& name, const QString& mapType); + Q_INVOKABLE void updateForCurrentView(double lon0, double lat0, double lon1, double lat1, int minZoom, int maxZoom, + const QString& mapName); - Q_INVOKABLE static QString loadSetting(const QString &key, const QString &defaultValue); - Q_INVOKABLE static QStringList mapTypeList(const QString &provider); - Q_INVOKABLE static void saveSetting(const QString &key, const QString &value); + Q_INVOKABLE static QStringList mapTypeList(const QString& provider); ImportAction importAction() const { return _importAction; } + int actionProgress() const { return _actionProgress; } + int selectedCount() const; - QmlObjectListModel *tileSets() { return _tileSets; } + + QmlObjectListModel* tileSets() { return _tileSets; } + QString errorMessage() const { return _errorMessage; } + QString tileCountStr() const; QString tileSizeStr() const; + quint64 tileCount() const { return (_imageSet.tileCount + _elevationSet.tileCount); } + quint64 tileSize() const { return (_imageSet.tileSize + _elevationSet.tileSize); } - void setActionProgress(int percentage) { if (percentage != _actionProgress) { _actionProgress = percentage; emit actionProgressChanged(); } } - void setErrorMessage(const QString &error) { if (error != _errorMessage) { _errorMessage = error; emit errorMessageChanged(); } } - void setImportAction(ImportAction action) { if (action != _importAction) { _importAction = action; emit importActionChanged(); } } + void setActionProgress(int percentage) + { + if (percentage != _actionProgress) { + _actionProgress = percentage; + emit actionProgressChanged(); + } + } + + void setErrorMessage(const QString& error) + { + if (error != _errorMessage) { + _errorMessage = error; + emit errorMessageChanged(); + } + } + + void setImportAction(ImportAction action) + { + if (action != _importAction) { + _importAction = action; + emit importActionChanged(); + } + } static QStringList mapList(); static QStringList mapProviderList(); @@ -98,21 +126,24 @@ class QGCMapEngineManager : public QObject void tileSizeChanged(); public slots: - void taskError(QGCMapTask::TaskType type, const QString &error); + void taskError(QGCMapTask::TaskType type, const QString& error); private slots: void _actionCompleted(); + void _actionProgressHandler(int percentage) { setActionProgress(percentage); } + void _resetCompleted() { loadTileSets(); } + void _tileSetDeleted(quint64 setID); - void _tileSetFetched(QGCCachedTileSet *tileSets); - void _tileSetSaved(QGCCachedTileSet *set); + void _tileSetFetched(QGCCachedTileSet* tileSets); + void _tileSetSaved(QGCCachedTileSet* set); void _updateTotals(quint32 totaltiles, quint64 totalsize, quint32 defaulttiles, quint64 defaultsize); void _handleExtractionProgress(qreal progress); void _handleExtractionFinished(bool success); private: - QmlObjectListModel *_tileSets = nullptr; + QmlObjectListModel* _tileSets = nullptr; QGCTileSet _imageSet; QGCTileSet _elevationSet; ImportAction _importAction = ImportAction::ActionNone; @@ -127,8 +158,6 @@ private slots: QString _errorMessage; bool _fetchElevation = true; bool _importReplace = false; - QGCCompressionJob *_extractionJob = nullptr; + QGCCompressionJob* _extractionJob = nullptr; QString _extractionOutputDir; - - static constexpr const char *kQmlOfflineMapKeyName = "QGCOfflineMap"; }; diff --git a/src/QtLocationPlugin/QGCMapTaskBase.h b/src/QtLocationPlugin/QGCMapTaskBase.h index 9c57a2118532..84fdbc8ff079 100644 --- a/src/QtLocationPlugin/QGCMapTaskBase.h +++ b/src/QtLocationPlugin/QGCMapTaskBase.h @@ -3,15 +3,24 @@ #include #include +class QGCCacheWorker; + +// Thread affinity: tasks are created/connected on the main thread and never moved +// to the worker; execute() runs on the worker thread, so result signals cross +// threads via queued AutoConnection. Raw-pointer result signals require a +// matching qRegisterMetaType before any Qt::QueuedConnection delivery. class QGCMapTask : public QObject { Q_OBJECT public: - enum class TaskType { + enum class TaskType + { taskInit, taskCacheTile, + taskRefreshTileValidators, taskFetchTile, + taskFetchFallbackTile, taskFetchTileSets, taskCreateTileSet, taskGetTileDownloadList, @@ -25,21 +34,19 @@ class QGCMapTask : public QObject }; Q_ENUM(TaskType); - explicit QGCMapTask(TaskType type, QObject *parent = nullptr) - : QObject(parent) - , m_type(type) - {} + explicit QGCMapTask(TaskType type, QObject* parent = nullptr) : QObject(parent), m_type(type) {} + ~QGCMapTask() override = default; TaskType type() const { return m_type; } - void setError(const QString &errorString = QString()) - { - emit error(m_type, errorString); - } + // Runs on the worker thread. + virtual void execute(QGCCacheWorker& worker) { Q_UNUSED(worker); } + + void setError(const QString& errorString = QString()) { emit error(m_type, errorString); } signals: - void error(QGCMapTask::TaskType type, const QString &errorString); + void error(QGCMapTask::TaskType type, const QString& errorString); private: const TaskType m_type = TaskType::taskInit; diff --git a/src/QtLocationPlugin/QGCMapTasks.cpp b/src/QtLocationPlugin/QGCMapTasks.cpp index 5e77325e53ad..adfb076669cc 100644 --- a/src/QtLocationPlugin/QGCMapTasks.cpp +++ b/src/QtLocationPlugin/QGCMapTasks.cpp @@ -1,7 +1,16 @@ #include "QGCMapTasks.h" -#include "QGCCachedTileSet.h" +#include +#include +#include +#include + #include "QGCCacheTile.h" +#include "QGCCachedTileSet.h" +#include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" +#include "QGCTileCacheWorker.h" +#include "QGCTileFallback.h" QGCCreateTileSetTask::~QGCCreateTileSetTask() { @@ -14,3 +23,202 @@ QGCSaveTileTask::~QGCSaveTileTask() { delete m_tile; } + +void QGCSaveTileTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + const QGCCacheTile* t = tile(); + if (!worker.database()->saveTile(t->hash, t->format, t->img, t->type, t->tileSet, t->etag, t->lastModified, + t->expiresAt, t->mustRevalidate)) { + setError("Error saving tile to cache"); + } +} + +void QGCFetchTileTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + auto tile = worker.database()->getTile(hash()); + if (tile) { + (void) worker.database()->bumpTileAccessed(hash()); + setTileFetched(QSharedPointer(tile.release())); + } else { + setError("Tile not in cache database"); + } +} + +void QGCFetchFallbackTileTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + const QSize tileSizePx(tileSize(), tileSize()); + for (int d = 1; d <= maxLevelsUp(); ++d) { + const int az = z() - d; + if (az < 0) { + break; + } + const int ax = x() >> d; + const int ay = y() >> d; + const QString hash = UrlFactory::getTileHash(providerName(), ax, ay, az); + auto tile = worker.database()->getTile(hash); + if (!tile) { + continue; + } + + QImage ancestor; + if (!ancestor.loadFromData(tile->img)) { + continue; + } + const QImage scaled = scaleAncestorToChild(ancestor, x(), y(), d, tileSizePx); + if (scaled.isNull()) { + continue; + } + const QByteArray bytes = encodeFallbackTile(scaled); + if (bytes.isEmpty()) { + continue; + } + (void) worker.database()->bumpTileAccessed(hash); + setFallbackFetched(bytes, QStringLiteral("png"), d); + return; + } + + setError("No ancestor tile available for fallback"); +} + +void QGCFetchTileSetTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + const QList records = worker.database()->getTileSets(); + for (const auto& rec : records) { + QGCCachedTileSet* set = new QGCCachedTileSet(rec.name); + set->blockSignals(true); + + set->setId(rec.setID); + set->setMapTypeStr(rec.mapTypeStr); + set->setTopleftLat(rec.topleftLat); + set->setTopleftLon(rec.topleftLon); + set->setBottomRightLat(rec.bottomRightLat); + set->setBottomRightLon(rec.bottomRightLon); + set->setMinZoom(rec.minZoom); + set->setMaxZoom(rec.maxZoom); + set->setType(rec.mapTypeStr); + set->setTotalTileCount(rec.numTiles); + set->setDefaultSet(rec.defaultSet); + set->setCreationDate(QDateTime::fromSecsSinceEpoch(rec.date)); + + const SetTotalsResult totals = + worker.database()->computeSetTotals(rec.setID, rec.defaultSet, rec.numTiles, set->type()); + set->setSavedTileCount(totals.savedTileCount); + set->setSavedTileSize(totals.savedTileSize); + set->setTotalTileSize(totals.totalTileSize); + set->setUniqueTileCount(totals.uniqueTileCount); + set->setUniqueTileSize(totals.uniqueTileSize); + + set->blockSignals(false); + (void) set->moveToThread(QCoreApplication::instance()->thread()); + setTileSetFetched(set); + } +} + +void QGCCreateTileSetTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + const auto setID = worker.database()->createTileSet( + tileSet()->name(), tileSet()->mapTypeStr(), tileSet()->topleftLat(), tileSet()->topleftLon(), + tileSet()->bottomRightLat(), tileSet()->bottomRightLon(), tileSet()->minZoom(), tileSet()->maxZoom(), + tileSet()->type(), tileSet()->totalTileCount()); + + if (!setID.has_value()) { + setError("Error saving tile set"); + return; + } + + tileSet()->blockSignals(true); + tileSet()->setId(setID.value()); + + const SetTotalsResult totals = worker.database()->computeSetTotals(setID.value(), tileSet()->defaultSet(), + tileSet()->totalTileCount(), tileSet()->type()); + tileSet()->setSavedTileCount(totals.savedTileCount); + tileSet()->setSavedTileSize(totals.savedTileSize); + tileSet()->setTotalTileSize(totals.totalTileSize); + tileSet()->setUniqueTileCount(totals.uniqueTileCount); + tileSet()->setUniqueTileSize(totals.uniqueTileSize); + tileSet()->blockSignals(false); + + setTileSetSaved(); +} + +void QGCGetTileDownloadListTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + const QList tileValues = worker.database()->getTileDownloadList(setID(), count()); + QQueue tiles; + for (const auto& t : tileValues) { + tiles.enqueue(new QGCTile(t)); + } + setTileListFetched(tiles); +} + +void QGCImportTileTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + auto progressCb = [this](int pct) { setProgress(pct); }; + + DatabaseResult result; + if (replace()) { + result = worker.database()->importSetsReplace(path(), progressCb); + } else { + result = worker.database()->importSetsMerge(path(), progressCb); + } + + worker.setDatabaseValid(worker.database()->isValid()); + + if (!result.success) { + setError(result.errorString); + return; + } + + if (replace() && worker.database()->isValid()) { + QSettings settings; + settings.remove(QLatin1String(QGCTileCacheDatabase::kBingNoTileDoneKey)); + worker.database()->deleteBingNoTileTiles(); + } + + setImportCompleted(); +} + +void QGCExportTileTask::execute(QGCCacheWorker& worker) +{ + if (!worker.validateDatabase(this)) { + return; + } + + auto progressCb = [this](int pct) { setProgress(pct); }; + DatabaseResult result = worker.database()->exportSets(sets(), path(), progressCb); + + if (!result.success) { + setError(result.errorString); + return; + } + + setExportCompleted(); +} diff --git a/src/QtLocationPlugin/QGCMapTasks.h b/src/QtLocationPlugin/QGCMapTasks.h index c4764db41ed0..9a92cea74ff4 100644 --- a/src/QtLocationPlugin/QGCMapTasks.h +++ b/src/QtLocationPlugin/QGCMapTasks.h @@ -1,35 +1,69 @@ #pragma once +#include #include #include +#include #include +#include #include "QGCMapTaskBase.h" -#include "QGCTileCacheTypes.h" #include "QGCTile.h" +#include "QGCTileCacheTypes.h" struct QGCCacheTile; class QGCCachedTileSet; +class QGCCacheWorker; //----------------------------------------------------------------------------- -class QGCFetchTileSetTask : public QGCMapTask +// Fire-and-forget command task: carries its TaskType plus the work captured as a +// callable, and emits a single payload-free completed() when the work succeeds. +// The callable returns false (after calling setError) to suppress completed(). +class QGCCommandTask : public QGCMapTask { Q_OBJECT public: - explicit QGCFetchTileSetTask(QObject *parent = nullptr) - : QGCMapTask(TaskType::taskFetchTileSets, parent) + using Work = std::function; + + QGCCommandTask(TaskType type, Work work, QObject* parent = nullptr) + : QGCMapTask(type, parent), m_work(std::move(work)) {} - ~QGCFetchTileSetTask() = default; - void setTileSetFetched(QGCCachedTileSet *tileSet) + ~QGCCommandTask() override = default; + + void execute(QGCCacheWorker& worker) override { - emit tileSetFetched(tileSet); + if (m_work && m_work(worker, *this)) { + emit completed(); + } } signals: - void tileSetFetched(QGCCachedTileSet *tileSet); + void completed(); + +private: + const Work m_work; +}; + +//----------------------------------------------------------------------------- + +class QGCFetchTileSetTask : public QGCMapTask +{ + Q_OBJECT + +public: + explicit QGCFetchTileSetTask(QObject* parent = nullptr) : QGCMapTask(TaskType::taskFetchTileSets, parent) {} + + ~QGCFetchTileSetTask() = default; + + void execute(QGCCacheWorker& worker) override; + + void setTileSetFetched(QGCCachedTileSet* tileSet) { emit tileSetFetched(tileSet); } + +signals: + void tileSetFetched(QGCCachedTileSet* tileSet); }; //----------------------------------------------------------------------------- @@ -39,14 +73,15 @@ class QGCCreateTileSetTask : public QGCMapTask Q_OBJECT public: - explicit QGCCreateTileSetTask(QGCCachedTileSet *tileSet, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskCreateTileSet, parent) - , m_tileSet(tileSet) - , m_saved(false) + explicit QGCCreateTileSetTask(QGCCachedTileSet* tileSet, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskCreateTileSet, parent), m_tileSet(tileSet), m_saved(false) {} + ~QGCCreateTileSetTask(); - QGCCachedTileSet *tileSet() { return m_tileSet; } + void execute(QGCCacheWorker& worker) override; + + QGCCachedTileSet* tileSet() { return m_tileSet; } void setTileSetSaved() { @@ -55,7 +90,7 @@ class QGCCreateTileSetTask : public QGCMapTask } signals: - void tileSetSaved(QGCCachedTileSet *tileSet); + void tileSetSaved(QGCCachedTileSet* tileSet); private: QGCCachedTileSet* const m_tileSet = nullptr; @@ -69,21 +104,20 @@ class QGCFetchTileTask : public QGCMapTask Q_OBJECT public: - explicit QGCFetchTileTask(const QString &hash, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskFetchTile, parent) - , m_hash(hash) + explicit QGCFetchTileTask(const QString& hash, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskFetchTile, parent), m_hash(hash) {} + ~QGCFetchTileTask() = default; - void setTileFetched(QGCCacheTile *tile) - { - emit tileFetched(tile); - } + void execute(QGCCacheWorker& worker) override; + + void setTileFetched(const QSharedPointer& tile) { emit tileFetched(tile); } QString hash() const { return m_hash; } signals: - void tileFetched(QGCCacheTile *tile); + void tileFetched(QSharedPointer tile); private: const QString m_hash; @@ -91,202 +125,132 @@ class QGCFetchTileTask : public QGCMapTask //----------------------------------------------------------------------------- -class QGCSaveTileTask : public QGCMapTask +// R6 lower-zoom fallback: when a tile cannot be served from cache or network the +// worker scans cached ancestor tiles (z-1 .. z-maxLevelsUp). On a hit it crops and +// scales the ancestor sub-square covering this tile and returns it as a placeholder. +class QGCFetchFallbackTileTask : public QGCMapTask { Q_OBJECT public: - explicit QGCSaveTileTask(QGCCacheTile *tile, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskCacheTile, parent) - , m_tile(tile) + QGCFetchFallbackTileTask(const QString& providerName, int x, int y, int z, int tileSize, int maxLevelsUp = 5, + QObject* parent = nullptr) + : QGCMapTask(TaskType::taskFetchFallbackTile, parent), + m_providerName(providerName), + m_x(x), + m_y(y), + m_z(z), + m_tileSize(tileSize), + m_maxLevelsUp(maxLevelsUp) {} - ~QGCSaveTileTask(); - const QGCCacheTile *tile() const { return m_tile; } - QGCCacheTile *tile() { return m_tile; } + ~QGCFetchFallbackTileTask() = default; -private: - QGCCacheTile* const m_tile = nullptr; -}; + void execute(QGCCacheWorker& worker) override; -//----------------------------------------------------------------------------- + QString providerName() const { return m_providerName; } -class QGCGetTileDownloadListTask : public QGCMapTask -{ - Q_OBJECT + int x() const { return m_x; } -public: - QGCGetTileDownloadListTask(quint64 setID, int count, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskGetTileDownloadList, parent) - , m_setID(setID) - , m_count(count) - {} - ~QGCGetTileDownloadListTask() = default; + int y() const { return m_y; } - quint64 setID() const { return m_setID; } - int count() const { return m_count; } + int z() const { return m_z; } + + int tileSize() const { return m_tileSize; } + + int maxLevelsUp() const { return m_maxLevelsUp; } - void setTileListFetched(const QQueue &tiles) + void setFallbackFetched(const QByteArray& image, const QString& format, int levelDelta) { - emit tileListFetched(tiles); + emit fallbackFetched(image, format, levelDelta); } signals: - void tileListFetched(QQueue tiles); + void fallbackFetched(QByteArray image, QString format, int levelDelta); private: - const quint64 m_setID = 0; - const int m_count = 0; + const QString m_providerName; + const int m_x; + const int m_y; + const int m_z; + const int m_tileSize; + const int m_maxLevelsUp; }; //----------------------------------------------------------------------------- -class QGCUpdateTileDownloadStateTask : public QGCMapTask +class QGCSaveTileTask : public QGCMapTask { Q_OBJECT public: - QGCUpdateTileDownloadStateTask(quint64 setID, QGCTile::TileState state, const QString &hash, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskUpdateTileDownloadState, parent) - , m_setID(setID) - , m_state(state) - , m_hash(hash) + explicit QGCSaveTileTask(QGCCacheTile* tile, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskCacheTile, parent), m_tile(tile) {} - ~QGCUpdateTileDownloadStateTask() = default; - - QString hash() const { return m_hash; } - quint64 setID() const { return m_setID; } - QGCTile::TileState state() const { return m_state; } - -private: - const quint64 m_setID = 0; - const QGCTile::TileState m_state = QGCTile::StatePending; - const QString m_hash; -}; - -//----------------------------------------------------------------------------- -class QGCDeleteTileSetTask : public QGCMapTask -{ - Q_OBJECT + ~QGCSaveTileTask(); -public: - explicit QGCDeleteTileSetTask(quint64 setID, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskDeleteTileSet, parent) - , m_setID(setID) - {} - ~QGCDeleteTileSetTask() = default; + void execute(QGCCacheWorker& worker) override; - quint64 setID() const { return m_setID; } + const QGCCacheTile* tile() const { return m_tile; } - void setTileSetDeleted() - { - emit tileSetDeleted(m_setID); - } - -signals: - void tileSetDeleted(quint64 setID); + QGCCacheTile* tile() { return m_tile; } private: - const quint64 m_setID = 0; + QGCCacheTile* const m_tile = nullptr; }; //----------------------------------------------------------------------------- -class QGCRenameTileSetTask : public QGCMapTask +class QGCGetTileDownloadListTask : public QGCMapTask { Q_OBJECT public: - QGCRenameTileSetTask(quint64 setID, const QString &newName, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskRenameTileSet, parent) - , m_setID(setID) - , m_newName(newName) + QGCGetTileDownloadListTask(quint64 setID, int count, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskGetTileDownloadList, parent), m_setID(setID), m_count(count) {} - ~QGCRenameTileSetTask() = default; - - quint64 setID() const { return m_setID; } - QString newName() const { return m_newName; } -private: - const quint64 m_setID = 0; - const QString m_newName; -}; - -//----------------------------------------------------------------------------- + ~QGCGetTileDownloadListTask() = default; -class QGCPruneCacheTask : public QGCMapTask -{ - Q_OBJECT + void execute(QGCCacheWorker& worker) override; -public: - explicit QGCPruneCacheTask(quint64 amount, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskPruneCache, parent) - , m_amount(amount) - {} - ~QGCPruneCacheTask() = default; + quint64 setID() const { return m_setID; } - quint64 amount() const { return m_amount; } + int count() const { return m_count; } - void setPruned() - { - emit pruned(); - } + void setTileListFetched(const QQueue& tiles) { emit tileListFetched(tiles); } signals: - void pruned(); + void tileListFetched(QQueue tiles); private: - const quint64 m_amount = 0; + const quint64 m_setID = 0; + const int m_count = 0; }; //----------------------------------------------------------------------------- -class QGCResetTask : public QGCMapTask +class QGCExportTileTask : public QGCMapTask { Q_OBJECT public: - explicit QGCResetTask(QObject *parent = nullptr) - : QGCMapTask(TaskType::taskReset, parent) + explicit QGCExportTileTask(const QList& sets, const QString& path, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskExport, parent), m_sets(sets), m_path(path) {} - ~QGCResetTask() = default; - - void setResetCompleted() - { - emit resetCompleted(); - } -signals: - void resetCompleted(); -}; + ~QGCExportTileTask() = default; -//----------------------------------------------------------------------------- + void execute(QGCCacheWorker& worker) override; -class QGCExportTileTask : public QGCMapTask -{ - Q_OBJECT + const QList& sets() const { return m_sets; } -public: - explicit QGCExportTileTask(const QList &sets, const QString &path, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskExport, parent) - , m_sets(sets) - , m_path(path) - {} - ~QGCExportTileTask() = default; - - const QList &sets() const { return m_sets; } QString path() const { return m_path; } - void setExportCompleted() - { - emit actionCompleted(); - } + void setExportCompleted() { emit actionCompleted(); } - void setProgress(int percentage) - { - emit actionProgress(percentage); - } + void setProgress(int percentage) { emit actionProgress(percentage); } signals: void actionCompleted(); @@ -304,21 +268,21 @@ class QGCImportTileTask : public QGCMapTask Q_OBJECT public: - QGCImportTileTask(const QString &path, bool replace, QObject *parent = nullptr) - : QGCMapTask(TaskType::taskImport, parent) - , m_path(path) - , m_replace(replace) + QGCImportTileTask(const QString& path, bool replace, QObject* parent = nullptr) + : QGCMapTask(TaskType::taskImport, parent), m_path(path), m_replace(replace) {} + ~QGCImportTileTask() = default; + void execute(QGCCacheWorker& worker) override; + QString path() const { return m_path; } + bool replace() const { return m_replace; } + int progress() const { return m_progress; } - void setImportCompleted() - { - emit actionCompleted(); - } + void setImportCompleted() { emit actionCompleted(); } void setProgress(int percentage) { diff --git a/src/QtLocationPlugin/QGCMapUrlEngine.cpp b/src/QtLocationPlugin/QGCMapUrlEngine.cpp index 455594bf84fa..4108ffcc79c8 100644 --- a/src/QtLocationPlugin/QGCMapUrlEngine.cpp +++ b/src/QtLocationPlugin/QGCMapUrlEngine.cpp @@ -1,9 +1,9 @@ #include "QGCMapUrlEngine.h" -#include "QGCTileSet.h" - -#include +#include #include +#include +#include #include "BingMapProvider.h" #include "ElevationMapProvider.h" @@ -11,67 +11,190 @@ #include "GenericMapProvider.h" #include "GoogleMapProvider.h" #include "MapboxMapProvider.h" -#include "TianDiTuProvider.h" #include "QGCLoggingCategory.h" +#include "QGCTileSet.h" +#include "TianDiTuProvider.h" QGC_LOGGING_CATEGORY(QGCMapUrlEngineLog, "QtLocationPlugin.QGCMapUrlEngine") const QList UrlFactory::_providers = { #ifndef QGC_NO_GOOGLE_MAPS - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), + std::make_shared(QStringLiteral("Google Street Map"), QStringLiteral("lyrs"), + QStringLiteral("m"), QStringLiteral("png"), AVERAGE_GOOGLE_STREET_MAP, + MapProvider::StreetMap), + std::make_shared(QStringLiteral("Google Satellite"), QStringLiteral("lyrs"), QStringLiteral("s"), + QStringLiteral("jpg"), AVERAGE_GOOGLE_SAT_MAP, MapProvider::SatelliteMapDay), + std::make_shared(QStringLiteral("Google Terrain"), QStringLiteral("v"), QStringLiteral("t,r"), + QStringLiteral("png"), AVERAGE_GOOGLE_TERRAIN_MAP, MapProvider::TerrainMap), + std::make_shared(QStringLiteral("Google Hybrid"), QStringLiteral("lyrs"), QStringLiteral("y"), + QStringLiteral("png"), AVERAGE_GOOGLE_SAT_MAP, MapProvider::HybridMap), + std::make_shared(QStringLiteral("Google Labels"), QStringLiteral("lyrs"), QStringLiteral("h"), + QStringLiteral("png"), QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap), #endif - std::make_shared(), - std::make_shared(), - std::make_shared(), - - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - - std::make_shared(), - - std::make_shared(), - std::make_shared(), - std::make_shared(), - - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - - std::make_shared(), - std::make_shared(), + std::make_shared(QStringLiteral("Bing Road"), QStringLiteral("r"), QStringLiteral("png"), + AVERAGE_BING_STREET_MAP, MapProvider::StreetMap), + std::make_shared(QStringLiteral("Bing Satellite"), QStringLiteral("a"), QStringLiteral("jpg"), + AVERAGE_BING_SAT_MAP, MapProvider::SatelliteMapDay), + std::make_shared(QStringLiteral("Bing Hybrid"), QStringLiteral("h"), QStringLiteral("jpg"), + AVERAGE_BING_SAT_MAP, MapProvider::HybridMap), + + std::make_shared(QObject::tr("TianDiTu Road"), QStringLiteral("cia_w"), QStringLiteral("png"), + AVERAGE_TIANDITU_STREET_MAP, MapProvider::StreetMap), + std::make_shared(QObject::tr("TianDiTu Satellite"), QStringLiteral("img_w"), + QStringLiteral("jpg"), AVERAGE_TIANDITU_SAT_MAP, MapProvider::SatelliteMapDay), + std::make_shared(MapProviderConfig{ + .name = QStringLiteral("Statkart Topo"), + .referrer = QStringLiteral("https://norgeskart.no/"), + .urlTemplate = QStringLiteral("https://cache.kartverket.no/v1/wmts/1.0.0/%1/default/webmercator/%2/%3/%4.png"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("topo4"), + .axisOrder = MapProviderConfig::ZYX}), + std::make_shared(MapProviderConfig{ + .name = QStringLiteral("Statkart Basemap"), + .referrer = QStringLiteral("https://norgeskart.no/"), + .urlTemplate = QStringLiteral("https://cache.kartverket.no/v1/wmts/1.0.0/%1/default/webmercator/%2/%3/%4.png"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("norgeskart_bakgrunn"), + .axisOrder = MapProviderConfig::ZYX}), + std::make_shared(MapProviderConfig{ + .name = QStringLiteral("Svalbard Topo"), + .referrer = QStringLiteral("https://www.npolar.no/"), + .urlTemplate = QStringLiteral( + "https://geodata.npolar.no/arcgis/rest/services/Basisdata/NP_Basiskart_Svalbard_WMTS_3857/MapServer/WMTS/" + "tile/1.0.0/Basisdata_NP_Basiskart_Svalbard_WMTS_3857/default/default028mm/%1/%2/%3"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .axisOrder = MapProviderConfig::ZYX}), + + std::make_shared(MapProviderConfig{ + .name = QStringLiteral("Eniro Topo"), + .referrer = QStringLiteral("https://www.eniro.se/"), + .urlTemplate = QStringLiteral("https://map.eniro.com/geowebcache/service/tms1.0.0/map/%1/%2/%3.%4"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .flipY = true, + .appendImageFormat = true}), + + std::make_shared(QStringLiteral("Esri World Street"), QStringLiteral("World_Street_Map"), + QGC_AVERAGE_TILE_SIZE, MapProvider::StreetMap), + std::make_shared(QStringLiteral("Esri World Satellite"), QStringLiteral("World_Imagery"), + QGC_AVERAGE_TILE_SIZE, MapProvider::SatelliteMapDay), + std::make_shared(QStringLiteral("Esri Terrain"), QStringLiteral("World_Terrain_Base"), + QGC_AVERAGE_TILE_SIZE, MapProvider::TerrainMap), + + std::make_shared(QStringLiteral("Mapbox Streets"), QStringLiteral("streets-v10"), + AVERAGE_MAPBOX_STREET_MAP, MapProvider::StreetMap), + std::make_shared(QStringLiteral("Mapbox Light"), QStringLiteral("light-v9"), + QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap), + std::make_shared(QStringLiteral("Mapbox Dark"), QStringLiteral("dark-v9"), QGC_AVERAGE_TILE_SIZE, + MapProvider::CustomMap), + std::make_shared(QStringLiteral("Mapbox Satellite"), QStringLiteral("satellite-v9"), + AVERAGE_MAPBOX_SAT_MAP, MapProvider::SatelliteMapDay), + std::make_shared(QStringLiteral("Mapbox Hybrid"), QStringLiteral("satellite-streets-v10"), + AVERAGE_MAPBOX_SAT_MAP, MapProvider::HybridMap), + std::make_shared(QStringLiteral("Mapbox StreetsBasic"), QStringLiteral("basic-v9"), + QGC_AVERAGE_TILE_SIZE, MapProvider::StreetMap), + std::make_shared(QStringLiteral("Mapbox Outdoors"), QStringLiteral("outdoors-v10"), + QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap), + std::make_shared(QStringLiteral("Mapbox Bright"), QStringLiteral("bright-v9"), + QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap), + std::make_shared(QStringLiteral("Mapbox Custom"), QStringLiteral("mapbox.custom"), + QGC_AVERAGE_TILE_SIZE, MapProvider::CustomMap), + + std::make_shared( + MapProviderConfig{.name = QStringLiteral("MapQuest Map"), + .referrer = QStringLiteral("https://mapquest.com"), + .urlTemplate = QStringLiteral("https://otile%1.mqcdn.com/tiles/1.0.0/%2/%3/%4/%5.%6"), + .imageFormat = QStringLiteral("jpg"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("map"), + .appendImageFormat = true, + .serverCount = 4}), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("MapQuest Sat"), + .referrer = QStringLiteral("https://mapquest.com"), + .urlTemplate = QStringLiteral("https://otile%1.mqcdn.com/tiles/1.0.0/%2/%3/%4/%5.%6"), + .imageFormat = QStringLiteral("jpg"), + .mapStyle = MapProvider::SatelliteMapDay, + .mapTypeId = QStringLiteral("sat"), + .appendImageFormat = true, + .serverCount = 4}), std::make_shared(), std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), - std::make_shared(), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Japan-GSI Contour"), + .referrer = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), + .urlTemplate = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("std"), + .appendImageFormat = true}), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Japan-GSI Seamless"), + .referrer = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), + .urlTemplate = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"), + .imageFormat = QStringLiteral("jpg"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("seamlessphoto"), + .appendImageFormat = true}), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Japan-GSI Anaglyph"), + .referrer = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), + .urlTemplate = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("anaglyphmap_color"), + .appendImageFormat = true}), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Japan-GSI Slope"), + .referrer = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), + .urlTemplate = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("slopemap"), + .appendImageFormat = true}), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Japan-GSI Relief"), + .referrer = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std"), + .urlTemplate = QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/%1/%2/%3/%4.%5"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .mapTypeId = QStringLiteral("relief"), + .appendImageFormat = true}), std::make_shared(), - std::make_shared(), + std::make_shared( + MapProviderConfig{.name = QStringLiteral("Street Map"), + .referrer = QStringLiteral("https://www.openstreetmap.org"), + .urlTemplate = QStringLiteral("https://tile.openstreetmap.org/%1/%2/%3.png"), + .imageFormat = QStringLiteral("png"), + .mapStyle = MapProvider::StreetMap, + .isOSM = true}), std::make_shared(), std::make_shared(), - std::make_shared() -}; + std::make_shared()}; + +QHash UrlFactory::_providersByMapId; +QHash UrlFactory::_providersByName; +std::once_flag UrlFactory::_initFlag; + +void UrlFactory::_ensureInitialized() +{ + std::call_once(_initFlag, [] { + for (const SharedMapProvider& provider : _providers) { + _providersByMapId.insert(provider->getMapId(), provider); + _providersByName.insert(provider->getMapName(), provider); + } + }); +} QString UrlFactory::getImageFormat(int qtMapId, QByteArrayView image) { @@ -126,11 +249,7 @@ quint32 UrlFactory::averageSizeForType(QStringView type) bool UrlFactory::isElevation(int qtMapId) { const SharedMapProvider provider = getMapProviderFromQtMapId(qtMapId); - if (provider) { - return provider->isElevationProvider(); - } - - return false; + return provider && std::dynamic_pointer_cast(provider) != nullptr; } int UrlFactory::long2tileX(QStringView mapType, double lon, int z) @@ -153,11 +272,13 @@ int UrlFactory::lat2tileY(QStringView mapType, double lat, int z) return 0; } -QGCTileSet UrlFactory::getTileCount(int zoom, double topleftLon, double topleftLat, double bottomRightLon, double bottomRightLat, QStringView mapType) +QGCTileSet UrlFactory::getTileCount(int zoom, double topleftLon, double topleftLat, double bottomRightLon, + double bottomRightLat, QStringView mapType) { const SharedMapProvider provider = getMapProviderFromProviderType(mapType); if (provider) { - // TODO: zoom = qBound(QGeoCameraCapabilities.minimumZoomLevel(), zoom, QGeoCameraCapabilities.maximumZoomLevel()); + // TODO: zoom = qBound(QGeoCameraCapabilities.minimumZoomLevel(), zoom, + // QGeoCameraCapabilities.maximumZoomLevel()); zoom = qBound(1, zoom, QGC_MAX_MAP_ZOOM); return provider->getTileCount(zoom, topleftLon, topleftLat, bottomRightLon, bottomRightLat); } @@ -167,33 +288,36 @@ QGCTileSet UrlFactory::getTileCount(int zoom, double topleftLon, double topleftL QString UrlFactory::getProviderTypeFromQtMapId(int qtMapId) { - if (qtMapId == -1) { + if (qtMapId == defaultSetMapId()) { return QString(); } - for (const SharedMapProvider &provider : _providers) { - if (provider->getMapId() == qtMapId) { - return provider->getMapName(); - } + const SharedMapProvider provider = getMapProviderFromQtMapId(qtMapId); + if (provider) { + return provider->getMapName(); } - qCWarning(QGCMapUrlEngineLog) << "map id not found:" << qtMapId; return QString(); } SharedMapProvider UrlFactory::getMapProviderFromQtMapId(int qtMapId) { - if (qtMapId == -1) { + if (qtMapId == defaultSetMapId()) { return nullptr; } - for (const SharedMapProvider &provider : _providers) { - if (provider->getMapId() == qtMapId) { - return provider; - } + _ensureInitialized(); + + const auto it = _providersByMapId.constFind(qtMapId); + if (it != _providersByMapId.constEnd()) { + return it.value(); } - qCWarning(QGCMapUrlEngineLog) << "provider not found from id:" << qtMapId; + thread_local QSet s_warnedIds; + if (!s_warnedIds.contains(qtMapId)) { + s_warnedIds.insert(qtMapId); + qCWarning(QGCMapUrlEngineLog) << "provider not found from id:" << qtMapId; + } return nullptr; } @@ -203,13 +327,19 @@ SharedMapProvider UrlFactory::getMapProviderFromProviderType(QStringView type) return nullptr; } - for (const SharedMapProvider &provider : _providers) { - if (provider->getMapName() == type) { - return provider; - } + _ensureInitialized(); + + const QString typeStr = type.toString(); + const auto it = _providersByName.constFind(typeStr); + if (it != _providersByName.constEnd()) { + return it.value(); } - qCWarning(QGCMapUrlEngineLog) << "type not found:" << type; + thread_local QSet s_warnedTypes; + if (!s_warnedTypes.contains(typeStr)) { + s_warnedTypes.insert(typeStr); + qCWarning(QGCMapUrlEngineLog) << "type not found:" << type; + } return nullptr; } @@ -219,21 +349,19 @@ int UrlFactory::getQtMapIdFromProviderType(QStringView type) return -1; } - for (const SharedMapProvider &provider : _providers) { - if (provider->getMapName() == type) { - return provider->getMapId(); - } + const SharedMapProvider provider = getMapProviderFromProviderType(type); + if (provider) { + return provider->getMapId(); } - qCWarning(QGCMapUrlEngineLog) << "type not found:" << type; return -1; } QStringList UrlFactory::getElevationProviderTypes() { QStringList types; - for (const SharedMapProvider &provider : _providers) { - if (provider->isElevationProvider()) { + for (const SharedMapProvider& provider : _providers) { + if (std::dynamic_pointer_cast(provider)) { types.append(provider->getMapName()); } } @@ -244,20 +372,43 @@ QStringList UrlFactory::getElevationProviderTypes() QStringList UrlFactory::getProviderTypes() { QStringList types; - for (const SharedMapProvider &provider : _providers) { + for (const SharedMapProvider& provider : _providers) { types.append(provider->getMapName()); } return types; } +namespace { +// Retina tiles share the SQLite cache with 1x tiles, so their hashes must not +// collide. The 10-digit provider-hash field has ample headroom above the small +// sequential mapIds, so @2x tiles reserve a high offset added to the mapId. +// providerTypeFromHash() strips it to recover the provider, keeping the mapping +// reversible for the importer / tileHashToType. +constexpr int kRetinaHashOffset = 1000000000; +} // namespace + +bool UrlFactory::useRetinaTiles() +{ + const QGuiApplication* const app = qobject_cast(QCoreApplication::instance()); + return app && (app->devicePixelRatio() > 1.0); +} + +int UrlFactory::tilePixelScale() +{ + return useRetinaTiles() ? 2 : 1; +} + QString UrlFactory::providerTypeFromHash(int hash) { - for (const SharedMapProvider &provider : _providers) { - const QString mapName = provider->getMapName(); - if (hashFromProviderType(mapName) == hash) { - return mapName; - } + _ensureInitialized(); + + if (hash >= kRetinaHashOffset) { + hash -= kRetinaHashOffset; + } + const auto it = _providersByMapId.constFind(hash); + if (it != _providersByMapId.constEnd()) { + return it.value()->getMapName(); } qCWarning(QGCMapUrlEngineLog) << "provider not found from hash:" << hash; @@ -266,24 +417,29 @@ QString UrlFactory::providerTypeFromHash(int hash) int UrlFactory::hashFromProviderType(QStringView type) { - for (const SharedMapProvider &provider : _providers) { - if (provider->getMapName() == type) { - return provider->getMapId(); - } + const SharedMapProvider provider = getMapProviderFromProviderType(type); + if (provider) { + return provider->getMapId(); } - qCWarning(QGCMapUrlEngineLog) << "provider not found for type:" << type; return -1; } QString UrlFactory::tileHashToType(QStringView tileHash) { - const int providerHash = tileHash.mid(0,10).toInt(); + if (tileHash.size() < 10) { + return QString(); + } + const int providerHash = tileHash.mid(0, 10).toInt(); return providerTypeFromHash(providerHash); } QString UrlFactory::getTileHash(QStringView type, int x, int y, int z) { - const int hash = hashFromProviderType(type); + int hash = hashFromProviderType(type); + // Keep @2x tiles in a disjoint hash space so they never collide with 1x tiles. + if ((hash >= 0) && useRetinaTiles()) { + hash += kRetinaHashOffset; + } return QString::asprintf("%010d%08d%08d%03d", hash, x, y, z); } diff --git a/src/QtLocationPlugin/QGCMapUrlEngine.h b/src/QtLocationPlugin/QGCMapUrlEngine.h index e722690d4945..e0c2323d7cb7 100644 --- a/src/QtLocationPlugin/QGCMapUrlEngine.h +++ b/src/QtLocationPlugin/QGCMapUrlEngine.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include "QGCTileSet.h" @@ -37,6 +39,8 @@ class UrlFactory static const QList>& getProviders() { return _providers; } static QStringList getElevationProviderTypes(); static QStringList getProviderTypes(); + static constexpr int defaultSetMapId() { return -1; } + static int getQtMapIdFromProviderType(QStringView type); static QString getProviderTypeFromQtMapId(int qtMapId); static std::shared_ptr getMapProviderFromQtMapId(int qtMapId); @@ -47,8 +51,21 @@ class UrlFactory static QString tileHashToType(QStringView tileHash); static QString getTileHash(QStringView type, int x, int y, int z); + // Retina/HiDPI (R4): when the display reports a device-pixel-ratio > 1 the + // engine requests 512px (@2x) tiles. Computed once from QGuiApplication and + // cached; returns false when no QGuiApplication exists (e.g. unit tests). + static bool useRetinaTiles(); + // Pixel scale for the active display: 2 when useRetinaTiles(), else 1. Used by + // providers that build @2x / scale=2 URLs. + static int tilePixelScale(); + private: + static void _ensureInitialized(); + static const QList> _providers; + static QHash> _providersByMapId; + static QHash> _providersByName; + static std::once_flag _initFlag; }; typedef std::shared_ptr SharedMapProvider; diff --git a/src/QtLocationPlugin/QGCTile.h b/src/QtLocationPlugin/QGCTile.h index 4db2e7620f31..06560cbc01ca 100644 --- a/src/QtLocationPlugin/QGCTile.h +++ b/src/QtLocationPlugin/QGCTile.h @@ -10,7 +10,8 @@ struct QGCTile StatePending = 0, StateDownloading, StateError, - StateComplete + StateComplete, + StatePaused }; int x = 0; @@ -18,7 +19,7 @@ struct QGCTile int z = 0; quint64 tileSet = UINT64_MAX; QString hash; - int type = -1; + QString type; }; Q_DECLARE_METATYPE(QGCTile) Q_DECLARE_METATYPE(QGCTile*) diff --git a/src/QtLocationPlugin/QGCTileCache.cpp b/src/QtLocationPlugin/QGCTileCache.cpp new file mode 100644 index 000000000000..fd55373c8947 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileCache.cpp @@ -0,0 +1,202 @@ +#include "QGCTileCache.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "AppMessages.h" +#include "AppSettings.h" +#include "MapsSettings.h" +#include "QGCCacheTile.h" +#include "QGCFileHelper.h" +#include "QGCLoggingCategory.h" +#include "QGCMapEngine.h" +#include "QGCMapTasks.h" +#include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" +#include "QGCTileCacheWorker.h" +#include "SettingsManager.h" + +QGC_LOGGING_CATEGORY(QGCTileCacheLog, "QtLocationPlugin.QGCTileCache") + +namespace { +QString s_databaseFilePath; +QString s_cachePath; +std::atomic s_cacheWasReset = false; + +bool wipeDirectory(const QString& dirPath) +{ + bool result = true; + + const QDir dir(dirPath); + if (dir.exists(dirPath)) { + s_cacheWasReset = true; + + const QFileInfoList fileList = dir.entryInfoList( + QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::AllDirs | QDir::Files, QDir::DirsFirst); + for (const QFileInfo& info : fileList) { + if (info.isDir()) { + result = wipeDirectory(info.absoluteFilePath()); + } else { + result = QFile::remove(info.absoluteFilePath()); + } + + if (!result) { + return result; + } + } + result = dir.rmdir(dirPath); + } + + return result; +} + +void wipeOldCaches() +{ + // Versioned caches are named "QGCMapCache" (e.g. QGCMapCache55). The live + // cache is the bare "QGCMapCache" directory; remove only the numbered variants. + static const QRegularExpression versionSuffix(QStringLiteral("^QGCMapCache\\d+$")); + + QStringList searchLocations; +#if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) + searchLocations << QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); +#else + searchLocations << QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + << QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation); +#endif + + for (const QString& location : searchLocations) { + if (location.isEmpty()) { + continue; + } + const QDir baseDir(location); + const QFileInfoList entries = + baseDir.entryInfoList({QStringLiteral("QGCMapCache*")}, QDir::Dirs | QDir::NoDotAndDotDot); + for (const QFileInfo& entry : entries) { + if (versionSuffix.match(entry.fileName()).hasMatch()) { + wipeDirectory(entry.absoluteFilePath()); + } + } + } +} + +void initCache() +{ + wipeOldCaches(); + + // QString cacheDir = QAbstractGeoTileCache::baseCacheDirectory() +#if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) + QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); + cacheDir += QStringLiteral("/QGCMapCache"); +#else + QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::CacheLocation); + cacheDir += QStringLiteral("/QGCMapCache"); +#endif + if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { + qCWarning(QGCTileCacheLog) << "Could not create mapping disk cache directory:" << cacheDir; + + cacheDir = QDir::homePath() + QStringLiteral("/.qgcmapscache/"); + if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { + qCWarning(QGCTileCacheLog) << "Could not create mapping disk cache directory:" << cacheDir; + cacheDir.clear(); + } + } + + s_cachePath = cacheDir; + if (!s_cachePath.isEmpty()) { + s_databaseFilePath = QString(s_cachePath + QStringLiteral("/qgcMapCache.db")); + + qCDebug(QGCTileCacheLog) << "Map Cache in:" << s_databaseFilePath; + } else { + qCCritical(QGCTileCacheLog) << "Could not find suitable map cache directory."; + } + + if (s_cacheWasReset) { + QGC::showAppMessage(QCoreApplication::translate("QGCTileCache", + "The Offline Map Cache database has been upgraded. " + "Your old map cache sets have been reset.")); + } +} +} // namespace + +void QGCTileCache::ensureInitialized() +{ + static std::once_flag cacheInit; + std::call_once(cacheInit, []() { initCache(); }); +} + +quint32 QGCTileCache::getMaxDiskCacheSetting() +{ + return SettingsManager::instance()->mapsSettings()->maxCacheDiskSize()->rawValue().toUInt(); +} + +void QGCTileCache::cacheTile(const QString& type, int x, int y, int z, const QByteArray& image, const QString& format, + qulonglong set, const QByteArray& etag, const QByteArray& lastModified, qint64 expiresAt, + bool mustRevalidate) +{ + const QString hash = UrlFactory::getTileHash(type, x, y, z); + cacheTile(type, hash, image, format, set, etag, lastModified, expiresAt, mustRevalidate); +} + +void QGCTileCache::cacheTile(const QString& type, const QString& hash, const QByteArray& image, const QString& format, + qulonglong set, const QByteArray& etag, const QByteArray& lastModified, qint64 expiresAt, + bool mustRevalidate) +{ + AppSettings* appSettings = SettingsManager::instance()->appSettings(); + if (!appSettings->disableAllPersistence()->rawValue().toBool()) { + QGCCacheTile* tile = new QGCCacheTile(hash, image, format, type, set); + tile->etag = etag; + tile->lastModified = lastModified; + tile->expiresAt = expiresAt; + tile->mustRevalidate = mustRevalidate; + QGCSaveTileTask* task = new QGCSaveTileTask(tile); + if (!getQGCMapEngine()->addTask(task)) { + task->deleteLater(); + } + } +} + +void QGCTileCache::refreshTileValidators(const QString& type, const QString& hash, const QByteArray& etag, + const QByteArray& lastModified, qint64 expiresAt) +{ + Q_UNUSED(type); + AppSettings* appSettings = SettingsManager::instance()->appSettings(); + if (appSettings->disableAllPersistence()->rawValue().toBool()) { + return; + } + QGCCommandTask* task = new QGCCommandTask( + QGCMapTask::TaskType::taskRefreshTileValidators, [=](QGCCacheWorker& worker, QGCMapTask& self) { + if (!worker.validateDatabase(&self)) { + return false; + } + if (!worker.database()->refreshTileValidators(hash, etag, lastModified, expiresAt)) { + self.setError("Error refreshing tile validators"); + return false; + } + return true; + }); + if (!getQGCMapEngine()->addTask(task)) { + task->deleteLater(); + } +} + +QGCFetchTileTask* QGCTileCache::createFetchTileTask(const QString& type, int x, int y, int z) +{ + const QString hash = UrlFactory::getTileHash(type, x, y, z); + QGCFetchTileTask* task = new QGCFetchTileTask(hash); + return task; +} + +QString QGCTileCache::getDatabaseFilePath() +{ + return s_databaseFilePath; +} + +QString QGCTileCache::getCachePath() +{ + return s_cachePath; +} diff --git a/src/QtLocationPlugin/QGCTileCache.h b/src/QtLocationPlugin/QGCTileCache.h new file mode 100644 index 000000000000..30b43d90e01e --- /dev/null +++ b/src/QtLocationPlugin/QGCTileCache.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include + +class QGCFetchTileTask; + +// Static facade over the SQLite tile store (QGCTileCacheDatabase via the cache +// worker): cache paths, tile-save tasks and validator refresh. Independent of +// the QtLocation QAbstractGeoTileCache vtable (see QGeoFileTileCacheQGC). +namespace QGCTileCache { +// Initialize cache paths once. Must run before the cache worker thread starts. +void ensureInitialized(); + +quint32 getMaxDiskCacheSetting(); + +void cacheTile(const QString& type, int x, int y, int z, const QByteArray& image, const QString& format, + qulonglong set = UINT64_MAX, const QByteArray& etag = QByteArray(), + const QByteArray& lastModified = QByteArray(), qint64 expiresAt = 0, bool mustRevalidate = false); +void cacheTile(const QString& type, const QString& hash, const QByteArray& image, const QString& format, + qulonglong set = UINT64_MAX, const QByteArray& etag = QByteArray(), + const QByteArray& lastModified = QByteArray(), qint64 expiresAt = 0, bool mustRevalidate = false); + +QGCFetchTileTask* createFetchTileTask(const QString& type, int x, int y, int z); + +// Refresh cache-validation metadata for an already-cached tile (e.g. after a +// 304 Not Modified) without rewriting the blob. +void refreshTileValidators(const QString& type, const QString& hash, const QByteArray& etag, + const QByteArray& lastModified, qint64 expiresAt); + +QString getDatabaseFilePath(); +QString getCachePath(); +} // namespace QGCTileCache diff --git a/src/QtLocationPlugin/QGCTileCacheDatabase.cpp b/src/QtLocationPlugin/QGCTileCacheDatabase.cpp index 32dc88064d18..804955e12353 100644 --- a/src/QtLocationPlugin/QGCTileCacheDatabase.cpp +++ b/src/QtLocationPlugin/QGCTileCacheDatabase.cpp @@ -4,12 +4,12 @@ #include #include #include +#include #include #include #include #include #include - #include #include "QGCCacheTile.h" @@ -17,17 +17,17 @@ #include "QGCMapUrlEngine.h" #include "QGCSqlHelper.h" #include "QGCTile.h" +#include "QGCTileDatabaseSchema.h" #include "QGCTileSet.h" QGC_LOGGING_CATEGORY(QGCTileCacheDatabaseLog, "QtLocationPlugin.QGCTileCacheDatabase") static std::atomic s_connectionCounter{0}; -QGCTileCacheDatabase::QGCTileCacheDatabase(const QString &databasePath) - : _databasePath(databasePath) - , _connectionName(QStringLiteral("QGCTileCache_%1").arg(s_connectionCounter.fetch_add(1))) -{ -} +QGCTileCacheDatabase::QGCTileCacheDatabase(const QString& databasePath) + : _databasePath(databasePath), + _connectionName(QStringLiteral("QGCTileCache_%1").arg(s_connectionCounter.fetch_add(1))) +{} QGCTileCacheDatabase::~QGCTileCacheDatabase() { @@ -53,59 +53,22 @@ bool QGCTileCacheDatabase::_ensureConnected() const return true; } -bool QGCTileCacheDatabase::_checkSchemaVersion() -{ - QSqlDatabase db = _database(); - const auto current = QGCSqlHelper::userVersion(db); - if (!current) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to read schema version"; - return false; - } - - const int version = *current; - if (version == kSchemaVersion) { - return true; - } - - QSqlQuery query(db); - - if (version == 0) { - // Either a fresh database or a legacy database created before versioning. - // Check for existing data — if Tiles table exists with rows, it's legacy. - // Legacy DBs stored map type as text; migration is not supported so the cache is rebuilt. - if (query.exec("SELECT COUNT(*) FROM Tiles") && query.next() && query.value(0).toInt() > 0) { - qCWarning(QGCTileCacheDatabaseLog) << "Legacy database detected (no schema version). Discarding cached tiles and rebuilding."; - _defaultSet = kInvalidTileSet; - query.exec("DROP TABLE IF EXISTS TilesDownload"); - query.exec("DROP TABLE IF EXISTS SetTiles"); - query.exec("DROP TABLE IF EXISTS Tiles"); - query.exec("DROP TABLE IF EXISTS TileSets"); - } - return true; - } - - // Future: handle incremental migrations here (version < kSchemaVersion). - qCWarning(QGCTileCacheDatabaseLog) << "Unknown schema version" << version << "(expected" << kSchemaVersion << "). Resetting cache."; - _defaultSet = kInvalidTileSet; - query.exec("DROP TABLE IF EXISTS TilesDownload"); - query.exec("DROP TABLE IF EXISTS SetTiles"); - query.exec("DROP TABLE IF EXISTS Tiles"); - query.exec("DROP TABLE IF EXISTS TileSets"); - return true; -} - -bool QGCTileCacheDatabase::init() +bool QGCTileCacheDatabase::init(bool keepConnected) { _failed = false; if (!_databasePath.isEmpty()) { qCDebug(QGCTileCacheDatabaseLog) << "Mapping cache directory:" << _databasePath; if (connectDB()) { - if (!_checkSchemaVersion()) { + bool didReset = false; + if (!QGCTileDatabaseSchema::checkSchemaVersion(_database(), &didReset)) { _failed = true; disconnectDB(); return false; } - _valid = _createDB(_database()); + if (didReset) { + _defaultSet = kInvalidTileSet; + } + _valid = QGCTileDatabaseSchema::createSchema(_database()); if (!_valid) { _failed = true; (void) QFile::remove(_databasePath); @@ -113,7 +76,11 @@ bool QGCTileCacheDatabase::init() } else { _failed = true; } - disconnectDB(); + // keepConnected lets the import path reuse the connection init() opened + // instead of disconnecting and immediately reconnecting. + if (!keepConnected || _failed) { + disconnectDB(); + } } else { qCCritical(QGCTileCacheDatabaseLog) << "Could not find suitable cache directory."; _failed = true; @@ -130,9 +97,14 @@ bool QGCTileCacheDatabase::connectDB() QSqlDatabase db = QSqlDatabase::addDatabase("QSQLITE", _connectionName); db.setDatabaseName(_databasePath); + // Block (up to 5s) rather than fail with SQLITE_BUSY when the WAL writer lock is + // contended across the worker thread and other tile-cache connections. + db.setConnectOptions(QStringLiteral("QSQLITE_BUSY_TIMEOUT=5000")); _valid = db.open(); if (_valid) { - QGCSqlHelper::applySqlitePragmas(db); + // The tile cache is large and read-heavy — opt into a 256 MB mmap window, an + // 8 KiB page size (better BLOB locality) and a 64 MiB page cache for bulk import. + QGCSqlHelper::applySqlitePragmas(db, 256LL * 1024 * 1024, -65536, 8192); _connected = true; } else { qCCritical(QGCTileCacheDatabaseLog) << "Map Cache SQL error (open db):" << db.lastError(); @@ -155,13 +127,21 @@ void QGCTileCacheDatabase::disconnectDB() { QSqlDatabase db = QSqlDatabase::database(_connectionName, false); if (db.isOpen()) { + QGCSqlHelper::runOptimize(db); db.close(); } } QSqlDatabase::removeDatabase(_connectionName); } -bool QGCTileCacheDatabase::saveTile(const QString &hash, const QString &format, const QByteArray &img, const QString &type, quint64 tileSet) +static QVariant _validatorOrNull(const QByteArray& value) +{ + return value.isEmpty() ? QVariant(QMetaType(QMetaType::QString)) : QVariant(QString::fromLatin1(value)); +} + +bool QGCTileCacheDatabase::saveTile(const QString& hash, const QString& format, const QByteArray& img, + const QString& type, quint64 tileSet, const QByteArray& etag, + const QByteArray& lastModified, qint64 expiresAt, bool mustRevalidate) { if (!_ensureConnected()) { return false; @@ -173,28 +153,19 @@ bool QGCTileCacheDatabase::saveTile(const QString &hash, const QString &format, return false; } + const qint64 nowSecs = QDateTime::currentSecsSinceEpoch(); QSqlQuery query(_database()); - if (!query.prepare("INSERT OR IGNORE INTO Tiles(hash, format, tile, size, type, date) VALUES(?, ?, ?, ?, ?, ?)")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare saveTile):" << query.lastError().text(); - return false; - } - query.addBindValue(hash); - query.addBindValue(format); - query.addBindValue(img); - query.addBindValue(img.size()); - query.addBindValue(UrlFactory::getQtMapIdFromProviderType(type)); - query.addBindValue(QDateTime::currentSecsSinceEpoch()); - if (!query.exec()) { + if (!QGCSqlHelper::execPrepared( + query, + "INSERT OR IGNORE INTO Tiles(hash, format, tile, size, typeStr, date, etag, lastModified, expiresAt, " + "accessed, mustRevalidate) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + hash, format, img, img.size(), type, nowSecs, _validatorOrNull(etag), _validatorOrNull(lastModified), + expiresAt, QDateTime::currentMSecsSinceEpoch(), mustRevalidate ? 1 : 0)) { qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (saveTile INSERT):" << query.lastError().text(); return false; } - if (!query.prepare("SELECT tileID FROM Tiles WHERE hash = ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare tile lookup):" << query.lastError().text(); - return false; - } - query.addBindValue(hash); - if (!query.exec() || !query.next()) { + if (!QGCSqlHelper::execPrepared(query, "SELECT tileID FROM Tiles WHERE hash = ?", hash) || !query.next()) { qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (tile lookup):" << query.lastError().text(); return false; } @@ -205,14 +176,10 @@ bool QGCTileCacheDatabase::saveTile(const QString &hash, const QString &format, qCWarning(QGCTileCacheDatabaseLog) << "Cannot save tile: no valid tile set"; return false; } - if (!query.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare SetTiles):" << query.lastError().text(); - return false; - } - query.addBindValue(tileID); - query.addBindValue(setID); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (add tile into SetTiles):" << query.lastError().text(); + if (!QGCSqlHelper::execPrepared(query, "INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)", tileID, + setID)) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (add tile into SetTiles):" << query.lastError().text(); return false; } @@ -225,47 +192,185 @@ bool QGCTileCacheDatabase::saveTile(const QString &hash, const QString &format, return true; } -std::unique_ptr QGCTileCacheDatabase::getTile(const QString &hash) +bool QGCTileCacheDatabase::saveTileBatch(const QList& tiles) { + if (tiles.isEmpty()) { + return true; + } + if (!_ensureConnected()) { - return nullptr; + return false; + } + + QGCSqlHelper::Transaction txn(_database()); + if (!txn.ok()) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to start transaction for saveTileBatch"; + return false; + } + + QSqlQuery insertQuery(_database()); + if (!insertQuery.prepare("INSERT OR IGNORE INTO Tiles(hash, format, tile, size, typeStr, date, etag, lastModified, " + "expiresAt, accessed, mustRevalidate) VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (prepare saveTileBatch insert):" << insertQuery.lastError().text(); + return false; + } + + QSqlQuery lookupQuery(_database()); + if (!lookupQuery.prepare("SELECT tileID FROM Tiles WHERE hash = ?")) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (prepare saveTileBatch lookup):" << lookupQuery.lastError().text(); + return false; + } + + QSqlQuery linkQuery(_database()); + if (!linkQuery.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (prepare saveTileBatch link):" << linkQuery.lastError().text(); + return false; + } + + const quint64 defaultSetID = _getDefaultTileSet(); + const qint64 now = QDateTime::currentSecsSinceEpoch(); + const qint64 nowMs = QDateTime::currentMSecsSinceEpoch(); + + for (const QGCCacheTile* tile : tiles) { + if (!tile) { + continue; + } + + insertQuery.addBindValue(tile->hash); + insertQuery.addBindValue(tile->format); + insertQuery.addBindValue(tile->img); + insertQuery.addBindValue(tile->img.size()); + insertQuery.addBindValue(tile->type); + insertQuery.addBindValue(now); + insertQuery.addBindValue(_validatorOrNull(tile->etag)); + insertQuery.addBindValue(_validatorOrNull(tile->lastModified)); + insertQuery.addBindValue(tile->expiresAt); + insertQuery.addBindValue(nowMs); + insertQuery.addBindValue(tile->mustRevalidate ? 1 : 0); + if (!insertQuery.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (saveTileBatch INSERT):" << insertQuery.lastError().text(); + return false; + } + + quint64 tileID = 0; + if (insertQuery.numRowsAffected() > 0) { + tileID = insertQuery.lastInsertId().toULongLong(); + } else { + lookupQuery.addBindValue(tile->hash); + if (!lookupQuery.exec() || !lookupQuery.next()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (saveTileBatch lookup):" << lookupQuery.lastError().text(); + return false; + } + tileID = lookupQuery.value(0).toULongLong(); + lookupQuery.finish(); + } + + const quint64 setID = (tile->tileSet == kInvalidTileSet) ? defaultSetID : tile->tileSet; + if (setID == kInvalidTileSet) { + qCWarning(QGCTileCacheDatabaseLog) << "Cannot save tile: no valid tile set"; + return false; + } + + linkQuery.addBindValue(tileID); + linkQuery.addBindValue(setID); + if (!linkQuery.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (saveTileBatch link):" << linkQuery.lastError().text(); + return false; + } + } + + if (!txn.commit()) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to commit saveTileBatch transaction"; + return false; + } + + return true; +} + +bool QGCTileCacheDatabase::refreshTileValidators(const QString& hash, const QByteArray& etag, + const QByteArray& lastModified, qint64 expiresAt) +{ + if (!_ensureConnected()) { + return false; } QSqlQuery query(_database()); - if (!query.prepare("SELECT tile, format, type FROM Tiles WHERE hash = ?")) { + if (!QGCSqlHelper::execPrepared( + query, "UPDATE Tiles SET etag = ?, lastModified = ?, expiresAt = ?, date = ? WHERE hash = ?", + _validatorOrNull(etag), _validatorOrNull(lastModified), expiresAt, QDateTime::currentSecsSinceEpoch(), + hash)) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (refreshTileValidators):" << query.lastError().text(); + return false; + } + + return true; +} + +std::unique_ptr QGCTileCacheDatabase::getTile(const QString& hash) +{ + if (!_ensureConnected()) { return nullptr; } - query.addBindValue(hash); - if (query.exec() && query.next()) { + + QSqlQuery query(_database()); + if (QGCSqlHelper::execPrepared( + query, + "SELECT tile, format, typeStr, etag, lastModified, expiresAt, mustRevalidate FROM Tiles WHERE hash = ?", + hash) && + query.next()) { const QByteArray tileData = query.value(0).toByteArray(); const QString format = query.value(1).toString(); - const QString type = UrlFactory::getProviderTypeFromQtMapId(query.value(2).toInt()); + const QString type = query.value(2).toString(); + auto tile = std::make_unique(hash, tileData, format, type); + tile->etag = query.value(3).toString().toLatin1(); + tile->lastModified = query.value(4).toString().toLatin1(); + tile->expiresAt = query.value(5).toLongLong(); + tile->mustRevalidate = (query.value(6).toInt() != 0); qCDebug(QGCTileCacheDatabaseLog) << "(Found in DB) HASH:" << hash; - return std::make_unique(hash, tileData, format, type); + return tile; } qCDebug(QGCTileCacheDatabaseLog) << "(NOT in DB) HASH:" << hash; return nullptr; } -std::optional QGCTileCacheDatabase::findTile(const QString &hash) +std::optional QGCTileCacheDatabase::findTile(const QString& hash) { if (!_ensureConnected()) { return std::nullopt; } QSqlQuery query(_database()); - if (!query.prepare("SELECT tileID FROM Tiles WHERE hash = ?")) { - return std::nullopt; - } - query.addBindValue(hash); - if (query.exec() && query.next()) { + if (QGCSqlHelper::execPrepared(query, "SELECT tileID FROM Tiles WHERE hash = ?", hash) && query.next()) { return query.value(0).toULongLong(); } return std::nullopt; } +bool QGCTileCacheDatabase::bumpTileAccessed(const QString& hash) +{ + if (!_ensureConnected()) { + return false; + } + + QSqlQuery query(_database()); + if (!QGCSqlHelper::execPrepared(query, "UPDATE Tiles SET accessed = ? WHERE hash = ?", + QDateTime::currentMSecsSinceEpoch(), hash)) { + qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (bumpTileAccessed):" << query.lastError().text(); + return false; + } + + return true; +} + QList QGCTileCacheDatabase::getTileSets() { QList records; @@ -276,8 +381,8 @@ QList QGCTileCacheDatabase::getTileSets() QSqlQuery query(_database()); query.setForwardOnly(true); if (!query.exec("SELECT setID, name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, " - "minZoom, maxZoom, type, numTiles, defaultSet, date " - "FROM TileSets ORDER BY defaultSet DESC, name ASC")) { + "minZoom, maxZoom, numTiles, defaultSet, date " + "FROM TileSets ORDER BY defaultSet DESC, name ASC")) { return records; } @@ -292,20 +397,40 @@ QList QGCTileCacheDatabase::getTileSets() rec.bottomRightLon = query.value(6).toDouble(); rec.minZoom = query.value(7).toInt(); rec.maxZoom = query.value(8).toInt(); - rec.type = query.value(9).toInt(); - rec.numTiles = query.value(10).toUInt(); - rec.defaultSet = (query.value(11).toInt() != 0); - rec.date = query.value(12).toULongLong(); + rec.numTiles = query.value(9).toUInt(); + rec.defaultSet = (query.value(10).toInt() != 0); + rec.date = query.value(11).toULongLong(); records.append(rec); } return records; } -std::optional QGCTileCacheDatabase::createTileSet(const QString &name, const QString &mapTypeStr, - double topleftLat, double topleftLon, - double bottomRightLat, double bottomRightLon, - int minZoom, int maxZoom, const QString &type, quint32 numTiles) +std::optional QGCTileCacheDatabase::createImportedTileSet(const QString& name, const QString& type, + int minZoom, int maxZoom) +{ + if (!_ensureConnected()) { + return std::nullopt; + } + + const QString uniqueName = _deduplicateSetName(name); + + QSqlQuery query(_database()); + if (!QGCSqlHelper::execPrepared( + query, "INSERT INTO TileSets(name, typeStr, minZoom, maxZoom, numTiles, date) VALUES(?, ?, ?, ?, ?, ?)", + uniqueName, type, minZoom, maxZoom, 0, QDateTime::currentSecsSinceEpoch())) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (createImportedTileSet):" << query.lastError().text(); + return std::nullopt; + } + + return query.lastInsertId().toULongLong(); +} + +std::optional QGCTileCacheDatabase::createTileSet(const QString& name, const QString& mapTypeStr, + double topleftLat, double topleftLon, double bottomRightLat, + double bottomRightLon, int minZoom, int maxZoom, + const QString& type, quint32 numTiles) { if (!_ensureConnected()) { return std::nullopt; @@ -318,25 +443,15 @@ std::optional QGCTileCacheDatabase::createTileSet(const QString &name, } QSqlQuery query(_database()); - if (!query.prepare("INSERT INTO TileSets(" - "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, maxZoom, type, numTiles, date" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare createTileSet):" << query.lastError().text(); - return std::nullopt; - } - query.addBindValue(name); - query.addBindValue(mapTypeStr); - query.addBindValue(topleftLat); - query.addBindValue(topleftLon); - query.addBindValue(bottomRightLat); - query.addBindValue(bottomRightLon); - query.addBindValue(minZoom); - query.addBindValue(maxZoom); - query.addBindValue(UrlFactory::getQtMapIdFromProviderType(type)); - query.addBindValue(numTiles); - query.addBindValue(QDateTime::currentSecsSinceEpoch()); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (add tileSet into TileSets):" << query.lastError().text(); + if (!QGCSqlHelper::execPrepared(query, + "INSERT INTO TileSets(" + "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, " + "maxZoom, numTiles, date" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + name, mapTypeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, + maxZoom, numTiles, QDateTime::currentSecsSinceEpoch())) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (add tileSet into TileSets):" << query.lastError().text(); return std::nullopt; } @@ -344,50 +459,66 @@ std::optional QGCTileCacheDatabase::createTileSet(const QString &name, // Process tiles in streaming batches to avoid holding all coordinates in memory constexpr int kHashBatchSize = 500; - const int mapTypeId = UrlFactory::getQtMapIdFromProviderType(type); - struct TileCoord { int x, y; QString hash; }; + struct TileCoord + { + int x, y; + QString hash; + }; + + QSqlQuery insertSetTile(_database()); + if (!insertSetTile.prepare(QStringLiteral("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)"))) { + return std::nullopt; + } + QSqlQuery insertDownload(_database()); + if (!insertDownload.prepare(QStringLiteral( + "INSERT OR IGNORE INTO TilesDownload(setID, hash, typeStr, x, y, z, state) VALUES(?, ?, ?, ?, ?, ?, ?)"))) { + return std::nullopt; + } - auto processBatch = [&](const QList &tiles, int z) -> bool { + auto processBatch = [&](const QList& tiles, int z) -> bool { QHash existingTiles; QSqlQuery lookup(_database()); lookup.setForwardOnly(true); - if (lookup.prepare(QStringLiteral("SELECT hash, tileID FROM Tiles WHERE hash IN (%1)").arg(QGCSqlHelper::placeholders(tiles.size())))) { - for (const auto &tc : tiles) { - lookup.addBindValue(tc.hash); - } - if (lookup.exec()) { - while (lookup.next()) { - existingTiles.insert(lookup.value(0).toString(), lookup.value(1).toULongLong()); - } - } + if (!lookup.prepare(QStringLiteral("SELECT hash, tileID FROM Tiles WHERE hash IN (%1)") + .arg(QGCSqlHelper::placeholders(tiles.size())))) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (prepare existing-tile lookup):" << lookup.lastError().text(); + return false; + } + for (const auto& tc : tiles) { + lookup.addBindValue(tc.hash); + } + if (!lookup.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (exec existing-tile lookup):" << lookup.lastError().text(); + return false; + } + while (lookup.next()) { + existingTiles.insert(lookup.value(0).toString(), lookup.value(1).toULongLong()); } - for (const auto &tc : tiles) { + for (const auto& tc : tiles) { auto it = existingTiles.find(tc.hash); if (it != existingTiles.end()) { - if (!query.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { - return false; - } - query.addBindValue(it.value()); - query.addBindValue(setID); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (add tile into SetTiles):" << query.lastError().text(); + insertSetTile.bindValue(0, it.value()); + insertSetTile.bindValue(1, setID); + if (!insertSetTile.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (add tile into SetTiles):" << insertSetTile.lastError().text(); return false; } } else { - if (!query.prepare("INSERT OR IGNORE INTO TilesDownload(setID, hash, type, x, y, z, state) VALUES(?, ?, ?, ?, ?, ?, ?)")) { - return false; - } - query.addBindValue(setID); - query.addBindValue(tc.hash); - query.addBindValue(mapTypeId); - query.addBindValue(tc.x); - query.addBindValue(tc.y); - query.addBindValue(z); - query.addBindValue(static_cast(QGCTile::StatePending)); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (add tile into TilesDownload):" << query.lastError().text(); + insertDownload.bindValue(0, setID); + insertDownload.bindValue(1, tc.hash); + insertDownload.bindValue(2, type); + insertDownload.bindValue(3, tc.x); + insertDownload.bindValue(4, tc.y); + insertDownload.bindValue(5, z); + insertDownload.bindValue(6, static_cast(QGCTile::StatePending)); + if (!insertDownload.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (add tile into TilesDownload):" << insertDownload.lastError().text(); return false; } } @@ -396,7 +527,8 @@ std::optional QGCTileCacheDatabase::createTileSet(const QString &name, }; for (int z = minZoom; z <= maxZoom; z++) { - const QGCTileSet set = UrlFactory::getTileCount(z, topleftLon, topleftLat, bottomRightLon, bottomRightLat, type); + const QGCTileSet set = + UrlFactory::getTileCount(z, topleftLon, topleftLat, bottomRightLon, bottomRightLat, type); QList batch; batch.reserve(kHashBatchSize); @@ -406,14 +538,16 @@ std::optional QGCTileCacheDatabase::createTileSet(const QString &name, batch.append({x, y, UrlFactory::getTileHash(type, x, y, z)}); if (batch.size() >= kHashBatchSize) { - if (!processBatch(batch, z)) return std::nullopt; + if (!processBatch(batch, z)) + return std::nullopt; batch.clear(); } } } if (!batch.isEmpty()) { - if (!processBatch(batch, z)) return std::nullopt; + if (!processBatch(batch, z)) + return std::nullopt; } } @@ -440,40 +574,31 @@ bool QGCTileCacheDatabase::deleteTileSet(quint64 id) QSqlQuery query(_database()); // Delete download queue entries first - if (!query.prepare("DELETE FROM TilesDownload WHERE setID = ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare download delete:" << query.lastError().text(); - return false; - } - query.addBindValue(id); - if (!query.exec()) { + if (!QGCSqlHelper::execPrepared(query, "DELETE FROM TilesDownload WHERE setID = ?", id)) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to delete download queue:" << query.lastError().text(); return false; } // Find tiles unique to this set (not shared with other sets) // Must collect IDs before deleting SetTiles links QList uniqueTileIDs; - if (query.prepare(QStringLiteral("SELECT tileID FROM SetTiles WHERE tileID IN (%1)").arg(kUniqueTilesSubquery))) { - query.addBindValue(id); - if (query.exec()) { - while (query.next()) { - uniqueTileIDs.append(query.value(0).toULongLong()); - } + if (QGCSqlHelper::execPrepared( + query, QStringLiteral("SELECT tileID FROM SetTiles WHERE tileID IN (%1)").arg(kUniqueTilesSubquery), id)) { + while (query.next()) { + uniqueTileIDs.append(query.value(0).toULongLong()); } } // Remove set-tile links - if (!query.prepare("DELETE FROM SetTiles WHERE setID = ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare SetTiles delete:" << query.lastError().text(); - return false; - } - query.addBindValue(id); - if (!query.exec()) { + if (!QGCSqlHelper::execPrepared(query, "DELETE FROM SetTiles WHERE setID = ?", id)) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to delete SetTiles links:" << query.lastError().text(); return false; } // Delete unique tiles (no longer referenced by any set) if (!uniqueTileIDs.isEmpty()) { - if (query.prepare(QStringLiteral("DELETE FROM Tiles WHERE tileID IN (%1)").arg(QGCSqlHelper::placeholders(uniqueTileIDs.size())))) { + if (query.prepare(QStringLiteral("DELETE FROM Tiles WHERE tileID IN (%1)") + .arg(QGCSqlHelper::placeholders(uniqueTileIDs.size())))) { for (const quint64 tileID : uniqueTileIDs) { query.addBindValue(tileID); } @@ -485,12 +610,8 @@ bool QGCTileCacheDatabase::deleteTileSet(quint64 id) } // Delete the tile set itself - if (!query.prepare("DELETE FROM TileSets WHERE setID = ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare TileSets delete:" << query.lastError().text(); - return false; - } - query.addBindValue(id); - if (!query.exec()) { + if (!QGCSqlHelper::execPrepared(query, "DELETE FROM TileSets WHERE setID = ?", id)) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to delete TileSet:" << query.lastError().text(); return false; } @@ -501,33 +622,24 @@ bool QGCTileCacheDatabase::deleteTileSet(quint64 id) return txn.commit(); } -bool QGCTileCacheDatabase::renameTileSet(quint64 setID, const QString &newName) +bool QGCTileCacheDatabase::renameTileSet(quint64 setID, const QString& newName) { if (!_ensureConnected()) { return false; } QSqlQuery query(_database()); - if (!query.prepare("UPDATE TileSets SET name = ? WHERE setID = ?")) { - return false; - } - query.addBindValue(newName); - query.addBindValue(setID); - return query.exec(); + return QGCSqlHelper::execPrepared(query, "UPDATE TileSets SET name = ? WHERE setID = ?", newName, setID); } -std::optional QGCTileCacheDatabase::findTileSetID(const QString &name) +std::optional QGCTileCacheDatabase::findTileSetID(const QString& name) { if (!_ensureConnected()) { return std::nullopt; } QSqlQuery query(_database()); - if (!query.prepare("SELECT setID FROM TileSets WHERE name = ?")) { - return std::nullopt; - } - query.addBindValue(name); - if (query.exec() && query.next()) { + if (QGCSqlHelper::execPrepared(query, "SELECT setID FROM TileSets WHERE name = ?", name) && query.next()) { return query.value(0).toULongLong(); } @@ -542,24 +654,10 @@ bool QGCTileCacheDatabase::resetDatabase() _defaultSet = kInvalidTileSet; - QGCSqlHelper::Transaction txn(_database()); - if (!txn.ok()) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to start transaction for resetDatabase"; - return false; - } - QSqlQuery query(_database()); - if (!query.exec("DROP TABLE IF EXISTS TilesDownload") || - !query.exec("DROP TABLE IF EXISTS SetTiles") || - !query.exec("DROP TABLE IF EXISTS Tiles") || - !query.exec("DROP TABLE IF EXISTS TileSets")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to drop tables:" << query.lastError().text(); - return false; - } - if (!txn.commit()) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to commit table drops in resetDatabase"; + if (!QGCTileDatabaseSchema::dropSchemaTables(_database())) { return false; } - _valid = _createDB(_database()); + _valid = QGCTileDatabaseSchema::createSchema(_database()); return _valid; } @@ -577,21 +675,17 @@ QList QGCTileCacheDatabase::getTileDownloadList(quint64 setID, int coun } QSqlQuery query(_database()); - if (!query.prepare("SELECT hash, type, x, y, z FROM TilesDownload WHERE setID = ? AND state = ? LIMIT ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare tile download list query:" << query.lastError().text(); - return tiles; - } - query.addBindValue(setID); - query.addBindValue(static_cast(QGCTile::StatePending)); - query.addBindValue(count); - if (!query.exec()) { + if (!QGCSqlHelper::execPrepared( + query, "SELECT hash, typeStr, x, y, z FROM TilesDownload WHERE setID = ? AND state = ? LIMIT ?", setID, + static_cast(QGCTile::StatePending), count)) { + qCWarning(QGCTileCacheDatabaseLog) << "Failed to query tile download list:" << query.lastError().text(); return tiles; } while (query.next()) { QGCTile tile; tile.hash = query.value(0).toString(); - tile.type = query.value(1).toInt(); + tile.type = query.value(1).toString(); tile.x = query.value(2).toInt(); tile.y = query.value(3).toInt(); tile.z = query.value(4).toInt(); @@ -599,17 +693,25 @@ QList QGCTileCacheDatabase::getTileDownloadList(quint64 setID, int coun } if (!tiles.isEmpty()) { - if (query.prepare(QStringLiteral("UPDATE TilesDownload SET state = ? WHERE setID = ? AND hash IN (%1)").arg(QGCSqlHelper::placeholders(tiles.size())))) { - query.addBindValue(static_cast(QGCTile::StateDownloading)); - query.addBindValue(setID); - for (qsizetype i = 0; i < tiles.size(); i++) { - query.addBindValue(tiles[i].hash); - } - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (batch set TilesDownload state):" << query.lastError().text(); - tiles.clear(); - return tiles; - } + if (!query.prepare(QStringLiteral("UPDATE TilesDownload SET state = ? WHERE setID = ? AND hash IN (%1)") + .arg(QGCSqlHelper::placeholders(tiles.size())))) { + // Returning the tiles without marking them StateDownloading would re-hand + // them out on the next call (duplicate downloads), so fail closed. + qCWarning(QGCTileCacheDatabaseLog) + << "Failed to prepare TilesDownload state update:" << query.lastError().text(); + tiles.clear(); + return tiles; + } + query.addBindValue(static_cast(QGCTile::StateDownloading)); + query.addBindValue(setID); + for (qsizetype i = 0; i < tiles.size(); i++) { + query.addBindValue(tiles[i].hash); + } + if (!query.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (batch set TilesDownload state):" << query.lastError().text(); + tiles.clear(); + return tiles; } } @@ -621,29 +723,19 @@ QList QGCTileCacheDatabase::getTileDownloadList(quint64 setID, int coun return tiles; } -bool QGCTileCacheDatabase::updateTileDownloadState(quint64 setID, int state, const QString &hash) +bool QGCTileCacheDatabase::updateTileDownloadState(quint64 setID, int state, const QString& hash) { if (!_ensureConnected()) { return false; } QSqlQuery query(_database()); - if (state == QGCTile::StateComplete) { - if (!query.prepare("DELETE FROM TilesDownload WHERE setID = ? AND hash = ?")) { - return false; - } - query.addBindValue(setID); - query.addBindValue(hash); - } else { - if (!query.prepare("UPDATE TilesDownload SET state = ? WHERE setID = ? AND hash = ?")) { - return false; - } - query.addBindValue(state); - query.addBindValue(setID); - query.addBindValue(hash); - } - - if (!query.exec()) { + const bool ok = + (state == QGCTile::StateComplete) + ? QGCSqlHelper::execPrepared(query, "DELETE FROM TilesDownload WHERE setID = ? AND hash = ?", setID, hash) + : QGCSqlHelper::execPrepared(query, "UPDATE TilesDownload SET state = ? WHERE setID = ? AND hash = ?", + state, setID, hash); + if (!ok) { qCWarning(QGCTileCacheDatabaseLog) << "Error:" << query.lastError().text(); return false; } @@ -651,20 +743,19 @@ bool QGCTileCacheDatabase::updateTileDownloadState(quint64 setID, int state, con return true; } -bool QGCTileCacheDatabase::updateAllTileDownloadStates(quint64 setID, int state) +bool QGCTileCacheDatabase::updateAllTileDownloadStates(quint64 setID, int state, int fromState) { if (!_ensureConnected()) { return false; } QSqlQuery query(_database()); - if (!query.prepare("UPDATE TilesDownload SET state = ? WHERE setID = ?")) { - return false; - } - query.addBindValue(state); - query.addBindValue(setID); - - if (!query.exec()) { + const bool ok = + (fromState >= 0) + ? QGCSqlHelper::execPrepared(query, "UPDATE TilesDownload SET state = ? WHERE setID = ? AND state = ?", + state, setID, fromState) + : QGCSqlHelper::execPrepared(query, "UPDATE TilesDownload SET state = ? WHERE setID = ?", state, setID); + if (!ok) { qCWarning(QGCTileCacheDatabaseLog) << "Error:" << query.lastError().text(); return false; } @@ -682,7 +773,10 @@ bool QGCTileCacheDatabase::pruneCache(quint64 amount) while (remaining > 0) { QSqlQuery query(_database()); query.setForwardOnly(true); - if (!query.prepare(QStringLiteral("SELECT tileID, size, hash FROM Tiles WHERE tileID IN (%1) ORDER BY date ASC LIMIT ?").arg(kUniqueTilesSubquery))) { + if (!query.prepare( + QStringLiteral( + "SELECT tileID, size, hash FROM Tiles WHERE tileID IN (%1) ORDER BY accessed ASC LIMIT ?") + .arg(kUniqueTilesSubquery))) { qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare prune query:" << query.lastError().text(); return false; } @@ -718,6 +812,12 @@ bool QGCTileCacheDatabase::pruneCache(quint64 amount) } } + // Reclaim pages freed by the deletes (auto_vacuum=INCREMENTAL DBs only). + QSqlDatabase db = _database(); + if (!QGCSqlHelper::incrementalVacuum(db)) { + qCWarning(QGCTileCacheDatabaseLog) << "Incremental vacuum failed after prune"; + } + return true; } @@ -799,7 +899,8 @@ TotalsResult QGCTileCacheDatabase::computeTotals() result.totalSize = query.value(1).toULongLong(); } - if (!query.prepare(QStringLiteral("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (%1)").arg(kUniqueTilesSubquery))) { + if (!query.prepare(QStringLiteral("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (%1)") + .arg(kUniqueTilesSubquery))) { return result; } query.addBindValue(_getDefaultTileSet()); @@ -811,7 +912,8 @@ TotalsResult QGCTileCacheDatabase::computeTotals() return result; } -SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDefault, quint32 totalTileCount, const QString &type) +SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDefault, quint32 totalTileCount, + const QString& type) { SetTotalsResult result; @@ -830,7 +932,8 @@ SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDef } QSqlQuery subquery(_database()); - if (!subquery.prepare("SELECT COUNT(size), SUM(size) FROM Tiles A INNER JOIN SetTiles B ON A.tileID = B.tileID WHERE B.setID = ?")) { + if (!subquery.prepare("SELECT COUNT(size), SUM(size) FROM Tiles A INNER JOIN SetTiles B ON A.tileID = B.tileID " + "WHERE B.setID = ?")) { return result; } subquery.addBindValue(setID); @@ -856,7 +959,8 @@ SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDef quint32 dbUniqueCount = 0; quint64 dbUniqueSize = 0; - if (subquery.prepare(QStringLiteral("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (%1)").arg(kUniqueTilesSubquery))) { + if (subquery.prepare(QStringLiteral("SELECT COUNT(size), SUM(size) FROM Tiles WHERE tileID IN (%1)") + .arg(kUniqueTilesSubquery))) { subquery.addBindValue(setID); if (subquery.exec() && subquery.next()) { dbUniqueCount = subquery.value(0).toUInt(); @@ -870,7 +974,8 @@ SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDef result.uniqueTileCount = dbUniqueCount; result.uniqueTileSize = dbUniqueSize; } else { - const quint32 estimatedCount = (totalTileCount > result.savedTileCount) ? (totalTileCount - result.savedTileCount) : 0; + const quint32 estimatedCount = + (totalTileCount > result.savedTileCount) ? (totalTileCount - result.savedTileCount) : 0; result.uniqueTileCount = estimatedCount; result.uniqueTileSize = estimatedCount * avg; } @@ -878,7 +983,7 @@ SetTotalsResult QGCTileCacheDatabase::computeSetTotals(quint64 setID, bool isDef return result; } -DatabaseResult QGCTileCacheDatabase::importSetsReplace(const QString &path, ProgressCallback progressCb) +DatabaseResult QGCTileCacheDatabase::importSetsReplace(const QString& path, ProgressCallback progressCb) { DatabaseResult result; if (QFileInfo(path).canonicalFilePath() == QFileInfo(_databasePath).canonicalFilePath()) { @@ -903,23 +1008,23 @@ DatabaseResult QGCTileCacheDatabase::importSetsReplace(const QString &path, Prog return result; } (void) QFile::remove(backupPath); - if (progressCb) progressCb(25); - init(); + if (progressCb) + progressCb(25); + // keepConnected=true leaves the SQLite connection open for the caller; the + // import flow reuses it immediately. The owning worker calls disconnectDB(). + init(/*keepConnected=*/true); if (!_valid) { result.errorString = QStringLiteral("Failed to initialize tile cache database after import"); - } else { - if (progressCb) progressCb(50); - connectDB(); - if (!_valid) { - result.errorString = QStringLiteral("Failed to connect to tile cache database after import"); - } + } else if (progressCb) { + progressCb(50); } - if (progressCb) progressCb(100); + if (progressCb) + progressCb(100); result.success = _valid; return result; } -DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, ProgressCallback progressCb) +DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString& path, ProgressCallback progressCb) { DatabaseResult result; if (QFileInfo(path).canonicalFilePath() == QFileInfo(_databasePath).canonicalFilePath()) { @@ -931,8 +1036,9 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre return result; } - QGCSqlHelper::ScopedConnection importDB(path, /*readOnly=*/true, - QStringLiteral("QGeoTileImportSession")); + // Bulk read of a potentially large tile DB — opt into a 64 MB mmap window. + QGCSqlHelper::ScopedConnection importDB(path, /*readOnly=*/true, QStringLiteral("QGeoTileImportSession"), + 64LL * 1024 * 1024); if (!importDB.isValid()) { result.errorString = "Error opening import database"; return result; @@ -960,7 +1066,6 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre const double bottomRightLon = query.value("bottomRightLon").toDouble(); const int minZoom = query.value("minZoom").toInt(); const int maxZoom = query.value("maxZoom").toInt(); - const int type = query.value("type").toInt(); const quint32 numTiles = query.value("numTiles").toUInt(); const int defaultSet = query.value("defaultSet").toInt(); quint64 insertSetID = _getDefaultTileSet(); @@ -976,8 +1081,9 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre name = _deduplicateSetName(name); QSqlQuery cQuery(_database()); if (!cQuery.prepare("INSERT INTO TileSets(" - "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, maxZoom, type, numTiles, defaultSet, date" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, " + "minZoom, maxZoom, numTiles, defaultSet, date" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { result.errorString = "Error preparing tile set insert"; break; } @@ -989,7 +1095,6 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre cQuery.addBindValue(bottomRightLon); cQuery.addBindValue(minZoom); cQuery.addBindValue(maxZoom); - cQuery.addBindValue(type); cQuery.addBindValue(numTiles); cQuery.addBindValue(defaultSet); cQuery.addBindValue(QDateTime::currentSecsSinceEpoch()); @@ -1001,14 +1106,13 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre } quint64 tilesIterated = 0; - const quint64 tilesSaved = _copyTilesForSet(importDB.database(), setID, insertSetID, - currentCount, tileCount, - lastProgress, progressCb, - &tilesIterated, false); + const quint64 tilesSaved = _copyTilesForSet(importDB.database(), setID, insertSetID, currentCount, + tileCount, lastProgress, progressCb, &tilesIterated, false); if (tilesSaved > 0) { tilesImported = true; QSqlQuery cQuery(_database()); - if (cQuery.prepare("SELECT COUNT(size) FROM Tiles A INNER JOIN SetTiles B ON A.tileID = B.tileID WHERE B.setID = ?")) { + if (cQuery.prepare("SELECT COUNT(size) FROM Tiles A INNER JOIN SetTiles B ON A.tileID = B.tileID " + "WHERE B.setID = ?")) { cQuery.addBindValue(insertSetID); if (cQuery.exec() && cQuery.next()) { const quint64 count = cQuery.value(0).toULongLong(); @@ -1031,7 +1135,7 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre tileCount = (alreadyExisting < tileCount) ? tileCount - alreadyExisting : 0; } - if ((tilesSaved == 0) && (defaultSet == 0)) { + if ((tilesSaved == 0) && (defaultSet == 0) && (insertSetID != _getDefaultTileSet())) { qCDebug(QGCTileCacheDatabaseLog) << "No unique tiles in" << name << "Removing it."; deleteTileSet(insertSetID); } @@ -1048,7 +1152,8 @@ DatabaseResult QGCTileCacheDatabase::importSetsMerge(const QString &path, Progre return result; } -DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets, const QString &path, ProgressCallback progressCb) +DatabaseResult QGCTileCacheDatabase::exportSets(const QList& sets, const QString& path, + ProgressCallback progressCb) { DatabaseResult result; if (!_ensureConnected()) { @@ -1061,15 +1166,15 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets } (void) QFile::remove(path); - QGCSqlHelper::ScopedConnection exportDB(path, /*readOnly=*/false, - QStringLiteral("QGeoTileExportSession")); + QGCSqlHelper::ScopedConnection exportDB(path, /*readOnly=*/false, QStringLiteral("QGeoTileExportSession")); if (!exportDB.isValid()) { - qCCritical(QGCTileCacheDatabaseLog) << "Map Cache SQL error (create export database):" << exportDB.database().lastError(); + qCCritical(QGCTileCacheDatabaseLog) + << "Map Cache SQL error (create export database):" << exportDB.database().lastError(); result.errorString = "Error opening export database"; return result; } - if (!_createDB(exportDB.database(), false)) { + if (!QGCTileDatabaseSchema::createSchema(exportDB.database(), false)) { result.errorString = "Error creating export database"; return result; } @@ -1077,10 +1182,11 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets quint64 tileCount = 0; quint64 currentCount = 0; int lastProgress = -1; - for (const auto &set : sets) { + for (const auto& set : sets) { QSqlQuery countQuery(_database()); quint64 actualCount = 0; - if (countQuery.prepare("SELECT COUNT(*) FROM Tiles T INNER JOIN SetTiles S ON T.tileID = S.tileID WHERE S.setID = ?")) { + if (countQuery.prepare( + "SELECT COUNT(*) FROM Tiles T INNER JOIN SetTiles S ON T.tileID = S.tileID WHERE S.setID = ?")) { countQuery.addBindValue(set.setID); if (countQuery.exec() && countQuery.next()) { actualCount = countQuery.value(0).toULongLong(); @@ -1093,10 +1199,10 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets tileCount = 1; } - for (const auto &set : sets) { + for (const auto& set : sets) { QSqlQuery query(_database()); query.setForwardOnly(true); - if (!query.prepare("SELECT T.hash, T.format, T.tile, T.type, T.date FROM Tiles T " + if (!query.prepare("SELECT T.hash, T.format, T.tile, T.typeStr, T.date FROM Tiles T " "INNER JOIN SetTiles S ON T.tileID = S.tileID WHERE S.setID = ?")) { qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare tile query for export set" << set.name; continue; @@ -1116,8 +1222,9 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets QSqlQuery exportQuery(exportDB.database()); if (!exportQuery.prepare("INSERT INTO TileSets(" - "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, maxZoom, type, numTiles, defaultSet, date" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { + "name, typeStr, topleftLat, topleftLon, bottomRightLat, bottomRightLon, minZoom, " + "maxZoom, numTiles, defaultSet, date" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")) { result.errorString = "Error preparing tile set insert for export"; break; } @@ -1129,7 +1236,6 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets exportQuery.addBindValue(set.bottomRightLon); exportQuery.addBindValue(set.minZoom); exportQuery.addBindValue(set.maxZoom); - exportQuery.addBindValue(set.type); exportQuery.addBindValue(set.numTiles); exportQuery.addBindValue(set.defaultSet); exportQuery.addBindValue(set.date); @@ -1140,52 +1246,61 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets const quint64 exportSetID = exportQuery.lastInsertId().toULongLong(); + // Prepare once per set, bind+exec per tile (re-preparing inside the loop + // recompiled the SQL for every tile). + QSqlQuery tileInsert(exportDB.database()); + QSqlQuery tileLookup(exportDB.database()); + QSqlQuery setTileInsert(exportDB.database()); + if (!tileInsert.prepare( + "INSERT INTO Tiles(hash, format, tile, size, typeStr, date) VALUES(?, ?, ?, ?, ?, ?)") || + !tileLookup.prepare("SELECT tileID FROM Tiles WHERE hash = ?") || + !setTileInsert.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { + qCWarning(QGCTileCacheDatabaseLog) + << "Failed to prepare tile export statements:" << tileInsert.lastError().text(); + result.errorString = "Error preparing tile export statements"; + break; + } + quint64 skippedTiles = 0; while (query.next()) { const QString hash = query.value(0).toString(); const QString format = query.value(1).toString(); const QByteArray img = query.value(2).toByteArray(); - const int tileType = query.value(3).toInt(); + const QString tileType = query.value(3).toString(); const quint64 tileDate = query.value(4).toULongLong(); quint64 exportTileID = 0; - if (!exportQuery.prepare("INSERT INTO Tiles(hash, format, tile, size, type, date) VALUES(?, ?, ?, ?, ?, ?)")) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to prepare tile INSERT for export:" << exportQuery.lastError().text(); - skippedTiles++; - continue; - } - exportQuery.addBindValue(hash); - exportQuery.addBindValue(format); - exportQuery.addBindValue(img); - exportQuery.addBindValue(img.size()); - exportQuery.addBindValue(tileType); - exportQuery.addBindValue(tileDate); - if (exportQuery.exec()) { - exportTileID = exportQuery.lastInsertId().toULongLong(); + tileInsert.addBindValue(hash); + tileInsert.addBindValue(format); + tileInsert.addBindValue(img); + tileInsert.addBindValue(img.size()); + tileInsert.addBindValue(tileType); + tileInsert.addBindValue(tileDate); + if (tileInsert.exec()) { + exportTileID = tileInsert.lastInsertId().toULongLong(); } else { - QSqlQuery lookup(exportDB.database()); - if (lookup.prepare("SELECT tileID FROM Tiles WHERE hash = ?")) { - lookup.addBindValue(hash); - if (lookup.exec() && lookup.next()) { - exportTileID = lookup.value(0).toULongLong(); - } + tileLookup.addBindValue(hash); + if (tileLookup.exec() && tileLookup.next()) { + exportTileID = tileLookup.value(0).toULongLong(); } + tileLookup.finish(); } if (exportTileID > 0) { - if (exportQuery.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { - exportQuery.addBindValue(exportTileID); - exportQuery.addBindValue(exportSetID); - if (!exportQuery.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to link tile to set in export:" << exportQuery.lastError().text(); - } + setTileInsert.addBindValue(exportTileID); + setTileInsert.addBindValue(exportSetID); + if (!setTileInsert.exec()) { + qCWarning(QGCTileCacheDatabaseLog) + << "Failed to link tile to set in export:" << setTileInsert.lastError().text(); } } else { skippedTiles++; } currentCount++; if (progressCb) { - const int progress = qMin(100, static_cast((static_cast(currentCount) / static_cast(tileCount)) * 100.0)); + const int progress = qMin( + 100, + static_cast((static_cast(currentCount) / static_cast(tileCount)) * 100.0)); if (lastProgress != progress) { lastProgress = progress; progressCb(progress); @@ -1204,119 +1319,6 @@ DatabaseResult QGCTileCacheDatabase::exportSets(const QList &sets return result; } -bool QGCTileCacheDatabase::_createDB(QSqlDatabase db, bool createDefault) -{ - // applySqlitePragmas (in connectDB / ScopedConnection ctor) already - // enabled foreign_keys; nothing to redo here. - QSqlQuery query(db); - - if (!query.exec( - "CREATE TABLE IF NOT EXISTS Tiles (" - "tileID INTEGER PRIMARY KEY NOT NULL, " - "hash TEXT NOT NULL UNIQUE, " - "format TEXT NOT NULL, " - "tile BLOB NULL, " - "size INTEGER, " - "type INTEGER, " - "date INTEGER DEFAULT 0)")) - { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (create Tiles db):" << query.lastError().text(); - return false; - } - - if (!query.exec( - "CREATE TABLE IF NOT EXISTS TileSets (" - "setID INTEGER PRIMARY KEY NOT NULL, " - "name TEXT NOT NULL UNIQUE, " - "typeStr TEXT, " - "topleftLat REAL DEFAULT 0.0, " - "topleftLon REAL DEFAULT 0.0, " - "bottomRightLat REAL DEFAULT 0.0, " - "bottomRightLon REAL DEFAULT 0.0, " - "minZoom INTEGER DEFAULT 3, " - "maxZoom INTEGER DEFAULT 3, " - "type INTEGER DEFAULT -1, " - "numTiles INTEGER DEFAULT 0, " - "defaultSet INTEGER DEFAULT 0, " - "date INTEGER DEFAULT 0)")) - { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (create TileSets db):" << query.lastError().text(); - return false; - } - - if (!query.exec( - "CREATE TABLE IF NOT EXISTS SetTiles (" - "setID INTEGER NOT NULL REFERENCES TileSets(setID) ON DELETE CASCADE, " - "tileID INTEGER NOT NULL REFERENCES Tiles(tileID) ON DELETE CASCADE)")) - { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (create SetTiles db):" << query.lastError().text(); - return false; - } - - if (!query.exec( - "CREATE TABLE IF NOT EXISTS TilesDownload (" - "setID INTEGER NOT NULL REFERENCES TileSets(setID) ON DELETE CASCADE, " - "hash TEXT NOT NULL, " - "type INTEGER, " - "x INTEGER, " - "y INTEGER, " - "z INTEGER, " - "state INTEGER DEFAULT 0)")) - { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (create TilesDownload db):" << query.lastError().text(); - return false; - } - - static const char *indexStatements[] = { - "CREATE UNIQUE INDEX IF NOT EXISTS idx_settiles_unique ON SetTiles(tileID, setID)", - "CREATE INDEX IF NOT EXISTS idx_settiles_setid ON SetTiles(setID)", - "CREATE INDEX IF NOT EXISTS idx_settiles_tileid ON SetTiles(tileID)", - "CREATE UNIQUE INDEX IF NOT EXISTS idx_tilesdownload_setid_hash ON TilesDownload(setID, hash)", - "CREATE INDEX IF NOT EXISTS idx_tilesdownload_setid_state ON TilesDownload(setID, state)", - "CREATE INDEX IF NOT EXISTS idx_tiles_date ON Tiles(date)", - }; - for (const char *sql : indexStatements) { - if (!query.exec(QLatin1String(sql))) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to create index:" << sql << query.lastError().text(); - } - } - - if (!QGCSqlHelper::setUserVersion(db, kSchemaVersion)) { - qCWarning(QGCTileCacheDatabaseLog) << "Failed to set schema version"; - } - - if (!createDefault) { - return true; - } - - if (!query.prepare("SELECT name FROM TileSets WHERE name = ?")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare default set check):" << db.lastError(); - return false; - } - query.addBindValue(QStringLiteral("Default Tile Set")); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (Looking for default tile set):" << db.lastError(); - return true; - } - if (query.next()) { - return true; - } - - if (!query.prepare("INSERT INTO TileSets(name, defaultSet, date) VALUES(?, ?, ?)")) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (prepare default tile set):" << db.lastError(); - return false; - } - query.addBindValue(QStringLiteral("Default Tile Set")); - query.addBindValue(1); - query.addBindValue(QDateTime::currentSecsSinceEpoch()); - if (!query.exec()) { - qCWarning(QGCTileCacheDatabaseLog) << "Map Cache SQL error (Creating default tile set):" << db.lastError(); - return false; - } - - return true; -} - quint64 QGCTileCacheDatabase::_getDefaultTileSet() { if (_defaultSet != kInvalidTileSet) { @@ -1337,14 +1339,15 @@ quint64 QGCTileCacheDatabase::_getDefaultTileSet() return kInvalidTileSet; } -bool QGCTileCacheDatabase::_deleteTilesByIDs(const QList &ids) +bool QGCTileCacheDatabase::_deleteTilesByIDs(const QList& ids) { if (ids.isEmpty()) { return true; } QSqlQuery query(_database()); - if (!query.prepare(QStringLiteral("DELETE FROM Tiles WHERE tileID IN (%1)").arg(QGCSqlHelper::placeholders(ids.size())))) { + if (!query.prepare( + QStringLiteral("DELETE FROM Tiles WHERE tileID IN (%1)").arg(QGCSqlHelper::placeholders(ids.size())))) { return false; } for (const quint64 id : ids) { @@ -1353,7 +1356,7 @@ bool QGCTileCacheDatabase::_deleteTilesByIDs(const QList &ids) return query.exec(); } -QString QGCTileCacheDatabase::_deduplicateSetName(const QString &name) +QString QGCTileCacheDatabase::_deduplicateSetName(const QString& name) { if (!findTileSetID(name).has_value()) { return name; @@ -1362,16 +1365,11 @@ QString QGCTileCacheDatabase::_deduplicateSetName(const QString &name) QSet existing; existing.insert(name); QSqlQuery query(_database()); - QString escaped = name; - escaped.replace(QLatin1Char('\\'), QStringLiteral("\\\\")); - escaped.replace(QLatin1Char('%'), QStringLiteral("\\%")); - escaped.replace(QLatin1Char('_'), QStringLiteral("\\_")); - if (query.prepare(QStringLiteral("SELECT name FROM TileSets WHERE name LIKE ? || ' %' ESCAPE '\\'"))) { - query.addBindValue(escaped); - if (query.exec()) { - while (query.next()) { - existing.insert(query.value(0).toString()); - } + const QString escaped = QGCSqlHelper::escapeLikePattern(name); + if (QGCSqlHelper::execPrepared( + query, QStringLiteral("SELECT name FROM TileSets WHERE name LIKE ? || ' %' ESCAPE '\\'"), escaped)) { + while (query.next()) { + existing.insert(query.value(0).toString()); } } @@ -1386,20 +1384,22 @@ QString QGCTileCacheDatabase::_deduplicateSetName(const QString &name) } quint64 QGCTileCacheDatabase::_copyTilesForSet(QSqlDatabase srcDB, quint64 srcSetID, quint64 dstSetID, - quint64 ¤tCount, quint64 tileCount, - int &lastProgress, ProgressCallback progressCb, - quint64 *tilesIteratedOut, bool useTransaction) + quint64& currentCount, quint64 tileCount, int& lastProgress, + ProgressCallback progressCb, quint64* tilesIteratedOut, + bool useTransaction) { QSqlQuery subQuery(srcDB); subQuery.setForwardOnly(true); - if (!subQuery.prepare("SELECT T.hash, T.format, T.tile, T.type, T.date FROM Tiles T " + if (!subQuery.prepare("SELECT T.hash, T.format, T.tile, T.typeStr, T.date FROM Tiles T " "INNER JOIN SetTiles S ON T.tileID = S.tileID WHERE S.setID = ?")) { - if (tilesIteratedOut) *tilesIteratedOut = 0; + if (tilesIteratedOut) + *tilesIteratedOut = 0; return 0; } subQuery.addBindValue(srcSetID); if (!subQuery.exec()) { - if (tilesIteratedOut) *tilesIteratedOut = 0; + if (tilesIteratedOut) + *tilesIteratedOut = 0; return 0; } @@ -1411,53 +1411,63 @@ quint64 QGCTileCacheDatabase::_copyTilesForSet(QSqlDatabase srcDB, quint64 srcSe txn = std::make_unique(_database()); if (!txn->ok()) { qCWarning(QGCTileCacheDatabaseLog) << "Failed to start transaction for merge import"; - if (tilesIteratedOut) *tilesIteratedOut = 0; + if (tilesIteratedOut) + *tilesIteratedOut = 0; return 0; } } - QSqlQuery cQuery(_database()); + // Prepare once, bind+exec per tile (re-preparing inside the loop recompiled + // the SQL for every tile). + QSqlQuery tileInsert(_database()); + QSqlQuery tileLookup(_database()); + QSqlQuery setTileInsert(_database()); + if (!tileInsert.prepare("INSERT INTO Tiles(hash, format, tile, size, typeStr, date) VALUES(?, ?, ?, ?, ?, ?)") || + !tileLookup.prepare("SELECT tileID FROM Tiles WHERE hash = ?") || + !setTileInsert.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { + qCWarning(QGCTileCacheDatabaseLog) + << "Failed to prepare merge import statements:" << tileInsert.lastError().text(); + if (tilesIteratedOut) + *tilesIteratedOut = 0; + return 0; + } while (subQuery.next()) { tilesFound++; const QString hash = subQuery.value(0).toString(); const QString format = subQuery.value(1).toString(); const QByteArray img = subQuery.value(2).toByteArray(); - const int tileType = subQuery.value(3).toInt(); + const QString tileType = subQuery.value(3).toString(); const quint64 tileDate = subQuery.value(4).toULongLong(); quint64 importTileID = 0; - if (cQuery.prepare("INSERT INTO Tiles(hash, format, tile, size, type, date) VALUES(?, ?, ?, ?, ?, ?)")) { - cQuery.addBindValue(hash); - cQuery.addBindValue(format); - cQuery.addBindValue(img); - cQuery.addBindValue(img.size()); - cQuery.addBindValue(tileType); - cQuery.addBindValue(tileDate); - if (cQuery.exec()) { - importTileID = cQuery.lastInsertId().toULongLong(); - } else { - if (cQuery.prepare("SELECT tileID FROM Tiles WHERE hash = ?")) { - cQuery.addBindValue(hash); - if (cQuery.exec() && cQuery.next()) { - importTileID = cQuery.value(0).toULongLong(); - } - } + tileInsert.addBindValue(hash); + tileInsert.addBindValue(format); + tileInsert.addBindValue(img); + tileInsert.addBindValue(img.size()); + tileInsert.addBindValue(tileType); + tileInsert.addBindValue(tileDate); + if (tileInsert.exec()) { + importTileID = tileInsert.lastInsertId().toULongLong(); + } else { + tileLookup.addBindValue(hash); + if (tileLookup.exec() && tileLookup.next()) { + importTileID = tileLookup.value(0).toULongLong(); } + tileLookup.finish(); } if (importTileID > 0) { - if (cQuery.prepare("INSERT OR IGNORE INTO SetTiles(tileID, setID) VALUES(?, ?)")) { - cQuery.addBindValue(importTileID); - cQuery.addBindValue(dstSetID); - if (cQuery.exec() && cQuery.numRowsAffected() > 0) { - tilesLinked++; - } + setTileInsert.addBindValue(importTileID); + setTileInsert.addBindValue(dstSetID); + if (setTileInsert.exec() && setTileInsert.numRowsAffected() > 0) { + tilesLinked++; } } currentCount++; if (tileCount > 0 && progressCb) { - const int progress = qMin(100, static_cast((static_cast(currentCount) / static_cast(tileCount)) * 100.0)); + const int progress = qMin( + 100, static_cast((static_cast(currentCount) / static_cast(tileCount)) * 100.0)); if (lastProgress != progress) { lastProgress = progress; progressCb(progress); @@ -1467,10 +1477,12 @@ quint64 QGCTileCacheDatabase::_copyTilesForSet(QSqlDatabase srcDB, quint64 srcSe if (txn && !txn->commit()) { qCWarning(QGCTileCacheDatabaseLog) << "Failed to commit merge import transaction"; - if (tilesIteratedOut) *tilesIteratedOut = tilesFound; + if (tilesIteratedOut) + *tilesIteratedOut = tilesFound; return 0; } - if (tilesIteratedOut) *tilesIteratedOut = tilesFound; + if (tilesIteratedOut) + *tilesIteratedOut = tilesFound; return tilesLinked; } diff --git a/src/QtLocationPlugin/QGCTileCacheDatabase.h b/src/QtLocationPlugin/QGCTileCacheDatabase.h index f1c6b588cd05..1ea803bea1ab 100644 --- a/src/QtLocationPlugin/QGCTileCacheDatabase.h +++ b/src/QtLocationPlugin/QGCTileCacheDatabase.h @@ -3,12 +3,11 @@ #include #include #include - #include #include -#include "QGCTileCacheTypes.h" #include "QGCTile.h" +#include "QGCTileCacheTypes.h" struct QGCCacheTile; class QSqlDatabase; @@ -17,38 +16,52 @@ class QGCTileCacheDatabase { public: static constexpr quint64 kInvalidTileSet = UINT64_MAX; - static constexpr int kSchemaVersion = 1; + static constexpr int kSchemaVersion = 5; - explicit QGCTileCacheDatabase(const QString &databasePath); + explicit QGCTileCacheDatabase(const QString& databasePath); ~QGCTileCacheDatabase(); - bool init(); + bool init(bool keepConnected = false); bool connectDB(); void disconnectDB(); bool isValid() const { return _valid; } + bool hasFailed() const { return _failed; } // Tiles - bool saveTile(const QString &hash, const QString &format, const QByteArray &img, const QString &type, quint64 tileSet); - std::unique_ptr getTile(const QString &hash); - std::optional findTile(const QString &hash); + bool saveTile(const QString& hash, const QString& format, const QByteArray& img, const QString& type, + quint64 tileSet, const QByteArray& etag = QByteArray(), const QByteArray& lastModified = QByteArray(), + qint64 expiresAt = 0, bool mustRevalidate = false); + // Refresh only the validation metadata for an existing tile after a 304 Not + // Modified, without rewriting the (unchanged) blob. + bool refreshTileValidators(const QString& hash, const QByteArray& etag, const QByteArray& lastModified, + qint64 expiresAt); + bool saveTileBatch(const QList& tiles); + std::unique_ptr getTile(const QString& hash); + std::optional findTile(const QString& hash); + // Updates the tile's LRU access timestamp (epoch ms) so pruneCache evicts genuinely cold tiles. + bool bumpTileAccessed(const QString& hash); // Tile Sets QList getTileSets(); - std::optional createTileSet(const QString &name, const QString &mapTypeStr, - double topleftLat, double topleftLon, - double bottomRightLat, double bottomRightLon, - int minZoom, int maxZoom, const QString &type, quint32 numTiles); + std::optional createTileSet(const QString& name, const QString& mapTypeStr, double topleftLat, + double topleftLon, double bottomRightLat, double bottomRightLon, int minZoom, + int maxZoom, const QString& type, quint32 numTiles); + // Creates an empty tile-set row only (no TilesDownload queueing) for bulk imports + // (MBTiles/PMTiles); tiles are inserted directly via saveTile against the returned setID. + std::optional createImportedTileSet(const QString& name, const QString& type, int minZoom, int maxZoom); bool deleteTileSet(quint64 id); - bool renameTileSet(quint64 setID, const QString &newName); - std::optional findTileSetID(const QString &name); + bool renameTileSet(quint64 setID, const QString& newName); + std::optional findTileSetID(const QString& name); bool resetDatabase(); // Downloads QList getTileDownloadList(quint64 setID, int count); - bool updateTileDownloadState(quint64 setID, int state, const QString &hash); - bool updateAllTileDownloadStates(quint64 setID, int state); + bool updateTileDownloadState(quint64 setID, int state, const QString& hash); + // When fromState >= 0, only rows currently in that state are transitioned — + // used by pause (Downloading/Pending -> Paused) and resume (Paused -> Pending). + bool updateAllTileDownloadStates(quint64 setID, int state, int fromState = -1); // Cache bool pruneCache(quint64 amount); @@ -56,30 +69,27 @@ class QGCTileCacheDatabase // Stats TotalsResult computeTotals(); - SetTotalsResult computeSetTotals(quint64 setID, bool isDefault, quint32 totalTileCount, const QString &type); + SetTotalsResult computeSetTotals(quint64 setID, bool isDefault, quint32 totalTileCount, const QString& type); // Import/Export - DatabaseResult importSetsReplace(const QString &path, ProgressCallback progressCb); - DatabaseResult importSetsMerge(const QString &path, ProgressCallback progressCb); - DatabaseResult exportSets(const QList &sets, const QString &path, ProgressCallback progressCb); + DatabaseResult importSetsReplace(const QString& path, ProgressCallback progressCb); + DatabaseResult importSetsMerge(const QString& path, ProgressCallback progressCb); + DatabaseResult exportSets(const QList& sets, const QString& path, ProgressCallback progressCb); // Exposed for unit tests only QSqlDatabase database() const; - static constexpr const char *kBingNoTileDoneKey = "_deleteBingNoTileTilesDone"; + static constexpr const char* kBingNoTileDoneKey = "_deleteBingNoTileTilesDone"; private: bool _ensureConnected() const; QSqlDatabase _database() const; - bool _checkSchemaVersion(); - bool _createDB(QSqlDatabase db, bool createDefault = true); quint64 _getDefaultTileSet(); - bool _deleteTilesByIDs(const QList &ids); - QString _deduplicateSetName(const QString &name); - quint64 _copyTilesForSet(QSqlDatabase srcDB, quint64 srcSetID, quint64 dstSetID, - quint64 ¤tCount, quint64 tileCount, - int &lastProgress, ProgressCallback progressCb, - quint64 *tilesIteratedOut, bool useTransaction = true); + bool _deleteTilesByIDs(const QList& ids); + QString _deduplicateSetName(const QString& name); + quint64 _copyTilesForSet(QSqlDatabase srcDB, quint64 srcSetID, quint64 dstSetID, quint64& currentCount, + quint64 tileCount, int& lastProgress, ProgressCallback progressCb, + quint64* tilesIteratedOut, bool useTransaction = true); QString _databasePath; QString _connectionName; @@ -88,7 +98,7 @@ class QGCTileCacheDatabase bool _valid = false; bool _failed = false; static constexpr int kPruneBatchSize = 128; - static constexpr const char *kUniqueTilesSubquery = + static constexpr const char* kUniqueTilesSubquery = "SELECT A.tileID FROM SetTiles A JOIN SetTiles B ON A.tileID = B.tileID " "WHERE B.setID = ? GROUP BY A.tileID HAVING COUNT(A.tileID) = 1"; }; diff --git a/src/QtLocationPlugin/QGCTileCacheTypes.h b/src/QtLocationPlugin/QGCTileCacheTypes.h index e2195404d39a..17357e1370ef 100644 --- a/src/QtLocationPlugin/QGCTileCacheTypes.h +++ b/src/QtLocationPlugin/QGCTileCacheTypes.h @@ -14,7 +14,6 @@ struct TileSetRecord { double bottomRightLon = 0.; int minZoom = 3; int maxZoom = 3; - int type = -1; quint32 numTiles = 0; bool defaultSet = false; quint64 date = 0; diff --git a/src/QtLocationPlugin/QGCTileCacheWorker.cpp b/src/QtLocationPlugin/QGCTileCacheWorker.cpp index 77c6a453b238..97b06fc90aa7 100644 --- a/src/QtLocationPlugin/QGCTileCacheWorker.cpp +++ b/src/QtLocationPlugin/QGCTileCacheWorker.cpp @@ -1,46 +1,50 @@ #include "QGCTileCacheWorker.h" -#include "QGCTileCacheDatabase.h" - -#include -#include #include "QGCCacheTile.h" -#include "QGCCachedTileSet.h" #include "QGCLoggingCategory.h" #include "QGCMapTasks.h" -#include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" QGC_LOGGING_CATEGORY(QGCTileCacheWorkerLog, "QtLocationPlugin.QGCTileCacheWorker") -QGCCacheWorker::QGCCacheWorker(QObject *parent) - : QThread(parent) +QGCCacheWorker::QGCCacheWorker() : QObject(nullptr) { + // No QObject parent: the worker is moved to its own thread, which a parented + // QObject forbids. QGCMapEngine owns and deletes it explicitly. + moveToThread(&_thread); + (void) connect(&_thread, &QThread::started, this, &QGCCacheWorker::_init); + // Cleanup at loop-exit (finished, on the worker thread) — not a queued + // _shutdown, which app teardown's missing event loop would never deliver. + (void) connect(&_thread, &QThread::finished, this, &QGCCacheWorker::_shutdown, Qt::DirectConnection); qCDebug(QGCTileCacheWorkerLog) << this; } QGCCacheWorker::~QGCCacheWorker() { stop(); - wait(); + (void) _thread.wait(); qCDebug(QGCTileCacheWorkerLog) << this; } void QGCCacheWorker::stop() { _stopRequested = true; + + if (_thread.isRunning()) { + // quit() directly, not a queued _shutdown: at app teardown QCoreApplication + // is gone, so a cross-thread posted event never runs and wait() would hang. + _thread.quit(); + return; + } + QMutexLocker lock(&_taskQueueMutex); qDeleteAll(_taskQueue); _taskQueue.clear(); - lock.unlock(); - - if (isRunning()) { - _waitc.wakeAll(); - } } -bool QGCCacheWorker::enqueueTask(QGCMapTask *task) +bool QGCCacheWorker::enqueueTask(QGCMapTask* task) { - if (!_dbValid && !isRunning() && (task->type() != QGCMapTask::TaskType::taskInit)) { + if (!_dbValid && !_thread.isRunning() && (task->type() != QGCMapTask::TaskType::taskInit)) { task->setError(tr("Database Not Initialized")); task->deleteLater(); return false; @@ -50,387 +54,156 @@ bool QGCCacheWorker::enqueueTask(QGCMapTask *task) _taskQueue.enqueue(task); lock.unlock(); - if (isRunning()) { - _waitc.wakeAll(); - } else { - start(QThread::NormalPriority); - } - - return true; -} - -void QGCCacheWorker::run() -{ - _stopRequested = false; - _database = std::make_unique(_databasePath); - - if (!_database->init()) { - qCWarning(QGCTileCacheWorkerLog) << "Failed To Init Database"; - _database.reset(); - - QMutexLocker lock(&_taskQueueMutex); - for (QGCMapTask *orphan : _taskQueue) { - orphan->setError(tr("Database Init Failed")); - orphan->deleteLater(); - } - _taskQueue.clear(); - return; - } - - if (_database->isValid()) { - if (_database->connectDB()) { - _database->deleteBingNoTileTiles(); - } - } - - _dbValid = _database->isValid(); - - _updateTimer.start(); - - QMutexLocker lock(&_taskQueueMutex); - while (!_stopRequested) { - if (!_taskQueue.isEmpty()) { - QGCMapTask* const task = _taskQueue.dequeue(); - lock.unlock(); - _runTask(task); - lock.relock(); - task->deleteLater(); - - const qsizetype count = _taskQueue.count(); - if (count > 100) { - _updateTimeout = kLongTimeoutMs; - } else if (count < 25) { - _updateTimeout = kShortTimeoutMs; - } - - if ((count == 0) || _updateTimer.hasExpired(_updateTimeout)) { - if (_database && _database->isValid()) { - lock.unlock(); - _emitTotals(); - lock.relock(); - } - } - } else { - (void) _waitc.wait(lock.mutex(), 5000); - } + if (!_thread.isRunning()) { + _thread.start(QThread::NormalPriority); } - for (QGCMapTask *orphan : _taskQueue) { - orphan->setError(tr("Worker shutting down")); - orphan->deleteLater(); + if (!_drainScheduled.exchange(true)) { + (void) QMetaObject::invokeMethod(this, &QGCCacheWorker::_drainQueue, Qt::QueuedConnection); } - _taskQueue.clear(); - lock.unlock(); - - _dbValid = false; - if (_database) { - _database->disconnectDB(); - _database.reset(); - } -} -void QGCCacheWorker::_runTask(QGCMapTask *task) -{ - switch (task->type()) { - case QGCMapTask::TaskType::taskInit: - break; - case QGCMapTask::TaskType::taskCacheTile: - _saveTile(task); - break; - case QGCMapTask::TaskType::taskFetchTile: - _getTile(task); - break; - case QGCMapTask::TaskType::taskFetchTileSets: - _getTileSets(task); - break; - case QGCMapTask::TaskType::taskCreateTileSet: - _createTileSet(task); - break; - case QGCMapTask::TaskType::taskGetTileDownloadList: - _getTileDownloadList(task); - break; - case QGCMapTask::TaskType::taskUpdateTileDownloadState: - _updateTileDownloadState(task); - break; - case QGCMapTask::TaskType::taskDeleteTileSet: - _deleteTileSet(task); - break; - case QGCMapTask::TaskType::taskRenameTileSet: - _renameTileSet(task); - break; - case QGCMapTask::TaskType::taskPruneCache: - _pruneCache(task); - break; - case QGCMapTask::TaskType::taskReset: - _resetCacheDatabase(task); - break; - case QGCMapTask::TaskType::taskExport: - _exportSets(task); - break; - case QGCMapTask::TaskType::taskImport: - _importSets(task); - break; - default: - qCWarning(QGCTileCacheWorkerLog) << "given unhandled task type" << task->type(); - break; - } + return true; } -bool QGCCacheWorker::_testTask(QGCMapTask *mtask) +bool QGCCacheWorker::validateDatabase(QGCMapTask* task) { if (!_database || !_database->isValid()) { - mtask->setError("No Cache Database"); + task->setError("No Cache Database"); return false; } return true; } -void QGCCacheWorker::_emitTotals() +void QGCCacheWorker::emitTotals() { TotalsResult t = _database->computeTotals(); emit updateTotals(t.totalCount, t.totalSize, t.defaultCount, t.defaultSize); - _updateTimer.restart(); } -void QGCCacheWorker::_saveTile(QGCMapTask *mtask) +void QGCCacheWorker::_saveTileBatch(const QList& tasks) { - if (!_testTask(mtask)) { + if (tasks.isEmpty()) { return; } - QGCSaveTileTask *task = static_cast(mtask); - if (!_database->saveTile(task->tile()->hash, task->tile()->format, - task->tile()->img, task->tile()->type, task->tile()->tileSet)) { - mtask->setError("Error saving tile to cache"); - } -} - -void QGCCacheWorker::_getTile(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { - return; - } - - QGCFetchTileTask *task = static_cast(mtask); - auto tile = _database->getTile(task->hash()); - if (tile) { - task->setTileFetched(tile.release()); - } else { - task->setError("Tile not in cache database"); - } -} - -void QGCCacheWorker::_getTileSets(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { + if (!_database || !_database->isValid()) { + for (QGCMapTask* mtask : tasks) { + mtask->setError("No Cache Database"); + } return; } - QGCFetchTileSetTask *task = static_cast(mtask); - const QList records = _database->getTileSets(); - - for (const auto &rec : records) { - QGCCachedTileSet *set = new QGCCachedTileSet(rec.name); - set->blockSignals(true); - - set->setId(rec.setID); - set->setMapTypeStr(rec.mapTypeStr); - set->setTopleftLat(rec.topleftLat); - set->setTopleftLon(rec.topleftLon); - set->setBottomRightLat(rec.bottomRightLat); - set->setBottomRightLon(rec.bottomRightLon); - set->setMinZoom(rec.minZoom); - set->setMaxZoom(rec.maxZoom); - set->setType(UrlFactory::getProviderTypeFromQtMapId(rec.type)); - set->setTotalTileCount(rec.numTiles); - set->setDefaultSet(rec.defaultSet); - set->setCreationDate(QDateTime::fromSecsSinceEpoch(rec.date)); - - const SetTotalsResult totals = _database->computeSetTotals(rec.setID, rec.defaultSet, rec.numTiles, set->type()); - set->setSavedTileCount(totals.savedTileCount); - set->setSavedTileSize(totals.savedTileSize); - set->setTotalTileSize(totals.totalTileSize); - set->setUniqueTileCount(totals.uniqueTileCount); - set->setUniqueTileSize(totals.uniqueTileSize); - - set->blockSignals(false); - (void) set->moveToThread(QCoreApplication::instance()->thread()); - task->setTileSetFetched(set); - } -} - -void QGCCacheWorker::_createTileSet(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { - return; + QList tiles; + tiles.reserve(tasks.size()); + for (QGCMapTask* mtask : tasks) { + tiles.append(static_cast(mtask)->tile()); } - QGCCreateTileSetTask *task = static_cast(mtask); - const auto setID = _database->createTileSet( - task->tileSet()->name(), task->tileSet()->mapTypeStr(), - task->tileSet()->topleftLat(), task->tileSet()->topleftLon(), - task->tileSet()->bottomRightLat(), task->tileSet()->bottomRightLon(), - task->tileSet()->minZoom(), task->tileSet()->maxZoom(), - task->tileSet()->type(), task->tileSet()->totalTileCount()); - - if (!setID.has_value()) { - mtask->setError("Error saving tile set"); - return; + if (!_database->saveTileBatch(tiles)) { + for (QGCMapTask* mtask : tasks) { + mtask->setError("Error saving tile to cache"); + } } - - task->tileSet()->blockSignals(true); - task->tileSet()->setId(setID.value()); - - const SetTotalsResult totals = _database->computeSetTotals( - setID.value(), task->tileSet()->defaultSet(), - task->tileSet()->totalTileCount(), task->tileSet()->type()); - task->tileSet()->setSavedTileCount(totals.savedTileCount); - task->tileSet()->setSavedTileSize(totals.savedTileSize); - task->tileSet()->setTotalTileSize(totals.totalTileSize); - task->tileSet()->setUniqueTileCount(totals.uniqueTileCount); - task->tileSet()->setUniqueTileSize(totals.uniqueTileSize); - task->tileSet()->blockSignals(false); - - task->setTileSetSaved(); } -void QGCCacheWorker::_getTileDownloadList(QGCMapTask *mtask) +void QGCCacheWorker::_init() { - if (!_testTask(mtask)) { - return; - } + _stopRequested = false; + // A _drainQueue invocation queued before the previous stop() may have been + // discarded when the thread's event loop exited, leaving _drainScheduled + // stuck true; clear it so enqueueTask re-posts a drain after restart. + _drainScheduled = false; + _database = std::make_unique(_databasePath); - QGCGetTileDownloadListTask *task = static_cast(mtask); - const QList tileValues = _database->getTileDownloadList(task->setID(), task->count()); - QQueue tiles; - for (const auto &t : tileValues) { - tiles.enqueue(new QGCTile(t)); - } - task->setTileListFetched(tiles); -} + if (!_database->init()) { + qCWarning(QGCTileCacheWorkerLog) << "Failed To Init Database"; + _database.reset(); -void QGCCacheWorker::_updateTileDownloadState(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { + QMutexLocker lock(&_taskQueueMutex); + for (QGCMapTask* orphan : _taskQueue) { + orphan->setError(tr("Database Init Failed")); + orphan->deleteLater(); + } + _taskQueue.clear(); return; } - QGCUpdateTileDownloadStateTask *task = static_cast(mtask); - bool ok; - if (task->hash() == QStringLiteral("*")) { - ok = _database->updateAllTileDownloadStates(task->setID(), static_cast(task->state())); - } else { - ok = _database->updateTileDownloadState(task->setID(), static_cast(task->state()), task->hash()); - } - if (!ok) { - mtask->setError("Error updating tile download state"); + if (_database->isValid()) { + if (_database->connectDB()) { + _database->deleteBingNoTileTiles(); + } } -} -void QGCCacheWorker::_pruneCache(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { - return; - } + _dbValid = _database->isValid(); - QGCPruneCacheTask *task = static_cast(mtask); - if (!_database->pruneCache(task->amount())) { - mtask->setError("Error pruning cache"); - return; + if (!_updateTimer) { + _updateTimer = new QTimer(this); + _updateTimer->setInterval(kUpdateTimerIntervalMs); + (void) connect(_updateTimer, &QTimer::timeout, this, [this]() { + if (_database && _database->isValid()) { + emitTotals(); + } + }); } - task->setPruned(); + _updateTimer->start(); } -void QGCCacheWorker::_deleteTileSet(QGCMapTask *mtask) +void QGCCacheWorker::_drainQueue() { - if (!_testTask(mtask)) { - return; - } - - QGCDeleteTileSetTask *task = static_cast(mtask); - if (!_database->deleteTileSet(task->setID())) { - mtask->setError("Error deleting tile set"); - return; - } - _emitTotals(); - task->setTileSetDeleted(); -} + _drainScheduled = false; -void QGCCacheWorker::_renameTileSet(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { + if (_stopRequested) { return; } - QGCRenameTileSetTask *task = static_cast(mtask); - if (!_database->renameTileSet(task->setID(), task->newName())) { - task->setError("Error renaming tile set"); - } -} - -void QGCCacheWorker::_resetCacheDatabase(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { - return; + QMutexLocker lock(&_taskQueueMutex); + while (!_stopRequested && !_taskQueue.isEmpty()) { + QGCMapTask* const task = _taskQueue.dequeue(); + if (task->type() == QGCMapTask::TaskType::taskCacheTile) { + QList batch; + batch.append(task); + while ((batch.size() < kMaxSaveBatch) && !_taskQueue.isEmpty() && + (_taskQueue.head()->type() == QGCMapTask::TaskType::taskCacheTile)) { + batch.append(_taskQueue.dequeue()); + } + lock.unlock(); + _saveTileBatch(batch); + for (QGCMapTask* batched : batch) { + batched->deleteLater(); + } + lock.relock(); + } else { + lock.unlock(); + task->execute(*this); + task->deleteLater(); + lock.relock(); + } } + lock.unlock(); - QGCResetTask *task = static_cast(mtask); - if (!_database->resetDatabase()) { - mtask->setError("Error resetting cache database"); - return; + if (!_stopRequested && _database && _database->isValid()) { + emitTotals(); } - _dbValid = _database->isValid(); - task->setResetCompleted(); } -void QGCCacheWorker::_importSets(QGCMapTask *mtask) +void QGCCacheWorker::_shutdown() { - if (!_testTask(mtask)) { - return; - } - - QGCImportTileTask *task = static_cast(mtask); - auto progress = [task](int pct) { task->setProgress(pct); }; - - DatabaseResult result; - if (task->replace()) { - result = _database->importSetsReplace(task->path(), progress); - } else { - result = _database->importSetsMerge(task->path(), progress); - } - - _dbValid = _database->isValid(); - - if (!result.success) { - task->setError(result.errorString); - return; - } - - if (task->replace() && _database->isValid()) { - QSettings settings; - settings.remove(QLatin1String(QGCTileCacheDatabase::kBingNoTileDoneKey)); - _database->deleteBingNoTileTiles(); + if (_updateTimer) { + _updateTimer->stop(); } - task->setImportCompleted(); -} - -void QGCCacheWorker::_exportSets(QGCMapTask *mtask) -{ - if (!_testTask(mtask)) { - return; + QMutexLocker lock(&_taskQueueMutex); + for (QGCMapTask* orphan : _taskQueue) { + orphan->setError(tr("Worker shutting down")); + orphan->deleteLater(); } + _taskQueue.clear(); + lock.unlock(); - QGCExportTileTask *task = static_cast(mtask); - - auto progress = [task](int pct) { task->setProgress(pct); }; - DatabaseResult result = _database->exportSets(task->sets(), task->path(), progress); - - if (!result.success) { - task->setError(result.errorString); - return; + _dbValid = false; + if (_database) { + _database->disconnectDB(); + _database.reset(); } - - task->setExportCompleted(); } diff --git a/src/QtLocationPlugin/QGCTileCacheWorker.h b/src/QtLocationPlugin/QGCTileCacheWorker.h index 6e3e025a1835..a671d22236a9 100644 --- a/src/QtLocationPlugin/QGCTileCacheWorker.h +++ b/src/QtLocationPlugin/QGCTileCacheWorker.h @@ -1,65 +1,74 @@ #pragma once -#include #include +#include #include #include #include -#include - +#include +#include #include class QGCMapTask; class QGCTileCacheDatabase; -class QGCCacheWorker : public QThread +// QObject worker on an owned QThread (never subclass QThread). A Qt event loop drains the +// mutex-protected task queue on the worker thread, coalescing consecutive cache-tile saves. +class QGCCacheWorker : public QObject { Q_OBJECT public: - explicit QGCCacheWorker(QObject *parent = nullptr); - ~QGCCacheWorker(); + QGCCacheWorker(); + ~QGCCacheWorker() override; + + void setDatabaseFile(const QString& path) + { + if (isRunning()) { + return; + } + _databasePath = path; + } + + bool isRunning() const { return _thread.isRunning(); } + + // Bounded wait only: callers must pass an explicit timeout. The destructor waits + // for the thread itself, so an unbounded ULONG_MAX default is never the intent. + bool wait(unsigned long timeMs) { return _thread.wait(timeMs); } + + // Accessors used by QGCMapTask::execute() implementations running on this thread. + QGCTileCacheDatabase* database() const { return _database.get(); } + + bool validateDatabase(QGCMapTask* task); + void emitTotals(); - void setDatabaseFile(const QString &path) { if (isRunning()) { return; } _databasePath = path; } + void setDatabaseValid(bool valid) { _dbValid = valid; } public slots: - bool enqueueTask(QGCMapTask *task); + bool enqueueTask(QGCMapTask* task); void stop(); signals: void updateTotals(quint32 totaltiles, quint64 totalsize, quint32 defaulttiles, quint64 defaultsize); -protected: - void run() final; +private slots: + void _init(); + void _drainQueue(); + void _shutdown(); private: - void _runTask(QGCMapTask *task); - - void _saveTile(QGCMapTask *task); - void _getTile(QGCMapTask *task); - void _getTileSets(QGCMapTask *task); - void _createTileSet(QGCMapTask *task); - void _getTileDownloadList(QGCMapTask *task); - void _updateTileDownloadState(QGCMapTask *task); - void _pruneCache(QGCMapTask *task); - void _deleteTileSet(QGCMapTask *task); - void _renameTileSet(QGCMapTask *task); - void _resetCacheDatabase(QGCMapTask *task); - void _importSets(QGCMapTask *task); - void _exportSets(QGCMapTask *task); - bool _testTask(QGCMapTask *task); - void _emitTotals(); + void _saveTileBatch(const QList& tasks); + QThread _thread; std::unique_ptr _database; + QTimer* _updateTimer = nullptr; QMutex _taskQueueMutex; QQueue _taskQueue; - QWaitCondition _waitc; QString _databasePath; - QElapsedTimer _updateTimer; - int _updateTimeout = kShortTimeoutMs; std::atomic_bool _dbValid = false; std::atomic_bool _stopRequested = false; + std::atomic_bool _drainScheduled = false; - static constexpr int kShortTimeoutMs = 2000; - static constexpr int kLongTimeoutMs = 5000; + static constexpr int kUpdateTimerIntervalMs = 3000; + static constexpr int kMaxSaveBatch = 64; }; diff --git a/src/QtLocationPlugin/QGCTileDatabaseSchema.cpp b/src/QtLocationPlugin/QGCTileDatabaseSchema.cpp new file mode 100644 index 000000000000..760acb49a2ce --- /dev/null +++ b/src/QtLocationPlugin/QGCTileDatabaseSchema.cpp @@ -0,0 +1,385 @@ +#include "QGCTileDatabaseSchema.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "QGCLoggingCategory.h" +#include "QGCMapUrlEngine.h" +#include "QGCSqlHelper.h" +#include "QGCTileCacheDatabase.h" + +QGC_LOGGING_CATEGORY(QGCTileDatabaseSchemaLog, "QtLocationPlugin.QGCTileDatabaseSchema") + +namespace QGCTileDatabaseSchema { + +bool dropSchemaTables(QSqlDatabase db) +{ + QGCSqlHelper::Transaction txn(db); + if (!txn.ok()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to start transaction for table drops"; + return false; + } + QSqlQuery query(db); + if (!query.exec("DROP TABLE IF EXISTS TilesDownload") || !query.exec("DROP TABLE IF EXISTS SetTiles") || + !query.exec("DROP TABLE IF EXISTS Tiles") || !query.exec("DROP TABLE IF EXISTS TileSets")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to drop tables:" << query.lastError().text(); + return false; + } + if (!txn.commit()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to commit table drops"; + return false; + } + return true; +} + +bool migrateSchema(QSqlDatabase db, int fromVersion) +{ + QGCSqlHelper::Transaction txn(db); + if (!txn.ok()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to start transaction for schema migration"; + return false; + } + + if (fromVersion < 2) { + // v1 -> v2: add HTTP cache-validation columns to Tiles. Existing rows + // get NULL/0 and simply revalidate on next fetch. + QSet columns; + QSqlQuery info(db); + if (info.exec(QStringLiteral("PRAGMA table_info(Tiles)"))) { + while (info.next()) { + columns.insert(info.value(1).toString()); + } + } + + struct ColumnDef + { + const char* name; + const char* ddl; + }; + + static const ColumnDef newColumns[] = { + {"etag", "ALTER TABLE Tiles ADD COLUMN etag TEXT"}, + {"lastModified", "ALTER TABLE Tiles ADD COLUMN lastModified TEXT"}, + {"expiresAt", "ALTER TABLE Tiles ADD COLUMN expiresAt INTEGER DEFAULT 0"}, + }; + for (const ColumnDef& col : newColumns) { + if (columns.contains(QLatin1String(col.name))) { + continue; + } + QSqlQuery alter(db); + if (!alter.exec(QLatin1String(col.ddl))) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Failed to add column" << col.name << ":" << alter.lastError().text(); + return false; + } + } + } + + if (fromVersion < 3) { + // v2 -> v3: add LRU access tracking + per-tile must-revalidate flag. + QSet columns; + QSqlQuery info(db); + if (info.exec(QStringLiteral("PRAGMA table_info(Tiles)"))) { + while (info.next()) { + columns.insert(info.value(1).toString()); + } + } + + if (!columns.contains(QStringLiteral("accessed"))) { + QSqlQuery alter(db); + if (!alter.exec(QStringLiteral("ALTER TABLE Tiles ADD COLUMN accessed INTEGER NOT NULL DEFAULT 0"))) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to add column accessed:" << alter.lastError().text(); + return false; + } + // Seed LRU order from insertion date so pre-v3 tiles are not all equal. + if (!alter.exec(QStringLiteral("UPDATE Tiles SET accessed = date"))) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to backfill accessed:" << alter.lastError().text(); + return false; + } + } + + if (!columns.contains(QStringLiteral("mustRevalidate"))) { + QSqlQuery alter(db); + if (!alter.exec(QStringLiteral("ALTER TABLE Tiles ADD COLUMN mustRevalidate INTEGER NOT NULL DEFAULT 0"))) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Failed to add column mustRevalidate:" << alter.lastError().text(); + return false; + } + } + + QSqlQuery index(db); + if (!index.exec(QStringLiteral("CREATE INDEX IF NOT EXISTS idx_tiles_accessed ON Tiles(accessed)"))) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to create idx_tiles_accessed:" << index.lastError().text(); + return false; + } + } + + if (fromVersion < 4) { + // v3 -> v4: covering index on (accessed, size) so pruneCache's LRU scan + + // size sum run index-only, without faulting BLOB pages into the cache. + QSqlQuery index(db); + if (!index.exec( + QStringLiteral("CREATE INDEX IF NOT EXISTS idx_tiles_accessed_size ON Tiles(accessed, size)"))) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Failed to create idx_tiles_accessed_size:" << index.lastError().text(); + return false; + } + } + + if (fromVersion < 5) { + // v4 -> v5: persist the stable provider-name string. The old int `type` is a + // positional registry index that mis-attributes tiles if providers are reordered. + auto addTypeStrColumn = [&db](const char* table) -> bool { + QSet columns; + QSqlQuery info(db); + if (info.exec(QStringLiteral("PRAGMA table_info(%1)").arg(QLatin1String(table)))) { + while (info.next()) { + columns.insert(info.value(1).toString()); + } + } + if (columns.contains(QStringLiteral("typeStr"))) { + return true; + } + QSqlQuery alter(db); + if (!alter.exec(QStringLiteral("ALTER TABLE %1 ADD COLUMN typeStr TEXT").arg(QLatin1String(table)))) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Failed to add typeStr to" << table << ":" << alter.lastError().text(); + return false; + } + return true; + }; + + auto backfillTypeStr = [&db](const char* table) -> bool { + QList mapIds; + QSqlQuery distinct(db); + if (distinct.exec(QStringLiteral("SELECT DISTINCT type FROM %1 WHERE typeStr IS NULL OR typeStr = ''") + .arg(QLatin1String(table)))) { + while (distinct.next()) { + mapIds.append(distinct.value(0).toInt()); + } + } + QSqlQuery update(db); + if (!update.prepare(QStringLiteral("UPDATE %1 SET typeStr = ? WHERE type = ? AND (typeStr IS NULL OR " + "typeStr = '')") + .arg(QLatin1String(table)))) { + return false; + } + for (const int mapId : mapIds) { + // Provider mapIds start at 1; skip 0/-1 sentinels so backfill doesn't log a + // spurious "provider not found". Those rows keep NULL typeStr and revalidate. + if (mapId < 1) { + continue; + } + const QString name = UrlFactory::getProviderTypeFromQtMapId(mapId); + if (name.isEmpty()) { + continue; + } + update.bindValue(0, name); + update.bindValue(1, mapId); + if (!update.exec()) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Failed to backfill typeStr on" << table << ":" << update.lastError().text(); + return false; + } + } + return true; + }; + + if (!addTypeStrColumn("Tiles") || !addTypeStrColumn("TilesDownload")) { + return false; + } + if (!backfillTypeStr("Tiles") || !backfillTypeStr("TilesDownload") || !backfillTypeStr("TileSets")) { + return false; + } + } + + if (!QGCSqlHelper::setUserVersion(db, QGCTileCacheDatabase::kSchemaVersion)) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to set schema version after migration"; + return false; + } + + if (!txn.commit()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to commit schema migration"; + return false; + } + + qCDebug(QGCTileDatabaseSchemaLog) << "Migrated tile cache schema from version" << fromVersion << "to" + << QGCTileCacheDatabase::kSchemaVersion; + return true; +} + +bool createSchema(QSqlDatabase db, bool createDefault) +{ + // applySqlitePragmas (in connectDB / ScopedConnection ctor) already + // enabled foreign_keys; nothing to redo here. + QSqlQuery query(db); + + if (!query.exec("CREATE TABLE IF NOT EXISTS Tiles (" + "tileID INTEGER PRIMARY KEY NOT NULL, " + "hash TEXT NOT NULL UNIQUE, " + "format TEXT NOT NULL, " + "tile BLOB NULL, " + "size INTEGER, " + "typeStr TEXT, " + "date INTEGER DEFAULT 0, " + "etag TEXT, " + "lastModified TEXT, " + "expiresAt INTEGER DEFAULT 0, " + "accessed INTEGER NOT NULL DEFAULT 0, " + "mustRevalidate INTEGER NOT NULL DEFAULT 0)")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (create Tiles db):" << query.lastError().text(); + return false; + } + + if (!query.exec("CREATE TABLE IF NOT EXISTS TileSets (" + "setID INTEGER PRIMARY KEY NOT NULL, " + "name TEXT NOT NULL UNIQUE, " + "typeStr TEXT, " + "topleftLat REAL DEFAULT 0.0, " + "topleftLon REAL DEFAULT 0.0, " + "bottomRightLat REAL DEFAULT 0.0, " + "bottomRightLon REAL DEFAULT 0.0, " + "minZoom INTEGER DEFAULT 3, " + "maxZoom INTEGER DEFAULT 3, " + "numTiles INTEGER DEFAULT 0, " + "defaultSet INTEGER DEFAULT 0, " + "date INTEGER DEFAULT 0)")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (create TileSets db):" << query.lastError().text(); + return false; + } + + if (!query.exec("CREATE TABLE IF NOT EXISTS SetTiles (" + "setID INTEGER NOT NULL REFERENCES TileSets(setID) ON DELETE CASCADE, " + "tileID INTEGER NOT NULL REFERENCES Tiles(tileID) ON DELETE CASCADE)")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (create SetTiles db):" << query.lastError().text(); + return false; + } + + if (!query.exec("CREATE TABLE IF NOT EXISTS TilesDownload (" + "setID INTEGER NOT NULL REFERENCES TileSets(setID) ON DELETE CASCADE, " + "hash TEXT NOT NULL, " + "typeStr TEXT, " + "x INTEGER, " + "y INTEGER, " + "z INTEGER, " + "state INTEGER DEFAULT 0)")) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Map Cache SQL error (create TilesDownload db):" << query.lastError().text(); + return false; + } + + static const char* indexStatements[] = { + "CREATE UNIQUE INDEX IF NOT EXISTS idx_settiles_unique ON SetTiles(tileID, setID)", + "CREATE INDEX IF NOT EXISTS idx_settiles_setid ON SetTiles(setID)", + "CREATE INDEX IF NOT EXISTS idx_settiles_tileid ON SetTiles(tileID)", + "CREATE UNIQUE INDEX IF NOT EXISTS idx_tilesdownload_setid_hash ON TilesDownload(setID, hash)", + "CREATE INDEX IF NOT EXISTS idx_tilesdownload_setid_state ON TilesDownload(setID, state)", + "CREATE INDEX IF NOT EXISTS idx_tiles_date ON Tiles(date)", + "CREATE INDEX IF NOT EXISTS idx_tiles_accessed ON Tiles(accessed)", + "CREATE INDEX IF NOT EXISTS idx_tiles_accessed_size ON Tiles(accessed, size)", + "CREATE INDEX IF NOT EXISTS idx_tilesets_default ON TileSets(defaultSet)", + }; + for (const char* sql : indexStatements) { + if (!query.exec(QLatin1String(sql))) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to create index:" << sql << query.lastError().text(); + } + } + + if (!QGCSqlHelper::setUserVersion(db, QGCTileCacheDatabase::kSchemaVersion)) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to set schema version"; + } + + if (!createDefault) { + return true; + } + + if (!query.prepare("SELECT name FROM TileSets WHERE name = ?")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (prepare default set check):" << db.lastError(); + return false; + } + query.addBindValue(QStringLiteral("Default Tile Set")); + if (!query.exec()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (Looking for default tile set):" << db.lastError(); + return true; + } + if (query.next()) { + return true; + } + + if (!query.prepare("INSERT INTO TileSets(name, defaultSet, date) VALUES(?, ?, ?)")) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (prepare default tile set):" << db.lastError(); + return false; + } + query.addBindValue(QStringLiteral("Default Tile Set")); + query.addBindValue(1); + query.addBindValue(QDateTime::currentSecsSinceEpoch()); + if (!query.exec()) { + qCWarning(QGCTileDatabaseSchemaLog) << "Map Cache SQL error (Creating default tile set):" << db.lastError(); + return false; + } + + return true; +} + +bool checkSchemaVersion(QSqlDatabase db, bool* didReset) +{ + const auto current = QGCSqlHelper::userVersion(db); + if (!current) { + qCWarning(QGCTileDatabaseSchemaLog) << "Failed to read schema version"; + return false; + } + + const int version = *current; + if (version == QGCTileCacheDatabase::kSchemaVersion) { + return true; + } + + if (version == 0) { + // Either a fresh database or a legacy database created before versioning. + // Check for existing data — if Tiles table exists with rows, it's legacy. + // Legacy DBs stored map type as text; migration is not supported so the cache is rebuilt. + bool isLegacy = false; + { + QSqlQuery query(db); + isLegacy = query.exec("SELECT COUNT(*) FROM Tiles") && query.next() && (query.value(0).toInt() > 0); + // In WAL mode the live SELECT cursor holds a read lock; finalize it before + // the DROPs or SQLite reports "database table is locked". + query.finish(); + } + if (isLegacy) { + qCWarning(QGCTileDatabaseSchemaLog) + << "Legacy database detected (no schema version). Discarding cached tiles and rebuilding."; + if (didReset) { + *didReset = true; + } + if (!dropSchemaTables(db)) { + return false; + } + } + return true; + } + + if ((version > 0) && (version < QGCTileCacheDatabase::kSchemaVersion)) { + if (migrateSchema(db, version)) { + return true; + } + qCWarning(QGCTileDatabaseSchemaLog) << "Schema migration from version" << version << "failed. Resetting cache."; + } else { + qCWarning(QGCTileDatabaseSchemaLog) << "Unknown schema version" << version << "(expected" + << QGCTileCacheDatabase::kSchemaVersion << "). Resetting cache."; + } + + if (didReset) { + *didReset = true; + } + return dropSchemaTables(db); +} + +} // namespace QGCTileDatabaseSchema diff --git a/src/QtLocationPlugin/QGCTileDatabaseSchema.h b/src/QtLocationPlugin/QGCTileDatabaseSchema.h new file mode 100644 index 000000000000..eb8a0b31fa27 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileDatabaseSchema.h @@ -0,0 +1,20 @@ +#pragma once + +class QSqlDatabase; + +// Schema DDL and version migration for the tile cache database, split out of +// QGCTileCacheDatabase so create/migrate/reset can be read and unit-tested +// independently of tile CRUD, stats and import/export. Every function operates +// on the caller's already-open QSqlDatabase connection and manages its own +// transaction; none touch QGCTileCacheDatabase instance state. +namespace QGCTileDatabaseSchema { +bool createSchema(QSqlDatabase db, bool createDefault = true); +bool dropSchemaTables(QSqlDatabase db); +bool migrateSchema(QSqlDatabase db, int fromVersion); + +// Brings an open connection's schema to the current version: migrates in +// place when possible, otherwise drops the tables and reports the reset via +// *didReset (the caller must then invalidate any cached default-set id). +// Returns false only on hard failure. +bool checkSchemaVersion(QSqlDatabase db, bool* didReset = nullptr); +} // namespace QGCTileDatabaseSchema diff --git a/src/QtLocationPlugin/QGCTileFallback.cpp b/src/QtLocationPlugin/QGCTileFallback.cpp new file mode 100644 index 000000000000..06c31b00df54 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileFallback.cpp @@ -0,0 +1,50 @@ +#include "QGCTileFallback.h" + +#include +#include +#include + +QImage scaleAncestorToChild(const QImage &ancestor, int x, int y, int levelDelta, QSize tileSize) +{ + if (ancestor.isNull() || (levelDelta <= 0) || tileSize.isEmpty()) { + return QImage(); + } + + const int divisions = 1 << levelDelta; // 2^levelDelta cells per ancestor side + const int cellX = x & (divisions - 1); // x mod 2^levelDelta + const int cellY = y & (divisions - 1); // y mod 2^levelDelta + + const int subW = ancestor.width() / divisions; + const int subH = ancestor.height() / divisions; + if ((subW <= 0) || (subH <= 0)) { + return QImage(); + } + + // Scale the ancestor sub-square straight into the target tile in one pass, + // avoiding the intermediate heap allocation that copy()+scaled() would make. + QImage result(tileSize, QImage::Format_ARGB32_Premultiplied); + result.fill(Qt::transparent); + QPainter painter(&result); + painter.setRenderHint(QPainter::SmoothPixmapTransform); + painter.drawImage(QRectF(QPointF(0, 0), QSizeF(tileSize)), + ancestor, + QRectF(cellX * subW, cellY * subH, subW, subH)); + painter.end(); + return result; +} + +QByteArray encodeFallbackTile(const QImage &image) +{ + if (image.isNull()) { + return QByteArray(); + } + QByteArray bytes; + QBuffer buffer(&bytes); + if (!buffer.open(QIODevice::WriteOnly)) { + return QByteArray(); + } + if (!image.save(&buffer, "PNG")) { + return QByteArray(); + } + return bytes; +} diff --git a/src/QtLocationPlugin/QGCTileFallback.h b/src/QtLocationPlugin/QGCTileFallback.h new file mode 100644 index 000000000000..61c3aa26f539 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileFallback.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +class QImage; + +// R6 lower-zoom fallback helpers. + +// Crop the sub-square of an ancestor tile that covers descendant tile (x, y) and +// scale it up to tileSize. For level delta d the child occupies a 1/2^d sub-square +// at cell ((x mod 2^d), (y mod 2^d)) of the ancestor. Returns a null QImage on +// invalid input (null ancestor, levelDelta <= 0, or out-of-range cell). +QImage scaleAncestorToChild(const QImage &ancestor, int x, int y, int levelDelta, QSize tileSize); + +// Encode a fallback QImage to PNG bytes. Returns empty on failure. +QByteArray encodeFallbackTile(const QImage &image); diff --git a/src/QtLocationPlugin/QGCTileSetImporter.cpp b/src/QtLocationPlugin/QGCTileSetImporter.cpp new file mode 100644 index 000000000000..a362eb002b71 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileSetImporter.cpp @@ -0,0 +1,729 @@ +#include "QGCTileSetImporter.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "QGCCacheTile.h" +#include "QGCCompression.h" +#include "QGCLoggingCategory.h" +#include "QGCMapUrlEngine.h" +#include "QGCSqlHelper.h" +#include "QGCTileCacheDatabase.h" + +QGC_LOGGING_CATEGORY(QGCTileSetImporterLog, "QtLocationPlugin.QGCTileSetImporter") + +namespace QGCTileSetImporter { + +namespace { + +// Flush imported tiles to the cache in chunks; each chunk is one transaction with +// reused prepared queries (QGCTileCacheDatabase::saveTileBatch) instead of a +// per-tile transaction + redundant SELECT. +constexpr int kImportBatchSize = 4096; + +// Decompressed-size caps for untrusted PMTiles blobs. Tile payloads are bounded +// generously (256 MiB) while directory blobs are tiny in practice (16 MiB cap). +constexpr qint64 kMaxTileDecompressedBytes = 256LL * 1024 * 1024; +constexpr qint64 kMaxDirDecompressedBytes = 16LL * 1024 * 1024; + +// TMS (MBTiles) -> XYZ row flip. zoom in [0, 30]; 1 << zoom stays within int range. +int tmsRowToXyz(int zoom, int tmsRow) +{ + return ((1 << zoom) - 1) - tmsRow; +} + +} // namespace + +Result importMBTiles(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, + const QString &providerType) +{ + Result result; + + if (!db.isValid()) { + result.errorString = QStringLiteral("Tile cache database is not valid"); + return result; + } + + // Read-only connection to the source archive; never mutated by the importer. + QGCSqlHelper::ScopedConnection conn(archivePath, /*readOnly*/ true, QStringLiteral("QGCMBTiles")); + if (!conn.isValid()) { + result.errorString = QStringLiteral("Failed to open MBTiles archive: %1").arg(archivePath); + return result; + } + + QSqlDatabase srcDB = conn.database(); + + // mb-util/tippecanoe emit a deduplicated schema (map + images) and may omit the + // classic `tiles` view. Prefer `tiles` when it has rows, else fall back to the + // map/images join. Both expose zoom_level/tile_column/tile_row/tile_data. + bool useDedupSchema = false; + { + QSqlQuery probe(srcDB); + const bool tilesHasRows = probe.exec(QStringLiteral("SELECT 1 FROM tiles LIMIT 1")) && probe.next(); + probe.finish(); + if (!tilesHasRows) { + QSqlQuery mapProbe(srcDB); + const bool mapHasRows = mapProbe.exec(QStringLiteral("SELECT 1 FROM map LIMIT 1")) && mapProbe.next(); + mapProbe.finish(); + if (!mapHasRows) { + result.errorString = QStringLiteral("MBTiles archive has no tiles to import"); + return result; + } + useDedupSchema = true; + } + } + + const QString zoomSql = useDedupSchema + ? QStringLiteral("SELECT MIN(zoom_level), MAX(zoom_level) FROM map") + : QStringLiteral("SELECT MIN(zoom_level), MAX(zoom_level) FROM tiles"); + const QString tileSql = useDedupSchema + ? QStringLiteral("SELECT m.zoom_level, m.tile_column, m.tile_row, i.tile_data " + "FROM map m JOIN images i ON m.tile_id = i.tile_id") + : QStringLiteral("SELECT zoom_level, tile_column, tile_row, tile_data FROM tiles"); + + int minZoom = 0; + int maxZoom = 0; + { + QSqlQuery zoomQuery(srcDB); + if (!zoomQuery.exec(zoomSql) || !zoomQuery.next()) { + result.errorString = QStringLiteral("Failed to read MBTiles zoom range: %1").arg(zoomQuery.lastError().text()); + return result; + } + minZoom = zoomQuery.value(0).toInt(); + maxZoom = zoomQuery.value(1).toInt(); + } + + const auto setID = db.createImportedTileSet(setName, providerType, minZoom, maxZoom); + if (!setID.has_value()) { + result.errorString = QStringLiteral("Failed to create imported tile set"); + return result; + } + + QSqlQuery tileQuery(srcDB); + tileQuery.setForwardOnly(true); + if (!tileQuery.exec(tileSql)) { + result.errorString = QStringLiteral("Failed to read MBTiles tiles: %1").arg(tileQuery.lastError().text()); + (void) db.deleteTileSet(setID.value()); + return result; + } + + quint32 imported = 0; + // Accumulate tiles and flush via saveTileBatch (one transaction, reused prepared + // queries) every kImportBatchSize, instead of a per-tile transaction + SELECT. + std::vector chunk; + chunk.reserve(kImportBatchSize); + const auto flush = [&]() -> bool { + if (chunk.empty()) { + return true; + } + QList ptrs; + ptrs.reserve(static_cast(chunk.size())); + for (const QGCCacheTile &t : chunk) { + ptrs.append(&t); + } + const bool ok = db.saveTileBatch(ptrs); + chunk.clear(); + return ok; + }; + + while (tileQuery.next()) { + const int zoom = tileQuery.value(0).toInt(); + const int column = tileQuery.value(1).toInt(); + const int tmsRow = tileQuery.value(2).toInt(); + const QByteArray tileData = tileQuery.value(3).toByteArray(); + if (tileData.isEmpty()) { + continue; + } + + const int xyzRow = tmsRowToXyz(zoom, tmsRow); + // Reuse the cache's canonical hashing so imported tiles dedupe with cached ones. + const QString hash = UrlFactory::getTileHash(providerType, column, xyzRow, zoom); + const QString format = UrlFactory::getImageFormat(providerType, tileData); + + chunk.emplace_back(hash, tileData, format, providerType, setID.value()); + ++imported; + if ((chunk.size() >= static_cast(kImportBatchSize)) && !flush()) { + qCWarning(QGCTileSetImporterLog) << "Failed to import MBTiles tile batch"; + result.errorString = QStringLiteral("Failed to import MBTiles tile batch"); + (void) db.deleteTileSet(setID.value()); + return result; + } + } + + if (!flush()) { + qCWarning(QGCTileSetImporterLog) << "Failed to import MBTiles tile batch"; + result.errorString = QStringLiteral("Failed to import MBTiles tile batch"); + (void) db.deleteTileSet(setID.value()); + return result; + } + + // Reflect the imported count in the set row (createImportedTileSet seeds numTiles=0). + { + QSqlQuery update(db.database()); + if (update.prepare(QStringLiteral("UPDATE TileSets SET numTiles = ? WHERE setID = ?"))) { + update.addBindValue(imported); + update.addBindValue(setID.value()); + (void) update.exec(); + } + } + + // Bulk import inflates the WAL; fold it back into the main DB and shrink the -wal. + QSqlDatabase cacheDB = db.database(); + QGCSqlHelper::walCheckpointTruncate(cacheDB); + + qCDebug(QGCTileSetImporterLog) << "Imported" << imported << "tiles from" << archivePath << "into set" << setID.value(); + + result.success = true; + result.setID = setID.value(); + result.tileCount = imported; + return result; +} + +namespace { + +// PMTiles v3 fixed header is exactly 127 bytes; magic is "PMTiles" + version byte. +constexpr int kPMTilesHeaderSize = 127; +constexpr char kPMTilesMagic[] = {'P', 'M', 'T', 'i', 'l', 'e', 's'}; + +// internal_compression / tile_compression codes (spec section: compression). +enum class PMCompression : quint8 { Unknown = 0, None = 1, Gzip = 2, Brotli = 3, Zstd = 4 }; + +// tile_type codes. +enum class PMTileType : quint8 { Unknown = 0, Mvt = 1, Png = 2, Jpeg = 3, Webp = 4, Avif = 5 }; + +struct PMHeader { + quint64 rootDirOffset = 0; + quint64 rootDirBytes = 0; + quint64 jsonMetadataOffset = 0; + quint64 jsonMetadataBytes = 0; + quint64 leafDirsOffset = 0; + quint64 leafDirsBytes = 0; + quint64 tileDataOffset = 0; + quint64 tileDataBytes = 0; + quint64 addressedTilesCount = 0; + PMCompression internalCompression = PMCompression::Unknown; + PMCompression tileCompression = PMCompression::Unknown; + PMTileType tileType = PMTileType::Unknown; + quint8 minZoom = 0; + quint8 maxZoom = 0; +}; + +struct PMEntry { + quint64 tileId = 0; + quint64 offset = 0; + quint32 length = 0; + quint32 runLength = 0; +}; + +QString compressionName(PMCompression c) +{ + switch (c) { + case PMCompression::None: return QStringLiteral("None"); + case PMCompression::Gzip: return QStringLiteral("Gzip"); + case PMCompression::Brotli: return QStringLiteral("Brotli"); + case PMCompression::Zstd: return QStringLiteral("Zstd"); + default: return QStringLiteral("Unknown"); + } +} + +// Decompress a PMTiles blob per its declared compression. None -> passthrough, +// Gzip -> libarchive inflate. Brotli/Zstd are unsupported and return nullopt +// (caller surfaces a codec-named error). +std::optional pmDecompress(const QByteArray &data, PMCompression compression, qint64 maxBytes) +{ + switch (compression) { + case PMCompression::None: + // Even uncompressed sections must respect the cap on untrusted input. + if (maxBytes > 0 && data.size() > maxBytes) { + return std::nullopt; + } + return data; + case PMCompression::Gzip: { + const QByteArray out = QGCCompression::decompressData(data, QGCCompression::Format::GZIP, maxBytes); + if (out.isEmpty() && !data.isEmpty()) { + return std::nullopt; + } + return out; + } + default: + return std::nullopt; + } +} + +bool parsePMHeader(const QByteArray &bytes, PMHeader &out, QString &error) +{ + if (bytes.size() < kPMTilesHeaderSize) { + error = QStringLiteral("PMTiles header is truncated (%1 bytes)").arg(bytes.size()); + return false; + } + if (std::memcmp(bytes.constData(), kPMTilesMagic, sizeof(kPMTilesMagic)) != 0) { + error = QStringLiteral("Not a PMTiles archive (bad magic)"); + return false; + } + const quint8 version = static_cast(bytes.at(7)); + if (version != 3) { + error = QStringLiteral("Unsupported PMTiles version %1 (only v3 is supported)").arg(version); + return false; + } + + const auto *p = reinterpret_cast(bytes.constData()); + auto u64 = [p](int off) { return qFromLittleEndian(p + off); }; + + out.rootDirOffset = u64(8); + out.rootDirBytes = u64(16); + out.jsonMetadataOffset = u64(24); + out.jsonMetadataBytes = u64(32); + out.leafDirsOffset = u64(40); + out.leafDirsBytes = u64(48); + out.tileDataOffset = u64(56); + out.tileDataBytes = u64(64); + out.addressedTilesCount = u64(72); + // u64 tile_entries_count(80), tile_contents_count(88), bool clustered(96). + out.internalCompression = static_cast(static_cast(bytes.at(97))); + out.tileCompression = static_cast(static_cast(bytes.at(98))); + out.tileType = static_cast(static_cast(bytes.at(99))); + out.minZoom = static_cast(bytes.at(100)); + out.maxZoom = static_cast(bytes.at(101)); + return true; +} + +// Reads a base-128 varint. Returns false if the buffer is exhausted. +bool readVarint(const uchar *data, qint64 size, qint64 &pos, quint64 &value) +{ + value = 0; + int shift = 0; + while (pos < size && shift < 64) { + const quint8 b = data[pos++]; + value |= static_cast(b & 0x7F) << shift; + if ((b & 0x80) == 0) { + return true; + } + shift += 7; + } + return false; +} + +// Deserialize a (decompressed) directory blob: delta-encoded tile_ids, then +// offsets (0 == consecutive after previous entry), lengths, run_lengths. +bool deserializeDirectory(const QByteArray &blob, std::vector &entries, QString &error) +{ + const auto *data = reinterpret_cast(blob.constData()); + const qint64 size = blob.size(); + qint64 pos = 0; + + quint64 numEntries = 0; + if (!readVarint(data, size, pos, numEntries)) { + error = QStringLiteral("Corrupt PMTiles directory (entry count)"); + return false; + } + // Each entry contributes at least one byte across its four varint streams + // (tile_id, run_length, length, offset). A numEntries larger than the bytes + // left in the blob is impossible — reject before allocating to bound memory. + const quint64 remainingBytes = static_cast(size - pos); + if (numEntries > remainingBytes) { + error = QStringLiteral("Corrupt PMTiles directory (entry count %1 exceeds %2 remaining bytes)") + .arg(numEntries) + .arg(remainingBytes); + return false; + } + entries.assign(static_cast(numEntries), PMEntry{}); + + quint64 lastId = 0; + for (quint64 i = 0; i < numEntries; ++i) { + quint64 delta = 0; + if (!readVarint(data, size, pos, delta)) { + error = QStringLiteral("Corrupt PMTiles directory (tile id)"); + return false; + } + lastId += delta; + entries[i].tileId = lastId; + } + for (quint64 i = 0; i < numEntries; ++i) { + quint64 rl = 0; + if (!readVarint(data, size, pos, rl)) { + error = QStringLiteral("Corrupt PMTiles directory (run length)"); + return false; + } + entries[i].runLength = static_cast(rl); + } + for (quint64 i = 0; i < numEntries; ++i) { + quint64 len = 0; + if (!readVarint(data, size, pos, len)) { + error = QStringLiteral("Corrupt PMTiles directory (length)"); + return false; + } + entries[i].length = static_cast(len); + } + for (quint64 i = 0; i < numEntries; ++i) { + quint64 ofs = 0; + if (!readVarint(data, size, pos, ofs)) { + error = QStringLiteral("Corrupt PMTiles directory (offset)"); + return false; + } + if (ofs == 0 && i > 0) { + entries[i].offset = entries[i - 1].offset + entries[i - 1].length; + } else if (ofs == 0) { + // First entry with delta 0 has no predecessor; offset is 0 (guard ofs-1 underflow). + entries[i].offset = 0; + } else { + entries[i].offset = ofs - 1; + } + } + return true; +} + +std::optional readSection(QFile &file, quint64 offset, quint64 length) +{ + if (length == 0) { + return QByteArray(); + } + if (!file.seek(static_cast(offset))) { + return std::nullopt; + } + QByteArray out = file.read(static_cast(length)); + if (static_cast(out.size()) != length) { + return std::nullopt; + } + return out; +} + +// Derive the cache image format string from tile_type, falling back to magic-byte +// sniffing for the generic/unknown case. +QString formatForTileType(PMTileType type, const QString &providerType, const QByteArray &tileData) +{ + switch (type) { + case PMTileType::Png: return QStringLiteral("png"); + case PMTileType::Jpeg: return QStringLiteral("jpg"); + case PMTileType::Webp: return QStringLiteral("webp"); + case PMTileType::Avif: return QStringLiteral("avif"); + default: return UrlFactory::getImageFormat(providerType, tileData); + } +} + +} // namespace + +quint64 zxyToTileId(quint8 z, quint32 x, quint32 y) +{ + // Number of tiles in all levels below z: sum(4^i) for i in [0, z) == (4^z - 1) / 3. + quint64 acc = 0; + for (quint8 i = 0; i < z; ++i) { + acc += (1ULL << (2 * i)); + } + + // Hilbert d2xy inverse (xy2d) over an n = 2^z grid. + quint64 n = 1ULL << z; + quint64 rx = 0; + quint64 ry = 0; + quint64 d = 0; + quint64 tx = x; + quint64 ty = y; + for (quint64 s = n / 2; s > 0; s /= 2) { + rx = (tx & s) > 0 ? 1 : 0; + ry = (ty & s) > 0 ? 1 : 0; + d += s * s * ((3 * rx) ^ ry); + // rotate + if (ry == 0) { + if (rx == 1) { + tx = s - 1 - tx; + ty = s - 1 - ty; + } + std::swap(tx, ty); + } + } + return acc + d; +} + +Result importPMTiles(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, + const QString &providerType) +{ + Result result; + + if (!db.isValid()) { + result.errorString = QStringLiteral("Tile cache database is not valid"); + return result; + } + + QFile file(archivePath); + if (!file.open(QIODevice::ReadOnly)) { + result.errorString = QStringLiteral("Failed to open PMTiles archive: %1").arg(archivePath); + return result; + } + + const QByteArray headerBytes = file.read(kPMTilesHeaderSize); + PMHeader header; + if (!parsePMHeader(headerBytes, header, result.errorString)) { + return result; + } + + if (header.tileType == PMTileType::Mvt) { + result.errorString = QStringLiteral("PMTiles vector (MVT) archives are not supported; raster tiles only"); + return result; + } + if (header.tileCompression == PMCompression::Brotli || header.tileCompression == PMCompression::Zstd) { + result.errorString = QStringLiteral("Unsupported PMTiles tile compression: %1") + .arg(compressionName(header.tileCompression)); + return result; + } + if (header.internalCompression == PMCompression::Brotli || header.internalCompression == PMCompression::Zstd) { + result.errorString = QStringLiteral("Unsupported PMTiles directory compression: %1") + .arg(compressionName(header.internalCompression)); + return result; + } + + const auto rootRaw = readSection(file, header.rootDirOffset, header.rootDirBytes); + if (!rootRaw) { + result.errorString = QStringLiteral("Failed to read PMTiles root directory"); + return result; + } + const auto rootBlob = pmDecompress(*rootRaw, header.internalCompression, kMaxDirDecompressedBytes); + if (!rootBlob) { + result.errorString = QStringLiteral("Failed to decompress PMTiles root directory (%1)") + .arg(compressionName(header.internalCompression)); + return result; + } + + std::vector rootEntries; + if (!deserializeDirectory(*rootBlob, rootEntries, result.errorString)) { + return result; + } + + const auto setID = db.createImportedTileSet(setName, providerType, header.minZoom, header.maxZoom); + if (!setID.has_value()) { + result.errorString = QStringLiteral("Failed to create imported tile set"); + return result; + } + + quint32 imported = 0; + bool failed = false; + + // Tile reads must stay inside the declared tile-data region; leaf reads inside the + // leaf-dirs region. Both checks reject wraps and out-of-region [offset, offset+len). + const auto inRegion = [](quint64 base, quint64 regionLen, quint64 relOffset, quint32 length) -> bool { + if (relOffset > regionLen) { + return false; + } + const quint64 end = relOffset + length; + if (end < relOffset || end > regionLen) { // wrap or past region end + return false; + } + quint64 absEnd = 0; + if (qAddOverflow(base, relOffset, &absEnd)) { // absolute start wraps + return false; + } + quint64 dummy = 0; + return !qAddOverflow(absEnd, static_cast(length), &dummy); // absolute end wraps + }; + + // Accumulated import buffer flushed in transactions of kImportBatchSize tiles via + // saveTileBatch (one transaction, reused prepared queries) instead of per-tile. + std::vector chunk; + chunk.reserve(kImportBatchSize); + const auto flush = [&]() -> bool { + if (chunk.empty()) { + return true; + } + QList ptrs; + ptrs.reserve(static_cast(chunk.size())); + for (const QGCCacheTile &t : chunk) { + ptrs.append(&t); + } + const bool ok = db.saveTileBatch(ptrs); + chunk.clear(); + if (!ok) { + result.errorString = QStringLiteral("Failed to import PMTiles tile batch"); + } + return ok; + }; + + // Read + decompress + queue one tile entry. PMTiles uses XYZ addressing (no TMS flip). + const auto saveEntry = [&](quint8 z, quint32 x, quint32 y, const PMEntry &entry) -> bool { + if (!inRegion(header.tileDataOffset, header.tileDataBytes, entry.offset, entry.length)) { + result.errorString = QStringLiteral("PMTiles tile read outside tile-data region"); + return false; + } + const auto tileRaw = readSection(file, header.tileDataOffset + entry.offset, entry.length); + if (!tileRaw) { + result.errorString = QStringLiteral("Failed to read PMTiles tile data"); + return false; + } + const auto tileBlob = pmDecompress(*tileRaw, header.tileCompression, kMaxTileDecompressedBytes); + if (!tileBlob) { + result.errorString = QStringLiteral("Unsupported PMTiles tile compression: %1") + .arg(compressionName(header.tileCompression)); + return false; + } + if (tileBlob->isEmpty()) { + return true; + } + const QString hash = UrlFactory::getTileHash(providerType, static_cast(x), static_cast(y), + static_cast(z)); + const QString format = formatForTileType(header.tileType, providerType, *tileBlob); + chunk.emplace_back(hash, *tileBlob, format, providerType, setID.value()); + ++imported; + if (chunk.size() >= static_cast(kImportBatchSize)) { + return flush(); + } + return true; + }; + + // Expand a tile id into z/x/y. Inverse of zxyToTileId over the run. Returns false on + // a zoom that would overflow the 1<<(2*zoom) shift (UB) — i.e. a corrupt tile id. + const auto tileIdToZxy = [](quint64 tileId, quint8 &z, quint32 &x, quint32 &y) -> bool { + quint64 acc = 0; + quint8 zoom = 0; + while (true) { + if (zoom > 31) { // 1ULL << (2*32) is UB; PMTiles zoom never reaches here + return false; + } + const quint64 levelTiles = 1ULL << (2 * zoom); + if (acc + levelTiles > tileId) { + break; + } + acc += levelTiles; + ++zoom; + } + quint64 pos = tileId - acc; + quint64 n = 1ULL << zoom; + quint64 tx = 0; + quint64 ty = 0; + for (quint64 s = 1; s < n; s *= 2) { + quint64 rx = 1 & (pos / 2); + quint64 ry = 1 & (pos ^ rx); + if (ry == 0) { + if (rx == 1) { + tx = s - 1 - tx; + ty = s - 1 - ty; + } + std::swap(tx, ty); + } + tx += s * rx; + ty += s * ry; + pos /= 4; + } + z = zoom; + x = static_cast(tx); + y = static_cast(ty); + return true; + }; + + // Bound total run-length expansion against the header's addressed-tile count so a + // crafted run_length can't fan out into billions of saveEntry calls (0 == unknown). + const quint64 maxExpanded = header.addressedTilesCount > 0 ? header.addressedTilesCount : UINT64_MAX; + quint64 expanded = 0; + + // Walk a directory; recurse into leaf dirs (run_length == 0). Depth is capped and a + // visited-offset set breaks cyclic leaf pointers (PMTiles trees are shallow). + constexpr int kMaxLeafDepth = 4; + std::set visitedLeaves; + std::function &, int)> walk = + [&](const std::vector &entries, int depth) -> bool { + for (const PMEntry &entry : entries) { + if (entry.runLength == 0) { + if (depth >= kMaxLeafDepth) { + result.errorString = QStringLiteral("PMTiles leaf directory nesting too deep"); + return false; + } + if (!visitedLeaves.insert(entry.offset).second) { + result.errorString = QStringLiteral("PMTiles leaf directory cycle detected"); + return false; + } + if (!inRegion(header.leafDirsOffset, header.leafDirsBytes, entry.offset, entry.length)) { + result.errorString = QStringLiteral("PMTiles leaf read outside leaf-dirs region"); + return false; + } + const auto leafRaw = readSection(file, header.leafDirsOffset + entry.offset, entry.length); + if (!leafRaw) { + result.errorString = QStringLiteral("Failed to read PMTiles leaf directory"); + return false; + } + const auto leafBlob = pmDecompress(*leafRaw, header.internalCompression, kMaxDirDecompressedBytes); + if (!leafBlob) { + result.errorString = QStringLiteral("Failed to decompress PMTiles leaf directory (%1)") + .arg(compressionName(header.internalCompression)); + return false; + } + std::vector leafEntries; + if (!deserializeDirectory(*leafBlob, leafEntries, result.errorString)) { + return false; + } + if (!walk(leafEntries, depth + 1)) { + return false; + } + continue; + } + for (quint32 run = 0; run < entry.runLength; ++run) { + if (++expanded > maxExpanded) { + result.errorString = QStringLiteral("PMTiles run-length expansion exceeds addressed-tile count"); + return false; + } + quint8 z = 0; + quint32 x = 0; + quint32 y = 0; + if (!tileIdToZxy(entry.tileId + run, z, x, y)) { + result.errorString = QStringLiteral("Corrupt PMTiles tile id"); + return false; + } + if (!saveEntry(z, x, y, entry)) { + return false; + } + } + } + return true; + }; + + failed = !walk(rootEntries, 0) || !flush(); + if (failed) { + qCWarning(QGCTileSetImporterLog) << "PMTiles import failed:" << result.errorString; + (void) db.deleteTileSet(setID.value()); + return result; + } + + { + QSqlQuery update(db.database()); + if (update.prepare(QStringLiteral("UPDATE TileSets SET numTiles = ? WHERE setID = ?"))) { + update.addBindValue(imported); + update.addBindValue(setID.value()); + (void) update.exec(); + } + } + + // Bulk import inflates the WAL; fold it back into the main DB and shrink the -wal. + QSqlDatabase cacheDB = db.database(); + QGCSqlHelper::walCheckpointTruncate(cacheDB); + + qCDebug(QGCTileSetImporterLog) << "Imported" << imported << "tiles from" << archivePath << "into set" + << setID.value(); + + result.success = true; + result.setID = setID.value(); + result.tileCount = imported; + return result; +} + +Result import(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, const QString &providerType) +{ + const QString suffix = QFileInfo(archivePath).suffix().toLower(); + if (suffix == QStringLiteral("mbtiles")) { + return importMBTiles(db, archivePath, setName, providerType); + } + if (suffix == QStringLiteral("pmtiles")) { + return importPMTiles(db, archivePath, setName, providerType); + } + + Result result; + result.errorString = QStringLiteral("Unsupported tile archive format: .%1").arg(suffix); + return result; +} + +} // namespace QGCTileSetImporter diff --git a/src/QtLocationPlugin/QGCTileSetImporter.h b/src/QtLocationPlugin/QGCTileSetImporter.h new file mode 100644 index 000000000000..5a0eba51ce25 --- /dev/null +++ b/src/QtLocationPlugin/QGCTileSetImporter.h @@ -0,0 +1,51 @@ +#pragma once + +#include + +#include + +class QGCTileCacheDatabase; + +/// Imports local offline tile archives (MBTiles, PMTiles) into the QGC SQLite +/// tile cache as a new offline tile set. Tiles are written through the existing +/// QGCTileCacheDatabase::saveTile path so they dedupe against already-cached +/// tiles and are protected from prune like any other offline set. +/// +/// Y-axis note: MBTiles store rows TMS-style (origin bottom-left); QGC hashes +/// tiles XYZ-style (origin top-left), so the importer flips the row before +/// hashing: y_xyz = (2^zoom - 1) - tile_row. +namespace QGCTileSetImporter { + +struct Result { + bool success = false; + QString errorString; + quint64 setID = 0; + quint32 tileCount = 0; +}; + +/// Default provider type imported tiles are hashed/tagged against. The generic +/// custom-URL provider is used because imported archives have no live source. +inline constexpr const char *kDefaultProviderType = "CustomURL Custom"; + +/// Imports \a archivePath (.mbtiles or .pmtiles, dispatched by extension) into +/// \a db. \a setName is the offline-set name (deduplicated if it collides). +/// \a providerType is the cache provider type used for tile hashing. +Result import(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, + const QString &providerType = QString::fromLatin1(kDefaultProviderType)); + +/// Imports a standard MBTiles SQLite file (tiles(zoom_level, tile_column, tile_row, tile_data)). +Result importMBTiles(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, + const QString &providerType); + +/// Imports a PMTiles v3 raster archive (https://github.com/protomaps/PMTiles). +/// Tiles flow through the same saveTile path as MBTiles. PMTiles uses XYZ tile +/// addressing, so no Y-flip is applied. Gzip and uncompressed directories/tiles +/// are supported; Brotli/Zstd payloads yield a clear "unsupported codec" error. +Result importPMTiles(QGCTileCacheDatabase &db, const QString &archivePath, const QString &setName, + const QString &providerType); + +/// Maps a PMTiles (z, x, y) coordinate to its v3 Hilbert-curve tile id. +/// Exposed for testing against the published spec vectors. +quint64 zxyToTileId(quint8 z, quint32 x, quint32 y); + +} // namespace QGCTileSetImporter diff --git a/src/QtLocationPlugin/QGeoFileTileCacheQGC.cpp b/src/QtLocationPlugin/QGeoFileTileCacheQGC.cpp index 0165e1eaa3e4..2cae302df53c 100644 --- a/src/QtLocationPlugin/QGeoFileTileCacheQGC.cpp +++ b/src/QtLocationPlugin/QGeoFileTileCacheQGC.cpp @@ -1,212 +1,38 @@ #include "QGeoFileTileCacheQGC.h" -#include -#include -#include +#include -#include "AppSettings.h" -#include "MapsSettings.h" -#include "AppMessages.h" -#include "QGCCacheTile.h" -#include "QGCFileHelper.h" #include "QGCLoggingCategory.h" -#include "QGCMapEngine.h" -#include "QGCMapTasks.h" -#include "QGCMapUrlEngine.h" -#include "SettingsManager.h" +#include "QGCTileCache.h" QGC_LOGGING_CATEGORY(QGeoFileTileCacheQGCLog, "QtLocationPlugin.QGeoFileTileCacheQGC") -QString QGeoFileTileCacheQGC::_databaseFilePath; -QString QGeoFileTileCacheQGC::_cachePath; -std::atomic QGeoFileTileCacheQGC::_cacheWasReset = false; - -QGeoFileTileCacheQGC::QGeoFileTileCacheQGC(const QVariantMap ¶meters, QObject *parent) - : QGeoFileTileCache(baseCacheDirectory(), parent) +QGeoFileTileCacheQGC::QGeoFileTileCacheQGC(const QVariantMap& parameters, QObject* parent) + : QAbstractGeoTileCache(parent) { + Q_UNUSED(parameters); qCDebug(QGeoFileTileCacheQGCLog) << this; - setCostStrategyDisk(QGeoFileTileCache::ByteSize); - setMaxDiskUsage(_getDefaultMaxDiskCache()); - setCostStrategyMemory(QGeoFileTileCache::ByteSize); - setMaxMemoryUsage(_getMemLimit(parameters)); - setCostStrategyTexture(QGeoFileTileCache::ByteSize); - setMinTextureUsage(_getDefaultMinTexture()); - setExtraTextureUsage(_getDefaultExtraTexture() - minTextureUsage()); - - static std::once_flag cacheInit; - std::call_once(cacheInit, [this]() { - _initCache(); - }); - - directory_ = _getCachePath(parameters); + // Init cache paths once before the cache worker thread starts. + QGCTileCache::ensureInitialized(); } QGeoFileTileCacheQGC::~QGeoFileTileCacheQGC() { - if (QGeoFileTileCacheQGCLog().isDebugEnabled()) { - printStats(); - } - qCDebug(QGeoFileTileCacheQGCLog) << this; } -uint32_t QGeoFileTileCacheQGC::_getMemLimit(const QVariantMap ¶meters) -{ - uint32_t memLimit = 0; - if (parameters.contains(QStringLiteral("mapping.cache.memory.size"))) { - bool ok = false; - memLimit = parameters.value(QStringLiteral("mapping.cache.memory.size")).toString().toUInt(&ok); - if (!ok) { - memLimit = 0; - } - } - - if (memLimit == 0) { - // Value saved in MB - memLimit = _getMaxMemCacheSetting() * qPow(1024, 2); - } - if (memLimit == 0) { - memLimit = _getDefaultMaxMemLimit(); - } - - // 1MB Minimum Memory Cache Required - // MaxMemoryUsage is 32bit Integer, Round down to 1GB - memLimit = qBound(static_cast(qPow(1024, 2)), memLimit, static_cast(qPow(1024, 3))); - return memLimit; -} - -quint32 QGeoFileTileCacheQGC::_getMaxMemCacheSetting() -{ - return SettingsManager::instance()->mapsSettings()->maxCacheMemorySize()->rawValue().toUInt(); -} - -quint32 QGeoFileTileCacheQGC::getMaxDiskCacheSetting() -{ - return SettingsManager::instance()->mapsSettings()->maxCacheDiskSize()->rawValue().toUInt(); -} - -void QGeoFileTileCacheQGC::cacheTile(const QString &type, int x, int y, int z, const QByteArray &image, const QString &format, qulonglong set) -{ - const QString hash = UrlFactory::getTileHash(type, x, y, z); - cacheTile(type, hash, image, format, set); -} - -void QGeoFileTileCacheQGC::cacheTile(const QString &type, const QString &hash, const QByteArray &image, const QString &format, qulonglong set) -{ - AppSettings *appSettings = SettingsManager::instance()->appSettings(); - if (!appSettings->disableAllPersistence()->rawValue().toBool()) { - QGCCacheTile *tile = new QGCCacheTile(hash, image, format, type, set); - QGCSaveTileTask *task = new QGCSaveTileTask(tile); - if (!getQGCMapEngine()->addTask(task)) { - task->deleteLater(); - } - } -} - -QGCFetchTileTask* QGeoFileTileCacheQGC::createFetchTileTask(const QString &type, int x, int y, int z) +QSharedPointer QGeoFileTileCacheQGC::get(const QGeoTileSpec& spec) { - const QString hash = UrlFactory::getTileHash(type, x, y, z); - QGCFetchTileTask *task = new QGCFetchTileTask(hash); - return task; + Q_UNUSED(spec); + return QSharedPointer(); } -QString QGeoFileTileCacheQGC::_getCachePath(const QVariantMap ¶meters) +void QGeoFileTileCacheQGC::insert(const QGeoTileSpec& spec, const QByteArray& bytes, const QString& format, + QAbstractGeoTileCache::CacheAreas areas) { - QString cacheDir; - if (parameters.contains(QStringLiteral("mapping.cache.directory"))) { - cacheDir = parameters.value(QStringLiteral("mapping.cache.directory")).toString(); - } else { - cacheDir = _cachePath + QLatin1String("/providers"); - if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { - qCWarning(QGeoFileTileCacheQGCLog) << "Could not create mapping disk cache directory:" << cacheDir; - cacheDir = QDir::homePath() + QStringLiteral("/.qgcmapscache/"); - } - } - - if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { - qCWarning(QGeoFileTileCacheQGCLog) << "Could not create mapping disk cache directory:" << cacheDir; - cacheDir.clear(); - } - - return cacheDir; -} - -bool QGeoFileTileCacheQGC::_wipeDirectory(const QString &dirPath) -{ - bool result = true; - - const QDir dir(dirPath); - if (dir.exists(dirPath)) { - _cacheWasReset = true; - - const QFileInfoList fileList = dir.entryInfoList(QDir::NoDotAndDotDot | QDir::System | QDir::Hidden | QDir::AllDirs | QDir::Files, QDir::DirsFirst); - for (const QFileInfo &info : fileList) { - if (info.isDir()) { - result = _wipeDirectory(info.absoluteFilePath()); - } else { - result = QFile::remove(info.absoluteFilePath()); - } - - if (!result) { - return result; - } - } - result = dir.rmdir(dirPath); - } - - return result; -} - -void QGeoFileTileCacheQGC::_wipeOldCaches() -{ - const QStringList oldCaches = {"/QGCMapCache55", "/QGCMapCache100", "/QGCMapCache300"}; - for (const QString &cache : oldCaches) { - QString oldCacheDir; - #if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) - oldCacheDir = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); - #else - oldCacheDir = QStandardPaths::writableLocation(QStandardPaths::GenericCacheLocation); - #endif - oldCacheDir += cache; - _wipeDirectory(oldCacheDir); - } -} - -void QGeoFileTileCacheQGC::_initCache() -{ - _wipeOldCaches(); - - // QString cacheDir = QAbstractGeoTileCache::baseCacheDirectory() -#if defined(Q_OS_ANDROID) || defined(Q_OS_IOS) - QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::AppDataLocation); - cacheDir += QStringLiteral("/QGCMapCache"); -#else - QString cacheDir = QStandardPaths::writableLocation(QStandardPaths::CacheLocation); - cacheDir += QStringLiteral("/QGCMapCache"); -#endif - if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { - qCWarning(QGeoFileTileCacheQGCLog) << "Could not create mapping disk cache directory:" << cacheDir; - - cacheDir = QDir::homePath() + QStringLiteral("/.qgcmapscache/"); - if (!QGCFileHelper::ensureDirectoryExists(cacheDir)) { - qCWarning(QGeoFileTileCacheQGCLog) << "Could not create mapping disk cache directory:" << cacheDir; - cacheDir.clear(); - } - } - - _cachePath = cacheDir; - if (!_cachePath.isEmpty()) { - _databaseFilePath = QString(_cachePath + QStringLiteral("/qgcMapCache.db")); - - qCDebug(QGeoFileTileCacheQGCLog) << "Map Cache in:" << _databaseFilePath; - } else { - qCCritical(QGeoFileTileCacheQGCLog) << "Could not find suitable map cache directory."; - } - - if (_cacheWasReset) { - QGC::showAppMessage(tr( - "The Offline Map Cache database has been upgraded. " - "Your old map cache sets have been reset.")); - } + Q_UNUSED(spec); + Q_UNUSED(bytes); + Q_UNUSED(format); + Q_UNUSED(areas); } diff --git a/src/QtLocationPlugin/QGeoFileTileCacheQGC.h b/src/QtLocationPlugin/QGeoFileTileCacheQGC.h index 00fde481cd8f..bfa6e5258b6c 100644 --- a/src/QtLocationPlugin/QGeoFileTileCacheQGC.h +++ b/src/QtLocationPlugin/QGeoFileTileCacheQGC.h @@ -1,46 +1,57 @@ #pragma once -#include - -#include - -class QGCFetchTileTask; - -class QGeoFileTileCacheQGC : public QGeoFileTileCache +#include + +// SQLite-backed tile cache. The flat-file QGeoFileTileCache is deliberately NOT +// used as a base: the SQLite tile store (QGCTileCacheDatabase via the cache +// worker) is the sole authoritative cache and owns expiry, ETag/Last-Modified +// revalidation and LRU. This subclasses QAbstractGeoTileCache directly so none +// of QGeoFileTileCache's memory/disk/texture QCache3Q caches or its startup +// loadTiles() disk scan are allocated. get() returns a null texture so every +// visible tile is routed through the fetcher -> reply -> SQLite path; insert() +// is a no-op so the engine never writes a duplicate flat-file copy. +// +// The cache facade (paths, tile-save tasks, validator refresh) lives in the +// free QGCTileCache namespace; this class holds only the QtLocation vtable. +class QGeoFileTileCacheQGC : public QAbstractGeoTileCache { Q_OBJECT public: - explicit QGeoFileTileCacheQGC(const QVariantMap ¶meters, QObject *parent = nullptr); + explicit QGeoFileTileCacheQGC(const QVariantMap& parameters, QObject* parent = nullptr); ~QGeoFileTileCacheQGC(); - static quint32 getMaxDiskCacheSetting(); - static void cacheTile(const QString &type, int x, int y, int z, const QByteArray &image, const QString &format, qulonglong set = UINT64_MAX); - static void cacheTile(const QString &type, const QString &hash, const QByteArray &image, const QString &format, qulonglong set = UINT64_MAX); - static QGCFetchTileTask *createFetchTileTask(const QString &type, int x, int y, int z); - static QString getDatabaseFilePath() { return _databaseFilePath; } - static QString getCachePath() { return _cachePath; } + QSharedPointer get(const QGeoTileSpec& spec) override; + void insert(const QGeoTileSpec& spec, const QByteArray& bytes, const QString& format, + QAbstractGeoTileCache::CacheAreas areas = QAbstractGeoTileCache::AllCaches) override; + + void init() override {} + + // No in-process caches to size, so the texture/cost-strategy knobs are inert. + void setMinTextureUsage(int) override {} + + void setExtraTextureUsage(int) override {} + + int maxTextureUsage() const override { return 0; } + + int minTextureUsage() const override { return 0; } + + int textureUsage() const override { return 0; } + + void clearAll() override {} + + void setCostStrategyDisk(CostStrategy) override {} -private: - // QString tileSpecToFilename(const QGeoTileSpec &spec, const QString &format, const QString &directory) const final; - // QGeoTileSpec filenameToTileSpec(const QString &filename) const final; + CostStrategy costStrategyDisk() const override { return ByteSize; } - static void _initCache(); - static bool _wipeDirectory(const QString &dirPath); - static void _wipeOldCaches(); + void setCostStrategyMemory(CostStrategy) override {} - static QString _getCachePath(const QVariantMap ¶meters); - static uint32_t _getMemLimit(const QVariantMap &Parameters); + CostStrategy costStrategyMemory() const override { return ByteSize; } - static uint32_t _getDefaultMaxMemLimit() { return (3 * qPow(1024, 2)); } - static uint32_t _getDefaultMaxDiskCache() { return 0; } // (50 * pow(1024, 2)); - static uint32_t _getDefaultExtraTexture() { return (6 * qPow(1024, 2)); } - static uint32_t _getDefaultMinTexture() { return 0; } + void setCostStrategyTexture(CostStrategy) override {} - static quint32 _getMaxMemCacheSetting(); + CostStrategy costStrategyTexture() const override { return ByteSize; } - // Initialized once via std::call_once in constructor before worker thread starts - static QString _databaseFilePath; - static QString _cachePath; - static std::atomic _cacheWasReset; +protected: + void printStats() override {} }; diff --git a/src/QtLocationPlugin/QGeoMapReplyQGC.cpp b/src/QtLocationPlugin/QGeoMapReplyQGC.cpp index cb3c79662d52..558b9468f178 100644 --- a/src/QtLocationPlugin/QGeoMapReplyQGC.cpp +++ b/src/QtLocationPlugin/QGeoMapReplyQGC.cpp @@ -1,29 +1,106 @@ #include "QGeoMapReplyQGC.h" +#include #include +#include +#include +#include +#include #include #include #include +#include +#include +#include +#include "BingMapProvider.h" #include "ElevationMapProvider.h" #include "MapProvider.h" #include "QGCCacheTile.h" +#include "QGCHostCircuitBreaker.h" #include "QGCLoggingCategory.h" #include "QGCMapEngine.h" #include "QGCMapTasks.h" #include "QGCMapUrlEngine.h" #include "QGCNetworkHelper.h" -#include "QGeoFileTileCacheQGC.h" +#include "QGCTileCache.h" +#include "QGeoTileFetcherQGC.h" +#include "TileFetchMetrics.h" QGC_LOGGING_CATEGORY(QGeoTiledMapReplyQGCLog, "QtLocationPlugin.QGeoTiledMapReplyQGC") +namespace { +constexpr int kHttpNotModified = 304; + +// Self-owned background conditional GET (R5). Outlives the QGeoTiledMapReplyQGC +// that spawned it (which has already finished with stale bytes), writes the fresh +// tile into the SQLite cache on 200 / refreshes validators on 304, and deletes +// itself. Holds no pointer back to the reply, so it is safe after the reply dies. +class DetachedRevalidator : public QObject +{ +public: + DetachedRevalidator(QNetworkAccessManager* nam, const QNetworkRequest& request, QString providerName, int x, int y, + int z, QByteArray etag, QByteArray lastModified, qint64 staleExpiresAt) + : _providerName(std::move(providerName)), + _x(x), + _y(y), + _z(z), + _etag(std::move(etag)), + _lastModified(std::move(lastModified)), + _staleExpiresAt(staleExpiresAt) + { + QNetworkReply* const reply = nam->get(request); + reply->setParent(this); + QGCNetworkHelper::ignoreSslErrorsIfNeeded(reply); + connect(reply, &QNetworkReply::finished, this, [this, reply]() { _finished(reply); }); + } + +private: + void _finished(QNetworkReply* reply) + { + const QGeoTileFetcherQGC::FetchResult result = QGeoTileFetcherQGC::snapshot(reply); + const QString hash = UrlFactory::getTileHash(_providerName, _x, _y, _z); + const QString host = reply->request().url().host(); + + if (result.error == QNetworkReply::NoError) { + HostCircuitBreaker::instance().recordSuccess(host); + if (result.statusCode == kHttpNotModified) { + const QByteArray etag = result.etag.isEmpty() ? _etag : result.etag; + const QByteArray lm = result.lastModified.isEmpty() ? _lastModified : result.lastModified; + const qint64 expiresAt = (result.expiresAt == 0) ? _staleExpiresAt : result.expiresAt; + QGCTileCache::refreshTileValidators(_providerName, hash, etag, lm, expiresAt); + } else if (QGCNetworkHelper::isHttpSuccess(result.statusCode) && !result.body.isEmpty()) { + const SharedMapProvider provider = UrlFactory::getMapProviderFromProviderType(_providerName); + const QString format = provider ? provider->getImageFormat(result.body) : QString(); + if (!format.isEmpty()) { + QGCTileCache::cacheTile(_providerName, hash, result.body, format, UINT64_MAX, result.etag, + result.lastModified, result.expiresAt, result.mustRevalidate); + } + } + // Non-2xx (incl. 404): leave the stale copy in place; do not poison the cache. + } else if (QGCNetworkHelper::isTransientError(result.error, result.statusCode)) { + HostCircuitBreaker::instance().recordFailure(host); + } + deleteLater(); + } + + const QString _providerName; + const int _x; + const int _y; + const int _z; + const QByteArray _etag; + const QByteArray _lastModified; + const qint64 _staleExpiresAt; +}; + +} // namespace + QByteArray QGeoTiledMapReplyQGC::_bingNoTileImage; QByteArray QGeoTiledMapReplyQGC::_badTile; -QGeoTiledMapReplyQGC::QGeoTiledMapReplyQGC(QNetworkAccessManager *networkManager, const QNetworkRequest &request, const QGeoTileSpec &spec, QObject *parent) - : QGeoTiledMapReply(spec, parent) - , _networkManager(networkManager) - , _request(request) +QGeoTiledMapReplyQGC::QGeoTiledMapReplyQGC(QNetworkAccessManager* networkManager, const QNetworkRequest& request, + const QGeoTileSpec& spec, QObject* parent, QGeoTileFetcherQGC* fetcher) + : QGeoTiledMapReply(spec, parent), _fetcher(fetcher), _networkManager(networkManager), _request(request) { qCDebug(QGeoTiledMapReplyQGCLog) << this; } @@ -43,14 +120,19 @@ bool QGeoTiledMapReplyQGC::init() _initDataFromResources(); - (void) connect(this, &QGeoTiledMapReplyQGC::errorOccurred, this, [this](QGeoTiledMapReply::Error error, const QString &errorString) { - qCWarning(QGeoTiledMapReplyQGCLog) << error << errorString; - setMapImageData(_badTile); - setMapImageFormat(QStringLiteral("png")); - setCached(false); - }, Qt::AutoConnection); - - QGCFetchTileTask *task = QGeoFileTileCacheQGC::createFetchTileTask(UrlFactory::getProviderTypeFromQtMapId(tileSpec().mapId()), tileSpec().x(), tileSpec().y(), tileSpec().zoom()); + (void) connect( + this, &QGeoTiledMapReplyQGC::errorOccurred, this, + [this](QGeoTiledMapReply::Error error, const QString& errorString) { + // Single-tile failures are routine; debug-level keeps a network burst from flooding warnings. + qCDebug(QGeoTiledMapReplyQGCLog) << error << errorString; + setMapImageData(_badTile); + setMapImageFormat(QStringLiteral("png")); + setCached(false); + }, + Qt::AutoConnection); + + QGCFetchTileTask* task = QGCTileCache::createFetchTileTask( + UrlFactory::getProviderTypeFromQtMapId(tileSpec().mapId()), tileSpec().x(), tileSpec().y(), tileSpec().zoom()); if (!task) { qCWarning(QGeoTiledMapReplyQGCLog) << "Failed to create fetch tile task"; m_initialized = false; @@ -95,22 +177,68 @@ void QGeoTiledMapReplyQGC::_networkReplyFinished() } reply->deleteLater(); + // The error path is owned by _networkReplyError (errorOccurred fires before + // finished); bail so we don't clobber it. if (reply->error() != QNetworkReply::NoError) { return; } - if (!reply->isOpen()) { - setError(QGeoTiledMapReply::ParseError, tr("Empty Reply")); + _handleNetworkResult(QGeoTileFetcherQGC::snapshot(reply)); +} + +void QGeoTiledMapReplyQGC::_handleNetworkResult(const QGeoTileFetcherQGC::FetchResult& result) +{ + if (isFinished()) { + return; + } + + if (result.error != QNetworkReply::NoError) { + // Reached only via the coalescing path: the non-coalesced path bails on + // error in _networkReplyFinished before delivering here. A delivered + // cancel is therefore a leader timeout/abort (a follower that aborted + // itself was already removed from the group), so retry rather than + // finishing with no tile. + if (_scheduleRetryIfNeeded(result.error, result.statusCode, result.retryAfter)) { + return; + } + TileFetchMetrics::instance().recordNetworkError(_fetchTimer.isValid() ? _fetchTimer.elapsed() : 0); + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, result.errorString); + return; + } + + const int statusCode = result.statusCode; + + // 304 Not Modified: the stale cached body is still valid. Refresh its + // validators/expiry in the cache and serve it without re-downloading. + if (statusCode == kHttpNotModified) { + if (_cachedTile) { + const QByteArray newEtag = result.etag.isEmpty() ? _etag : result.etag; + const QByteArray newLastModified = result.lastModified.isEmpty() ? _lastModified : result.lastModified; + // A bare 304 has no freshness headers (_parseExpiry defaults to now+7d); keep the cached expiry instead. + const qint64 newExpiresAt = result.hasExplicitExpiry ? result.expiresAt : _cachedTile->expiresAt; + + const SharedMapProvider provider = UrlFactory::getMapProviderFromQtMapId(tileSpec().mapId()); + if (provider) { + QGCTileCache::refreshTileValidators( + provider->getMapName(), + UrlFactory::getTileHash(provider->getMapName(), tileSpec().x(), tileSpec().y(), tileSpec().zoom()), + newEtag, newLastModified, newExpiresAt); + } + HostCircuitBreaker::instance().recordSuccess(_request.url().host()); + TileFetchMetrics::instance().recordNotModified(_fetchTimer.isValid() ? _fetchTimer.elapsed() : 0); + _serveCachedTile(); + } else { + setError(QGeoTiledMapReply::UnknownError, tr("Unexpected 304 Not Modified")); + } return; } - const int statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); if (!QGCNetworkHelper::isHttpSuccess(statusCode)) { - setError(QGeoTiledMapReply::CommunicationError, reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString()); + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, result.reasonPhrase); return; } - QByteArray image = reply->readAll(); + QByteArray image = result.body; if (image.isEmpty()) { setError(QGeoTiledMapReply::ParseError, tr("Image is Empty")); return; @@ -122,13 +250,13 @@ void QGeoTiledMapReplyQGC::_networkReplyFinished() return; } - if (mapProvider->isBingProvider() && (image == _bingNoTileImage)) { + if (std::dynamic_pointer_cast(mapProvider) && (image == _bingNoTileImage)) { setError(QGeoTiledMapReply::CommunicationError, tr("Bing Tile Above Zoom Level")); return; } - if (mapProvider->isElevationProvider()) { - const SharedElevationProvider elevationProvider = std::dynamic_pointer_cast(mapProvider); + if (const SharedElevationProvider elevationProvider = + std::dynamic_pointer_cast(mapProvider)) { image = elevationProvider->serialize(image); if (image.isEmpty()) { setError(QGeoTiledMapReply::ParseError, tr("Failed to Serialize Terrain Tile")); @@ -144,29 +272,104 @@ void QGeoTiledMapReplyQGC::_networkReplyFinished() } setMapImageFormat(format); - QGeoFileTileCacheQGC::cacheTile(mapProvider->getMapName(), tileSpec().x(), tileSpec().y(), tileSpec().zoom(), image, format); + QGCTileCache::cacheTile(mapProvider->getMapName(), tileSpec().x(), tileSpec().y(), tileSpec().zoom(), image, format, + UINT64_MAX, result.etag, result.lastModified, result.expiresAt, result.mustRevalidate); + HostCircuitBreaker::instance().recordSuccess(_request.url().host()); + TileFetchMetrics::instance().recordNetworkSuccess(static_cast(result.body.size()), + _fetchTimer.isValid() ? _fetchTimer.elapsed() : 0); setFinished(true); } void QGeoTiledMapReplyQGC::_networkReplyError(QNetworkReply::NetworkError error) { - if (error != QNetworkReply::OperationCanceledError) { - const QNetworkReply* const reply = qobject_cast(sender()); - if (!reply) { - setError(QGeoTiledMapReply::CommunicationError, tr("Invalid Reply")); - } else { - setError(QGeoTiledMapReply::CommunicationError, reply->errorString()); - } - } else { + if (error == QNetworkReply::OperationCanceledError) { setFinished(true); + return; + } + + const QNetworkReply* const reply = qobject_cast(sender()); + const int statusCode = reply ? reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt() : 0; + + if (_scheduleRetryIfNeeded(reply, error, statusCode)) { + return; + } + + if (!reply) { + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, tr("Invalid Reply")); + } else { + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, reply->errorString()); } } -void QGeoTiledMapReplyQGC::_networkReplySslErrors(const QList &errors) +bool QGeoTiledMapReplyQGC::_scheduleRetryIfNeeded(const QNetworkReply* reply, QNetworkReply::NetworkError error, + int statusCode) +{ + if (!QGCNetworkHelper::isTransientError(error, statusCode)) { + return false; + } + // Transient failure: feed the per-host breaker regardless of whether retries remain. + HostCircuitBreaker::instance().recordFailure(_request.url().host()); + if (_retryAttempt >= kMaxRetryAttempts) { + return false; + } + + std::chrono::milliseconds delay = QGCNetworkHelper::retryBackoff(_retryAttempt); + if (statusCode == QGCNetworkHelper::kHttpTooManyRequests) { + delay = QGCNetworkHelper::retryAfterFromReply(reply).value_or(QGCNetworkHelper::kDefaultRateLimitDelay); + } + + _retryAttempt++; + qCDebug(QGeoTiledMapReplyQGCLog) << "Retrying tile fetch, attempt" << _retryAttempt << "in" << delay.count() + << "ms"; + + QTimer::singleShot(delay, this, [this]() { + if (!isFinished()) { + _startNetworkRequest(); + } + }); + + return true; +} + +bool QGeoTiledMapReplyQGC::_scheduleRetryIfNeeded(QNetworkReply::NetworkError error, int statusCode, + std::optional retryAfter) +{ + // Coalesced-only overload: a leader transfer timeout surfaces as + // OperationCanceledError here (per-reply user aborts never reach this path), + // so treat it as transient alongside the usual retriable errors. + const bool transient = + (error == QNetworkReply::OperationCanceledError) || QGCNetworkHelper::isTransientError(error, statusCode); + if (!transient) { + return false; + } + HostCircuitBreaker::instance().recordFailure(_request.url().host()); + if (_retryAttempt >= kMaxRetryAttempts) { + return false; + } + + std::chrono::milliseconds delay = QGCNetworkHelper::retryBackoff(_retryAttempt); + if (statusCode == QGCNetworkHelper::kHttpTooManyRequests) { + delay = retryAfter.value_or(QGCNetworkHelper::kDefaultRateLimitDelay); + } + + _retryAttempt++; + qCDebug(QGeoTiledMapReplyQGCLog) << "Retrying coalesced tile fetch, attempt" << _retryAttempt << "in" + << delay.count() << "ms"; + + QTimer::singleShot(delay, this, [this]() { + if (!isFinished()) { + _startNetworkRequest(); + } + }); + + return true; +} + +void QGeoTiledMapReplyQGC::_networkReplySslErrors(const QList& errors) { QString errorString; - for (const QSslError &error : errors) { + for (const QSslError& error : errors) { if (!errorString.isEmpty()) { (void) errorString.append('\n'); } @@ -178,31 +381,241 @@ void QGeoTiledMapReplyQGC::_networkReplySslErrors(const QList &errors } } -void QGeoTiledMapReplyQGC::_cacheReply(QGCCacheTile *tile) +void QGeoTiledMapReplyQGC::_cacheReply(QSharedPointer tile) { - if (tile) { - setMapImageData(tile->img); - setMapImageFormat(tile->format); - setCached(true); - setFinished(true); - delete tile; - } else { + if (!tile) { + setError(QGeoTiledMapReply::UnknownError, tr("Invalid Cache Tile")); + return; + } + + _cachedTile = tile; + + const qint64 now = QDateTime::currentSecsSinceEpoch(); + // expiresAt == 0 (no-store/no-cache/max-age=0) means immediately stale, not fresh-forever. + const bool expired = (_cachedTile->expiresAt == 0) || (_cachedTile->expiresAt <= now); + const bool revalidatable = !_cachedTile->etag.isEmpty() || !_cachedTile->lastModified.isEmpty(); + // A tile stored with Cache-Control no-cache/must-revalidate must never be + // served without a fresh check; treat it like an expired tile. + const bool needsRevalidation = expired || _cachedTile->mustRevalidate; + + // Fresh, or no network to revalidate over: serve what we cached. + if (!needsRevalidation || !QGCNetworkHelper::isInternetAvailable()) { + TileFetchMetrics::instance().recordCacheHit(); + _serveCachedTile(); + return; + } + + // must-revalidate (RFC 7234 §5.2.2.1): a stale tile MUST NOT be served without a + // successful check, so block on the network — a conditional GET when we have + // validators, else a full refetch. No stale-while-revalidate for these. + if (_cachedTile->mustRevalidate) { + if (revalidatable) { + _etag = _cachedTile->etag; + _lastModified = _cachedTile->lastModified; + } + _startNetworkRequest(); + return; + } + + // Plain expiry, online, with validators: stale-while-revalidate (R5) — hand the + // stale bytes to the renderer NOW so the map never blanks, then refresh in the + // background. QtLocation's QGeoTiledMapReply is single-shot, so the fresh tile + // lands on the next pass. + if (revalidatable) { + _etag = _cachedTile->etag; + _lastModified = _cachedTile->lastModified; + qCDebug(QGeoTiledMapReplyQGCLog) << "Serving stale tile while revalidating" << tileSpec().x() << tileSpec().y() + << tileSpec().zoom(); + _backgroundRevalidate(); + TileFetchMetrics::instance().recordCacheHit(); + _serveCachedTile(); + return; + } + + // No validators: a full refetch so tiles lacking etag/Last-Modified don't stay + // stale forever. Here the live reply IS the refetch (no stale copy to serve early + // without validators to confirm it). + qCDebug(QGeoTiledMapReplyQGCLog) << "Refetching expired tile (no validators)" << tileSpec().x() << tileSpec().y() + << tileSpec().zoom(); + _startNetworkRequest(); +} + +void QGeoTiledMapReplyQGC::_serveCachedTile() +{ + if (!_cachedTile) { setError(QGeoTiledMapReply::UnknownError, tr("Invalid Cache Tile")); + return; + } + setMapImageData(_cachedTile->img); + setMapImageFormat(_cachedTile->format); + setCached(true); + setFinished(true); + _cachedTile.reset(); +} + +void QGeoTiledMapReplyQGC::_serveAncestorFallbackOr(QGeoTiledMapReply::Error err, const QString& errStr) +{ + if (isFinished()) { + return; + } + + // Attempt the fallback scan at most once; on a second terminal failure raise the + // original error directly so we never loop enqueuing fallback tasks. + if (_fallbackAttempted) { + setError(err, errStr); + return; + } + _fallbackAttempted = true; + + const QString provider = UrlFactory::getProviderTypeFromQtMapId(tileSpec().mapId()); + if (provider.isEmpty()) { + setError(err, errStr); + return; + } + + _fallbackPendingError = err; + _fallbackPendingErrorString = errStr; + + const int tileSize = UrlFactory::useRetinaTiles() ? 512 : 256; + auto* task = new QGCFetchFallbackTileTask(provider, tileSpec().x(), tileSpec().y(), tileSpec().zoom(), tileSize); + (void) connect(task, &QGCFetchFallbackTileTask::fallbackFetched, this, &QGeoTiledMapReplyQGC::_fallbackTileFetched); + (void) connect(task, &QGCMapTask::error, this, &QGeoTiledMapReplyQGC::_fallbackTileError); + if (!getQGCMapEngine()->addTask(task)) { + task->deleteLater(); + setError(err, errStr); + } +} + +void QGeoTiledMapReplyQGC::_fallbackTileFetched(const QByteArray& image, const QString& format, int levelDelta) +{ + if (isFinished()) { + return; + } + if (image.isEmpty()) { + setError(_fallbackPendingError, _fallbackPendingErrorString); + return; + } + qCDebug(QGeoTiledMapReplyQGCLog) << "Serving lower-zoom fallback tile" << tileSpec().x() << tileSpec().y() + << tileSpec().zoom() << "from ancestor" << levelDelta << "levels up"; + setMapImageData(image); + setMapImageFormat(format); + setCached(true); + setFinished(true); +} + +void QGeoTiledMapReplyQGC::_fallbackTileError(QGCMapTask::TaskType type, QStringView errorString) +{ + Q_UNUSED(type); + Q_UNUSED(errorString); + if (isFinished()) { + return; } + setError(_fallbackPendingError, _fallbackPendingErrorString); +} + +void QGeoTiledMapReplyQGC::_backgroundRevalidate() +{ + // Only spawn a detached GET on the fetcher path, where the QNAM is the engine's + // long-lived shared manager. Without a fetcher (bare per-reply manager) the + // manager may not outlive the detached reply, so serve stale only. + if (!_cachedTile || !_networkManager || !_fetcher) { + return; + } + const SharedMapProvider provider = UrlFactory::getMapProviderFromQtMapId(tileSpec().mapId()); + if (!provider) { + return; + } + + QNetworkRequest req = _request; + if (!_etag.isEmpty()) { + req.setRawHeader(QByteArrayLiteral("If-None-Match"), _etag); + } + if (!_lastModified.isEmpty()) { + req.setRawHeader(QByteArrayLiteral("If-Modified-Since"), _lastModified); + } + req.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + + // Skip the live GET entirely when the breaker is open for this host. + if (!HostCircuitBreaker::instance().allowRequest(req.url().host())) { + return; + } + + // Self-owned: outlives this reply, deletes itself when the GET completes. + new DetachedRevalidator(_networkManager, req, provider->getMapName(), tileSpec().x(), tileSpec().y(), + tileSpec().zoom(), _cachedTile->etag, _cachedTile->lastModified, _cachedTile->expiresAt); } void QGeoTiledMapReplyQGC::_cacheError(QGCMapTask::TaskType type, QStringView errorString) { Q_UNUSED(errorString); - Q_ASSERT(type == QGCMapTask::TaskType::taskFetchTile); + if (type != QGCMapTask::TaskType::taskFetchTile) { + qCWarning(QGeoTiledMapReplyQGCLog) << "Unexpected cache task type in _cacheError:" << static_cast(type); + return; + } if (!QGCNetworkHelper::isInternetAvailable()) { - setError(QGeoTiledMapReply::CommunicationError, tr("Network Not Available")); + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, tr("Network Not Available")); + return; + } + + TileFetchMetrics::instance().recordCacheMiss(); + _startNetworkRequest(); +} + +void QGeoTiledMapReplyQGC::_applyConditionalHeaders() +{ + // Force revalidation past QNAM's own cache so the conditional request + // actually reaches the server. + if (!_etag.isEmpty()) { + _request.setRawHeader(QByteArrayLiteral("If-None-Match"), _etag); + } + if (!_lastModified.isEmpty()) { + _request.setRawHeader(QByteArrayLiteral("If-Modified-Since"), _lastModified); + } + if (!_etag.isEmpty() || !_lastModified.isEmpty()) { + _request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::AlwaysNetwork); + } +} + +void QGeoTiledMapReplyQGC::_cancelDedupFetch() +{ + if (_fetcher) { + _fetcher->cancelFetch(this, _request); + } +} + +void QGeoTiledMapReplyQGC::_startNetworkRequest() +{ + // Circuit breaker (R2): fast-fail when the breaker for this host is OPEN so a + // dead/erroring endpoint doesn't keep spawning doomed GETs. A HALF-OPEN probe + // is allowed through to test recovery. + const QString host = _request.url().host(); + if (!HostCircuitBreaker::instance().allowRequest(host)) { + qCDebug(QGeoTiledMapReplyQGCLog) << "Circuit breaker open; failing fast for host" << host; + _serveAncestorFallbackOr(QGeoTiledMapReply::CommunicationError, tr("Host temporarily unavailable")); return; } _request.setOriginatingObject(this); + _applyConditionalHeaders(); + if (!_fetchTimer.isValid()) { + _fetchTimer.start(); + } + + if (_fetcher) { + // Coalesce identical in-flight fetches. The fetcher delivers the result by + // calling _handleNetworkResult directly, so no signal hookup is needed. + (void) connect(this, &QGeoTiledMapReplyQGC::aborted, this, &QGeoTiledMapReplyQGC::_cancelDedupFetch, + Qt::UniqueConnection); + _fetcher->dedupFetch(this, _request); + return; + } + + if (!_networkManager) { + setError(QGeoTiledMapReply::CommunicationError, tr("No network manager available")); + return; + } QNetworkReply* const reply = _networkManager->get(_request); reply->setParent(this); diff --git a/src/QtLocationPlugin/QGeoMapReplyQGC.h b/src/QtLocationPlugin/QGeoMapReplyQGC.h index 7aa878d18f1b..5764f6a0fdaa 100644 --- a/src/QtLocationPlugin/QGeoMapReplyQGC.h +++ b/src/QtLocationPlugin/QGeoMapReplyQGC.h @@ -4,18 +4,31 @@ #include #include +#include + +#include + +#include + #include "QGCMapTaskBase.h" +#include "QGeoTileFetcherQGC.h" struct QGCCacheTile; +class QGeoTileFetcherQGC; class QNetworkAccessManager; class QSslError; +class QUrl; class QGeoTiledMapReplyQGC : public QGeoTiledMapReply { Q_OBJECT + // Delivers coalesced fetch results straight into the reply. + friend class QGeoTileFetcherQGC; + public: - explicit QGeoTiledMapReplyQGC(QNetworkAccessManager *networkManager, const QNetworkRequest &request, const QGeoTileSpec &spec, QObject *parent = nullptr); + explicit QGeoTiledMapReplyQGC(QNetworkAccessManager* networkManager, const QNetworkRequest& request, + const QGeoTileSpec& spec, QObject* parent = nullptr, QGeoTileFetcherQGC* fetcher = nullptr); ~QGeoTiledMapReplyQGC(); bool init(); @@ -24,16 +37,54 @@ class QGeoTiledMapReplyQGC : public QGeoTiledMapReply private slots: void _networkReplyFinished(); void _networkReplyError(QNetworkReply::NetworkError error); - void _networkReplySslErrors(const QList &errors); - void _cacheReply(QGCCacheTile *tile); + void _networkReplySslErrors(const QList& errors); + void _cacheReply(QSharedPointer tile); void _cacheError(QGCMapTask::TaskType type, QStringView errorString); + void _handleNetworkResult(const QGeoTileFetcherQGC::FetchResult& result); + void _fallbackTileFetched(const QByteArray& image, const QString& format, int levelDelta); + void _fallbackTileError(QGCMapTask::TaskType type, QStringView errorString); private: static void _initDataFromResources(); + void _startNetworkRequest(); + // Qt::UniqueConnection needs a pointer-to-member slot, not a lambda. + void _cancelDedupFetch(); + bool _scheduleRetryIfNeeded(const QNetworkReply* reply, QNetworkReply::NetworkError error, int statusCode); + bool _scheduleRetryIfNeeded(QNetworkReply::NetworkError error, int statusCode, + std::optional retryAfter = std::nullopt); - QNetworkAccessManager *_networkManager = nullptr; + static constexpr int kMaxRetryAttempts = 3; + + void _applyConditionalHeaders(); + void _serveCachedTile(); + // R6 lower-zoom fallback: on a terminal network/offline failure, enqueue a + // worker task that scans cached ancestor tiles. On a hit the scaled ancestor is + // served as a placeholder; on a miss the original error is raised. Attempted at + // most once per reply to avoid loops. + void _serveAncestorFallbackOr(QGeoTiledMapReply::Error err, const QString& errStr); + // Stale-while-revalidate (R5): fire a detached conditional GET that updates the + // SQLite cache in the background. The reply is single-shot (already finished with + // stale bytes), so the fresh tile is picked up on the next viewport pass. + void _backgroundRevalidate(); + + QGeoTileFetcherQGC* _fetcher = nullptr; + QNetworkAccessManager* _networkManager = nullptr; QNetworkRequest _request; bool m_initialized = false; + bool _fallbackAttempted = false; + int _retryAttempt = 0; + + // Original terminal error, raised if the ancestor-fallback lookup misses. + QGeoTiledMapReply::Error _fallbackPendingError = QGeoTiledMapReply::NoError; + QString _fallbackPendingErrorString; + + // Validation metadata for the stale cached tile we are revalidating. When + // _cachedTile is non-null the body is reused on a 304 Not Modified and the + // etag/lastModified are sent as conditional request headers. + QSharedPointer _cachedTile; + QByteArray _etag; + QByteArray _lastModified; + QElapsedTimer _fetchTimer; static QByteArray _bingNoTileImage; static QByteArray _badTile; diff --git a/src/QtLocationPlugin/QGeoServiceProviderPluginQGC.cpp b/src/QtLocationPlugin/QGeoServiceProviderPluginQGC.cpp index de6991b2c6c2..747b70f0870f 100644 --- a/src/QtLocationPlugin/QGeoServiceProviderPluginQGC.cpp +++ b/src/QtLocationPlugin/QGeoServiceProviderPluginQGC.cpp @@ -36,16 +36,23 @@ QGeoCodingManagerEngine *QGeoServiceProviderFactoryQGC::createGeocodingManagerEn QGeoMappingManagerEngine *QGeoServiceProviderFactoryQGC::createMappingManagerEngine( const QVariantMap ¶meters, QGeoServiceProvider::Error *error, QString *errorString) const { + if (!m_engine || (m_engine->thread() != QThread::currentThread())) { + if (error) { + *error = QGeoServiceProvider::ConnectionError; + } + if (errorString) { + *errorString = QStringLiteral("Geo service plugin engine unavailable or accessed from the wrong thread"); + } + return nullptr; + } + if (error) { *error = QGeoServiceProvider::NoError; } if (errorString) { - *errorString = ""; + errorString->clear(); } - Q_ASSERT(m_engine); - Q_ASSERT(m_engine->thread() == QThread::currentThread()); - QNetworkAccessManager *networkManager = m_engine->networkAccessManager(); return new QGeoTiledMappingManagerEngineQGC(parameters, error, errorString, networkManager, nullptr); diff --git a/src/QtLocationPlugin/QGeoTileFetcherQGC.cpp b/src/QtLocationPlugin/QGeoTileFetcherQGC.cpp index 69a420522799..e32643fa646a 100644 --- a/src/QtLocationPlugin/QGeoTileFetcherQGC.cpp +++ b/src/QtLocationPlugin/QGeoTileFetcherQGC.cpp @@ -1,38 +1,420 @@ #include "QGeoTileFetcherQGC.h" +#include +#include +#include +#include #include +#include #include +#include +#include #include #include "MapProvider.h" +#include "QGCHostCircuitBreaker.h" #include "QGCLoggingCategory.h" #include "QGCMapUrlEngine.h" +#include "QGCNetworkHelper.h" #include "QGeoMapReplyQGC.h" #include "QGeoTiledMappingManagerEngineQGC.h" QGC_LOGGING_CATEGORY(QGeoTileFetcherQGCLog, "QtLocationPlugin.QGeoTileFetcherQGC") -QGeoTileFetcherQGC::QGeoTileFetcherQGC(QNetworkAccessManager *networkManager, const QVariantMap ¶meters, QGeoTiledMappingManagerEngineQGC *parent) - : QGeoTileFetcher(parent) - , m_networkManager(networkManager) +QByteArray QGeoTileFetcherQGC::s_userAgentOverride; +QMutex QGeoTileFetcherQGC::s_templateMutex; +QHash QGeoTileFetcherQGC::s_templateCache; + +void QGeoTileFetcherQGC::setUserAgentOverride(const QByteArray& userAgent) +{ + s_userAgentOverride = userAgent; +} + +namespace { +// Raster tile providers commonly omit Expires/Cache-Control; cap how long an +// undated tile is treated as fresh before revalidation. 7 days is conservative +// and within OSM tile-usage etiquette for client-side caching. +constexpr qint64 kDefaultTileExpirySecs = 7 * 24 * 60 * 60; +} // namespace + +uint32_t QGeoTileFetcherQGC::concurrentDownloads(const QString& type) +{ + Q_UNUSED(type); + + const QNetworkInformation* const networkInfo = QNetworkInformation::instance(); + if (!networkInfo) { + return kDefaultConcurrent; + } + + switch (networkInfo->transportMedium()) { + case QNetworkInformation::TransportMedium::Cellular: + case QNetworkInformation::TransportMedium::Bluetooth: + return kConstrainedConcurrent; + default: + return kDefaultConcurrent; + } +} + +QGeoTileFetcherQGC::QGeoTileFetcherQGC(QNetworkAccessManager* networkManager, const QVariantMap& parameters, + QGeoTiledMappingManagerEngineQGC* parent) + : QGeoTileFetcher(parent), m_networkManager(networkManager) { - Q_ASSERT(networkManager); + if (!networkManager) { + qCDebug(QGeoTileFetcherQGCLog) << "No network manager injected; using a fetcher-owned shared instance"; + } qCDebug(QGeoTileFetcherQGCLog) << this; - // TODO: Allow useragent override again - /*if (parameters.contains(QStringLiteral("useragent"))) { - setUserAgent(parameters.value(QStringLiteral("useragent")).toString().toLatin1()); - }*/ + m_concurrencyBudget = concurrentDownloads(QString()); + if (QNetworkInformation* const networkInfo = QNetworkInformation::instance()) { + (void) connect(networkInfo, &QNetworkInformation::transportMediumChanged, this, + [this](QNetworkInformation::TransportMedium) { + m_concurrencyBudget = concurrentDownloads(QString()); + _startNextPending(); + }); + } + + if (parameters.contains(QStringLiteral("useragent"))) { + setUserAgentOverride(parameters.value(QStringLiteral("useragent")).toString().toLatin1()); + } } QGeoTileFetcherQGC::~QGeoTileFetcherQGC() { qCDebug(QGeoTileFetcherQGCLog) << this; + + // The concurrency gate is a process-wide singleton: slots held by in-flight + // leader GETs leak unless released here on mid-flight teardown. + for (auto it = m_inFlight.constBegin(); it != m_inFlight.constEnd(); ++it) { + if (it->reply) { + disconnect(it->reply, nullptr, this, nullptr); + it->reply->abort(); + HostConcurrencyGate::instance().release(it->host); + } + } +} + +QNetworkAccessManager* QGeoTileFetcherQGC::_sharedNetworkManager() +{ + if (m_networkManager) { + return m_networkManager; + } + if (!m_ownedNetworkManager) { + m_ownedNetworkManager = new QNetworkAccessManager(this); + QGCNetworkHelper::configureProxy(m_ownedNetworkManager); + } + return m_ownedNetworkManager; +} + +size_t qHash(const QGeoTileFetcherQGC::FetchKey& k, size_t seed) noexcept +{ + return qHashMulti(seed, k.url, k.ifNoneMatch, k.ifModifiedSince); +} + +QGeoTileFetcherQGC::FetchKey QGeoTileFetcherQGC::_keyFor(const QNetworkRequest& request) +{ + return FetchKey{ + request.url(), + request.rawHeader(QByteArrayLiteral("If-None-Match")), + request.rawHeader(QByteArrayLiteral("If-Modified-Since")), + }; +} + +void QGeoTileFetcherQGC::dedupFetch(QGeoTiledMapReplyQGC* requester, const QNetworkRequest& request) +{ + if (!requester) { + return; + } + + const FetchKey key = _keyFor(request); + InFlight& entry = m_inFlight[key]; + entry.followers.append(QPointer(requester)); + + if (entry.reply) { + return; + } + + const SharedMapProvider provider = UrlFactory::getMapProviderFromQtMapId(requester->tileSpec().mapId()); + entry.isOSM = provider && provider->isOSMProvider(); + const QString host = request.url().host(); + + // Over the concurrency budget (or the shared per-host gate): park this request + // and let a finishing leader pull it in. Drained in arrival order, which is the + // viewport-priority order produced by updateTileRequests(). + if (!_canStart(entry.isOSM, host)) { + m_pendingFetches.append(PendingFetch{key, request, entry.isOSM, host}); + return; + } + + _startLeaderGet(key, request, entry.isOSM, host); +} + +void QGeoTileFetcherQGC::_startLeaderGet(const FetchKey& key, const QNetworkRequest& request, bool isOSM, + const QString& host) +{ + auto it = m_inFlight.find(key); + if ((it == m_inFlight.end()) || it->reply) { + return; + } + + // Shared cross-path gate is the hard cap (A2): another QNAM/path may already + // hold this host's slots. If so, re-park so a release elsewhere wakes us. + if (!HostConcurrencyGate::instance().tryAcquire(host, static_cast(_currentBudget()), isOSM)) { + m_pendingFetches.append(PendingFetch{key, request, isOSM, host}); + return; + } + + QNetworkAccessManager* nam = _sharedNetworkManager(); + QNetworkRequest req = request; + // Single timeout authority: the fetcher's leader timeout (chrono overload). + // getNetworkRequest() no longer sets one, so this always applies. + req.setTransferTimeout(kLeaderTransferTimeout); + QNetworkReply* reply = nam->get(req); + reply->setParent(this); + QGCNetworkHelper::ignoreSslErrorsIfNeeded(reply); + it->reply = reply; + it->host = host; + + ++m_activeNetworkFetches; + if (isOSM) { + ++m_activeOSMFetches; + } + m_peakConcurrentFetches = qMax(m_peakConcurrentFetches, m_activeNetworkFetches); + + // errorOccurred fires before finished for failures, but finished always fires + // afterwards (incl. timeout/cancel) — so completion is driven solely off + // finished() and the snapshot carries the error. ssl errors surface as a + // failing finished() once ignoreSslErrorsIfNeeded has had its say. + (void) connect(reply, &QNetworkReply::finished, this, [this, key, reply]() { _onLeaderFinished(key, reply); }); +} + +void QGeoTileFetcherQGC::cancelFetch(QGeoTiledMapReplyQGC* requester, const QNetworkRequest& request) +{ + const FetchKey key = _keyFor(request); + auto it = m_inFlight.find(key); + if (it == m_inFlight.end()) { + return; + } + it->followers.removeIf( + [requester](const QPointer& p) { return (p == requester) || p.isNull(); }); + + if (it->followers.isEmpty()) { + // No one left to receive this fetch: tear down the leader reply so it + // doesn't run on (and fire into a dead lambda) after the last follower left. + const bool hadReply = (it->reply != nullptr); + const bool wasOSM = it->isOSM; + const QString host = it->host; + _eraseEntry(key, it->reply); + if (hadReply) { + _onFetchFinished(wasOSM, host); + } else { + // Parked but never started: just drop it from the pending queue. + m_pendingFetches.removeIf([&key](const PendingFetch& p) { return p.key == key; }); + } + } +} + +void QGeoTileFetcherQGC::_eraseEntry(const FetchKey& key, QNetworkReply* reply) +{ + if (reply) { + disconnect(reply, nullptr, this, nullptr); + reply->abort(); + reply->deleteLater(); + } + (void) m_inFlight.remove(key); +} + +void QGeoTileFetcherQGC::_onLeaderFinished(const FetchKey& key, QNetworkReply* reply) +{ + // Ignore stale or superseded deliveries: the entry may have been erased on + // cancel, or recreated by a retry with a different reply. In both cases the + // reply's lifetime is owned by whoever replaced us, so don't touch it here. + const auto it = m_inFlight.constFind(key); + if ((it == m_inFlight.constEnd()) || (it->reply != reply)) { + return; + } + + const FetchResult result = snapshot(reply); + + // Detach before delivering so a follower re-requesting (e.g. retry) starts fresh. + const InFlight entry = m_inFlight.take(key); + disconnect(reply, nullptr, this, nullptr); + reply->deleteLater(); + + _onFetchFinished(entry.isOSM, entry.host); + + for (const QPointer& follower : entry.followers) { + if (follower) { + follower->_handleNetworkResult(result); + } + } } -QGeoTiledMapReply* QGeoTileFetcherQGC::getTileImage(const QGeoTileSpec &spec) +void QGeoTileFetcherQGC::_onFetchFinished(bool wasOSM, const QString& host) { + if (m_activeNetworkFetches > 0) { + --m_activeNetworkFetches; + } + if (wasOSM && (m_activeOSMFetches > 0)) { + --m_activeOSMFetches; + } + HostConcurrencyGate::instance().release(host); + _startNextPending(); +} + +bool QGeoTileFetcherQGC::_canStart(bool isOSM, const QString& host) const +{ + Q_UNUSED(host); + // Cheap local pre-filter only: the authoritative cross-path cap is claimed + // (and may still reject) in _startLeaderGet via HostConcurrencyGate. This + // keeps the in-thread budget/OSM accounting for scheduling fairness. + if (m_activeNetworkFetches >= static_cast(_currentBudget())) { + return false; + } + if (isOSM && (m_activeOSMFetches >= static_cast(kConstrainedConcurrent))) { + return false; + } + return true; +} + +void QGeoTileFetcherQGC::_startNextPending() +{ + // Scan rather than strict takeFirst: an OSM request blocked by its cap must not + // stall non-OSM (or other) requests queued behind it. Order is otherwise preserved. + bool started = true; + while (started && !m_pendingFetches.isEmpty()) { + started = false; + for (int i = 0; i < m_pendingFetches.size(); ++i) { + const PendingFetch& pending = m_pendingFetches.at(i); + if (!_canStart(pending.isOSM, pending.host)) { + continue; + } + const auto it = m_inFlight.constFind(pending.key); + if ((it == m_inFlight.constEnd()) || it->reply) { + // Cancelled/already started while parked: drop and keep scanning. + m_pendingFetches.removeAt(i); + started = true; + break; + } + const PendingFetch taken = m_pendingFetches.takeAt(i); + _startLeaderGet(taken.key, taken.request, taken.isOSM, taken.host); + started = true; + break; + } + } +} + +uint32_t QGeoTileFetcherQGC::_currentBudget() const +{ + return m_concurrencyBudget; +} + +qint64 QGeoTileFetcherQGC::_parseExpiry(const QNetworkReply* reply, bool *hasExplicitExpiry) +{ + if (hasExplicitExpiry) { + *hasExplicitExpiry = true; + } + + // Cache-Control: max-age wins over Expires per RFC 7234. + const QByteArray cacheControl = reply->rawHeader(QByteArrayLiteral("Cache-Control")); + if (!cacheControl.isEmpty()) { + qint64 sMaxAge = -1; + for (const QByteArray& directive : cacheControl.split(',')) { + const QByteArray trimmed = directive.trimmed(); + if (trimmed.startsWith("no-store") || trimmed.startsWith("no-cache")) { + return 0; + } + if (trimmed.startsWith("max-age=")) { + bool ok = false; + const qint64 maxAge = trimmed.mid(8).toLongLong(&ok); + return (ok && (maxAge > 0)) ? (QDateTime::currentSecsSinceEpoch() + maxAge) : 0; + } + // s-maxage is a fallback only: max-age above wins for a private cache. + if (trimmed.startsWith("s-maxage=")) { + bool ok = false; + const qint64 value = trimmed.mid(9).toLongLong(&ok); + if (ok) { + sMaxAge = value; + } + } + } + if (sMaxAge >= 0) { + return (sMaxAge > 0) ? (QDateTime::currentSecsSinceEpoch() + sMaxAge) : 0; + } + } + + // Expires has no KnownHeaders enum; parse the raw HTTP-date with the C locale + // (RFC 7231 IMF-fixdate), interpreted as UTC. Qt::RFC2822Date won't accept the + // trailing "GMT", so it can't be used here. + const QByteArray expires = reply->rawHeader(QByteArrayLiteral("Expires")); + if (!expires.isEmpty()) { + QDateTime expiresDt = + QLocale::c().toDateTime(QString::fromLatin1(expires), QStringLiteral("ddd, dd MMM yyyy HH:mm:ss 'GMT'")); + if (expiresDt.isValid()) { + expiresDt.setTimeZone(QTimeZone::UTC); + return expiresDt.toSecsSinceEpoch(); + } + } + + // No server-provided freshness lifetime: default to a conservative 7 days so + // undated tiles still get revalidated rather than cached forever. + if (hasExplicitExpiry) { + *hasExplicitExpiry = false; + } + return QDateTime::currentSecsSinceEpoch() + kDefaultTileExpirySecs; +} + +bool QGeoTileFetcherQGC::_parseMustRevalidate(const QNetworkReply* reply) +{ + // no-cache / must-revalidate => cache but always revalidate before serving. + // no-store => caller should not persist at all (treated as must-revalidate + // here so a stored copy is never trusted without a fresh check). + const QByteArray cacheControl = reply->rawHeader(QByteArrayLiteral("Cache-Control")); + if (cacheControl.isEmpty()) { + return false; + } + for (const QByteArray& directive : cacheControl.split(',')) { + const QByteArray trimmed = directive.trimmed(); + if (trimmed.startsWith("no-cache") || trimmed.startsWith("no-store") || trimmed.startsWith("must-revalidate")) { + return true; + } + } + return false; +} + +QGeoTileFetcherQGC::FetchResult QGeoTileFetcherQGC::snapshot(QNetworkReply* reply) +{ + FetchResult r; + r.error = reply->error(); + r.statusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + r.reasonPhrase = reply->attribute(QNetworkRequest::HttpReasonPhraseAttribute).toString(); + r.errorString = reply->errorString(); + r.isOpen = reply->isOpen(); + r.etag = reply->rawHeader(QByteArrayLiteral("ETag")); + // Round-trip Last-Modified as the canonical IMF-fixdate string for caching and + // for re-sending as If-Modified-Since. Qt parses it via the known-header enum. + const QVariant lastModifiedVar = reply->header(QNetworkRequest::LastModifiedHeader); + if (lastModifiedVar.isValid()) { + const QDateTime lm = lastModifiedVar.toDateTime().toUTC(); + r.lastModified = QLocale::c().toString(lm, QStringLiteral("ddd, dd MMM yyyy HH:mm:ss 'GMT'")).toLatin1(); + } else { + r.lastModified = reply->rawHeader(QByteArrayLiteral("Last-Modified")); + } + r.expiresAt = _parseExpiry(reply, &r.hasExplicitExpiry); + r.mustRevalidate = _parseMustRevalidate(reply); + r.retryAfter = QGCNetworkHelper::retryAfterFromReply(reply); + if (r.error == QNetworkReply::NoError) { + r.body = reply->readAll(); + } + return r; +} + +QGeoTiledMapReply* QGeoTileFetcherQGC::getTileImage(const QGeoTileSpec& spec) +{ + if ((spec.zoom() < 1) || (spec.zoom() > QGC_MAX_MAP_ZOOM)) { + return nullptr; + } + const SharedMapProvider provider = UrlFactory::getMapProviderFromQtMapId(spec.mapId()); if (!provider) { return nullptr; @@ -47,7 +429,7 @@ QGeoTiledMapReply* QGeoTileFetcherQGC::getTileImage(const QGeoTileSpec &spec) return nullptr; } - QGeoTiledMapReplyQGC *tileImage = new QGeoTiledMapReplyQGC(m_networkManager, request, spec); + QGeoTiledMapReplyQGC* tileImage = new QGeoTiledMapReplyQGC(_sharedNetworkManager(), request, spec, nullptr, this); if (!tileImage->init()) { tileImage->deleteLater(); return nullptr; @@ -58,7 +440,7 @@ QGeoTiledMapReply* QGeoTileFetcherQGC::getTileImage(const QGeoTileSpec &spec) bool QGeoTileFetcherQGC::initialized() const { - return (m_networkManager != nullptr); + return true; } bool QGeoTileFetcherQGC::fetchingEnabled() const @@ -66,12 +448,46 @@ bool QGeoTileFetcherQGC::fetchingEnabled() const return initialized(); } -void QGeoTileFetcherQGC::timerEvent(QTimerEvent *event) +void QGeoTileFetcherQGC::timerEvent(QTimerEvent* event) { QGeoTileFetcher::timerEvent(event); } -void QGeoTileFetcherQGC::handleReply(QGeoTiledMapReply *reply, const QGeoTileSpec &spec) +void QGeoTileFetcherQGC::updateTileRequests(const QSet& tilesAdded, + const QSet& tilesRemoved) +{ + // Reaches into Qt-private QGeoTileFetcherPrivate (queue_/timer_/queueMutex_) because + // the base queue is plain FIFO with no priority hook. This depends on the private + // layout of qgeotilefetcher_p_p.h and may need revisiting on Qt minor-version bumps. + QGeoTileFetcherPrivate* const d = static_cast(QObjectPrivate::get(this)); + + QMutexLocker ml(&d->queueMutex_); + + // Pan/zoom: drop still-pending (queued, no reply yet) removed tiles. Replies + // already created for removed tiles are deliberately left to finish into the + // cache, so we do NOT touch d->invmap_ (that is the base cancelTileRequests + // path, which aborts in-flight work). + for (const QGeoTileSpec& spec : tilesRemoved) { + d->queue_.removeAll(spec); + } + + // Float newly-visible tiles ahead of any prefetch/low-priority backlog: the + // base queue is plain FIFO, so prepend added tiles (skipping ones already + // queued, preserving their existing position). + int insertAt = 0; + for (const QGeoTileSpec& spec : tilesAdded) { + if (!d->queue_.contains(spec)) { + d->queue_.insert(insertAt, spec); + ++insertAt; + } + } + + if (d->enabled_ && initialized() && !d->queue_.isEmpty() && !d->timer_.isActive()) { + d->timer_.start(0, this); + } +} + +void QGeoTileFetcherQGC::handleReply(QGeoTiledMapReply* reply, const QGeoTileSpec& spec) { if (!reply) { return; @@ -86,13 +502,69 @@ void QGeoTileFetcherQGC::handleReply(QGeoTiledMapReply *reply, const QGeoTileSpe if (reply->error() == QGeoTiledMapReply::NoError) { emit tileFinished(spec, reply->mapImageData(), reply->mapImageFormat()); } else { - emit tileError(spec, reply->errorString()); + const QString error = reply->errorString().isEmpty() ? tr("Unknown tile fetch error") : reply->errorString(); + emit tileError(spec, error); } } QNetworkRequest QGeoTileFetcherQGC::getNetworkRequest(int mapId, int x, int y, int zoom) { - const SharedMapProvider mapProvider = UrlFactory::getMapProviderFromQtMapId(mapId); + const SharedMapProvider provider = UrlFactory::getMapProviderFromQtMapId(mapId); + if (!provider || (zoom < 1) || !provider->isValidTileCoordinate(x, y, zoom)) { + return QNetworkRequest(); + } + const CachedRequestTemplate tmpl = _templateForProvider(provider); + return _requestFromTemplate(tmpl, x, y, zoom); +} + +QNetworkRequest QGeoTileFetcherQGC::getNetworkRequest(QStringView providerType, int x, int y, int zoom) +{ + const SharedMapProvider provider = UrlFactory::getMapProviderFromProviderType(providerType); + if (!provider || (zoom < 1) || !provider->isValidTileCoordinate(x, y, zoom)) { + return QNetworkRequest(); + } + const CachedRequestTemplate tmpl = _templateForProvider(provider); + return _requestFromTemplate(tmpl, x, y, zoom); +} + +QGeoTileFetcherQGC::CachedRequestTemplate QGeoTileFetcherQGC::_templateForProvider(const SharedMapProvider& provider) +{ + if (!provider) { + return CachedRequestTemplate{}; + } + + const int mapId = provider->getMapId(); + { + QMutexLocker lock(&s_templateMutex); + const auto it = s_templateCache.constFind(mapId); + if (it != s_templateCache.constEnd()) { + return *it; + } + } + + // Build outside the lock: getTileURL(0,0,0) only seeds the host/scheme so the + // template carries provider state; per-tile URLs replace it in _requestFromTemplate. + CachedRequestTemplate tmpl; + tmpl.provider = provider; + tmpl.request = _buildTileRequest(provider, 0, 0, 0); + + QMutexLocker lock(&s_templateMutex); + // First writer wins; a racing builder produced an identical template. + return *s_templateCache.insert(mapId, tmpl); +} + +QNetworkRequest QGeoTileFetcherQGC::_requestFromTemplate(const CachedRequestTemplate& tmpl, int x, int y, int zoom) +{ + if (!tmpl.provider) { + return QNetworkRequest(); + } + QNetworkRequest request = tmpl.request; + request.setUrl(tmpl.provider->getTileURL(x, y, zoom)); + return request; +} + +QNetworkRequest QGeoTileFetcherQGC::_buildTileRequest(const SharedMapProvider& mapProvider, int x, int y, int zoom) +{ if (!mapProvider) { return QNetworkRequest(); } @@ -100,12 +572,21 @@ QNetworkRequest QGeoTileFetcherQGC::getNetworkRequest(int mapId, int x, int y, i QNetworkRequest request; request.setUrl(mapProvider->getTileURL(x, y, zoom)); request.setPriority(QNetworkRequest::NormalPriority); - request.setTransferTimeout(10000); + // Timeout is owned by the fetcher's leader-GET path (kLeaderTransferTimeout), + // not set here, so there is a single source of truth. // request.setOriginatingObject(this); // Headers request.setRawHeader(QByteArrayLiteral("Accept"), QByteArrayLiteral("*/*")); - request.setHeader(QNetworkRequest::UserAgentHeader, s_userAgent); + // OSM tile-usage policy requires an app-identifiable User-Agent. Commercial + // providers keep the browser-style UA for compatibility. + if (mapProvider->isOSMProvider()) { + request.setHeader(QNetworkRequest::UserAgentHeader, QGCNetworkHelper::defaultUserAgent()); + } else if (!s_userAgentOverride.isEmpty()) { + request.setHeader(QNetworkRequest::UserAgentHeader, s_userAgentOverride); + } else { + request.setHeader(QNetworkRequest::UserAgentHeader, QString::fromLatin1(s_userAgent)); + } const QByteArray referrer = mapProvider->getReferrer().toUtf8(); if (!referrer.isEmpty()) { request.setRawHeader(QByteArrayLiteral("Referer"), referrer); @@ -115,14 +596,15 @@ QNetworkRequest QGeoTileFetcherQGC::getNetworkRequest(int mapId, int x, int y, i request.setRawHeader(QByteArrayLiteral("User-Token"), token); } request.setRawHeader(QByteArrayLiteral("Connection"), QByteArrayLiteral("keep-alive")); - // request.setRawHeader(QByteArrayLiteral("Accept-Encoding"), QByteArrayLiteral("gzip, deflate, br")); + // Don't set Accept-Encoding: a manual value disables QNAM's automatic + // response decompression, so tiles would land in the cache still compressed. + // Never send Cache-Control: no-cache on the request (OSM etiquette); revalidation + // is driven by conditional headers in QGeoTiledMapReplyQGC instead. - // Attributes - request.setAttribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferCache); + // Attributes. No QNetworkDiskCache is attached to the QNAM, so cache-load / + // cache-save control would be no-ops; cache policy lives in QGeoFileTileCacheQGC. request.setAttribute(QNetworkRequest::BackgroundRequestAttribute, true); - request.setAttribute(QNetworkRequest::CacheSaveControlAttribute, true); request.setAttribute(QNetworkRequest::RedirectPolicyAttribute, QNetworkRequest::NoLessSafeRedirectPolicy); - request.setAttribute(QNetworkRequest::Http2AllowedAttribute, true); request.setAttribute(QNetworkRequest::DoNotBufferUploadDataAttribute, false); // request.setAttribute(QNetworkRequest::AutoDeleteReplyOnFinishAttribute, true); diff --git a/src/QtLocationPlugin/QGeoTileFetcherQGC.h b/src/QtLocationPlugin/QGeoTileFetcherQGC.h index 2159ac316eb5..ad99514073a2 100644 --- a/src/QtLocationPlugin/QGeoTileFetcherQGC.h +++ b/src/QtLocationPlugin/QGeoTileFetcherQGC.h @@ -1,8 +1,21 @@ #pragma once +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include +#include +#include +#include +class MapProvider; class QGeoTiledMappingManagerEngineQGC; class QGeoTiledMapReplyQGC; class QGeoTileSpec; @@ -13,27 +26,208 @@ class QGeoTileFetcherQGC : public QGeoTileFetcher Q_OBJECT public: - explicit QGeoTileFetcherQGC(QNetworkAccessManager *networkManager, const QVariantMap ¶meters, QGeoTiledMappingManagerEngineQGC *parent = nullptr); + explicit QGeoTileFetcherQGC(QNetworkAccessManager* networkManager, const QVariantMap& parameters, + QGeoTiledMappingManagerEngineQGC* parent = nullptr); ~QGeoTileFetcherQGC(); static QNetworkRequest getNetworkRequest(int mapId, int x, int y, int zoom); - /* Note: QNetworkAccessManager queues the requests it receives. The number of requests executed in parallel is dependent on the protocol. - * Currently, for the HTTP protocol on desktop platforms, 6 requests are executed in parallel for one host/port combination. */ - static uint32_t concurrentDownloads(const QString &type) { Q_UNUSED(type); return 6; } + // Overload for the bulk-download path, which already holds the stable provider + // name; avoids the name -> mapId -> name registry round-trip of the int overload. + static QNetworkRequest getNetworkRequest(QStringView providerType, int x, int y, int zoom); + // QNAM runs at most 6 HTTP requests in parallel per host:port; scale down on + // metered/high-latency transports (cellular, bluetooth). Falls back to the + // default when no QNetworkInformation backend is loaded (e.g. unit tests). + static uint32_t concurrentDownloads(const QString& type); + + static constexpr uint32_t kDefaultConcurrent = 6; + static constexpr uint32_t kConstrainedConcurrent = 2; + + // Optional per-provider UA override (from the "useragent" plugin parameter). + // Applied only to non-OSM providers; OSM always uses the policy-compliant UA. + static void setUserAgentOverride(const QByteArray& userAgent); + + // Completed-fetch snapshot shared with every reply coalesced onto a single + // network request. Copied out of the leader's QNetworkReply so followers, + // which never see the live reply, can run their own post-processing. + struct FetchResult + { + QNetworkReply::NetworkError error = QNetworkReply::NoError; + int statusCode = 0; + QString reasonPhrase; + QString errorString; + QByteArray body; + QByteArray etag; + QByteArray lastModified; + qint64 expiresAt = 0; + // expiresAt came from a server freshness header, not the 7-day fallback; lets a bare 304 keep its cached expiry. + bool hasExplicitExpiry = false; + // Response carried Cache-Control no-cache/no-store/must-revalidate: + // a stored copy must be revalidated before it is served. + bool mustRevalidate = false; + bool isOpen = false; + // Parsed Retry-After of the leader reply, so coalesced followers (which + // never see the live reply) can still honour rate-limit backoff. + std::optional retryAfter; + }; + + // Coalesce identical in-flight fetches. The fetcher owns a single QNetworkReply + // per {url + conditional headers}; the first caller triggers the GET, later + // header-identical callers attach. Requests with differing If-None-Match / + // If-Modified-Since are NOT merged (a follower without a cached body could not + // service a 304). Each follower is delivered its result individually. Must run + // on the fetcher's thread. requester must be a QGeoTiledMapReplyQGC. + void dedupFetch(QGeoTiledMapReplyQGC* requester, const QNetworkRequest& request); + // Drop a still-pending follower (e.g. on abort). If it was the last follower, + // the leader reply is aborted and the in-flight entry erased. + void cancelFetch(QGeoTiledMapReplyQGC* requester, const QNetworkRequest& request); + +#ifdef QGC_UNITTEST_BUILD + // Test-only: number of distinct in-flight (coalesced) fetch entries. + int inFlightUrlCount() const { return m_inFlight.size(); } + + // Followers across all entries whose key URL matches (ignores headers). + int followerCount(const QUrl& url) const + { + int n = 0; + for (auto it = m_inFlight.constBegin(); it != m_inFlight.constEnd(); ++it) { + if (it.key().url == url) { + n += it->followers.size(); + } + } + return n; + } +#endif + + // Capture a completed reply into a thread-detached snapshot (also used by the + // non-coalesced reply path). + static FetchResult snapshot(QNetworkReply* reply); + +#ifdef QGC_UNITTEST_BUILD + // Test-only: leader GETs currently outstanding against the network. + int activeNetworkFetches() const { return m_activeNetworkFetches; } + + // Test-only: high-water mark of concurrent leader GETs. + int peakConcurrentFetches() const { return m_peakConcurrentFetches; } + + void setConcurrencyBudgetForTest(uint32_t budget) { m_concurrencyBudget = budget; } +#endif + +public slots: + // Re-ordered drain over the base FIFO: drop still-pending removed tiles and + // float newly-visible added tiles ahead of the prefetch backlog (the base + // queue is plain FIFO, so the priority split lives here). In-flight replies + // for removed tiles are left to finish into the cache. + void updateTileRequests(const QSet& tilesAdded, const QSet& tilesRemoved); private: - QGeoTiledMapReply* getTileImage(const QGeoTileSpec &spec) final; + static QNetworkRequest _buildTileRequest(const std::shared_ptr& mapProvider, int x, int y, + int zoom); + + // P2: provider lookup + static header set are identical for every tile of a + // given mapId, so resolve them once and cache a header-applied request + // template; per tile only the URL is swapped. getNetworkRequest() runs on + // both the live (engine) and bulk (main) threads, so the cache is guarded. + struct CachedRequestTemplate + { + std::shared_ptr provider; + QNetworkRequest request; + }; + + static QNetworkRequest _requestFromTemplate(const CachedRequestTemplate& tmpl, int x, int y, int zoom); + // Resolve + memoize the template for a provider (keyed by its stable mapId). + // Returns a copy taken under the cache lock so callers never touch a shared + // entry that another thread could mutate. + static CachedRequestTemplate _templateForProvider(const std::shared_ptr& provider); + + QGeoTiledMapReply* getTileImage(const QGeoTileSpec& spec) final; bool initialized() const final; bool fetchingEnabled() const final; - void timerEvent(QTimerEvent *event) final; - void handleReply(QGeoTiledMapReply *reply, const QGeoTileSpec &spec) final; + void timerEvent(QTimerEvent* event) final; + void handleReply(QGeoTiledMapReply* reply, const QGeoTileSpec& spec) final; - QNetworkAccessManager *m_networkManager = nullptr; + // Single QNAM shared by every reply this fetcher spawns. Lives on the + // fetcher's thread, so replies (created on the same thread) reuse it without + // any cross-thread access. Falls back to a fetcher-owned instance when no + // manager was injected. + QNetworkAccessManager* _sharedNetworkManager(); + + // In-flight requests merge only when url AND both conditional headers match. + struct FetchKey + { + QUrl url; + QByteArray ifNoneMatch; + QByteArray ifModifiedSince; + + bool operator==(const FetchKey& o) const + { + return (url == o.url) && (ifNoneMatch == o.ifNoneMatch) && (ifModifiedSince == o.ifModifiedSince); + } + }; + + friend size_t qHash(const QGeoTileFetcherQGC::FetchKey& k, size_t seed) noexcept; + + static FetchKey _keyFor(const QNetworkRequest& request); + void _onLeaderFinished(const FetchKey& key, QNetworkReply* reply); + void _eraseEntry(const FetchKey& key, QNetworkReply* reply); + // Concurrency gate: start a leader GET now if under budget, else park the + // request so a finishing leader can pull it in (_startNextPending). + void _startLeaderGet(const FetchKey& key, const QNetworkRequest& request, bool isOSM, const QString& host); + // OSM tile-usage policy asks clients to stay at/under 2 concurrent connections; + // cap OSM leaders independently of (and below) the transport budget. + bool _canStart(bool isOSM, const QString& host) const; + void _onFetchFinished(bool wasOSM, const QString& host); + void _startNextPending(); + uint32_t _currentBudget() const; + static qint64 _parseExpiry(const QNetworkReply* reply, bool *hasExplicitExpiry = nullptr); + static bool _parseMustRevalidate(const QNetworkReply* reply); + + struct InFlight + { + QNetworkReply* reply = nullptr; + QList> followers; + bool isOSM = false; + // Host the shared concurrency slot was claimed against; released on + // completion. Empty until a leader GET actually starts. + QString host; + }; + + QNetworkAccessManager* m_networkManager = nullptr; + QNetworkAccessManager* m_ownedNetworkManager = nullptr; + QHash m_inFlight; + + // Leader GETs currently outstanding; capped at m_concurrencyBudget. Followers + // coalesced onto a leader do not count. Requests over budget wait in + // m_pendingFetches (FIFO, fed in drain order so viewport priority carries). + int m_activeNetworkFetches = 0; + int m_activeOSMFetches = 0; + int m_peakConcurrentFetches = 0; + uint32_t m_concurrencyBudget = kDefaultConcurrent; + + struct PendingFetch + { + FetchKey key; + QNetworkRequest request; + bool isOSM = false; + QString host; + }; + + QList m_pendingFetches; + + // Single timeout authority for tile fetches (see getNetworkRequest comment). + // Shortened under unit-test builds so UI tests fail fast against CI's dead network + // rather than stacking 20s timeouts past the CTest watchdog. +#ifdef QGC_UNITTEST_BUILD + static constexpr std::chrono::milliseconds kLeaderTransferTimeout = std::chrono::seconds(2); +#else + static constexpr std::chrono::milliseconds kLeaderTransferTimeout = std::chrono::seconds(20); +#endif #if defined Q_OS_MACOS - static constexpr const char* s_userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:125.0) Gecko/20100101 Firefox/125.0"; + static constexpr const char* s_userAgent = + "Mozilla/5.0 (Macintosh; Intel Mac OS X 14.5; rv:125.0) Gecko/20100101 Firefox/125.0"; #elif defined Q_OS_WIN - static constexpr const char* s_userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/112.0"; + static constexpr const char* s_userAgent = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:100.0) Gecko/20100101 Firefox/112.0"; #elif defined Q_OS_ANDROID static constexpr const char* s_userAgent = "Mozilla/5.0 (Android 13; Tablet; rv:68.0) Gecko/68.0 Firefox/112.0"; #elif defined Q_OS_LINUX @@ -42,4 +236,14 @@ class QGeoTileFetcherQGC : public QGeoTileFetcher #else static constexpr const char* s_userAgent = "Qt Location based application"; #endif + + static QByteArray s_userAgentOverride; + + // mapId -> resolved provider + prebuilt header template (P2). Guarded by + // s_templateMutex because getNetworkRequest is static and reached from the + // live-fetch (engine) and bulk-download (main) threads. + static QMutex s_templateMutex; + static QHash s_templateCache; }; + +Q_DECLARE_METATYPE(QGeoTileFetcherQGC::FetchResult) diff --git a/src/QtLocationPlugin/QGeoTiledMapQGC.cpp b/src/QtLocationPlugin/QGeoTiledMapQGC.cpp index f19c198362dc..cd0d2dca8bc0 100644 --- a/src/QtLocationPlugin/QGeoTiledMapQGC.cpp +++ b/src/QtLocationPlugin/QGeoTiledMapQGC.cpp @@ -20,7 +20,8 @@ QGeoMap::Capabilities QGeoTiledMapQGC::capabilities() const { return Capabilities(SupportsVisibleRegion | SupportsAnchoringCoordinate - | SupportsVisibleArea); + | SupportsVisibleArea + | SupportsSetBearing); } /*void QGeoTiledMapQGC::evaluateCopyrights(const QSet &visibleTiles) diff --git a/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.cpp b/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.cpp index ffa04fc2c0ae..0187b9a290ac 100644 --- a/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.cpp +++ b/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.cpp @@ -1,43 +1,49 @@ #include "QGeoTiledMappingManagerEngineQGC.h" -#include - #include -#include -#include -#include -#include #include +#include #include #include -#include +#include +#include +#include +#include #include "MapProvider.h" -#include "QGCNetworkHelper.h" #include "QGCApplication.h" #include "QGCLoggingCategory.h" #include "QGCMapEngine.h" #include "QGCMapEngineManager.h" #include "QGCMapUrlEngine.h" +#include "QGCNetworkHelper.h" +#include "QGCTileCache.h" #include "QGeoFileTileCacheQGC.h" -#include "QGeoTiledMapQGC.h" #include "QGeoTileFetcherQGC.h" +#include "QGeoTiledMapQGC.h" QGC_LOGGING_CATEGORY(QGeoTiledMappingManagerEngineQGCLog, "QtLocationPlugin.QGeoTiledMappingManagerEngineQGC") -QGeoTiledMappingManagerEngineQGC::QGeoTiledMappingManagerEngineQGC(const QVariantMap ¶meters, QGeoServiceProvider::Error *error, QString *errorString, QNetworkAccessManager *networkManager, QObject *parent) - : QGeoTiledMappingManagerEngine(parent) - , m_networkManager(networkManager) +QGeoTiledMappingManagerEngineQGC::QGeoTiledMappingManagerEngineQGC(const QVariantMap& parameters, + QGeoServiceProvider::Error* error, + QString* errorString, + QNetworkAccessManager* networkManager, + QObject* parent) + : QGeoTiledMappingManagerEngine(parent), m_networkManager(networkManager) { qCDebug(QGeoTiledMappingManagerEngineQGCLog) << this; setLocale(qgcApp()->getCurrentLanguage()); - (void) connect(qgcApp(), &QGCApplication::languageChanged, this, [this](const QLocale &locale) { - setLocale(locale); - }); + (void) connect(qgcApp(), &QGCApplication::languageChanged, this, + [this](const QLocale& locale) { setLocale(locale); }); + + // R4: request 512px (@2x) tiles on HiDPI displays so the map isn't a blurry + // upscale of 256px tiles. UrlFactory::useRetinaTiles() reads the display DPR + // (false without a QGuiApplication, e.g. unit tests). + const int tilePx = UrlFactory::useRetinaTiles() ? 512 : 256; QGeoCameraCapabilities cameraCaps{}; - cameraCaps.setTileSize(256); + cameraCaps.setTileSize(tilePx); cameraCaps.setMinimumZoomLevel(2.0); cameraCaps.setMaximumZoomLevel(QGC_MAX_MAP_ZOOM); cameraCaps.setSupportsBearing(true); @@ -51,50 +57,61 @@ QGeoTiledMappingManagerEngineQGC::QGeoTiledMappingManagerEngineQGC(const QVarian setCameraCapabilities(cameraCaps); setTileVersion(kTileVersion); - setTileSize(QSize(256, 256)); + setTileSize(QSize(tilePx, tilePx)); QList mapList; const QList providers = UrlFactory::getProviders(); - for (const SharedMapProvider &provider : providers) { + for (const SharedMapProvider& provider : providers) { const QGeoMapType map = QGeoMapType( - static_cast(provider->getMapStyle()), - provider->getMapName(), - provider->getMapName(), - false, - false, - provider->getMapId(), - QByteArrayLiteral("QGroundControl"), - cameraCapabilities() - ); + static_cast(provider->getMapStyle()), provider->getMapName(), provider->getMapName(), + false, false, provider->getMapId(), QByteArrayLiteral("QGroundControl"), cameraCapabilities()); mapList.append(map); } setSupportedMapTypes(mapList); setCacheHint(QAbstractGeoTileCache::CacheArea::AllCaches); - QGeoFileTileCacheQGC *fileTileCache = new QGeoFileTileCacheQGC(parameters, this); + QGeoFileTileCacheQGC* fileTileCache = new QGeoFileTileCacheQGC(parameters, this); setTileCache(fileTileCache); // MapEngine must be init after fileTileCache static std::once_flag mapEngineInit; - std::call_once(mapEngineInit, [fileTileCache]() { - getQGCMapEngine()->init(fileTileCache->getDatabaseFilePath()); - }); - - m_prefetchStyle = QGCNetworkHelper::isInternetAvailable() ? QGeoTiledMap::PrefetchTwoNeighbourLayers : QGeoTiledMap::NoPrefetching; - (void) connect(QNetworkInformation::instance(), &QNetworkInformation::reachabilityChanged, this, [this](QNetworkInformation::Reachability newReachability) { - if (newReachability == QNetworkInformation::Reachability::Online) { - m_prefetchStyle = QGeoTiledMap::PrefetchTwoNeighbourLayers; - } else { - m_prefetchStyle = QGeoTiledMap::NoPrefetching; + std::call_once(mapEngineInit, []() { getQGCMapEngine()->init(QGCTileCache::getDatabaseFilePath()); }); + + m_prefetchStyle = QGCNetworkHelper::isInternetAvailable() ? QGeoTiledMap::PrefetchTwoNeighbourLayers + : QGeoTiledMap::NoPrefetching; + (void) connect(QNetworkInformation::instance(), &QNetworkInformation::reachabilityChanged, this, + [this](QNetworkInformation::Reachability newReachability) { + const QGeoTiledMap::PrefetchStyle newStyle = + (newReachability == QNetworkInformation::Reachability::Online) + ? QGeoTiledMap::PrefetchTwoNeighbourLayers + : QGeoTiledMap::NoPrefetching; + if (newStyle == m_prefetchStyle) { + return; + } + m_prefetchStyle = newStyle; + _updatePrefetchStyles(); + }); + + if (!m_networkManager) { + qCCritical(QGeoTiledMappingManagerEngineQGCLog) << "No network manager provided"; + if (error) { + *error = QGeoServiceProvider::ConnectionError; } - }); + if (errorString) { + *errorString = QStringLiteral("No network manager available"); + } + return; + } - Q_ASSERT(m_networkManager); - QGeoTileFetcherQGC *tileFetcher = new QGeoTileFetcherQGC(m_networkManager, parameters, this); + QGeoTileFetcherQGC* tileFetcher = new QGeoTileFetcherQGC(m_networkManager, parameters, this); - *error = QGeoServiceProvider::NoError; - errorString->clear(); - setTileFetcher(tileFetcher); // Calls engineInitialized() + if (error) { + *error = QGeoServiceProvider::NoError; + } + if (errorString) { + errorString->clear(); + } + setTileFetcher(tileFetcher); // Calls engineInitialized() } QGeoTiledMappingManagerEngineQGC::~QGeoTiledMappingManagerEngineQGC() @@ -102,9 +119,32 @@ QGeoTiledMappingManagerEngineQGC::~QGeoTiledMappingManagerEngineQGC() qCDebug(QGeoTiledMappingManagerEngineQGCLog) << this; } -QGeoMap *QGeoTiledMappingManagerEngineQGC::createMap() +QGeoMap* QGeoTiledMappingManagerEngineQGC::createMap() { - QGeoTiledMapQGC *map = new QGeoTiledMapQGC(this, this); + QGeoTiledMapQGC* map = new QGeoTiledMapQGC(this, this); map->setPrefetchStyle(m_prefetchStyle); + + m_activeMaps.append(QPointer(map)); + // QPointer auto-nulls before destroyed() fires, so prune by null on any death. + (void) connect(map, &QObject::destroyed, this, [this]() { + for (int i = m_activeMaps.count() - 1; i >= 0; --i) { + if (!m_activeMaps[i]) { + m_activeMaps.removeAt(i); + } + } + }); + return map; } + +void QGeoTiledMappingManagerEngineQGC::_updatePrefetchStyles() +{ + for (int i = m_activeMaps.count() - 1; i >= 0; --i) { + const QPointer& mapPtr = m_activeMaps[i]; + if (!mapPtr) { + m_activeMaps.removeAt(i); + continue; + } + mapPtr->setPrefetchStyle(m_prefetchStyle); + } +} diff --git a/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.h b/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.h index 5c25cd138a9d..32cf9d8953bf 100644 --- a/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.h +++ b/src/QtLocationPlugin/QGeoTiledMappingManagerEngineQGC.h @@ -1,9 +1,13 @@ #pragma once +#include +#include #include +#include #include class QNetworkAccessManager; +class QGeoTiledMapQGC; class QGeoTiledMappingManagerEngineQGC : public QGeoTiledMappingManagerEngine { @@ -17,7 +21,11 @@ class QGeoTiledMappingManagerEngineQGC : public QGeoTiledMappingManagerEngine QNetworkAccessManager *networkManager() const { return m_networkManager; } private: + void _updatePrefetchStyles(); + QNetworkAccessManager *m_networkManager = nullptr; + QGeoTiledMap::PrefetchStyle m_prefetchStyle = QGeoTiledMap::PrefetchTwoNeighbourLayers; + QList> m_activeMaps; static constexpr int kTileVersion = 1; }; diff --git a/src/QtLocationPlugin/TileFetchMetrics.cpp b/src/QtLocationPlugin/TileFetchMetrics.cpp new file mode 100644 index 000000000000..4bad022ec8ee --- /dev/null +++ b/src/QtLocationPlugin/TileFetchMetrics.cpp @@ -0,0 +1,78 @@ +#include "TileFetchMetrics.h" + +QString TileFetchStats::toString() const +{ + return QStringLiteral("tiles=%1 cacheHits=%2 cacheMisses=%3 net=%4 errors=%5 notModified=%6 bytes=%7 avgLatencyMs=%8") + .arg(totalFetches()) + .arg(cacheHits) + .arg(cacheMisses) + .arg(networkSuccess) + .arg(networkErrors) + .arg(notModified) + .arg(bytesDownloaded) + .arg(averageLatencyMs(), 0, 'f', 1); +} + +TileFetchMetrics &TileFetchMetrics::instance() +{ + static TileFetchMetrics s_instance; + return s_instance; +} + +void TileFetchMetrics::recordCacheHit() +{ + _cacheHits.fetch_add(1, std::memory_order_relaxed); +} + +void TileFetchMetrics::recordCacheMiss() +{ + _cacheMisses.fetch_add(1, std::memory_order_relaxed); +} + +void TileFetchMetrics::recordNotModified(quint64 latencyMs) +{ + _notModified.fetch_add(1, std::memory_order_relaxed); + _totalLatencyMs.fetch_add(latencyMs, std::memory_order_relaxed); + _latencySamples.fetch_add(1, std::memory_order_relaxed); +} + +void TileFetchMetrics::recordNetworkSuccess(quint64 bytes, quint64 latencyMs) +{ + _networkSuccess.fetch_add(1, std::memory_order_relaxed); + _bytesDownloaded.fetch_add(bytes, std::memory_order_relaxed); + _totalLatencyMs.fetch_add(latencyMs, std::memory_order_relaxed); + _latencySamples.fetch_add(1, std::memory_order_relaxed); +} + +void TileFetchMetrics::recordNetworkError(quint64 latencyMs) +{ + _networkErrors.fetch_add(1, std::memory_order_relaxed); + _totalLatencyMs.fetch_add(latencyMs, std::memory_order_relaxed); + _latencySamples.fetch_add(1, std::memory_order_relaxed); +} + +TileFetchStats TileFetchMetrics::snapshot() const +{ + TileFetchStats s; + s.cacheHits = _cacheHits.load(std::memory_order_relaxed); + s.cacheMisses = _cacheMisses.load(std::memory_order_relaxed); + s.networkSuccess = _networkSuccess.load(std::memory_order_relaxed); + s.networkErrors = _networkErrors.load(std::memory_order_relaxed); + s.notModified = _notModified.load(std::memory_order_relaxed); + s.bytesDownloaded = _bytesDownloaded.load(std::memory_order_relaxed); + s.totalLatencyMs = _totalLatencyMs.load(std::memory_order_relaxed); + s.latencySamples = _latencySamples.load(std::memory_order_relaxed); + return s; +} + +void TileFetchMetrics::reset() +{ + _cacheHits.store(0, std::memory_order_relaxed); + _cacheMisses.store(0, std::memory_order_relaxed); + _networkSuccess.store(0, std::memory_order_relaxed); + _networkErrors.store(0, std::memory_order_relaxed); + _notModified.store(0, std::memory_order_relaxed); + _bytesDownloaded.store(0, std::memory_order_relaxed); + _totalLatencyMs.store(0, std::memory_order_relaxed); + _latencySamples.store(0, std::memory_order_relaxed); +} diff --git a/src/QtLocationPlugin/TileFetchMetrics.h b/src/QtLocationPlugin/TileFetchMetrics.h new file mode 100644 index 000000000000..35f97b892477 --- /dev/null +++ b/src/QtLocationPlugin/TileFetchMetrics.h @@ -0,0 +1,53 @@ +#pragma once + +#include + +#include +#include + +// Lightweight, lock-free counters for tile-fetch outcomes. Plain struct snapshot +// for reads (no Q_GADGET/QML exposure) plus a process-wide aggregator updated +// from any thread the fetch/reply paths run on. +struct TileFetchStats { + quint64 cacheHits = 0; + quint64 cacheMisses = 0; + quint64 networkSuccess = 0; + quint64 networkErrors = 0; + quint64 notModified = 0; + quint64 bytesDownloaded = 0; + quint64 totalLatencyMs = 0; + quint64 latencySamples = 0; + + double averageLatencyMs() const { + return (latencySamples > 0) ? (static_cast(totalLatencyMs) / static_cast(latencySamples)) : 0.0; + } + quint64 totalFetches() const { return cacheHits + networkSuccess + networkErrors + notModified; } + QString toString() const; +}; + +class TileFetchMetrics +{ +public: + static TileFetchMetrics &instance(); + + void recordCacheHit(); + void recordCacheMiss(); + void recordNotModified(quint64 latencyMs); + void recordNetworkSuccess(quint64 bytes, quint64 latencyMs); + void recordNetworkError(quint64 latencyMs); + + TileFetchStats snapshot() const; + void reset(); + +private: + TileFetchMetrics() = default; + + std::atomic _cacheHits{0}; + std::atomic _cacheMisses{0}; + std::atomic _networkSuccess{0}; + std::atomic _networkErrors{0}; + std::atomic _notModified{0}; + std::atomic _bytesDownloaded{0}; + std::atomic _totalLatencyMs{0}; + std::atomic _latencySamples{0}; +}; diff --git a/src/Settings/App.SettingsGroup.json b/src/Settings/App.SettingsGroup.json index 0a45092c81e6..f81929bb3836 100644 --- a/src/Settings/App.SettingsGroup.json +++ b/src/Settings/App.SettingsGroup.json @@ -258,6 +258,15 @@ "label": "TianDiTu", "keywords": "token,api key,tianditu" }, + { + "name": "linzToken", + "shortDesc": "Personal API token for accessing LINZ Basemaps map tiles.", + "longDesc": "Your personal access token for LINZ Basemaps", + "type": "string", + "default": "", + "label": "LINZ", + "keywords": "token,api key,linz" + }, { "name": "mapboxToken", "shortDesc": "Personal API token for accessing Mapbox map tiles and styling.", @@ -297,7 +306,7 @@ { "name": "customURL", "shortDesc": "URL pattern with {x}, {y}, {z} substitutions for a custom map tile server.", - "longDesc": "URL for X Y Z map with {x} {y} {z} or {zoom} substitutions. Eg: https://basemaps.linz.govt.nz/v1/tiles/aerial/EPSG:3857/{z}/{x}/{y}.png?api=d01ev80nqcjxddfvc6amyvkk1ka", + "longDesc": "URL for X Y Z map with {x} {y} {z} or {zoom} substitutions. Eg: https://basemaps.linz.govt.nz/v1/tiles/aerial/EPSG:3857/{z}/{x}/{y}.png?api=YOUR_LINZ_API_KEY", "type": "string", "default": "", "label": "Server URL", diff --git a/src/Settings/AppSettings.cc b/src/Settings/AppSettings.cc index c2efdc29b0c5..2d7412a43236 100644 --- a/src/Settings/AppSettings.cc +++ b/src/Settings/AppSettings.cc @@ -147,6 +147,7 @@ DECLARE_SETTINGSFACT(AppSettings, useChecklist) DECLARE_SETTINGSFACT(AppSettings, enforceChecklist) DECLARE_SETTINGSFACT(AppSettings, enableMultiVehiclePanel) DECLARE_SETTINGSFACT(AppSettings, tiandituToken) +DECLARE_SETTINGSFACT(AppSettings, linzToken) DECLARE_SETTINGSFACT(AppSettings, mapboxToken) DECLARE_SETTINGSFACT(AppSettings, mapboxAccount) DECLARE_SETTINGSFACT(AppSettings, mapboxStyle) diff --git a/src/Settings/AppSettings.h b/src/Settings/AppSettings.h index c5827a305fca..11459696d056 100644 --- a/src/Settings/AppSettings.h +++ b/src/Settings/AppSettings.h @@ -39,6 +39,7 @@ class AppSettings : public SettingsGroup DEFINE_SETTINGFACT(enforceChecklist) DEFINE_SETTINGFACT(enableMultiVehiclePanel) DEFINE_SETTINGFACT(tiandituToken) + DEFINE_SETTINGFACT(linzToken) DEFINE_SETTINGFACT(mapboxToken) DEFINE_SETTINGFACT(mapboxAccount) DEFINE_SETTINGFACT(mapboxStyle) diff --git a/src/Utilities/Database/QGCSqlHelper.cc b/src/Utilities/Database/QGCSqlHelper.cc index 78e3b9aad909..ded03394bf4e 100644 --- a/src/Utilities/Database/QGCSqlHelper.cc +++ b/src/Utilities/Database/QGCSqlHelper.cc @@ -30,12 +30,48 @@ QString placeholders(int n) return out; } -void applySqlitePragmas(QSqlDatabase& db) +void applySqlitePragmas(QSqlDatabase& db, qint64 mmapSize, qint64 cacheSizeKb, int pageSize) { QSqlQuery q(db); + // page_size only binds on a not-yet-created DB and must precede journal_mode=WAL; + // SQLite silently ignores it on an existing DB (would need VACUUM to change). + if (pageSize > 0) { + q.exec(QStringLiteral("PRAGMA page_size=%1").arg(pageSize)); + } + // auto_vacuum must be set before journal_mode=WAL to take effect on a fresh DB; + // on an existing DB SQLite silently ignores it unless followed by a full VACUUM. + q.exec(QStringLiteral("PRAGMA auto_vacuum=INCREMENTAL")); q.exec(QStringLiteral("PRAGMA journal_mode=WAL")); q.exec(QStringLiteral("PRAGMA synchronous=NORMAL")); q.exec(QStringLiteral("PRAGMA foreign_keys=ON")); + q.exec(QStringLiteral("PRAGMA cache_size=%1").arg(cacheSizeKb)); + q.exec(QStringLiteral("PRAGMA temp_store=MEMORY")); + // mmap_size reserves address space per connection; keep it modest by default and + // raise it only for large read-heavy DBs (e.g. the tile cache) via the argument. + if (mmapSize > 0) { + q.exec(QStringLiteral("PRAGMA mmap_size=%1").arg(mmapSize)); + } +} + +bool walCheckpointTruncate(QSqlDatabase& db) +{ + QSqlQuery q(db); + return q.exec(QStringLiteral("PRAGMA wal_checkpoint(TRUNCATE)")); +} + +bool runOptimize(QSqlDatabase& db) +{ + QSqlQuery q(db); + return q.exec(QStringLiteral("PRAGMA optimize")); +} + +bool incrementalVacuum(QSqlDatabase& db, int pages) +{ + QSqlQuery q(db); + const QString sql = (pages > 0) + ? QStringLiteral("PRAGMA incremental_vacuum(%1)").arg(pages) + : QStringLiteral("PRAGMA incremental_vacuum"); + return q.exec(sql); } std::optional userVersion(QSqlDatabase& db) @@ -57,7 +93,7 @@ bool setUserVersion(QSqlDatabase& db, int v) std::atomic ScopedConnection::s_connId{0}; -ScopedConnection::ScopedConnection(const QString& dbPath, bool readOnly, const QString& prefix) +ScopedConnection::ScopedConnection(const QString& dbPath, bool readOnly, const QString& prefix, qint64 mmapSize) { if (dbPath.isEmpty()) { return; @@ -67,12 +103,15 @@ ScopedConnection::ScopedConnection(const QString& dbPath, bool readOnly, const Q QSqlDatabase db = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), _connName); db.setDatabaseName(dbPath); + // Wait out a contended WAL writer lock instead of failing with SQLITE_BUSY. + QString connectOptions = QStringLiteral("QSQLITE_BUSY_TIMEOUT=5000"); if (readOnly) { - db.setConnectOptions(QStringLiteral("QSQLITE_OPEN_READONLY")); + connectOptions += QStringLiteral(";QSQLITE_OPEN_READONLY"); } + db.setConnectOptions(connectOptions); if (db.open()) { - applySqlitePragmas(db); + applySqlitePragmas(db, mmapSize); _valid = true; } } diff --git a/src/Utilities/Database/QGCSqlHelper.h b/src/Utilities/Database/QGCSqlHelper.h index f1dc9a7fa345..a29f4e0e184b 100644 --- a/src/Utilities/Database/QGCSqlHelper.h +++ b/src/Utilities/Database/QGCSqlHelper.h @@ -20,10 +20,47 @@ namespace QGCSqlHelper { /// E.g. `"DELETE FROM T WHERE id IN (" + placeholders(n) + ")"`. [[nodiscard]] QString placeholders(int n); +// ── Prepare + bind + exec ────────────────────────────────────────────── +/// Prepares \a sql on \a query, positionally binds \a binds in order, and +/// execs. Returns false if any step fails; on failure \a query retains the +/// failing error (query.lastError()). On success the caller may still use +/// \a query for result iteration / lastInsertId() / numRowsAffected(). +/// Not for prepared-once-reused loops — re-preparing per call defeats that. +template +[[nodiscard]] bool execPrepared(QSqlQuery& query, const QString& sql, const Args&... binds) +{ + if (!query.prepare(sql)) { + return false; + } + (query.addBindValue(binds), ...); + return query.exec(); +} + // ── SQLite PRAGMA helpers ────────────────────────────────────────────── -/// Applies standard QGC pragmas: WAL journal mode, NORMAL synchronous, -/// foreign_keys = ON. -void applySqlitePragmas(QSqlDatabase& db); +/// Applies standard QGC pragmas: page_size (empty DBs only), auto_vacuum=INCREMENTAL +/// (before WAL so it binds on fresh DBs), WAL journal mode, NORMAL synchronous, +/// foreign_keys=ON, cache_size, temp_store=MEMORY and mmap_size. auto_vacuum only +/// takes effect on DBs created with it (or after a full VACUUM). \a mmapSize bytes of +/// address space are reserved per connection; the default (0) skips the mmap pragma — +/// opt in only for large read-heavy DBs (e.g. the tile cache). \a cacheSizeKb sets +/// cache_size (negative = KiB of RAM, the SQLite convention); pass a larger magnitude +/// for write-heavy connections. \a pageSize, when > 0, is issued first; SQLite applies +/// it only to a not-yet-created DB and ignores it otherwise. +void applySqlitePragmas(QSqlDatabase& db, qint64 mmapSize = 0, qint64 cacheSizeKb = -10000, int pageSize = 0); + +/// Runs PRAGMA wal_checkpoint(TRUNCATE) to flush the WAL into the main DB and +/// shrink the -wal file back to zero. Returns the query success. +bool walCheckpointTruncate(QSqlDatabase& db); + +/// Runs PRAGMA optimize — lets SQLite refresh stale query-planner statistics. +/// Cheap; intended to run on connection close. Returns the query success. +bool runOptimize(QSqlDatabase& db); + +/// Runs PRAGMA incremental_vacuum(\a pages) to reclaim freed pages from an +/// auto_vacuum=INCREMENTAL database. Pass pages <= 0 (default) to reclaim all +/// free pages. No-op (returns true) on a DB without incremental auto_vacuum. +/// Returns the query success. +bool incrementalVacuum(QSqlDatabase& db, int pages = 0); /// PRAGMA user_version — read/write the SQLite schema version integer. /// Returns std::nullopt on failure. @@ -45,7 +82,7 @@ class ScopedConnection /// for distinguishing concurrent connections in logs / Qt's connection /// registry. If \a readOnly is true, QSQLITE_OPEN_READONLY is set. explicit ScopedConnection(const QString& dbPath, bool readOnly = false, - const QString& prefix = QStringLiteral("QGCSql")); + const QString& prefix = QStringLiteral("QGCSql"), qint64 mmapSize = 0); ~ScopedConnection(); ScopedConnection(const ScopedConnection&) = delete; diff --git a/src/Utilities/Network/QGCNetworkHelper.cc b/src/Utilities/Network/QGCNetworkHelper.cc index 3354f2263d55..9e2efa33b1dc 100644 --- a/src/Utilities/Network/QGCNetworkHelper.cc +++ b/src/Utilities/Network/QGCNetworkHelper.cc @@ -1,22 +1,27 @@ #include "QGCNetworkHelper.h" +#include #include +#include #include #include #include +#include +#include +#include #include +#include #include #include #include #include #include #include +#include #include "QGCCompression.h" #include "QGCLoggingCategory.h" -#include - QGC_LOGGING_CATEGORY(QGCNetworkHelperLog, "Utilities.QGCNetworkHelper") namespace QGCNetworkHelper { @@ -414,11 +419,9 @@ void configureRequest(QNetworkRequest& request, const RequestConfig& config) request.setHeader(QNetworkRequest::ContentTypeHeader, config.contentType); } - for (const auto &[attribute, value] : config.requestAttributes) { + for (const auto& [attribute, value] : config.requestAttributes) { request.setAttribute(attribute, value); } - - request.setRawHeader("Connection", "keep-alive"); } QNetworkRequest createRequest(const QUrl& url, const RequestConfig& config) @@ -434,7 +437,6 @@ void setStandardHeaders(QNetworkRequest& request, const QString& userAgent) request.setHeader(QNetworkRequest::UserAgentHeader, ua); request.setRawHeader("Accept", "*/*"); request.setRawHeader("Accept-Encoding", "gzip, deflate"); - request.setRawHeader("Connection", "keep-alive"); } void setJsonHeaders(QNetworkRequest& request) @@ -452,7 +454,7 @@ QString defaultUserAgent() { static QString userAgent; if (userAgent.isEmpty()) { - userAgent = QStringLiteral("%1/%2 (Qt %3)") + userAgent = QStringLiteral("%1/%2 (+https://qgroundcontrol.com; Qt %3)") .arg(QCoreApplication::applicationName()) .arg(QCoreApplication::applicationVersion()) .arg(QString::fromLatin1(qVersion())); @@ -544,19 +546,20 @@ QList loadCaCertificates(const QString& filePath, QString* erro return certs; } -bool loadClientCertAndKey(const QString& certPath, const QString& keyPath, - QSslCertificate& certOut, QSslKey& keyOut, +bool loadClientCertAndKey(const QString& certPath, const QString& keyPath, QSslCertificate& certOut, QSslKey& keyOut, QString* errorOut) { const auto certs = QSslCertificate::fromPath(certPath, QSsl::Pem); if (certs.isEmpty()) { - if (errorOut) *errorOut = QStringLiteral("No certificate found in %1").arg(certPath); + if (errorOut) + *errorOut = QStringLiteral("No certificate found in %1").arg(certPath); return false; } QFile keyFile(keyPath); if (!keyFile.open(QIODevice::ReadOnly)) { - if (errorOut) *errorOut = QStringLiteral("Cannot open key file %1").arg(keyPath); + if (errorOut) + *errorOut = QStringLiteral("Cannot open key file %1").arg(keyPath); return false; } @@ -566,7 +569,8 @@ bool loadClientCertAndKey(const QString& certPath, const QString& keyPath, key = QSslKey(&keyFile, QSsl::Ec); } if (key.isNull()) { - if (errorOut) *errorOut = QStringLiteral("Invalid key in %1").arg(keyPath); + if (errorOut) + *errorOut = QStringLiteral("Invalid key in %1").arg(keyPath); return false; } @@ -648,6 +652,81 @@ int httpStatusCode(const QNetworkReply* reply) return statusAttr.isValid() ? statusAttr.toInt() : -1; } +std::chrono::milliseconds retryBackoff(int attempt) +{ + constexpr qint64 kBaseMs = 1000; + constexpr qint64 kCapMs = 32000; + const int shift = qBound(0, attempt, 20); // cap shift so 2^attempt can't overflow + const qint64 base = kBaseMs * (qint64(1) << shift); + const qint64 jitter = QRandomGenerator::global()->bounded(1000); + return std::chrono::milliseconds(std::min(base + jitter, kCapMs)); +} + +bool isTransientError(QNetworkReply::NetworkError error, int statusCode) +{ + // 408 Request Timeout and 429 Too Many Requests are the standard transient + // HTTP signals; treat them as retriable so callers can back off / honour + // Retry-After. 401/403/404 are deliberately absent: they are permanent. + if ((statusCode == kHttpRequestTimeout) || (statusCode == kHttpTooManyRequests)) { + return true; + } + + if ((statusCode >= 500) && (statusCode < 600)) { + return true; + } + + switch (error) { + case QNetworkReply::TimeoutError: + case QNetworkReply::TemporaryNetworkFailureError: + case QNetworkReply::NetworkSessionFailedError: + case QNetworkReply::ProxyTimeoutError: + case QNetworkReply::ServiceUnavailableError: + case QNetworkReply::UnknownNetworkError: + case QNetworkReply::RemoteHostClosedError: + case QNetworkReply::ConnectionRefusedError: + case QNetworkReply::HostNotFoundError: + case QNetworkReply::ProxyConnectionRefusedError: + case QNetworkReply::ProxyConnectionClosedError: + case QNetworkReply::ProxyNotFoundError: + return true; + default: + return false; + } +} + +std::optional retryAfterFromReply(const QNetworkReply* reply) +{ + if (!reply) { + return std::nullopt; + } + + const QByteArray raw = reply->headers().combinedValue(QHttpHeaders::WellKnownHeader::RetryAfter).trimmed(); + if (raw.isEmpty()) { + return std::nullopt; + } + + bool ok = false; + const qint64 seconds = raw.toLongLong(&ok); + if (ok && (seconds >= 0)) { + // Clamp before the *1000 so a huge-but-parseable value can't overflow qint64. + const qint64 cappedSeconds = std::min(seconds, kMaxRateLimitDelay.count() / 1000); + return std::chrono::milliseconds(cappedSeconds * 1000); + } + + // Retry-After HTTP-date is an IMF-fixdate; Qt::RFC2822Date rejects its trailing "GMT", so parse with the C locale. + QDateTime when = QLocale::c().toDateTime(QString::fromLatin1(raw), QStringLiteral("ddd, dd MMM yyyy HH:mm:ss 'GMT'")); + if (when.isValid()) { + when.setTimeZone(QTimeZone::UTC); + const qint64 deltaMs = QDateTime::currentDateTimeUtc().msecsTo(when.toUTC()); + if (deltaMs <= 0) { + return std::chrono::milliseconds::zero(); + } + return std::min(std::chrono::milliseconds(deltaMs), kMaxRateLimitDelay); + } + + return std::nullopt; +} + QUrl redirectUrl(const QNetworkReply* reply) { if (!reply) { diff --git a/src/Utilities/Network/QGCNetworkHelper.h b/src/Utilities/Network/QGCNetworkHelper.h index da6d8ce7dea4..5039d3555814 100644 --- a/src/Utilities/Network/QGCNetworkHelper.h +++ b/src/Utilities/Network/QGCNetworkHelper.h @@ -14,6 +14,8 @@ #include #include #include +#include +#include class QIODevice; class QNetworkAccessManager; @@ -266,8 +268,7 @@ QList loadCaCertificates(const QString& filePath, QString* erro /// @param keyOut Receives the loaded key /// @param errorOut Optional pointer to receive error message /// @return true on success -bool loadClientCertAndKey(const QString& certPath, const QString& keyPath, - QSslCertificate& certOut, QSslKey& keyOut, +bool loadClientCertAndKey(const QString& certPath, const QString& keyPath, QSslCertificate& certOut, QSslKey& keyOut, QString* errorOut = nullptr); // ============================================================================ @@ -322,6 +323,38 @@ qint64 contentLength(const QNetworkReply* reply); /// Check if response is JSON based on Content-Type bool isJsonResponse(const QNetworkReply* reply); +// ============================================================================ +// Retry / Transient Error Helpers +// ============================================================================ + +/// HTTP 408 — server-side request timeout, safe to retry. +constexpr int kHttpRequestTimeout = 408; + +/// HTTP 429 — standard rate-limit signal from web services. +constexpr int kHttpTooManyRequests = 429; + +/// Upper bound on any server-provided Retry-After so a pathological value +/// cannot pin a request pipeline. +constexpr std::chrono::milliseconds kMaxRateLimitDelay = std::chrono::seconds(60); + +/// Fixed cooldown for a 429 when the server omits Retry-After; the exponential +/// schedule is too eager for an explicit rate-limit response. +constexpr std::chrono::milliseconds kDefaultRateLimitDelay = std::chrono::seconds(5); + +/// Classify whether a failed request is worth retrying: 408, 429, any 5xx, or a +/// transient network/connection-layer error (timeout, temporary failure, remote +/// close, refused/unreachable). 401/403/404 are permanent and return false. +bool isTransientError(QNetworkReply::NetworkError error, int statusCode); + +/// Parse a reply's Retry-After header (RFC 7231 §7.1.3 — delta-seconds or +/// HTTP-date), clamped to kMaxRateLimitDelay. nullopt if absent/unparseable. +std::optional retryAfterFromReply(const QNetworkReply* reply); + +/// Jittered exponential backoff for 0-based retry index `attempt`: +/// min(1000 * 2^attempt + [0,1000) jitter, 32000) ms. The additive jitter +/// de-syncs requests that fail together so they don't retry in lockstep. +std::chrono::milliseconds retryBackoff(int attempt); + // ============================================================================ // Network Availability // ============================================================================ diff --git a/src/Viewer3D/Viewer3DTileReply.cc b/src/Viewer3D/Viewer3DTileReply.cc index 9a043a1cdde4..e07d1b2e69d8 100644 --- a/src/Viewer3D/Viewer3DTileReply.cc +++ b/src/Viewer3D/Viewer3DTileReply.cc @@ -1,30 +1,30 @@ #include "Viewer3DTileReply.h" +#include +#include +#include +#include +#include +#include + +#include "BingMapProvider.h" #include "MapProvider.h" #include "QGCCacheTile.h" #include "QGCLoggingCategory.h" #include "QGCMapEngine.h" #include "QGCMapTasks.h" #include "QGCMapUrlEngine.h" -#include "QGeoFileTileCacheQGC.h" +#include "QGCTileCache.h" #include "QGeoTileFetcherQGC.h" -#include -#include -#include -#include -#include - -#include - QGC_LOGGING_CATEGORY(Viewer3DTileReplyLog, "Viewer3d.Viewer3DTileReply") QByteArray Viewer3DTileReply::_bingNoTileImage; static std::once_flag s_bingNoTileInitFlag; -Viewer3DTileReply::Viewer3DTileReply(int zoomLevel, int tileX, int tileY, int mapId, const QString &mapType, QNetworkAccessManager *networkManager, QObject *parent) - : QObject{parent} - , _networkManager(networkManager) +Viewer3DTileReply::Viewer3DTileReply(int zoomLevel, int tileX, int tileY, int mapId, const QString& mapType, + QNetworkAccessManager* networkManager, QObject* parent) + : QObject{parent}, _networkManager(networkManager) { std::call_once(s_bingNoTileInitFlag, []() { QFile file(QStringLiteral(":/res/BingNoTileBytes.dat")); @@ -42,9 +42,9 @@ Viewer3DTileReply::Viewer3DTileReply(int zoomLevel, int tileX, int tileY, int ma _tile.zoomLevel = zoomLevel; _tile.mapId = mapId; - QGCFetchTileTask *task = QGeoFileTileCacheQGC::createFetchTileTask(mapType, tileX, tileY, zoomLevel); + QGCFetchTileTask* task = QGCTileCache::createFetchTileTask(mapType, tileX, tileY, zoomLevel); connect(task, &QGCFetchTileTask::tileFetched, this, &Viewer3DTileReply::_onCacheHit); - connect(task, &QGCMapTask::error, this, [this](QGCMapTask::TaskType, const QString &) { _onCacheMiss(); }); + connect(task, &QGCMapTask::error, this, [this](QGCMapTask::TaskType, const QString&) { _onCacheMiss(); }); if (!getQGCMapEngine()->addTask(task)) { task->deleteLater(); _onCacheMiss(); @@ -56,20 +56,21 @@ Viewer3DTileReply::~Viewer3DTileReply() if (_reply) { _disconnectReply(); _reply->abort(); - delete _reply; + _reply->deleteLater(); _reply = nullptr; } } void Viewer3DTileReply::_prepareDownload() { - const QNetworkRequest request = QGeoTileFetcherQGC::getNetworkRequest(_tile.mapId, _tile.x, _tile.y, _tile.zoomLevel); + const QNetworkRequest request = + QGeoTileFetcherQGC::getNetworkRequest(_tile.mapId, _tile.x, _tile.y, _tile.zoomLevel); _reply = _networkManager->get(request); connect(_reply, &QNetworkReply::finished, this, &Viewer3DTileReply::_onRequestFinished); connect(_reply, &QNetworkReply::errorOccurred, this, &Viewer3DTileReply::_onRequestError); } -void Viewer3DTileReply::_onCacheHit(QGCCacheTile *tile) +void Viewer3DTileReply::_onCacheHit(QSharedPointer tile) { if (!tile) { _onCacheMiss(); @@ -77,7 +78,6 @@ void Viewer3DTileReply::_onCacheHit(QGCCacheTile *tile) } _tile.data = tile->img; - delete tile; if (_isBingEmptyTile()) { _tile.data.clear(); @@ -111,7 +111,7 @@ void Viewer3DTileReply::_onRequestFinished() const SharedMapProvider mapProvider = UrlFactory::getMapProviderFromQtMapId(_tile.mapId); if (mapProvider && !_tile.data.isEmpty()) { const QString format = mapProvider->getImageFormat(_tile.data); - QGeoFileTileCacheQGC::cacheTile(mapProvider->getMapName(), _tile.x, _tile.y, _tile.zoomLevel, _tile.data, format); + QGCTileCache::cacheTile(mapProvider->getMapName(), _tile.x, _tile.y, _tile.zoomLevel, _tile.data, format); } emit tileDone(_tile); @@ -119,7 +119,8 @@ void Viewer3DTileReply::_onRequestFinished() void Viewer3DTileReply::_onRequestError() { - qCWarning(Viewer3DTileReplyLog) << "Request error for tile x:" << _tile.x << "y:" << _tile.y << "zoom:" << _tile.zoomLevel; + qCWarning(Viewer3DTileReplyLog) << "Request error for tile x:" << _tile.x << "y:" << _tile.y + << "zoom:" << _tile.zoomLevel; _timeoutTimer->stop(); disconnect(_timeoutTimer, &QTimer::timeout, this, &Viewer3DTileReply::_onTimeout); _disconnectReply(); @@ -128,7 +129,8 @@ void Viewer3DTileReply::_onRequestError() void Viewer3DTileReply::_onTimeout() { - qCDebug(Viewer3DTileReplyLog) << "Timeout for tile x:" << _tile.x << "y:" << _tile.y << "retry:" << _timeoutCounter << "/" << kMaxRetries; + qCDebug(Viewer3DTileReplyLog) << "Timeout for tile x:" << _tile.x << "y:" << _tile.y << "retry:" << _timeoutCounter + << "/" << kMaxRetries; if (_timeoutCounter > kMaxRetries) { _disconnectReply(); disconnect(_timeoutTimer, &QTimer::timeout, this, &Viewer3DTileReply::_onTimeout); @@ -138,7 +140,7 @@ void Viewer3DTileReply::_onTimeout() if (_reply) { _disconnectReply(); _reply->abort(); - delete _reply; + _reply->deleteLater(); _reply = nullptr; } _prepareDownload(); @@ -155,5 +157,6 @@ void Viewer3DTileReply::_disconnectReply() bool Viewer3DTileReply::_isBingEmptyTile() const { const SharedMapProvider mapProvider = UrlFactory::getMapProviderFromQtMapId(_tile.mapId); - return mapProvider && mapProvider->isBingProvider() && !_tile.data.isEmpty() && _tile.data == _bingNoTileImage; + return mapProvider && std::dynamic_pointer_cast(mapProvider) && !_tile.data.isEmpty() && + _tile.data == _bingNoTileImage; } diff --git a/src/Viewer3D/Viewer3DTileReply.h b/src/Viewer3D/Viewer3DTileReply.h index 2c26cd711827..f10167825b26 100644 --- a/src/Viewer3D/Viewer3DTileReply.h +++ b/src/Viewer3D/Viewer3DTileReply.h @@ -30,7 +30,7 @@ class Viewer3DTileReply : public QObject void _onRequestFinished(); void _onRequestError(); void _onTimeout(); - void _onCacheHit(QGCCacheTile *tile); + void _onCacheHit(QSharedPointer tile); void _onCacheMiss(); void _disconnectReply(); bool _isBingEmptyTile() const; diff --git a/test/QtLocationPlugin/CMakeLists.txt b/test/QtLocationPlugin/CMakeLists.txt index d349808d0b86..d32701912ac3 100644 --- a/test/QtLocationPlugin/CMakeLists.txt +++ b/test/QtLocationPlugin/CMakeLists.txt @@ -7,8 +7,13 @@ target_sources(${CMAKE_PROJECT_NAME} PRIVATE MapProviderTest.cc MapProviderTest.h + MockMapProvider.h QGCMapEngineManagerArchiveTest.cc QGCMapEngineManagerArchiveTest.h + QGCMapEngineManagerTest.cc + QGCMapEngineManagerTest.h + QGCMapTaskTest.cc + QGCMapTaskTest.h QGCCachedTileSetTest.cc QGCCachedTileSetTest.h QGCCacheWorkerTest.cc @@ -17,6 +22,21 @@ target_sources(${CMAKE_PROJECT_NAME} QGCTileCacheDatabaseTest.h QGCTileSetTest.cc QGCTileSetTest.h + QGCTileSetImporterTest.cc + QGCTileSetImporterTest.h + QGCTileFallbackTest.cc + QGCTileFallbackTest.h + QGCHostCircuitBreakerTest.cc + QGCHostCircuitBreakerTest.h + QGeoFileTileCacheQGCTest.cc + QGeoFileTileCacheQGCTest.h + QGeoTileFetcherQGCTest.cc + QGeoTileFetcherQGCTest.h + QGeoTiledMapReplyQGCTest.cc + QGeoTiledMapReplyQGCTest.h + QtLocationTestBase.h + TileFetchMetricsTest.cc + TileFetchMetricsTest.h UrlFactoryTest.cc UrlFactoryTest.h ) @@ -27,6 +47,15 @@ add_qgc_test(MapProviderTest LABELS Unit) add_qgc_test(QGCCacheWorkerTest LABELS Unit) add_qgc_test(QGCCachedTileSetTest LABELS Unit) add_qgc_test(QGCMapEngineManagerArchiveTest LABELS Unit RESOURCE_LOCK TempFiles) +add_qgc_test(QGCMapEngineManagerTest LABELS Unit) +add_qgc_test(QGCMapTaskTest LABELS Unit) add_qgc_test(QGCTileCacheDatabaseTest LABELS Unit) add_qgc_test(QGCTileSetTest LABELS Unit) +add_qgc_test(QGCTileSetImporterTest LABELS Unit) +add_qgc_test(QGCTileFallbackTest LABELS Unit) +add_qgc_test(QGCHostCircuitBreakerTest LABELS Unit) +add_qgc_test(QGeoFileTileCacheQGCTest LABELS Unit) +add_qgc_test(QGeoTileFetcherQGCTest LABELS Unit) +add_qgc_test(QGeoTiledMapReplyQGCTest LABELS Unit) +add_qgc_test(TileFetchMetricsTest LABELS Unit) add_qgc_test(UrlFactoryTest LABELS Unit) diff --git a/test/QtLocationPlugin/MapProviderTest.cc b/test/QtLocationPlugin/MapProviderTest.cc index c5dd307eb54e..3d1a438eac44 100644 --- a/test/QtLocationPlugin/MapProviderTest.cc +++ b/test/QtLocationPlugin/MapProviderTest.cc @@ -3,6 +3,7 @@ #include #include "MapProvider.h" +#include "MockMapProvider.h" #include "QGCTileSet.h" class TestableMapProvider : public MapProvider @@ -12,17 +13,13 @@ class TestableMapProvider : public MapProvider const QString& imageFormat = QStringLiteral("png"), quint32 avgSize = QGC_AVERAGE_TILE_SIZE) : MapProvider(name, QString(), imageFormat, avgSize, MapProvider::CustomMap) - { - } + {} using MapProvider::_getServerNum; using MapProvider::_tileXYToQuadKey; private: - QString _getURL(int, int, int) const override - { - return {}; - } + QString _getURL(int, int, int) const override { return {}; } }; // --- Image format detection --- @@ -206,9 +203,27 @@ void MapProviderTest::_testGettersReturnConstructorValues() QCOMPARE(p.getMapName(), QStringLiteral("MyMap")); QCOMPARE(p.getAverageSize(), static_cast(7777)); QCOMPARE(p.getMapStyle(), MapProvider::CustomMap); - QVERIFY(!p.isElevationProvider()); - QVERIFY(!p.isBingProvider()); QVERIFY(p.getMapId() > 0); } +// --- Tile URL dispatch (via MockMapProvider) --- + +void MapProviderTest::_testGetTileURLDispatchesToGetURL() +{ + MockMapProvider p; + const QUrl url = p.getTileURL(3, 5, 7); + QCOMPARE(p.getURLCallCount(), 1); + QVERIFY(url.isValid()); + QCOMPARE(url, QUrl(QStringLiteral("https://mock.test/tile/7/3/5.png"))); +} + +void MapProviderTest::_testGetTileURLEmptyReturnsInvalid() +{ + MockMapProvider p; + p.setShouldReturnEmpty(true); + const QUrl url = p.getTileURL(1, 2, 3); + QCOMPARE(p.getURLCallCount(), 1); + QVERIFY(url.isEmpty()); +} + UT_REGISTER_TEST(MapProviderTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/MapProviderTest.h b/test/QtLocationPlugin/MapProviderTest.h index c6ed65a5e6ce..e77bd8f8230a 100644 --- a/test/QtLocationPlugin/MapProviderTest.h +++ b/test/QtLocationPlugin/MapProviderTest.h @@ -28,4 +28,6 @@ private slots: void _testGetServerNumBasic(); void _testGetServerNumWrap(); void _testGettersReturnConstructorValues(); + void _testGetTileURLDispatchesToGetURL(); + void _testGetTileURLEmptyReturnsInvalid(); }; diff --git a/test/QtLocationPlugin/MockMapProvider.h b/test/QtLocationPlugin/MockMapProvider.h new file mode 100644 index 000000000000..c1f4b99694c6 --- /dev/null +++ b/test/QtLocationPlugin/MockMapProvider.h @@ -0,0 +1,38 @@ +#pragma once + +#include "MapProvider.h" + +class MockMapProvider : public MapProvider +{ +public: + explicit MockMapProvider(const QString& name = QStringLiteral("MockProvider"), + const QString& imageFormat = QStringLiteral("png"), + quint32 avgSize = QGC_AVERAGE_TILE_SIZE, + const QString& referrer = QStringLiteral("https://mock.test"), + MapProvider::MapStyle mapStyle = MapProvider::CustomMap) + : MapProvider(name, referrer, imageFormat, avgSize, mapStyle) + {} + + using MapProvider::_getServerNum; + using MapProvider::_tileXYToQuadKey; + + int getURLCallCount() const { return _urlCallCount; } + + void resetCallCount() { _urlCallCount = 0; } + + void setShouldReturnEmpty(bool shouldReturnEmpty) { _shouldReturnEmpty = shouldReturnEmpty; } + +private: + QString _getURL(int x, int y, int zoom) const override + { + ++_urlCallCount; + if (_shouldReturnEmpty) { + return {}; + } + + return QStringLiteral("https://mock.test/tile/%1/%2/%3.%4").arg(zoom).arg(x).arg(y).arg(_imageFormat); + } + + mutable int _urlCallCount = 0; + bool _shouldReturnEmpty = false; +}; diff --git a/test/QtLocationPlugin/QGCCacheWorkerTest.cc b/test/QtLocationPlugin/QGCCacheWorkerTest.cc index 57f5dbe614de..99347d2fc67b 100644 --- a/test/QtLocationPlugin/QGCCacheWorkerTest.cc +++ b/test/QtLocationPlugin/QGCCacheWorkerTest.cc @@ -1,13 +1,15 @@ #include "QGCCacheWorkerTest.h" +#include +#include #include #include "QGCCacheTile.h" #include "QGCCachedTileSet.h" #include "QGCMapTasks.h" #include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" #include "QGCTileCacheWorker.h" -#include static const QString kTestProviderType = QStringLiteral("Bing Road"); @@ -64,6 +66,33 @@ void QGCCacheWorkerTest::_testEnqueueBeforeInit() QVERIFY_TRUE_WAIT(errorReceived, TestTimeout::shortMs()); } +void QGCCacheWorkerTest::_testEnqueueAfterStop() +{ + QTemporaryDir tempDir; + QGCCacheWorker worker; + worker.setDatabaseFile(tempDir.filePath("enqueue_after_stop.db")); + QVERIFY(_startWorker(worker)); + + worker.stop(); + QVERIFY(worker.wait(TestTimeout::mediumMs())); + QVERIFY(!worker.isRunning()); + + // A taskInit would restart the worker, so exercise post-stop rejection with a non-init + // task: enqueueTask() rejects with "Database Not Initialized" without restarting the thread. + auto* task = new QGCFetchTileTask(QStringLiteral("after_stop_hash")); + bool errorReceived = false; + QString errorString; + connect(task, &QGCMapTask::error, this, [&](QGCMapTask::TaskType, const QString& error) { + errorReceived = true; + errorString = error; + }); + + QVERIFY(!worker.enqueueTask(task)); + QVERIFY_TRUE_WAIT(errorReceived, TestTimeout::shortMs()); + QCOMPARE(errorString, QStringLiteral("Database Not Initialized")); + QVERIFY(!worker.isRunning()); +} + void QGCCacheWorkerTest::_testUpdateTotalsOnInit() { QTemporaryDir tempDir; @@ -91,6 +120,38 @@ void QGCCacheWorkerTest::_testUpdateTotalsOnInit() worker.wait(TestTimeout::mediumMs()); } +void QGCCacheWorkerTest::_testGetTotalsTask() +{ + QTemporaryDir tempDir; + QGCCacheWorker worker; + worker.setDatabaseFile(tempDir.filePath("get_totals.db")); + QVERIFY(_startWorker(worker)); + + auto* tile = + new QGCCacheTile(QStringLiteral("gt1"), QByteArray("totals_data"), QStringLiteral("png"), kTestProviderType); + QVERIFY(worker.enqueueTask(new QGCSaveTileTask(tile))); + + quint32 totalTiles = UINT32_MAX; + quint64 totalSize = UINT64_MAX; + auto conn = connect( + &worker, &QGCCacheWorker::updateTotals, this, + [&](quint32 tiles, quint64 size, quint32, quint64) { + totalTiles = tiles; + totalSize = size; + }, + Qt::QueuedConnection); + + QVERIFY(worker.enqueueTask(new QGCMapTask(QGCMapTask::TaskType::taskInit))); + QTRY_VERIFY_WITH_TIMEOUT(totalTiles != UINT32_MAX, TestTimeout::mediumMs()); + disconnect(conn); + + QVERIFY(totalTiles >= 1); + QVERIFY(totalSize > 0); + + worker.stop(); + worker.wait(TestTimeout::mediumMs()); +} + void QGCCacheWorkerTest::_testSaveAndFetchTile() { QTemporaryDir tempDir; @@ -104,10 +165,11 @@ void QGCCacheWorkerTest::_testSaveAndFetchTile() // Fetch — FIFO guarantees save completes first auto* fetchTask = new QGCFetchTileTask(QStringLiteral("h1")); - QGCCacheTile* fetched = nullptr; + QSharedPointer fetched; bool fetchError = false; connect( - fetchTask, &QGCFetchTileTask::tileFetched, this, [&](QGCCacheTile* t) { fetched = t; }, Qt::QueuedConnection); + fetchTask, &QGCFetchTileTask::tileFetched, this, [&](QSharedPointer t) { fetched = t; }, + Qt::QueuedConnection); connect( fetchTask, &QGCMapTask::error, this, [&](QGCMapTask::TaskType, const QString&) { fetchError = true; }, Qt::QueuedConnection); @@ -120,7 +182,6 @@ void QGCCacheWorkerTest::_testSaveAndFetchTile() QCOMPARE(fetched->hash, QStringLiteral("h1")); QCOMPARE(fetched->img, QByteArray("tile_data")); QCOMPARE(fetched->format, QStringLiteral("png")); - delete fetched; worker.stop(); worker.wait(TestTimeout::mediumMs()); @@ -134,10 +195,11 @@ void QGCCacheWorkerTest::_testFetchTileNotFound() QVERIFY(_startWorker(worker)); auto* fetchTask = new QGCFetchTileTask(QStringLiteral("nonexistent")); - QGCCacheTile* fetched = nullptr; + QSharedPointer fetched; bool fetchError = false; connect( - fetchTask, &QGCFetchTileTask::tileFetched, this, [&](QGCCacheTile* t) { fetched = t; }, Qt::QueuedConnection); + fetchTask, &QGCFetchTileTask::tileFetched, this, [&](QSharedPointer t) { fetched = t; }, + Qt::QueuedConnection); connect( fetchTask, &QGCMapTask::error, this, [&](QGCMapTask::TaskType, const QString&) { fetchError = true; }, Qt::QueuedConnection); @@ -183,6 +245,43 @@ void QGCCacheWorkerTest::_testFetchTileSets() worker.wait(TestTimeout::mediumMs()); } +void QGCCacheWorkerTest::_testFetchTileSetsMainThreadAffinity() +{ + QTemporaryDir tempDir; + QGCCacheWorker worker; + worker.setDatabaseFile(tempDir.filePath("tile_sets_affinity.db")); + QVERIFY(_startWorker(worker)); + + bool taskDone = false; + auto totalsConn = + connect(&worker, &QGCCacheWorker::updateTotals, this, [&]() { taskDone = true; }, Qt::QueuedConnection); + + bool fetchError = false; + QList sets; + auto* fetchTask = new QGCFetchTileSetTask(); + connect( + fetchTask, &QGCFetchTileSetTask::tileSetFetched, this, [&](QGCCachedTileSet* set) { sets.append(set); }, + Qt::QueuedConnection); + connect( + fetchTask, &QGCMapTask::error, this, [&](QGCMapTask::TaskType, const QString&) { fetchError = true; }, + Qt::QueuedConnection); + + QVERIFY(worker.enqueueTask(fetchTask)); + QTRY_VERIFY_WITH_TIMEOUT(taskDone, TestTimeout::mediumMs()); + disconnect(totalsConn); + + QVERIFY(!fetchError); + QVERIFY(!sets.isEmpty()); + for (QGCCachedTileSet* set : sets) { + QVERIFY(set != nullptr); + QCOMPARE(set->thread(), QCoreApplication::instance()->thread()); + } + qDeleteAll(sets); + + worker.stop(); + worker.wait(TestTimeout::mediumMs()); +} + void QGCCacheWorkerTest::_testCreateAndDeleteTileSet() { QTemporaryDir tempDir; @@ -219,15 +318,24 @@ void QGCCacheWorkerTest::_testCreateAndDeleteTileSet() // Delete it const quint64 setID = savedSet->id(); - auto* deleteTask = new QGCDeleteTileSetTask(setID); - quint64 deletedID = 0; - connect( - deleteTask, &QGCDeleteTileSetTask::tileSetDeleted, this, [&](quint64 id) { deletedID = id; }, - Qt::QueuedConnection); + auto* deleteTask = + new QGCCommandTask(QGCMapTask::TaskType::taskDeleteTileSet, [setID](QGCCacheWorker& w, QGCMapTask& self) { + if (!w.validateDatabase(&self)) { + return false; + } + if (!w.database()->deleteTileSet(setID)) { + self.setError("Error deleting tile set"); + return false; + } + w.emitTotals(); + return true; + }); + bool deleted = false; + connect(deleteTask, &QGCCommandTask::completed, this, [&]() { deleted = true; }, Qt::QueuedConnection); QVERIFY(worker.enqueueTask(deleteTask)); - QTRY_VERIFY_WITH_TIMEOUT(deletedID != 0, TestTimeout::mediumMs()); - QCOMPARE(deletedID, setID); + QTRY_VERIFY_WITH_TIMEOUT(deleted, TestTimeout::mediumMs()); + QVERIFY(deleted); delete savedSet; @@ -248,9 +356,18 @@ void QGCCacheWorkerTest::_testPruneCache() QVERIFY(worker.enqueueTask(new QGCSaveTileTask(tile))); } - auto* pruneTask = new QGCPruneCacheTask(500); + auto* pruneTask = new QGCCommandTask(QGCMapTask::TaskType::taskPruneCache, [](QGCCacheWorker& w, QGCMapTask& self) { + if (!w.validateDatabase(&self)) { + return false; + } + if (!w.database()->pruneCache(500)) { + self.setError("Error pruning cache"); + return false; + } + return true; + }); bool pruneDone = false; - connect(pruneTask, &QGCPruneCacheTask::pruned, this, [&]() { pruneDone = true; }, Qt::QueuedConnection); + connect(pruneTask, &QGCCommandTask::completed, this, [&]() { pruneDone = true; }, Qt::QueuedConnection); QVERIFY(worker.enqueueTask(pruneTask)); QTRY_VERIFY_WITH_TIMEOUT(pruneDone, TestTimeout::mediumMs()); @@ -280,9 +397,19 @@ void QGCCacheWorkerTest::_testResetDatabase() auto* tile = new QGCCacheTile(QStringLiteral("r1"), QByteArray("data"), QStringLiteral("png"), kTestProviderType); QVERIFY(worker.enqueueTask(new QGCSaveTileTask(tile))); - auto* resetTask = new QGCResetTask(); + auto* resetTask = new QGCCommandTask(QGCMapTask::TaskType::taskReset, [](QGCCacheWorker& w, QGCMapTask& self) { + if (!w.validateDatabase(&self)) { + return false; + } + if (!w.database()->resetDatabase()) { + self.setError("Error resetting cache database"); + return false; + } + w.setDatabaseValid(w.database()->isValid()); + return true; + }); bool resetDone = false; - connect(resetTask, &QGCResetTask::resetCompleted, this, [&]() { resetDone = true; }, Qt::QueuedConnection); + connect(resetTask, &QGCCommandTask::completed, this, [&]() { resetDone = true; }, Qt::QueuedConnection); QVERIFY(worker.enqueueTask(resetTask)); QTRY_VERIFY_WITH_TIMEOUT(resetDone, TestTimeout::mediumMs()); @@ -319,4 +446,39 @@ void QGCCacheWorkerTest::_testStopWhileProcessing() QVERIFY(!worker.isRunning()); } +void QGCCacheWorkerTest::_testCacheTileBatchCoalesce() +{ + QTemporaryDir tempDir; + QGCCacheWorker worker; + worker.setDatabaseFile(tempDir.filePath("batch_coalesce.db")); + QVERIFY(_startWorker(worker)); + + constexpr int kTileCount = 200; + for (int i = 0; i < kTileCount; i++) { + auto* tile = new QGCCacheTile(QStringLiteral("bc_%1").arg(i), QByteArray(64, 'B'), QStringLiteral("png"), + kTestProviderType); + QVERIFY(worker.enqueueTask(new QGCSaveTileTask(tile))); + } + + quint32 totalTiles = 0; + bool totalsReceived = false; + auto conn = connect( + &worker, &QGCCacheWorker::updateTotals, this, + [&](quint32 tiles, quint64, quint32, quint64) { + totalTiles = tiles; + totalsReceived = true; + }, + Qt::QueuedConnection); + QVERIFY(worker.enqueueTask(new QGCMapTask(QGCMapTask::TaskType::taskInit))); + QTRY_VERIFY_WITH_TIMEOUT(totalsReceived && (totalTiles >= static_cast(kTileCount)), + TestTimeout::mediumMs()); + disconnect(conn); + + QCOMPARE(totalTiles, static_cast(kTileCount)); + + worker.stop(); + QVERIFY(worker.wait(TestTimeout::mediumMs())); + QVERIFY(!worker.isRunning()); +} + UT_REGISTER_TEST(QGCCacheWorkerTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCCacheWorkerTest.h b/test/QtLocationPlugin/QGCCacheWorkerTest.h index 2b424ef018e0..921ca02ea810 100644 --- a/test/QtLocationPlugin/QGCCacheWorkerTest.h +++ b/test/QtLocationPlugin/QGCCacheWorkerTest.h @@ -12,14 +12,18 @@ private slots: void initTestCase() override; void _testStartAndStop(); void _testEnqueueBeforeInit(); + void _testEnqueueAfterStop(); void _testUpdateTotalsOnInit(); + void _testGetTotalsTask(); void _testSaveAndFetchTile(); void _testFetchTileNotFound(); void _testFetchTileSets(); + void _testFetchTileSetsMainThreadAffinity(); void _testCreateAndDeleteTileSet(); void _testPruneCache(); void _testResetDatabase(); void _testStopWhileProcessing(); + void _testCacheTileBatchCoalesce(); private: bool _startWorker(QGCCacheWorker& worker, int timeoutMs = TestTimeout::mediumMs()); diff --git a/test/QtLocationPlugin/QGCCachedTileSetTest.cc b/test/QtLocationPlugin/QGCCachedTileSetTest.cc index a1ecec840411..330e696d49df 100644 --- a/test/QtLocationPlugin/QGCCachedTileSetTest.cc +++ b/test/QtLocationPlugin/QGCCachedTileSetTest.cc @@ -245,4 +245,56 @@ void QGCCachedTileSetTest::_testSetSelectedEmitsSignal() QVERIFY(!ts.selected()); } +void QGCCachedTileSetTest::_testDownloadStatusStrings() +{ + QGCCachedTileSet defaultSet(QStringLiteral("Default")); + defaultSet.setDefaultSet(true); + defaultSet.setTotalTileSize(1024 * 1024); + QCOMPARE(defaultSet.downloadStatus(), defaultSet.totalTilesSizeStr()); + + // In-progress form ("saved / total") is chosen by tile COUNT, not size. + QGCCachedTileSet inProgressSet(QStringLiteral("InProgress")); + inProgressSet.setTotalTileCount(10); + inProgressSet.setSavedTileCount(4); + inProgressSet.setTotalTileSize(10 * 1024 * 1024); + inProgressSet.setSavedTileSize(4 * 1024 * 1024); + QVERIFY(inProgressSet.downloadStatus().contains(QStringLiteral("/"))); + + QGCCachedTileSet completeSet(QStringLiteral("Complete")); + completeSet.setTotalTileCount(50); + completeSet.setSavedTileCount(50); + completeSet.setSavedTileSize(2 * 1024 * 1024); + QCOMPARE(completeSet.downloadStatus(), completeSet.savedTileSizeStr()); +} + +void QGCCachedTileSetTest::_testErrorCountStr() +{ + QGCCachedTileSet ts(QStringLiteral("Error Test")); + ts.setErrorCount(3); + QVERIFY(ts.errorCountStr().contains(QStringLiteral("3"))); +} + +void QGCCachedTileSetTest::_testPausedDefaultsFalse() +{ + QGCCachedTileSet ts(QStringLiteral("test")); + QVERIFY(!ts.paused()); +} + +void QGCCachedTileSetTest::_testPauseSetsPausedAndStopsDownloading() +{ + QGCCachedTileSet ts(QStringLiteral("PauseSet")); + ts.setId(1); + ts.setDownloading(true); + + QSignalSpy pausedSpy(&ts, &QGCCachedTileSet::pausedChanged); + QSignalSpy downloadingSpy(&ts, &QGCCachedTileSet::downloadingChanged); + + ts.pauseDownloadTask(); + + QVERIFY(ts.paused()); + QVERIFY(!ts.downloading()); + QCOMPARE(pausedSpy.count(), 1); + QVERIFY(downloadingSpy.count() >= 1); +} + UT_REGISTER_TEST(QGCCachedTileSetTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCCachedTileSetTest.h b/test/QtLocationPlugin/QGCCachedTileSetTest.h index 5b374838f44b..f73efd13729f 100644 --- a/test/QtLocationPlugin/QGCCachedTileSetTest.h +++ b/test/QtLocationPlugin/QGCCachedTileSetTest.h @@ -22,4 +22,8 @@ private slots: void _testCompleteWhenDefaultSet(); void _testCompleteWhenAllSaved(); void _testSetSelectedEmitsSignal(); + void _testDownloadStatusStrings(); + void _testErrorCountStr(); + void _testPauseSetsPausedAndStopsDownloading(); + void _testPausedDefaultsFalse(); }; diff --git a/test/QtLocationPlugin/QGCHostCircuitBreakerTest.cc b/test/QtLocationPlugin/QGCHostCircuitBreakerTest.cc new file mode 100644 index 000000000000..f7bca27027d8 --- /dev/null +++ b/test/QtLocationPlugin/QGCHostCircuitBreakerTest.cc @@ -0,0 +1,58 @@ +#include "QGCHostCircuitBreakerTest.h" + +#include "QGCHostCircuitBreaker.h" + +// HostCircuitBreaker::instance() is a process-wide singleton, so each test uses a +// unique host key to avoid state leaking between cases. The 60s open / 30s window +// transitions are wall-clock based and not exercised here (would require time +// injection); these tests cover the failure-threshold and success-reset logic. + +void QGCHostCircuitBreakerTest::_testEmptyHostAlwaysAllowed() +{ + HostCircuitBreaker &cb = HostCircuitBreaker::instance(); + QVERIFY(cb.allowRequest(QString())); + cb.recordFailure(QString()); + QVERIFY(cb.allowRequest(QString())); +} + +void QGCHostCircuitBreakerTest::_testBelowThresholdStaysClosed() +{ + HostCircuitBreaker &cb = HostCircuitBreaker::instance(); + const QString host = QStringLiteral("cb-below.example"); + for (int i = 0; i < 4; ++i) { + cb.recordFailure(host); + QVERIFY(cb.allowRequest(host)); + } +} + +void QGCHostCircuitBreakerTest::_testFifthFailureOpens() +{ + HostCircuitBreaker &cb = HostCircuitBreaker::instance(); + const QString host = QStringLiteral("cb-open.example"); + for (int i = 0; i < 5; ++i) { + cb.recordFailure(host); + } + QVERIFY(!cb.allowRequest(host)); +} + +void QGCHostCircuitBreakerTest::_testSuccessResetsFailureCounter() +{ + HostCircuitBreaker &cb = HostCircuitBreaker::instance(); + const QString host = QStringLiteral("cb-reset.example"); + + for (int i = 0; i < 4; ++i) { + cb.recordFailure(host); + } + cb.recordSuccess(host); + + // Counter reset: four more failures alone must not trip the breaker. + for (int i = 0; i < 4; ++i) { + cb.recordFailure(host); + QVERIFY(cb.allowRequest(host)); + } + // The fifth still opens, proving the threshold is intact after reset. + cb.recordFailure(host); + QVERIFY(!cb.allowRequest(host)); +} + +UT_REGISTER_TEST(QGCHostCircuitBreakerTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCHostCircuitBreakerTest.h b/test/QtLocationPlugin/QGCHostCircuitBreakerTest.h new file mode 100644 index 000000000000..41676f3c68fa --- /dev/null +++ b/test/QtLocationPlugin/QGCHostCircuitBreakerTest.h @@ -0,0 +1,14 @@ +#pragma once + +#include "UnitTest.h" + +class QGCHostCircuitBreakerTest : public UnitTest +{ + Q_OBJECT + +private slots: + void _testEmptyHostAlwaysAllowed(); + void _testBelowThresholdStaysClosed(); + void _testFifthFailureOpens(); + void _testSuccessResetsFailureCounter(); +}; diff --git a/test/QtLocationPlugin/QGCMapEngineManagerTest.cc b/test/QtLocationPlugin/QGCMapEngineManagerTest.cc new file mode 100644 index 000000000000..631c778fc8d3 --- /dev/null +++ b/test/QtLocationPlugin/QGCMapEngineManagerTest.cc @@ -0,0 +1,135 @@ +#include "QGCMapEngineManagerTest.h" + +#include +#include +#include + +#include "QGCCachedTileSet.h" +#include "QGCMapEngineManager.h" +#include "QmlObjectListModel.h" + +void QGCMapEngineManagerTest::init() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + QVERIFY(manager->tileSets() != nullptr); + _initialTileSetCount = manager->tileSets()->count(); +} + +void QGCMapEngineManagerTest::cleanup() +{ + // Tests append throwaway sets to the singleton's model; trim them here so a failed + // assertion mid-test cannot leak state into the next test. + QmlObjectListModel* model = QGCMapEngineManager::instance()->tileSets(); + while (model->count() > _initialTileSetCount) { + delete model->removeAt(model->count() - 1); + } +} + +void QGCMapEngineManagerTest::_testInstanceAndModelAvailable() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + QVERIFY(manager->tileSets() != nullptr); +} + +void QGCMapEngineManagerTest::_testProviderListsAndMapTypes() +{ + const QStringList mapList = QGCMapEngineManager::mapList(); + const QStringList providerList = QGCMapEngineManager::mapProviderList(); + const QStringList elevationList = QGCMapEngineManager::elevationProviderList(); + + QVERIFY(!mapList.isEmpty()); + QVERIFY(!providerList.isEmpty()); + QVERIFY(!elevationList.isEmpty()); + + const QString mapProvider = providerList.first(); + const QStringList mapTypes = QGCMapEngineManager::mapTypeList(mapProvider); + QVERIFY2(!mapTypes.isEmpty(), qPrintable(QStringLiteral("No map types for provider: %1").arg(mapProvider))); +} + +void QGCMapEngineManagerTest::_testUpdateForCurrentViewUpdatesTotals() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + + const QStringList mapList = QGCMapEngineManager::mapList(); + QVERIFY(!mapList.isEmpty()); + const QString mapName = mapList.first(); + + QSignalSpy tileCountSpy(manager, &QGCMapEngineManager::tileCountChanged); + QSignalSpy tileSizeSpy(manager, &QGCMapEngineManager::tileSizeChanged); + + manager->updateForCurrentView(-122.50, 37.80, -122.30, 37.70, 1, 3, mapName); + + QVERIFY(tileCountSpy.count() > 0); + QVERIFY(tileSizeSpy.count() > 0); + QVERIFY(manager->tileCount() > 0); + QVERIFY(manager->tileSize() > 0); +} + +void QGCMapEngineManagerTest::_testFindNameAndSelectionHelpers() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + + QmlObjectListModel* model = manager->tileSets(); + QVERIFY(model != nullptr); + + auto* setA = new QGCCachedTileSet(QStringLiteral("__unit_test_set_a__")); + auto* setB = new QGCCachedTileSet(QStringLiteral("__unit_test_set_b__")); + setA->setManager(manager); + setB->setManager(manager); + + model->append(setA); + model->append(setB); + + QVERIFY(manager->findName(QStringLiteral("__unit_test_set_a__"))); + QVERIFY(manager->findName(QStringLiteral("__unit_test_set_b__"))); + QVERIFY(!manager->findName(QStringLiteral("__unit_test_set_missing__"))); + + manager->selectAll(); + QVERIFY(setA->selected()); + QVERIFY(setB->selected()); + + manager->selectNone(); + QVERIFY(!setA->selected()); + QVERIFY(!setB->selected()); + QCOMPARE(manager->selectedCount(), 0); +} + +void QGCMapEngineManagerTest::_testRenameOptimisticallyUpdatesName() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + + QmlObjectListModel* model = manager->tileSets(); + QVERIFY(model != nullptr); + + auto* set = new QGCCachedTileSet(QStringLiteral("__rename_source__")); + set->setManager(manager); + set->setId(std::numeric_limits::max()); + model->append(set); + + manager->renameTileSet(set, QStringLiteral("__rename_target__")); + QCOMPARE(set->name(), QStringLiteral("__rename_target__")); +} + +void QGCMapEngineManagerTest::_testTaskErrorLabels() +{ + QGCMapEngineManager* manager = QGCMapEngineManager::instance(); + QVERIFY(manager != nullptr); + + // not on this branch: taskError() labels only a subset of task types; rename/prune/import + // fall through to "Database Error". Assert against the types that ARE labeled here. + manager->taskError(QGCMapTask::TaskType::taskCreateTileSet, QStringLiteral("create failed")); + QVERIFY(manager->errorMessage().contains(QStringLiteral("Create Tile Set"))); + + manager->taskError(QGCMapTask::TaskType::taskDeleteTileSet, QStringLiteral("delete failed")); + QVERIFY(manager->errorMessage().contains(QStringLiteral("Delete Tile Set"))); + + manager->taskError(QGCMapTask::TaskType::taskReset, QStringLiteral("reset failed")); + QVERIFY(manager->errorMessage().contains(QStringLiteral("Reset Tile Sets"))); +} + +UT_REGISTER_TEST(QGCMapEngineManagerTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCMapEngineManagerTest.h b/test/QtLocationPlugin/QGCMapEngineManagerTest.h new file mode 100644 index 000000000000..b10c3371a830 --- /dev/null +++ b/test/QtLocationPlugin/QGCMapEngineManagerTest.h @@ -0,0 +1,22 @@ +#pragma once + +#include "UnitTest.h" + +class QGCMapEngineManagerTest : public UnitTest +{ + Q_OBJECT + +private slots: + void init(); + void cleanup(); + + void _testInstanceAndModelAvailable(); + void _testProviderListsAndMapTypes(); + void _testUpdateForCurrentViewUpdatesTotals(); + void _testFindNameAndSelectionHelpers(); + void _testRenameOptimisticallyUpdatesName(); + void _testTaskErrorLabels(); + +private: + int _initialTileSetCount = 0; +}; diff --git a/test/QtLocationPlugin/QGCMapTaskTest.cc b/test/QtLocationPlugin/QGCMapTaskTest.cc new file mode 100644 index 000000000000..f4375b4d9267 --- /dev/null +++ b/test/QtLocationPlugin/QGCMapTaskTest.cc @@ -0,0 +1,190 @@ +#include "QGCMapTaskTest.h" + +#include +#include +#include +#include + +#include "QGCCacheTile.h" +#include "QGCCachedTileSet.h" +#include "QGCMapTasks.h" +#include "QGCTile.h" +#include "QGCTileCacheWorker.h" + +void QGCMapTaskTest::_testBaseTypeAndError() +{ + QGCFetchTileTask task(QStringLiteral("abc")); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskFetchTile); + + QSignalSpy spy(&task, &QGCMapTask::error); + QVERIFY(spy.isValid()); + task.setError(QStringLiteral("boom")); + QCOMPARE(spy.count(), 1); + const QList args = spy.takeFirst(); + QCOMPARE(args.at(0).value(), QGCMapTask::TaskType::taskFetchTile); + QCOMPARE(args.at(1).toString(), QStringLiteral("boom")); +} + +void QGCMapTaskTest::_testFetchTileSetTask() +{ + QGCFetchTileSetTask task; + QCOMPARE(task.type(), QGCMapTask::TaskType::taskFetchTileSets); + + QGCCachedTileSet* captured = nullptr; + connect(&task, &QGCFetchTileSetTask::tileSetFetched, &task, [&captured](QGCCachedTileSet* s) { captured = s; }); + + QGCCachedTileSet set(QStringLiteral("set")); + task.setTileSetFetched(&set); + QCOMPARE(captured, &set); +} + +void QGCMapTaskTest::_testCreateTileSetTaskSavedTransfersOwnership() +{ + QPointer set = new QGCCachedTileSet(QStringLiteral("owned")); + { + QGCCreateTileSetTask task(set); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskCreateTileSet); + QCOMPARE(task.tileSet(), set.data()); + + QGCCachedTileSet* captured = nullptr; + connect(&task, &QGCCreateTileSetTask::tileSetSaved, &task, [&captured](QGCCachedTileSet* s) { captured = s; }); + task.setTileSetSaved(); + QCOMPARE(captured, set.data()); + } + QVERIFY(!set.isNull()); + delete set; +} + +void QGCMapTaskTest::_testCreateTileSetTaskUnsavedDeletesTileSet() +{ + QPointer set = new QGCCachedTileSet(QStringLiteral("owned")); + { + QGCCreateTileSetTask task(set); + } + QVERIFY(set.isNull()); +} + +void QGCMapTaskTest::_testFetchTileTask() +{ + QGCFetchTileTask task(QStringLiteral("hash123")); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskFetchTile); + QCOMPARE(task.hash(), QStringLiteral("hash123")); + + QSharedPointer captured; + connect(&task, &QGCFetchTileTask::tileFetched, &task, + [&captured](QSharedPointer t) { captured = t; }); + + auto tile = QSharedPointer::create(QStringLiteral("hash123"), 7); + task.setTileFetched(tile); + QCOMPARE(captured, tile); +} + +void QGCMapTaskTest::_testSaveTileTaskDeletesTileOnDestroy() +{ + auto* tile = new QGCCacheTile(QStringLiteral("h"), QByteArrayLiteral("img"), QStringLiteral("png"), + QStringLiteral("Bing Road")); + QGCSaveTileTask task(tile); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskCacheTile); + QCOMPARE(task.tile(), tile); + QCOMPARE(static_cast(task).tile(), tile); + // Task dtor owns and deletes `tile`; exercised on scope exit (sanitizer-verified). +} + +void QGCMapTaskTest::_testGetTileDownloadListTask() +{ + QGCGetTileDownloadListTask task(42, 16); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskGetTileDownloadList); + QCOMPARE(task.setID(), static_cast(42)); + QCOMPARE(task.count(), 16); + + QQueue captured; + bool fired = false; + connect(&task, &QGCGetTileDownloadListTask::tileListFetched, &task, [&](const QQueue& tiles) { + captured = tiles; + fired = true; + }); + + auto* tile = new QGCTile(); + QQueue queue; + queue.enqueue(tile); + task.setTileListFetched(queue); + + QVERIFY(fired); + QCOMPARE(captured.size(), 1); + QCOMPARE(captured.first(), tile); + delete tile; +} + +void QGCMapTaskTest::_testCommandTask() +{ + QGCCacheWorker worker; + + QGCCommandTask okTask(QGCMapTask::TaskType::taskReset, [](QGCCacheWorker&, QGCMapTask&) { return true; }); + QCOMPARE(okTask.type(), QGCMapTask::TaskType::taskReset); + + QSignalSpy completedSpy(&okTask, &QGCCommandTask::completed); + QVERIFY(completedSpy.isValid()); + okTask.execute(worker); + QCOMPARE(completedSpy.count(), 1); + + QGCCommandTask failTask(QGCMapTask::TaskType::taskDeleteTileSet, [](QGCCacheWorker&, QGCMapTask& self) { + self.setError(QStringLiteral("nope")); + return false; + }); + QSignalSpy failCompletedSpy(&failTask, &QGCCommandTask::completed); + QSignalSpy errorSpy(&failTask, &QGCMapTask::error); + failTask.execute(worker); + QCOMPARE(failCompletedSpy.count(), 0); + QCOMPARE(errorSpy.count(), 1); + QCOMPARE(errorSpy.takeFirst().at(0).value(), QGCMapTask::TaskType::taskDeleteTileSet); +} + +void QGCMapTaskTest::_testExportTileTask() +{ + TileSetRecord record; + record.setID = 11; + record.name = QStringLiteral("export"); + const QList sets{record}; + + QGCExportTileTask task(sets, QStringLiteral("/tmp/out.qgctiledb")); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskExport); + QCOMPARE(task.path(), QStringLiteral("/tmp/out.qgctiledb")); + QCOMPARE(task.sets().size(), 1); + QCOMPARE(task.sets().first().setID, static_cast(11)); + + QSignalSpy progressSpy(&task, &QGCExportTileTask::actionProgress); + QSignalSpy completedSpy(&task, &QGCExportTileTask::actionCompleted); + QVERIFY(progressSpy.isValid()); + QVERIFY(completedSpy.isValid()); + + task.setProgress(50); + QCOMPARE(progressSpy.count(), 1); + QCOMPARE(progressSpy.takeFirst().at(0).toInt(), 50); + + task.setExportCompleted(); + QCOMPARE(completedSpy.count(), 1); +} + +void QGCMapTaskTest::_testImportTileTask() +{ + QGCImportTileTask task(QStringLiteral("/tmp/in.qgctiledb"), true); + QCOMPARE(task.type(), QGCMapTask::TaskType::taskImport); + QCOMPARE(task.path(), QStringLiteral("/tmp/in.qgctiledb")); + QVERIFY(task.replace()); + QCOMPARE(task.progress(), 0); + + QSignalSpy progressSpy(&task, &QGCImportTileTask::actionProgress); + QSignalSpy completedSpy(&task, &QGCImportTileTask::actionCompleted); + QVERIFY(progressSpy.isValid()); + QVERIFY(completedSpy.isValid()); + + task.setProgress(75); + QCOMPARE(task.progress(), 75); + QCOMPARE(progressSpy.count(), 1); + QCOMPARE(progressSpy.takeFirst().at(0).toInt(), 75); + + task.setImportCompleted(); + QCOMPARE(completedSpy.count(), 1); +} + +UT_REGISTER_TEST(QGCMapTaskTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCMapTaskTest.h b/test/QtLocationPlugin/QGCMapTaskTest.h new file mode 100644 index 000000000000..9163a45fbf65 --- /dev/null +++ b/test/QtLocationPlugin/QGCMapTaskTest.h @@ -0,0 +1,20 @@ +#pragma once + +#include "UnitTest.h" + +class QGCMapTaskTest : public UnitTest +{ + Q_OBJECT + +private slots: + void _testBaseTypeAndError(); + void _testFetchTileSetTask(); + void _testCreateTileSetTaskSavedTransfersOwnership(); + void _testCreateTileSetTaskUnsavedDeletesTileSet(); + void _testFetchTileTask(); + void _testSaveTileTaskDeletesTileOnDestroy(); + void _testGetTileDownloadListTask(); + void _testCommandTask(); + void _testExportTileTask(); + void _testImportTileTask(); +}; diff --git a/test/QtLocationPlugin/QGCTileCacheDatabaseTest.cc b/test/QtLocationPlugin/QGCTileCacheDatabaseTest.cc index e4cfca6a643e..af8589bc2cee 100644 --- a/test/QtLocationPlugin/QGCTileCacheDatabaseTest.cc +++ b/test/QtLocationPlugin/QGCTileCacheDatabaseTest.cc @@ -3,7 +3,10 @@ #include #include #include +#include +#include #include +#include #include #include @@ -11,11 +14,10 @@ #include "QGCMapUrlEngine.h" #include "QGCTile.h" #include "QGCTileCacheDatabase.h" -#include static const QString kFixedProviderType = QStringLiteral("Bing Road"); -std::unique_ptr QGCTileCacheDatabaseTest::_createInitializedDB(QTemporaryDir &tempDir) +std::unique_ptr QGCTileCacheDatabaseTest::_createInitializedDB(QTemporaryDir& tempDir) { auto db = std::make_unique(tempDir.filePath("tiles.db")); if (!db->init() || !db->connectDB()) { @@ -39,10 +41,10 @@ void QGCTileCacheDatabaseTest::_insertDownloadRecord(QGCTileCacheDatabase* db, q { QSqlQuery query(db->database()); QVERIFY(query.prepare( - "INSERT OR IGNORE INTO TilesDownload(setID, hash, type, x, y, z, state) VALUES(?, ?, ?, ?, ?, ?, ?)")); + "INSERT OR IGNORE INTO TilesDownload(setID, hash, typeStr, x, y, z, state) VALUES(?, ?, ?, ?, ?, ?, ?)")); query.addBindValue(setID); query.addBindValue(hash); - query.addBindValue(0); + query.addBindValue(QStringLiteral("TestProvider")); query.addBindValue(0); query.addBindValue(0); query.addBindValue(1); @@ -65,7 +67,8 @@ void QGCTileCacheDatabaseTest::_testInitWithValidPath() void QGCTileCacheDatabaseTest::_testInitWithEmptyPath() { // Expected: critical about missing cache directory - expectLogMessage("QtLocationPlugin.QGCTileCacheDatabase", QtCriticalMsg, QRegularExpression("Could not find suitable cache directory")); + expectLogMessage("QtLocationPlugin.QGCTileCacheDatabase", QtCriticalMsg, + QRegularExpression("Could not find suitable cache directory")); const QString emptyPath; QGCTileCacheDatabase db(emptyPath); @@ -122,6 +125,68 @@ void QGCTileCacheDatabaseTest::_testSaveTileAndGetTile() QCOMPARE(tile->type, kFixedProviderType); } +void QGCTileCacheDatabaseTest::_testSaveTileBatch() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const QStringList providerTypes = UrlFactory::getProviderTypes(); + QVERIFY(!providerTypes.isEmpty()); + const QString type = providerTypes.first(); + + std::vector owned; + owned.reserve(3); + for (int i = 0; i < 3; ++i) { + owned.emplace_back(QStringLiteral("batch_hash_%1").arg(i), QByteArray("batch_tile_") + QByteArray::number(i), + QStringLiteral("png"), type, QGCTileCacheDatabase::kInvalidTileSet); + } + + QList tiles; + for (const QGCCacheTile& t : owned) { + tiles.append(&t); + } + + QVERIFY(db->saveTileBatch(tiles)); + + for (const QGCCacheTile& t : owned) { + auto fetched = db->getTile(t.hash); + QVERIFY(fetched != nullptr); + QCOMPARE(fetched->img, t.img); + QCOMPARE(fetched->format, t.format); + QCOMPARE(fetched->type, type); + } +} + +void QGCTileCacheDatabaseTest::_testSaveTileBatchEmpty() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db->saveTileBatch({})); +} + +void QGCTileCacheDatabaseTest::_testSaveTileBatchDuplicateHash() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const QStringList providerTypes = UrlFactory::getProviderTypes(); + QVERIFY(!providerTypes.isEmpty()); + const QString type = providerTypes.first(); + + const QGCCacheTile first(QStringLiteral("dup_hash"), QByteArray("first_bytes"), QStringLiteral("png"), type, + QGCTileCacheDatabase::kInvalidTileSet); + const QGCCacheTile second(QStringLiteral("dup_hash"), QByteArray("second_bytes"), QStringLiteral("png"), type, + QGCTileCacheDatabase::kInvalidTileSet); + + const QList tiles = {&first, &second}; + QVERIFY(db->saveTileBatch(tiles)); + + // INSERT OR IGNORE on the hash column keeps the first write; the duplicate is ignored. + auto fetched = db->getTile(QStringLiteral("dup_hash")); + QVERIFY(fetched != nullptr); + QCOMPARE(fetched->img, QByteArray("first_bytes")); +} + void QGCTileCacheDatabaseTest::_testGetTileNotFound() { QTemporaryDir tempDir; @@ -520,7 +585,7 @@ void QGCTileCacheDatabaseTest::_testOperationsAfterDisconnect() auto db = _createInitializedDB(tempDir); db->disconnectDB(); - const char *categoryPattern = "QtLocationPlugin.QGCTileCacheDatabase"; + const char* categoryPattern = "QtLocationPlugin.QGCTileCacheDatabase"; const QRegularExpression disconnectedPattern(QStringLiteral("Database not connected")); expectLogMessage(categoryPattern, QtWarningMsg, disconnectedPattern); @@ -823,7 +888,16 @@ void QGCTileCacheDatabaseTest::_testDeleteBingNoTileTiles() QVERIFY(db->findTile(QStringLiteral("normal_tile")).has_value()); QSettings settings; - settings.remove(QStringLiteral("_deleteBingNoTileTilesDone")); + const QString doneKey = QString::fromLatin1(QGCTileCacheDatabase::kBingNoTileDoneKey); + const QVariant savedDone = settings.value(doneKey); + const auto restoreDone = qScopeGuard([&settings, &doneKey, &savedDone]() { + if (savedDone.isValid()) { + settings.setValue(doneKey, savedDone); + } else { + settings.remove(doneKey); + } + }); + settings.remove(doneKey); db->deleteBingNoTileTiles(); @@ -918,7 +992,7 @@ void QGCTileCacheDatabaseTest::_testSchemaVersionResetsLegacyDB() // Open with the versioned code — should detect legacy and reset QGCTileCacheDatabase db(path); - expectLogMessage("QtLocationPlugin.QGCTileCacheDatabase", QtWarningMsg, + expectLogMessage("QtLocationPlugin.QGCTileDatabaseSchema", QtWarningMsg, QRegularExpression(QStringLiteral( "Legacy database detected \\(no schema version\\)\\. Discarding cached tiles and " "rebuilding\\."))); @@ -941,53 +1015,44 @@ void QGCTileCacheDatabaseTest::_testSchemaVersionResetsLegacyDB() QVERIFY(sets[0].defaultSet); } -void QGCTileCacheDatabaseTest::_testSaveTileTypeStoredAsInteger() +void QGCTileCacheDatabaseTest::_testSaveTileTypeStoredAsString() { QTemporaryDir tempDir; auto db = _createInitializedDB(tempDir); - const int expectedMapId = UrlFactory::getQtMapIdFromProviderType(kFixedProviderType); - QVERIFY(expectedMapId != -1); - - const QString hash = QStringLiteral("type_int_test"); + const QString hash = QStringLiteral("type_str_test"); QVERIFY(db->saveTile(hash, QStringLiteral("png"), QByteArray("data"), kFixedProviderType, QGCTileCacheDatabase::kInvalidTileSet)); - // Verify the raw DB stores the type as an integer mapId, not a string { QSqlQuery query(db->database()); - QVERIFY(query.prepare("SELECT type FROM Tiles WHERE hash = ?")); + QVERIFY(query.prepare("SELECT typeStr FROM Tiles WHERE hash = ?")); query.addBindValue(hash); QVERIFY(query.exec()); QVERIFY(query.next()); - QCOMPARE(query.value(0).toInt(), expectedMapId); + QCOMPARE(query.value(0).toString(), kFixedProviderType); } - // Verify getTile converts the integer back to the provider name string auto tile = db->getTile(hash); QVERIFY(tile != nullptr); QCOMPARE(tile->type, kFixedProviderType); } -void QGCTileCacheDatabaseTest::_testCreateTileSetTypeStoredAsInteger() +void QGCTileCacheDatabaseTest::_testCreateTileSetTypeStoredAsString() { QTemporaryDir tempDir; auto db = _createInitializedDB(tempDir); - const int expectedMapId = UrlFactory::getQtMapIdFromProviderType(kFixedProviderType); - QVERIFY(expectedMapId != -1); - - const auto setID = db->createTileSet(QStringLiteral("TypeInt Set"), QStringLiteral("TestMap"), 37.0, -122.0, 36.0, - -121.0, 5, 5, kFixedProviderType, 100); + const auto setID = db->createTileSet(QStringLiteral("TypeStr Set"), kFixedProviderType, 37.0, -122.0, 36.0, -121.0, + 5, 5, kFixedProviderType, 100); QVERIFY(setID.has_value()); - // Verify TileSets.type stores the integer mapId const auto sets = db->getTileSets(); bool found = false; for (const auto& rec : sets) { if (rec.setID == setID.value()) { found = true; - QCOMPARE(rec.type, expectedMapId); + QCOMPARE(rec.mapTypeStr, kFixedProviderType); break; } } @@ -1044,7 +1109,7 @@ void QGCTileCacheDatabaseTest::_testTilesTableColumns() query.value(4).toString(), query.value(5).toBool()}); } - QCOMPARE(cols.size(), 7); + QCOMPARE(cols.size(), 12); auto findCol = [&](const QString& name) -> const ColInfo* { for (const auto& c : cols) { @@ -1078,14 +1143,39 @@ void QGCTileCacheDatabaseTest::_testTilesTableColumns() QVERIFY(c); QCOMPARE(c->type, QStringLiteral("INTEGER")); - c = findCol(QStringLiteral("type")); + c = findCol(QStringLiteral("typeStr")); QVERIFY(c); - QCOMPARE(c->type, QStringLiteral("INTEGER")); + QCOMPARE(c->type, QStringLiteral("TEXT")); c = findCol(QStringLiteral("date")); QVERIFY(c); QCOMPARE(c->type, QStringLiteral("INTEGER")); QCOMPARE(c->dflt, QStringLiteral("0")); + + c = findCol(QStringLiteral("etag")); + QVERIFY(c); + QCOMPARE(c->type, QStringLiteral("TEXT")); + + c = findCol(QStringLiteral("lastModified")); + QVERIFY(c); + QCOMPARE(c->type, QStringLiteral("TEXT")); + + c = findCol(QStringLiteral("expiresAt")); + QVERIFY(c); + QCOMPARE(c->type, QStringLiteral("INTEGER")); + QCOMPARE(c->dflt, QStringLiteral("0")); + + c = findCol(QStringLiteral("accessed")); + QVERIFY(c); + QCOMPARE(c->type, QStringLiteral("INTEGER")); + QVERIFY(c->notnull); + QCOMPARE(c->dflt, QStringLiteral("0")); + + c = findCol(QStringLiteral("mustRevalidate")); + QVERIFY(c); + QCOMPARE(c->type, QStringLiteral("INTEGER")); + QVERIFY(c->notnull); + QCOMPARE(c->dflt, QStringLiteral("0")); } void QGCTileCacheDatabaseTest::_testTileSetsTableColumns() @@ -1112,7 +1202,7 @@ void QGCTileCacheDatabaseTest::_testTileSetsTableColumns() query.value(4).toString(), query.value(5).toBool()}); } - QCOMPARE(cols.size(), 13); + QCOMPARE(cols.size(), 12); auto findCol = [&](const QString& name) -> const ColInfo* { for (const auto& c : cols) { @@ -1167,11 +1257,6 @@ void QGCTileCacheDatabaseTest::_testTileSetsTableColumns() QCOMPARE(c->type, QStringLiteral("INTEGER")); QCOMPARE(c->dflt, QStringLiteral("3")); - c = findCol(QStringLiteral("type")); - QVERIFY(c); - QCOMPARE(c->type, QStringLiteral("INTEGER")); - QCOMPARE(c->dflt, QStringLiteral("-1")); - c = findCol(QStringLiteral("numTiles")); QVERIFY(c); QCOMPARE(c->type, QStringLiteral("INTEGER")); @@ -1263,9 +1348,9 @@ void QGCTileCacheDatabaseTest::_testTilesDownloadTableColumns() QCOMPARE(c->type, QStringLiteral("TEXT")); QVERIFY(c->notnull); - c = findCol(QStringLiteral("type")); + c = findCol(QStringLiteral("typeStr")); QVERIFY(c); - QCOMPARE(c->type, QStringLiteral("INTEGER")); + QCOMPARE(c->type, QStringLiteral("TEXT")); c = findCol(QStringLiteral("x")); QVERIFY(c); @@ -1300,9 +1385,15 @@ void QGCTileCacheDatabaseTest::_testIndexesExist() } const QStringList expected = { - QStringLiteral("idx_settiles_setid"), QStringLiteral("idx_settiles_tileid"), - QStringLiteral("idx_settiles_unique"), QStringLiteral("idx_tiles_date"), - QStringLiteral("idx_tilesdownload_setid_hash"), QStringLiteral("idx_tilesdownload_setid_state"), + QStringLiteral("idx_settiles_setid"), + QStringLiteral("idx_settiles_tileid"), + QStringLiteral("idx_settiles_unique"), + QStringLiteral("idx_tiles_accessed"), + QStringLiteral("idx_tiles_accessed_size"), + QStringLiteral("idx_tiles_date"), + QStringLiteral("idx_tilesdownload_setid_hash"), + QStringLiteral("idx_tilesdownload_setid_state"), + QStringLiteral("idx_tilesets_default"), }; QCOMPARE(indexes.size(), expected.size()); @@ -1320,15 +1411,14 @@ void QGCTileCacheDatabaseTest::_testForeignKeyCascadeDelete() QVERIFY(UrlFactory::getQtMapIdFromProviderType(kFixedProviderType) != -1); const QString hash = QStringLiteral("fk_cascade_hash"); - QVERIFY( - db->saveTile(hash, QStringLiteral("png"), QByteArray("data"), kFixedProviderType, - QGCTileCacheDatabase::kInvalidTileSet)); + QVERIFY(db->saveTile(hash, QStringLiteral("png"), QByteArray("data"), kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet)); const auto tileID = db->findTile(hash); QVERIFY(tileID.has_value()); - const auto setID = db->createTileSet(QStringLiteral("CascadeTestSet"), kFixedProviderType, 10.0, 20.0, 30.0, - 40.0, 5, 5, kFixedProviderType, 1); + const auto setID = db->createTileSet(QStringLiteral("CascadeTestSet"), kFixedProviderType, 10.0, 20.0, 30.0, 40.0, + 5, 5, kFixedProviderType, 1); QVERIFY(setID.has_value()); _linkTileToSet(db.get(), tileID.value(), setID.value()); @@ -1362,4 +1452,240 @@ void QGCTileCacheDatabaseTest::_testForeignKeyCascadeDelete() } } +void QGCTileCacheDatabaseTest::_testSaveTileWithValidators() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const QString hash = QStringLiteral("validator_hash"); + const QByteArray etag("\"abc123\""); + const QByteArray lastModified("Wed, 21 Oct 2015 07:28:00 GMT"); + const qint64 expiresAt = 1893456000; + + QVERIFY(db->saveTile(hash, QStringLiteral("png"), QByteArray("body"), kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet, etag, lastModified, expiresAt)); + + auto tile = db->getTile(hash); + QVERIFY(tile != nullptr); + QCOMPARE(tile->etag, etag); + QCOMPARE(tile->lastModified, lastModified); + QCOMPARE(tile->expiresAt, expiresAt); +} + +void QGCTileCacheDatabaseTest::_testRefreshTileValidators() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const QString hash = QStringLiteral("refresh_hash"); + QVERIFY(db->saveTile(hash, QStringLiteral("png"), QByteArray("body"), kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet, QByteArray("\"old\""), QByteArray(), 100)); + + const QByteArray newEtag("\"new-etag\""); + const qint64 newExpiry = 2000000000; + QVERIFY(db->refreshTileValidators(hash, newEtag, QByteArray("Thu, 01 Jan 2099 00:00:00 GMT"), newExpiry)); + + auto tile = db->getTile(hash); + QVERIFY(tile != nullptr); + QCOMPARE(tile->etag, newEtag); + QCOMPARE(tile->expiresAt, newExpiry); + QCOMPARE(tile->img, QByteArray("body")); +} + +void QGCTileCacheDatabaseTest::_testMigrateV1ToV2PreservesTiles() +{ + QTemporaryDir tempDir; + const QString path = tempDir.filePath("v1.db"); + + // Build a v1 schema (no validator columns) with one cached tile and set user_version=1. + { + QSqlDatabase v1 = QSqlDatabase::addDatabase("QSQLITE", "v1_setup"); + v1.setDatabaseName(path); + QVERIFY(v1.open()); + QSqlQuery q(v1); + QVERIFY( + q.exec("CREATE TABLE Tiles (tileID INTEGER PRIMARY KEY NOT NULL, hash TEXT NOT NULL UNIQUE, format TEXT " + "NOT NULL, tile BLOB NULL, size INTEGER, type INTEGER, date INTEGER DEFAULT 0)")); + QVERIFY( + q.exec("CREATE TABLE TileSets (setID INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL UNIQUE, typeStr " + "TEXT, topleftLat REAL, topleftLon REAL, bottomRightLat REAL, bottomRightLon REAL, minZoom " + "INTEGER, maxZoom INTEGER, type INTEGER, numTiles INTEGER, defaultSet INTEGER, date INTEGER)")); + QVERIFY(q.exec("CREATE TABLE SetTiles (setID INTEGER NOT NULL, tileID INTEGER NOT NULL)")); + QVERIFY( + q.exec("CREATE TABLE TilesDownload (setID INTEGER NOT NULL, hash TEXT NOT NULL, type INTEGER, x INTEGER, " + "y INTEGER, z INTEGER, state INTEGER DEFAULT 0)")); + QVERIFY(q.exec("INSERT INTO TileSets(name, defaultSet, date) VALUES('Default Tile Set', 1, 0)")); + QVERIFY(q.exec( + "INSERT INTO Tiles(hash, format, tile, size, type, date) VALUES('keep_hash', 'png', X'AA', 1, 0, 0)")); + QVERIFY(q.exec("INSERT INTO SetTiles(setID, tileID) VALUES(1, 1)")); + QVERIFY(q.exec("PRAGMA user_version = 1")); + v1.close(); + } + QSqlDatabase::removeDatabase("v1_setup"); + + QGCTileCacheDatabase db(path); + QVERIFY(db.init()); + QVERIFY(db.connectDB()); + + // In-place upgrade: the v1 tile must survive (no cache wipe). + QVERIFY(db.findTile(QStringLiteral("keep_hash")).has_value()); + + QSqlQuery query(db.database()); + QVERIFY(query.exec("PRAGMA user_version")); + QVERIFY(query.next()); + QCOMPARE(query.value(0).toInt(), QGCTileCacheDatabase::kSchemaVersion); + + // New validator columns must now exist. + QSet columns; + QVERIFY(query.exec("PRAGMA table_info(Tiles)")); + while (query.next()) { + columns.insert(query.value(1).toString()); + } + QVERIFY(columns.contains(QStringLiteral("etag"))); + QVERIFY(columns.contains(QStringLiteral("lastModified"))); + QVERIFY(columns.contains(QStringLiteral("expiresAt"))); +} + +void QGCTileCacheDatabaseTest::_testUpdateAllTileDownloadStatesFiltered() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const auto setID = db->findTileSetID(QStringLiteral("Default Tile Set")); + QVERIFY(setID.has_value()); + + _insertDownloadRecord(db.get(), setID.value(), QStringLiteral("pend"), QGCTile::StatePending); + _insertDownloadRecord(db.get(), setID.value(), QStringLiteral("down"), QGCTile::StateDownloading); + + // Pause: only Downloading -> Paused; Pending stays untouched by this call. + QVERIFY(db->updateAllTileDownloadStates(setID.value(), QGCTile::StatePaused, QGCTile::StateDownloading)); + + auto stateOf = [&](const QString& hash) { + QSqlQuery q(db->database()); + q.prepare("SELECT state FROM TilesDownload WHERE setID = ? AND hash = ?"); + q.addBindValue(setID.value()); + q.addBindValue(hash); + q.exec(); + return q.next() ? q.value(0).toInt() : -1; + }; + + QCOMPARE(stateOf(QStringLiteral("down")), static_cast(QGCTile::StatePaused)); + QCOMPARE(stateOf(QStringLiteral("pend")), static_cast(QGCTile::StatePending)); + + // Resume: Paused -> Pending. + QVERIFY(db->updateAllTileDownloadStates(setID.value(), QGCTile::StatePending, QGCTile::StatePaused)); + QCOMPARE(stateOf(QStringLiteral("down")), static_cast(QGCTile::StatePending)); +} + +void QGCTileCacheDatabaseTest::_testPruneCacheLruOrder() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + const QByteArray data(100, 'L'); + + // Insert an old tile first, then a newer tile. By insertion date the old one + // would be evicted first (FIFO). Then access the old tile so true LRU must + // keep it and evict the un-accessed newer tile instead. + QVERIFY(db->saveTile(QStringLiteral("lru_old"), QStringLiteral("png"), data, kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet)); + QVERIFY(db->saveTile(QStringLiteral("lru_new"), QStringLiteral("png"), data, kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet)); + + // Force the older row to have a smaller initial accessed value than the newer + // one, then bump the old tile so it becomes the most-recently-used. + { + QSqlQuery q(db->database()); + QVERIFY(q.exec(QStringLiteral("UPDATE Tiles SET accessed = 1000 WHERE hash = 'lru_old'"))); + QVERIFY(q.exec(QStringLiteral("UPDATE Tiles SET accessed = 2000 WHERE hash = 'lru_new'"))); + } + QVERIFY(db->bumpTileAccessed(QStringLiteral("lru_old"))); + + // Prune just enough to evict a single 100-byte tile. + QVERIFY(db->pruneCache(100)); + + // The freshly-accessed old tile survives; the cold newer tile is evicted. + QVERIFY(db->findTile(QStringLiteral("lru_old")).has_value()); + QVERIFY(!db->findTile(QStringLiteral("lru_new")).has_value()); +} + +void QGCTileCacheDatabaseTest::_testSaveTileMustRevalidate() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + + QVERIFY(db->saveTile(QStringLiteral("revalidate_yes"), QStringLiteral("png"), QByteArray("body"), + kFixedProviderType, QGCTileCacheDatabase::kInvalidTileSet, QByteArray("\"e\""), QByteArray(), + 0, /*mustRevalidate*/ true)); + QVERIFY(db->saveTile(QStringLiteral("revalidate_no"), QStringLiteral("png"), QByteArray("body"), kFixedProviderType, + QGCTileCacheDatabase::kInvalidTileSet)); + + auto must = db->getTile(QStringLiteral("revalidate_yes")); + QVERIFY(must != nullptr); + QVERIFY(must->mustRevalidate); + + auto plain = db->getTile(QStringLiteral("revalidate_no")); + QVERIFY(plain != nullptr); + QVERIFY(!plain->mustRevalidate); +} + +void QGCTileCacheDatabaseTest::_testMigrateV2ToV3AddsColumns() +{ + QTemporaryDir tempDir; + const QString path = tempDir.filePath("v2.db"); + + // Build a v2 schema (validators present, no LRU/mustRevalidate) and set user_version=2. + { + QSqlDatabase v2 = QSqlDatabase::addDatabase("QSQLITE", "v2_setup"); + v2.setDatabaseName(path); + QVERIFY(v2.open()); + QSqlQuery q(v2); + QVERIFY( + q.exec("CREATE TABLE Tiles (tileID INTEGER PRIMARY KEY NOT NULL, hash TEXT NOT NULL UNIQUE, format TEXT " + "NOT NULL, tile BLOB NULL, size INTEGER, type INTEGER, date INTEGER DEFAULT 0, etag TEXT, " + "lastModified TEXT, expiresAt INTEGER DEFAULT 0)")); + QVERIFY( + q.exec("CREATE TABLE TileSets (setID INTEGER PRIMARY KEY NOT NULL, name TEXT NOT NULL UNIQUE, typeStr " + "TEXT, topleftLat REAL, topleftLon REAL, bottomRightLat REAL, bottomRightLon REAL, minZoom " + "INTEGER, maxZoom INTEGER, type INTEGER, numTiles INTEGER, defaultSet INTEGER, date INTEGER)")); + QVERIFY(q.exec("CREATE TABLE SetTiles (setID INTEGER NOT NULL, tileID INTEGER NOT NULL)")); + QVERIFY( + q.exec("CREATE TABLE TilesDownload (setID INTEGER NOT NULL, hash TEXT NOT NULL, type INTEGER, x INTEGER, " + "y INTEGER, z INTEGER, state INTEGER DEFAULT 0)")); + QVERIFY(q.exec("INSERT INTO TileSets(name, defaultSet, date) VALUES('Default Tile Set', 1, 0)")); + QVERIFY( + q.exec("INSERT INTO Tiles(hash, format, tile, size, type, date) " + "VALUES('v2_hash', 'png', X'AA', 1, 0, 4242)")); + QVERIFY(q.exec("INSERT INTO SetTiles(setID, tileID) VALUES(1, 1)")); + QVERIFY(q.exec("PRAGMA user_version = 2")); + v2.close(); + } + QSqlDatabase::removeDatabase("v2_setup"); + + QGCTileCacheDatabase db(path); + QVERIFY(db.init()); + QVERIFY(db.connectDB()); + + // In-place upgrade: the v2 tile survives. + QVERIFY(db.findTile(QStringLiteral("v2_hash")).has_value()); + + QSqlQuery query(db.database()); + QVERIFY(query.exec("PRAGMA user_version")); + QVERIFY(query.next()); + QCOMPARE(query.value(0).toInt(), QGCTileCacheDatabase::kSchemaVersion); + + QSet columns; + QVERIFY(query.exec("PRAGMA table_info(Tiles)")); + while (query.next()) { + columns.insert(query.value(1).toString()); + } + QVERIFY(columns.contains(QStringLiteral("accessed"))); + QVERIFY(columns.contains(QStringLiteral("mustRevalidate"))); + + // accessed must be backfilled from date so pre-v3 rows retain relative order. + QVERIFY(query.exec("SELECT accessed FROM Tiles WHERE hash = 'v2_hash'")); + QVERIFY(query.next()); + QCOMPARE(query.value(0).toLongLong(), static_cast(4242)); +} + UT_REGISTER_TEST(QGCTileCacheDatabaseTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCTileCacheDatabaseTest.h b/test/QtLocationPlugin/QGCTileCacheDatabaseTest.h index 77a7d6e1866f..b5d31f6cf8b5 100644 --- a/test/QtLocationPlugin/QGCTileCacheDatabaseTest.h +++ b/test/QtLocationPlugin/QGCTileCacheDatabaseTest.h @@ -16,6 +16,9 @@ private slots: void _testConnectDisconnect(); void _testDefaultTileSetCreated(); void _testSaveTileAndGetTile(); + void _testSaveTileBatch(); + void _testSaveTileBatchEmpty(); + void _testSaveTileBatchDuplicateHash(); void _testGetTileNotFound(); void _testFindTile(); void _testGetTileSetsMultiple(); @@ -45,8 +48,8 @@ private slots: void _testPruneCacheMultipleBatches(); void _testSchemaVersionSetOnFreshDB(); void _testSchemaVersionResetsLegacyDB(); - void _testSaveTileTypeStoredAsInteger(); - void _testCreateTileSetTypeStoredAsInteger(); + void _testSaveTileTypeStoredAsString(); + void _testCreateTileSetTypeStoredAsString(); void _testTablesExist(); void _testTilesTableColumns(); void _testTileSetsTableColumns(); @@ -54,9 +57,16 @@ private slots: void _testTilesDownloadTableColumns(); void _testIndexesExist(); void _testForeignKeyCascadeDelete(); + void _testSaveTileWithValidators(); + void _testRefreshTileValidators(); + void _testMigrateV1ToV2PreservesTiles(); + void _testUpdateAllTileDownloadStatesFiltered(); + void _testPruneCacheLruOrder(); + void _testSaveTileMustRevalidate(); + void _testMigrateV2ToV3AddsColumns(); private: - std::unique_ptr _createInitializedDB(QTemporaryDir &tempDir); + std::unique_ptr _createInitializedDB(QTemporaryDir& tempDir); void _insertTileSet(QGCTileCacheDatabase* db, const QString& name, quint64& outSetID); void _insertDownloadRecord(QGCTileCacheDatabase* db, quint64 setID, const QString& hash, int state = 0); void _linkTileToSet(QGCTileCacheDatabase* db, quint64 tileID, quint64 setID); diff --git a/test/QtLocationPlugin/QGCTileFallbackTest.cc b/test/QtLocationPlugin/QGCTileFallbackTest.cc new file mode 100644 index 000000000000..83b7d1426fdd --- /dev/null +++ b/test/QtLocationPlugin/QGCTileFallbackTest.cc @@ -0,0 +1,99 @@ +#include "QGCTileFallbackTest.h" + +#include +#include + +#include "QGCTileFallback.h" + +namespace { +const QRgb kRed = qRgb(255, 0, 0); +const QRgb kGreen = qRgb(0, 255, 0); +const QRgb kBlue = qRgb(0, 0, 255); +const QRgb kYellow = qRgb(255, 255, 0); + +QImage makeQuadrantAncestor() +{ + QImage img(256, 256, QImage::Format_ARGB32); + { + QPainter p(&img); + p.fillRect(0, 0, 128, 128, QColor(kRed)); // cell (0,0) + p.fillRect(128, 0, 128, 128, QColor(kGreen)); // cell (1,0) + p.fillRect(0, 128, 128, 128, QColor(kBlue)); // cell (0,1) + p.fillRect(128, 128, 128, 128, QColor(kYellow)); // cell (1,1) + } + return img; +} + +QRgb centerPixel(const QImage &img) +{ + return img.pixel(img.width() / 2, img.height() / 2); +} +} // namespace + +void QGCTileFallbackTest::_testScaleLevelDelta1Quadrants() +{ + const QImage ancestor = makeQuadrantAncestor(); + const QSize tileSize(256, 256); + + struct Case { int x; int y; QRgb color; }; + const Case cases[] = { + {0, 0, kRed}, + {1, 0, kGreen}, + {0, 1, kBlue}, + {1, 1, kYellow}, + {2, 2, kRed}, // x%2,y%2 == (0,0) + {3, 0, kGreen}, // (1,0) + }; + + for (const Case &c : cases) { + const QImage out = scaleAncestorToChild(ancestor, c.x, c.y, 1, tileSize); + QVERIFY(!out.isNull()); + QCOMPARE(out.size(), tileSize); + QCOMPARE(centerPixel(out), c.color); + } +} + +void QGCTileFallbackTest::_testScaleLevelDelta2SubCell() +{ + // levelDelta=2 => divisions=4, each sub-cell is 64x64 of the 256 ancestor. + // cell (0,0) lies entirely inside the red top-left quadrant. + const QImage ancestor = makeQuadrantAncestor(); + const QSize tileSize(256, 256); + + const QImage topLeft = scaleAncestorToChild(ancestor, 0, 0, 2, tileSize); + QVERIFY(!topLeft.isNull()); + QCOMPARE(topLeft.size(), tileSize); + QCOMPARE(centerPixel(topLeft), kRed); + + // cell (3,3) lies inside the yellow bottom-right quadrant. + const QImage bottomRight = scaleAncestorToChild(ancestor, 3, 3, 2, tileSize); + QVERIFY(!bottomRight.isNull()); + QCOMPARE(bottomRight.size(), tileSize); + QCOMPARE(centerPixel(bottomRight), kYellow); +} + +void QGCTileFallbackTest::_testScaleInvalidInputs() +{ + const QImage ancestor = makeQuadrantAncestor(); + const QSize tileSize(256, 256); + + QVERIFY(scaleAncestorToChild(QImage(), 0, 0, 1, tileSize).isNull()); + QVERIFY(scaleAncestorToChild(ancestor, 0, 0, 0, tileSize).isNull()); + QVERIFY(scaleAncestorToChild(ancestor, 0, 0, -1, tileSize).isNull()); + QVERIFY(scaleAncestorToChild(ancestor, 0, 0, 1, QSize()).isNull()); + + // levelDelta so large the sub-square rounds to zero pixels => null. + QVERIFY(scaleAncestorToChild(ancestor, 0, 0, 20, tileSize).isNull()); +} + +void QGCTileFallbackTest::_testEncodeFallbackTile() +{ + const QImage ancestor = makeQuadrantAncestor(); + const QByteArray png = encodeFallbackTile(ancestor); + QVERIFY(!png.isEmpty()); + QCOMPARE(png.left(8), QByteArray::fromHex("89504e470d0a1a0a")); + + QVERIFY(encodeFallbackTile(QImage()).isEmpty()); +} + +UT_REGISTER_TEST(QGCTileFallbackTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCTileFallbackTest.h b/test/QtLocationPlugin/QGCTileFallbackTest.h new file mode 100644 index 000000000000..aa862faa49aa --- /dev/null +++ b/test/QtLocationPlugin/QGCTileFallbackTest.h @@ -0,0 +1,14 @@ +#pragma once + +#include "UnitTest.h" + +class QGCTileFallbackTest : public UnitTest +{ + Q_OBJECT + +private slots: + void _testScaleLevelDelta1Quadrants(); + void _testScaleLevelDelta2SubCell(); + void _testScaleInvalidInputs(); + void _testEncodeFallbackTile(); +}; diff --git a/test/QtLocationPlugin/QGCTileSetImporterTest.cc b/test/QtLocationPlugin/QGCTileSetImporterTest.cc new file mode 100644 index 000000000000..0c92eff4219f --- /dev/null +++ b/test/QtLocationPlugin/QGCTileSetImporterTest.cc @@ -0,0 +1,506 @@ +#include "QGCTileSetImporterTest.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "QGCMapUrlEngine.h" +#include "QGCTileCacheDatabase.h" +#include "QGCTileSetImporter.h" +#include "QGCCompression.h" +#include "QGCCompressionTypes.h" + +#include +#include + +namespace { +const QString kProviderType = QStringLiteral("CustomURL Custom"); +} + +std::unique_ptr QGCTileSetImporterTest::_createInitializedDB(QTemporaryDir &tempDir) +{ + auto db = std::make_unique(tempDir.filePath("tiles.db")); + if (!db->init() || !db->connectDB()) { + return nullptr; + } + return db; +} + +// Builds a minimal valid MBTiles file: tiles(zoom_level, tile_column, tile_row, tile_data). +// Two tiles at zoom 1 with distinct TMS rows so the Y-flip is observable. +QString QGCTileSetImporterTest::_writeMBTilesFixture(QTemporaryDir &tempDir, const QString &fileName) +{ + const QString path = tempDir.filePath(fileName); + { + QSqlDatabase src = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), QStringLiteral("mbtiles_fixture")); + src.setDatabaseName(path); + if (!src.open()) { + return QString(); + } + QSqlQuery q(src); + q.exec(QStringLiteral("CREATE TABLE tiles (zoom_level INTEGER, tile_column INTEGER, tile_row INTEGER, " + "tile_data BLOB)")); + // PNG magic so getImageFormat recognises the blobs. + const QByteArray png = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("tileA"); + const QByteArray png2 = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("tileB"); + + q.prepare(QStringLiteral("INSERT INTO tiles(zoom_level, tile_column, tile_row, tile_data) VALUES(?, ?, ?, ?)")); + q.addBindValue(1); q.addBindValue(0); q.addBindValue(0); q.addBindValue(png); q.exec(); + q.addBindValue(1); q.addBindValue(1); q.addBindValue(1); q.addBindValue(png2); q.exec(); + src.close(); + } + QSqlDatabase::removeDatabase(QStringLiteral("mbtiles_fixture")); + return path; +} + +void QGCTileSetImporterTest::_testImportMBTiles() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + QVERIFY(UrlFactory::getQtMapIdFromProviderType(kProviderType) != -1); + + const QString path = _writeMBTilesFixture(tempDir, QStringLiteral("fixture.mbtiles")); + QVERIFY(!path.isEmpty()); + + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("Imported MB"), kProviderType); + QVERIFY2(result.success, qPrintable(result.errorString)); + QCOMPARE(result.tileCount, static_cast(2)); + QVERIFY(result.setID != 0); + + // The named set must exist alongside the default set. + const auto setID = db->findTileSetID(QStringLiteral("Imported MB")); + QVERIFY(setID.has_value()); + QCOMPARE(setID.value(), result.setID); +} + +void QGCTileSetImporterTest::_testImportMBTilesYFlip() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QString path = _writeMBTilesFixture(tempDir, QStringLiteral("flip.mbtiles")); + QVERIFY(!path.isEmpty()); + + QVERIFY(QGCTileSetImporter::import(*db, path, QStringLiteral("Flip Set"), kProviderType).success); + + // zoom=1 tms_row=0 -> xyz y = (2^1 - 1) - 0 = 1 ; tms_row=1 -> xyz y = 0. + const QString hash0 = UrlFactory::getTileHash(kProviderType, 0, 1, 1); // column 0, flipped row + const QString hash1 = UrlFactory::getTileHash(kProviderType, 1, 0, 1); // column 1, flipped row + QVERIFY(db->findTile(hash0).has_value()); + QVERIFY(db->findTile(hash1).has_value()); + + // The un-flipped (raw TMS) hashes must NOT be present, proving the flip happened. + QVERIFY(!db->findTile(UrlFactory::getTileHash(kProviderType, 0, 0, 1)).has_value()); +} + +void QGCTileSetImporterTest::_testImportUnsupportedExtension() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, tempDir.filePath("foo.xyz"), QStringLiteral("Bad"), kProviderType); + QVERIFY(!result.success); + QVERIFY(result.errorString.contains(QStringLiteral("Unsupported"))); +} + +void QGCTileSetImporterTest::_testImportMissingFile() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, tempDir.filePath("nope.mbtiles"), QStringLiteral("Missing"), kProviderType); + QVERIFY(!result.success); +} + +namespace { +void appendVarint(QByteArray &out, quint64 v) +{ + do { + quint8 b = v & 0x7F; + v >>= 7; + if (v) { + b |= 0x80; + } + out.append(static_cast(b)); + } while (v); +} + + +// Build a 127-byte PMTiles v3 header. Compression fields default to None(1), tile_type png(2). +struct PMFields { + quint64 rootOffset = 0; + quint64 rootBytes = 0; + quint64 leafOffset = 0; + quint64 leafBytes = 0; + quint64 tileDataOffset = 0; + quint64 tileDataBytes = 0; + quint64 addressedTiles = 1; +}; + +QByteArray buildPMHeader(const PMFields &f) +{ + constexpr int kHeaderSize = 127; + QByteArray header(kHeaderSize, '\0'); + std::memcpy(header.data(), "PMTiles", 7); + header[7] = 3; + auto put = [&header](int off, quint64 v) { qToLittleEndian(v, header.data() + off); }; + put(8, f.rootOffset); + put(16, f.rootBytes); + put(24, 0); put(32, 0); + put(40, f.leafOffset); + put(48, f.leafBytes); + put(56, f.tileDataOffset); + put(64, f.tileDataBytes); + put(72, f.addressedTiles); put(80, 1); put(88, 1); + header[96] = 1; // clustered + header[97] = 1; // internal_compression None + header[98] = 1; // tile_compression None + header[99] = 2; // tile_type png + header[100] = 0; // min_zoom + header[101] = 0; // max_zoom + return header; +} + +} // namespace + +QString QGCTileSetImporterTest::_writeMalformedPMTiles( + QTemporaryDir &tempDir, const QString &fileName, + const std::function &mutate) +{ + // Caller fully owns header/rootDir/tile bytes (offsets included) via the mutate callback. + QByteArray header; + QByteArray rootDir; + QByteArray tile; + mutate(header, rootDir, tile); + + const QString path = tempDir.filePath(fileName); + QFile fout(path); + if (!fout.open(QIODevice::WriteOnly)) { + return QString(); + } + fout.write(header); + fout.write(rootDir); + fout.write(tile); + fout.close(); + return path; +} + +QString QGCTileSetImporterTest::_writePMTilesFixture(QTemporaryDir &tempDir, const QString &fileName) +{ + // Hand-build a minimal v3 archive: uncompressed root dir + uncompressed PNG tile, + // single entry at z/x/y = 0/0/0 (tile_id 0), run_length 1. + const QByteArray tile = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("pmtileA"); + + QByteArray rootDir; + appendVarint(rootDir, 1); // num entries + appendVarint(rootDir, 0); // tile_id delta (first id = 0) + appendVarint(rootDir, 1); // run_length + appendVarint(rootDir, tile.size()); // length + appendVarint(rootDir, 1); // offset (+1 framing -> real offset 0) + + constexpr int kHeaderSize = 127; + const quint64 rootOffset = kHeaderSize; + const quint64 rootBytes = static_cast(rootDir.size()); + const quint64 tileDataOffset = rootOffset + rootBytes; + + QByteArray header(kHeaderSize, '\0'); + std::memcpy(header.data(), "PMTiles", 7); + header[7] = 3; // version + auto put = [&header](int off, quint64 v) { qToLittleEndian(v, header.data() + off); }; + put(8, rootOffset); + put(16, rootBytes); + put(24, 0); put(32, 0); // json metadata + put(40, 0); put(48, 0); // leaf dirs + put(56, tileDataOffset); + put(64, static_cast(tile.size())); + put(72, 1); put(80, 1); put(88, 1); // addressed/entries/contents counts + header[96] = 1; // clustered + header[97] = 1; // internal_compression = None + header[98] = 1; // tile_compression = None + header[99] = 2; // tile_type = png + header[100] = 0; // min_zoom + header[101] = 0; // max_zoom + + const QString path = tempDir.filePath(fileName); + QFile f(path); + if (!f.open(QIODevice::WriteOnly)) { + return QString(); + } + f.write(header); + f.write(rootDir); + f.write(tile); + f.close(); + return path; +} + +void QGCTileSetImporterTest::_testPMTilesZxyToTileId() +{ + // Published PMTiles v3 spec vectors. + QCOMPARE(QGCTileSetImporter::zxyToTileId(0, 0, 0), static_cast(0)); + QCOMPARE(QGCTileSetImporter::zxyToTileId(1, 0, 0), static_cast(1)); + QCOMPARE(QGCTileSetImporter::zxyToTileId(1, 0, 1), static_cast(2)); + QCOMPARE(QGCTileSetImporter::zxyToTileId(1, 1, 1), static_cast(3)); + QCOMPARE(QGCTileSetImporter::zxyToTileId(1, 1, 0), static_cast(4)); + QCOMPARE(QGCTileSetImporter::zxyToTileId(2, 0, 0), static_cast(5)); +} + +void QGCTileSetImporterTest::_testImportPMTiles() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + QVERIFY(UrlFactory::getQtMapIdFromProviderType(kProviderType) != -1); + + const QString path = _writePMTilesFixture(tempDir, QStringLiteral("fixture.pmtiles")); + QVERIFY(!path.isEmpty()); + + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("Imported PM"), kProviderType); + QVERIFY2(result.success, qPrintable(result.errorString)); + QCOMPARE(result.tileCount, static_cast(1)); + QVERIFY(result.setID != 0); + + // tile_id 0 == z/x/y 0/0/0; PMTiles is XYZ so no flip. + const QString hash = UrlFactory::getTileHash(kProviderType, 0, 0, 0); + QVERIFY(db->findTile(hash).has_value()); + + const auto setID = db->findTileSetID(QStringLiteral("Imported PM")); + QVERIFY(setID.has_value()); + QCOMPARE(setID.value(), result.setID); +} + +void QGCTileSetImporterTest::_testPMTilesDirEntryCountTooLarge() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QByteArray tile = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("t"); + + const QString path = _writeMalformedPMTiles( + tempDir, QStringLiteral("s2.pmtiles"), + [&](QByteArray &header, QByteArray &rootDir, QByteArray &outTile) { + // numEntries varint claims a billion entries; blob is a few bytes. + appendVarint(rootDir, 1000000000ULL); + const quint64 rootOffset = 127; + PMFields f; + f.rootOffset = rootOffset; + f.rootBytes = static_cast(rootDir.size()); + f.tileDataOffset = rootOffset + f.rootBytes; + f.tileDataBytes = static_cast(tile.size()); + header = buildPMHeader(f); + outTile = tile; + }); + QVERIFY(!path.isEmpty()); + + // S2 is rejected in deserializeDirectory before the tile set is created, so the + // importer returns the error without emitting the "PMTiles import failed" warning. + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("S2"), kProviderType); + QVERIFY(!result.success); + QVERIFY(!result.errorString.isEmpty()); +} + +void QGCTileSetImporterTest::_testPMTilesTileOutsideDataRegion() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QByteArray tile = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("t"); + + const QString path = _writeMalformedPMTiles( + tempDir, QStringLiteral("s3.pmtiles"), + [&](QByteArray &header, QByteArray &rootDir, QByteArray &outTile) { + // One entry whose offset+length exceeds the declared tileDataBytes. + appendVarint(rootDir, 1); // num entries + appendVarint(rootDir, 0); // tile_id delta + appendVarint(rootDir, 1); // run_length + appendVarint(rootDir, 9999); // length far past the region + appendVarint(rootDir, 1); // offset (+1 framing -> 0) + + const quint64 rootOffset = 127; + PMFields f; + f.rootOffset = rootOffset; + f.rootBytes = static_cast(rootDir.size()); + f.tileDataOffset = rootOffset + f.rootBytes; + f.tileDataBytes = static_cast(tile.size()); // small region + header = buildPMHeader(f); + outTile = tile; + }); + QVERIFY(!path.isEmpty()); + + expectLogMessage("QtLocationPlugin.QGCTileSetImporter", QtWarningMsg, + QRegularExpression(QStringLiteral("PMTiles import failed"))); + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("S3"), kProviderType); + verifyExpectedLogMessage(); + QVERIFY(!result.success); + QVERIFY(result.errorString.contains(QStringLiteral("tile-data region"))); +} + +void QGCTileSetImporterTest::_testPMTilesLeafCycle() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + // Root has a single leaf entry (run_length 0) at leaf offset 0. The leaf directory + // itself is another leaf entry pointing back to leaf offset 0 -> cycle. Must return + // promptly with an error, not recurse forever. + QByteArray leafDir; + { + appendVarint(leafDir, 1); // num entries + appendVarint(leafDir, 0); // tile_id delta + appendVarint(leafDir, 0); // run_length 0 -> leaf pointer + appendVarint(leafDir, 0); // length placeholder (patched below) + appendVarint(leafDir, 1); // offset (+1 framing -> leaf offset 0, same as itself) + } + + QByteArray rootDir; + { + appendVarint(rootDir, 1); // num entries + appendVarint(rootDir, 0); // tile_id delta + appendVarint(rootDir, 0); // run_length 0 -> leaf pointer + appendVarint(rootDir, leafDir.size()); // length of leaf dir + appendVarint(rootDir, 1); // offset (+1 framing -> leaf offset 0) + } + + const QString path = _writeMalformedPMTiles( + tempDir, QStringLiteral("s4.pmtiles"), + [&](QByteArray &header, QByteArray &outRoot, QByteArray &outTile) { + const quint64 rootOffset = 127; + PMFields f; + f.rootOffset = rootOffset; + f.rootBytes = static_cast(rootDir.size()); + f.leafOffset = rootOffset + f.rootBytes; + f.leafBytes = static_cast(leafDir.size()); + f.tileDataOffset = f.leafOffset + f.leafBytes; + f.tileDataBytes = 0; + header = buildPMHeader(f); + outRoot = rootDir; + // Layout written is header + rootDir + tile; place the leaf bytes as the "tile" blob + // so they live at leafOffset. + outTile = leafDir; + }); + QVERIFY(!path.isEmpty()); + + expectLogMessage("QtLocationPlugin.QGCTileSetImporter", QtWarningMsg, + QRegularExpression(QStringLiteral("PMTiles import failed"))); + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("S4"), kProviderType); + verifyExpectedLogMessage(); + QVERIFY(!result.success); + QVERIFY(!result.errorString.isEmpty()); +} + +void QGCTileSetImporterTest::_testPMTilesZoomTooLarge() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QByteArray tile = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("t"); + + const QString path = _writeMalformedPMTiles( + tempDir, QStringLiteral("s5.pmtiles"), + [&](QByteArray &header, QByteArray &rootDir, QByteArray &outTile) { + // tile_id so large that decoding zoom exceeds 31 -> tileIdToZxy returns false. + appendVarint(rootDir, 1); // num entries + appendVarint(rootDir, 0xFFFFFFFFFFFFFFFFULL); // tile_id delta (huge) + appendVarint(rootDir, 1); // run_length + appendVarint(rootDir, static_cast(tile.size())); // length + appendVarint(rootDir, 1); // offset (+1 -> 0) + + const quint64 rootOffset = 127; + PMFields f; + f.rootOffset = rootOffset; + f.rootBytes = static_cast(rootDir.size()); + f.tileDataOffset = rootOffset + f.rootBytes; + f.tileDataBytes = static_cast(tile.size()); + f.addressedTiles = 1; + header = buildPMHeader(f); + outTile = tile; + }); + QVERIFY(!path.isEmpty()); + + expectLogMessage("QtLocationPlugin.QGCTileSetImporter", QtWarningMsg, + QRegularExpression(QStringLiteral("PMTiles import failed"))); + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("S5"), kProviderType); + verifyExpectedLogMessage(); + QVERIFY(!result.success); + QVERIFY(!result.errorString.isEmpty()); +} + +void QGCTileSetImporterTest::_testDecompressCapRejectsOversize() +{ + // 40-byte gzip stream that inflates to 4096 'A' bytes (deterministic, mtime=0). + const QByteArray gz = QByteArray::fromHex( + "1f8b08000000000002ffedc1010d000000c2a06cef5fca1e0e28000000e0dd004034a6fe00100000"); + + // libarchive logs an environment-internal size-limit warning; the behavior under + // test is the empty return, so suppress the noise. + ignoreLogMessage("Utilities.QGClibarchive", QtWarningMsg, + QRegularExpression(QStringLiteral("Size limit exceeded"))); + + // Cap below the inflated size -> rejected (empty). + const QByteArray capped = QGCCompression::decompressData(gz, QGCCompression::Format::GZIP, 1024); + QVERIFY(capped.isEmpty()); + + // Unlimited (0) -> full 4096-byte output. + const QByteArray full = QGCCompression::decompressData(gz, QGCCompression::Format::GZIP, 0); + QCOMPARE(full.size(), 4096); + QCOMPARE(full.at(0), 'A'); +} + +void QGCTileSetImporterTest::_testImportMBTilesDedupSchema() +{ + QTemporaryDir tempDir; + auto db = _createInitializedDB(tempDir); + QVERIFY(db); + + const QString path = tempDir.filePath(QStringLiteral("dedup.mbtiles")); + { + QSqlDatabase src = QSqlDatabase::addDatabase(QStringLiteral("QSQLITE"), QStringLiteral("mbtiles_dedup")); + src.setDatabaseName(path); + QVERIFY(src.open()); + QSqlQuery q(src); + // mb-util dedup schema: map(zoom,col,row,tile_id) JOIN images(tile_id,tile_data). No `tiles`. + QVERIFY(q.exec(QStringLiteral("CREATE TABLE map (zoom_level INTEGER, tile_column INTEGER, " + "tile_row INTEGER, tile_id TEXT)"))); + QVERIFY(q.exec(QStringLiteral("CREATE TABLE images (tile_id TEXT, tile_data BLOB)"))); + + const QByteArray pngA = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("dedupA"); + const QByteArray pngB = QByteArray::fromHex("89504e470d0a1a0a") + QByteArray("dedupB"); + q.prepare(QStringLiteral("INSERT INTO images(tile_id, tile_data) VALUES(?, ?)")); + q.addBindValue(QStringLiteral("a")); q.addBindValue(pngA); QVERIFY(q.exec()); + q.addBindValue(QStringLiteral("b")); q.addBindValue(pngB); QVERIFY(q.exec()); + + // Two map rows share image "a" (dedup), one uses "b" -> 3 imported tiles. + q.prepare(QStringLiteral("INSERT INTO map(zoom_level, tile_column, tile_row, tile_id) VALUES(?,?,?,?)")); + q.addBindValue(1); q.addBindValue(0); q.addBindValue(0); q.addBindValue(QStringLiteral("a")); QVERIFY(q.exec()); + q.addBindValue(1); q.addBindValue(1); q.addBindValue(0); q.addBindValue(QStringLiteral("a")); QVERIFY(q.exec()); + q.addBindValue(1); q.addBindValue(1); q.addBindValue(1); q.addBindValue(QStringLiteral("b")); QVERIFY(q.exec()); + src.close(); + } + QSqlDatabase::removeDatabase(QStringLiteral("mbtiles_dedup")); + + const QGCTileSetImporter::Result result = + QGCTileSetImporter::import(*db, path, QStringLiteral("Dedup MB"), kProviderType); + QVERIFY2(result.success, qPrintable(result.errorString)); + QCOMPARE(result.tileCount, static_cast(3)); +} + +UT_REGISTER_TEST(QGCTileSetImporterTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGCTileSetImporterTest.h b/test/QtLocationPlugin/QGCTileSetImporterTest.h new file mode 100644 index 000000000000..5b4771fe713a --- /dev/null +++ b/test/QtLocationPlugin/QGCTileSetImporterTest.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +#include "UnitTest.h" + +class QGCTileCacheDatabase; +class QTemporaryDir; + +class QGCTileSetImporterTest : public UnitTest +{ + Q_OBJECT + +private slots: + void _testImportMBTiles(); + void _testImportMBTilesYFlip(); + void _testImportUnsupportedExtension(); + void _testImportMissingFile(); + void _testPMTilesZxyToTileId(); + void _testImportPMTiles(); + void _testPMTilesDirEntryCountTooLarge(); + void _testPMTilesTileOutsideDataRegion(); + void _testPMTilesLeafCycle(); + void _testPMTilesZoomTooLarge(); + void _testDecompressCapRejectsOversize(); + void _testImportMBTilesDedupSchema(); + +private: + std::unique_ptr _createInitializedDB(QTemporaryDir &tempDir); + QString _writeMBTilesFixture(QTemporaryDir &tempDir, const QString &fileName); + QString _writePMTilesFixture(QTemporaryDir &tempDir, const QString &fileName); + QString _writeMalformedPMTiles(QTemporaryDir &tempDir, const QString &fileName, + const std::function &mutate); +}; diff --git a/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.cc b/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.cc new file mode 100644 index 000000000000..7dc796eb0a21 --- /dev/null +++ b/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.cc @@ -0,0 +1,67 @@ +#include "QGeoFileTileCacheQGCTest.h" + +#include +#include + +#include "QGCMapTasks.h" +#include "QGCMapUrlEngine.h" +#include "QGCTileCache.h" +#include "QGeoFileTileCacheQGC.h" + +class TestableQGeoFileTileCacheQGC : public QGeoFileTileCacheQGC +{ +public: + explicit TestableQGeoFileTileCacheQGC(const QVariantMap& parameters) : QGeoFileTileCacheQGC(parameters, nullptr) {} + + using QGeoFileTileCacheQGC::get; + using QGeoFileTileCacheQGC::insert; +}; + +void QGeoFileTileCacheQGCTest::_testCreateFetchTileTaskValidation() +{ + const QStringList providerTypes = UrlFactory::getProviderTypes(); + QVERIFY(!providerTypes.isEmpty()); + QGCFetchTileTask* task = QGCTileCache::createFetchTileTask(providerTypes.first(), 1, 2, 3); + QVERIFY(task != nullptr); + delete task; +} + +void QGeoFileTileCacheQGCTest::_testMemoryInsertAndGet() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + TestableQGeoFileTileCacheQGC cache(QVariantMap{}); + const QGeoTileSpec spec = tileSpec(mapId, 10, 20, 3); + + // QGeoFileTileCacheQGC overrides get()/insert() as no-ops so the base QGeoFileTileCache + // (memory + flat-file) never shadows the SQLite store: a base-cache hit would bypass + // expiry/ETag/must-revalidate. So even a decodable PNG inserted into the memory cache + // is not retrievable via get() — the SQLite cache is the sole authority. + QImage image(1, 1, QImage::Format_ARGB32); + image.fill(Qt::red); + QByteArray pngBytes; + QBuffer buffer(&pngBytes); + QVERIFY(buffer.open(QIODevice::WriteOnly)); + QVERIFY(image.save(&buffer, "PNG")); + buffer.close(); + + cache.insert(spec, pngBytes, QStringLiteral("png"), QAbstractGeoTileCache::MemoryCache); + const QSharedPointer texture = cache.get(spec); + QVERIFY(texture.isNull()); +} + +void QGeoFileTileCacheQGCTest::_testDiskOnlyInsertDoesNotPopulateMemory() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + TestableQGeoFileTileCacheQGC cache(QVariantMap{}); + const QGeoTileSpec spec = tileSpec(mapId, 11, 21, 3); + + cache.insert(spec, QByteArrayLiteral("img_data"), QStringLiteral("png"), QAbstractGeoTileCache::DiskCache); + const QSharedPointer texture = cache.get(spec); + QVERIFY(texture.isNull()); +} + +UT_REGISTER_TEST(QGeoFileTileCacheQGCTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.h b/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.h new file mode 100644 index 000000000000..778793e546a8 --- /dev/null +++ b/test/QtLocationPlugin/QGeoFileTileCacheQGCTest.h @@ -0,0 +1,13 @@ +#pragma once + +#include "QtLocationTestBase.h" + +class QGeoFileTileCacheQGCTest : public QtLocationTestBase +{ + Q_OBJECT + +private slots: + void _testCreateFetchTileTaskValidation(); + void _testMemoryInsertAndGet(); + void _testDiskOnlyInsertDoesNotPopulateMemory(); +}; diff --git a/test/QtLocationPlugin/QGeoTileFetcherQGCTest.cc b/test/QtLocationPlugin/QGeoTileFetcherQGCTest.cc new file mode 100644 index 000000000000..5e7e40b8d167 --- /dev/null +++ b/test/QtLocationPlugin/QGeoTileFetcherQGCTest.cc @@ -0,0 +1,219 @@ +#include "QGeoTileFetcherQGCTest.h" + +#include +#include +#include +#include +#include +#include + +#include "Fixtures/RAIIFixtures.h" +#include "QGCHostCircuitBreaker.h" +#include "QGeoMapReplyQGC.h" +#include "QGeoTileFetcherQGC.h" + +void QGeoTileFetcherQGCTest::init() +{ + // The host gate is a process-wide singleton; reset so leader GETs left + // in-flight by a prior test (unreachable port, never completes) don't carry + // held slots into the next one. + HostConcurrencyGate::instance().resetForTest(); +} + +void QGeoTileFetcherQGCTest::_testConcurrentDownloadsConstant() +{ + QCOMPARE(QGeoTileFetcherQGC::concurrentDownloads(QStringLiteral("AnyProvider")), 6u); +} + +void QGeoTileFetcherQGCTest::_testGetNetworkRequestInvalidMapId() +{ + const QNetworkRequest request = QGeoTileFetcherQGC::getNetworkRequest(-1, 0, 0, 1); + QVERIFY(request.url().isEmpty()); +} + +void QGeoTileFetcherQGCTest::_testGetNetworkRequestValidMapBasics() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QNetworkRequest request = QGeoTileFetcherQGC::getNetworkRequest(mapId, 10, 20, 5); + QVERIFY(!request.url().isEmpty()); + QVERIFY(request.url().isValid()); + QCOMPARE(request.priority(), QNetworkRequest::NormalPriority); + QCOMPARE(request.transferTimeout(), 0); // owned by the fetcher leader-GET path + + QCOMPARE(request.rawHeader(QByteArrayLiteral("Accept")), QByteArrayLiteral("*/*")); + QCOMPARE(request.rawHeader(QByteArrayLiteral("Connection")), QByteArrayLiteral("keep-alive")); +} + +void QGeoTileFetcherQGCTest::_testGetNetworkRequestAttributes() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QNetworkRequest request = QGeoTileFetcherQGC::getNetworkRequest(mapId, 1, 2, 3); + + QCOMPARE(request.attribute(QNetworkRequest::BackgroundRequestAttribute).toBool(), true); + QCOMPARE(request.attribute(QNetworkRequest::RedirectPolicyAttribute).toInt(), + static_cast(QNetworkRequest::NoLessSafeRedirectPolicy)); + QCOMPARE(request.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute).toBool(), false); + // Trimmed: Http2Allowed (default true in Qt 6) and Cache{Load,Save}Control + // (no QNetworkDiskCache attached, so they were no-ops). + QVERIFY(!request.attribute(QNetworkRequest::CacheLoadControlAttribute).isValid()); + QVERIFY(!request.attribute(QNetworkRequest::CacheSaveControlAttribute).isValid()); + QVERIFY(!request.attribute(QNetworkRequest::Http2AllowedAttribute).isValid()); +} + +void QGeoTileFetcherQGCTest::_testDedupCoalescesSameUrl() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + QNetworkAccessManager nam; + QGeoTileFetcherQGC fetcher(&nam, QVariantMap(), nullptr); + + const QNetworkRequest request(QUrl(QStringLiteral("http://127.0.0.1:9/same_tile.png"))); + QGeoTiledMapReplyQGC a(&nam, request, tileSpec(mapId, 1, 1, 2)); + QGeoTiledMapReplyQGC b(&nam, request, tileSpec(mapId, 1, 1, 2)); + QGeoTiledMapReplyQGC c(&nam, request, tileSpec(mapId, 1, 1, 2)); + + fetcher.dedupFetch(&a, request); + fetcher.dedupFetch(&b, request); + fetcher.dedupFetch(&c, request); + + QCOMPARE(fetcher.inFlightUrlCount(), 1); + QCOMPARE(fetcher.followerCount(request.url()), 3); + + fetcher.cancelFetch(&a, request); + QCOMPARE(fetcher.followerCount(request.url()), 2); +} + +void QGeoTileFetcherQGCTest::_testDedupSeparateUrlsDistinct() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + QNetworkAccessManager nam; + QGeoTileFetcherQGC fetcher(&nam, QVariantMap(), nullptr); + + const QNetworkRequest r1(QUrl(QStringLiteral("http://127.0.0.1:9/a.png"))); + const QNetworkRequest r2(QUrl(QStringLiteral("http://127.0.0.1:9/b.png"))); + QGeoTiledMapReplyQGC a(&nam, r1, tileSpec(mapId, 1, 1, 2)); + QGeoTiledMapReplyQGC b(&nam, r2, tileSpec(mapId, 2, 2, 2)); + + fetcher.dedupFetch(&a, r1); + fetcher.dedupFetch(&b, r2); + + QCOMPARE(fetcher.inFlightUrlCount(), 2); + QCOMPARE(fetcher.followerCount(r1.url()), 1); + QCOMPARE(fetcher.followerCount(r2.url()), 1); +} + +void QGeoTileFetcherQGCTest::_testParseExpirySMaxAgeFallback() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com/t.png"))); + reply.setHttpStatus(200); + reply.setRawHeader(QByteArrayLiteral("Cache-Control"), QByteArrayLiteral("public, s-maxage=3600")); + + const qint64 before = QDateTime::currentSecsSinceEpoch(); + const QGeoTileFetcherQGC::FetchResult result = QGeoTileFetcherQGC::snapshot(&reply); + const qint64 after = QDateTime::currentSecsSinceEpoch(); + + QVERIFY(result.expiresAt >= (before + 3600)); + QVERIFY(result.expiresAt <= (after + 3600)); +} + +void QGeoTileFetcherQGCTest::_testParseExpiryMaxAgeWinsOverSMaxAge() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com/t.png"))); + reply.setHttpStatus(200); + reply.setRawHeader(QByteArrayLiteral("Cache-Control"), QByteArrayLiteral("s-maxage=86400, max-age=60")); + + const qint64 before = QDateTime::currentSecsSinceEpoch(); + const QGeoTileFetcherQGC::FetchResult result = QGeoTileFetcherQGC::snapshot(&reply); + const qint64 after = QDateTime::currentSecsSinceEpoch(); + + QVERIFY(result.expiresAt >= (before + 60)); + QVERIFY(result.expiresAt <= (after + 60)); +} + +void QGeoTileFetcherQGCTest::_testParseExpirySMaxAgeZeroIsNotCached() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com/t.png"))); + reply.setHttpStatus(200); + reply.setRawHeader(QByteArrayLiteral("Cache-Control"), QByteArrayLiteral("s-maxage=0")); + + const QGeoTileFetcherQGC::FetchResult result = QGeoTileFetcherQGC::snapshot(&reply); + QCOMPARE(result.expiresAt, static_cast(0)); +} + +void QGeoTileFetcherQGCTest::_testConcurrencyGateCapsInFlight() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + QNetworkAccessManager nam; + QGeoTileFetcherQGC fetcher(&nam, QVariantMap(), nullptr); + fetcher.setConcurrencyBudgetForTest(2); + + constexpr int kRequests = 6; + std::vector> replies; + for (int i = 0; i < kRequests; ++i) { + const QUrl url(QStringLiteral("http://127.0.0.1:9/tile_%1.png").arg(i)); + const QNetworkRequest request(url); + auto reply = std::make_unique(&nam, request, tileSpec(mapId, i, i, 2)); + fetcher.dedupFetch(reply.get(), request); + replies.push_back(std::move(reply)); + QVERIFY(fetcher.activeNetworkFetches() <= 2); + } + + QCOMPARE(fetcher.inFlightUrlCount(), kRequests); + QCOMPARE(fetcher.activeNetworkFetches(), 2); + QCOMPARE(fetcher.peakConcurrentFetches(), 2); +} + +void QGeoTileFetcherQGCTest::_testHostConcurrencyGateSharedCap() +{ + HostConcurrencyGate& gate = HostConcurrencyGate::instance(); + const QString host = QStringLiteral("tile.example.org"); + + // OSM hosts are clamped to kOSMHostLimit (2) regardless of a larger budget, + // so live + bulk paths combined can never exceed the policy cap. + QVERIFY(gate.tryAcquire(host, 6, true)); + QVERIFY(gate.tryAcquire(host, 6, true)); + QVERIFY(!gate.tryAcquire(host, 6, true)); + QCOMPARE(gate.activeForTest(host), 2); + + gate.release(host); + QVERIFY(gate.tryAcquire(host, 6, true)); + QCOMPARE(gate.activeForTest(host), 2); + + gate.release(host); + gate.release(host); + QCOMPARE(gate.activeForTest(host), 0); + + // Empty host is never gated (unit-test URLs without a real host). + QVERIFY(gate.tryAcquire(QString(), 1, true)); + QVERIFY(gate.tryAcquire(QString(), 1, true)); +} + +void QGeoTileFetcherQGCTest::_testSnapshotParsesMustRevalidate() +{ + const auto mustRevalidateFor = [](const QByteArray& cacheControl) { + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com/t.png"))); + reply.setHttpStatus(200); + if (!cacheControl.isEmpty()) { + reply.setRawHeader(QByteArrayLiteral("Cache-Control"), cacheControl); + } + return QGeoTileFetcherQGC::snapshot(&reply).mustRevalidate; + }; + + QVERIFY(mustRevalidateFor(QByteArrayLiteral("no-cache"))); + QVERIFY(mustRevalidateFor(QByteArrayLiteral("must-revalidate"))); + QVERIFY(mustRevalidateFor(QByteArrayLiteral("no-store"))); + QVERIFY(mustRevalidateFor(QByteArrayLiteral("public, max-age=60, must-revalidate"))); + QVERIFY(!mustRevalidateFor(QByteArrayLiteral("public, max-age=3600"))); + QVERIFY(!mustRevalidateFor(QByteArray())); +} + +UT_REGISTER_TEST(QGeoTileFetcherQGCTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGeoTileFetcherQGCTest.h b/test/QtLocationPlugin/QGeoTileFetcherQGCTest.h new file mode 100644 index 000000000000..26456a87d21a --- /dev/null +++ b/test/QtLocationPlugin/QGeoTileFetcherQGCTest.h @@ -0,0 +1,24 @@ +#pragma once + +#include "QtLocationTestBase.h" + +class QGeoTileFetcherQGCTest : public QtLocationTestBase +{ + Q_OBJECT + +private slots: + void init(); + + void _testConcurrentDownloadsConstant(); + void _testGetNetworkRequestInvalidMapId(); + void _testGetNetworkRequestValidMapBasics(); + void _testGetNetworkRequestAttributes(); + void _testDedupCoalescesSameUrl(); + void _testDedupSeparateUrlsDistinct(); + void _testParseExpirySMaxAgeFallback(); + void _testParseExpiryMaxAgeWinsOverSMaxAge(); + void _testParseExpirySMaxAgeZeroIsNotCached(); + void _testConcurrencyGateCapsInFlight(); + void _testHostConcurrencyGateSharedCap(); + void _testSnapshotParsesMustRevalidate(); +}; diff --git a/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.cc b/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.cc new file mode 100644 index 000000000000..7440bc607fc2 --- /dev/null +++ b/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.cc @@ -0,0 +1,221 @@ +#include "QGeoTiledMapReplyQGCTest.h" + +#include +#include +#include + +#include + +#include "QGCCacheTile.h" +#include "QGCMapUrlEngine.h" +#include "QGCNetworkHelper.h" +#include "QGeoMapReplyQGC.h" +#include "QGeoTileFetcherQGC.h" +#include "TileFetchMetrics.h" + +void QGeoTiledMapReplyQGCTest::_testCacheReplyMarksCachedAndFinished() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 12, 34, 5); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/tile.png"))), spec); + + auto tile = QSharedPointer::create(QStringLiteral("test_hash"), QByteArrayLiteral("tile_bytes"), QStringLiteral("png"), + UrlFactory::getProviderTypeFromQtMapId(mapId)); + // Fresh so the tile is served directly regardless of network availability; + // a default (expired, no-validator) tile would trigger a refetch when online. + tile->expiresAt = 4102444800; // 2100-01-01 + + const bool invoked = + QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, tile)); + QVERIFY(invoked); + + QVERIFY(reply.isFinished()); + QVERIFY(reply.isCached()); + QCOMPARE(reply.mapImageData(), QByteArrayLiteral("tile_bytes")); + QCOMPARE(reply.mapImageFormat(), QStringLiteral("png")); +} + +void QGeoTiledMapReplyQGCTest::_testCacheReplyNullTileSetsError() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 12, 34, 5); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/tile.png"))), spec); + + const bool invoked = + QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, QSharedPointer())); + QVERIFY(invoked); + + QCOMPARE(reply.error(), QGeoTiledMapReply::UnknownError); +} + +void QGeoTiledMapReplyQGCTest::_testNonExpiredCacheServedWithoutNetworkManager() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 12, 34, 5); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/tile.png"))), spec); + + auto cachedTile = QSharedPointer::create(QStringLiteral("cached_hash"), QByteArrayLiteral("cached_bytes"), + QStringLiteral("png"), UrlFactory::getProviderTypeFromQtMapId(mapId)); + cachedTile->expiresAt = 4102444800; // 2100-01-01: fresh, so served without a network manager + QVERIFY(QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, cachedTile))); + + QVERIFY(reply.isFinished()); + QVERIFY(reply.isCached()); + QCOMPARE(reply.mapImageData(), QByteArrayLiteral("cached_bytes")); +} + +void QGeoTiledMapReplyQGCTest::_testHandleNetworkResult304WithoutCacheSetsError() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 5, 6, 4); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/t.png"))), spec); + + QGeoTileFetcherQGC::FetchResult result; + result.statusCode = 304; + result.isOpen = true; + + QVERIFY(QMetaObject::invokeMethod(&reply, "_handleNetworkResult", Qt::DirectConnection, + Q_ARG(QGeoTileFetcherQGC::FetchResult, result))); + + QCOMPARE(reply.error(), QGeoTiledMapReply::UnknownError); +} + +void QGeoTiledMapReplyQGCTest::_testHandleNetworkResultSuccessCachesAndFinishes() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 5, 6, 4); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/t.png"))), spec); + + const TileFetchStats before = TileFetchMetrics::instance().snapshot(); + + QGeoTileFetcherQGC::FetchResult result; + result.statusCode = 200; + result.isOpen = true; + result.body = QByteArray::fromHex("89504e470d0a1a0a") + QByteArrayLiteral("fakecontent"); + + QVERIFY(QMetaObject::invokeMethod(&reply, "_handleNetworkResult", Qt::DirectConnection, + Q_ARG(QGeoTileFetcherQGC::FetchResult, result))); + + QVERIFY(reply.isFinished()); + QCOMPARE(reply.error(), QGeoTiledMapReply::NoError); + + const TileFetchStats after = TileFetchMetrics::instance().snapshot(); + QCOMPARE(after.networkSuccess, before.networkSuccess + 1); + QVERIFY(after.bytesDownloaded >= before.bytesDownloaded + static_cast(result.body.size())); +} + +void QGeoTiledMapReplyQGCTest::_testHandleNetworkResultHttpErrorSetsError() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 5, 6, 4); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/t.png"))), spec); + + QGeoTileFetcherQGC::FetchResult result; + result.statusCode = 404; + result.isOpen = true; + result.reasonPhrase = QStringLiteral("Not Found"); + + QVERIFY(QMetaObject::invokeMethod(&reply, "_handleNetworkResult", Qt::DirectConnection, + Q_ARG(QGeoTileFetcherQGC::FetchResult, result))); + + QCOMPARE(reply.error(), QGeoTiledMapReply::CommunicationError); +} + +void QGeoTiledMapReplyQGCTest::_testFreshCacheServedWithoutRevalidation() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 5, 6, 4); + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/t.png"))), spec); + + const TileFetchStats before = TileFetchMetrics::instance().snapshot(); + + // expiresAt far in the future => fresh, served directly even though validators exist. + auto tile = QSharedPointer::create(QStringLiteral("fresh_hash"), QByteArrayLiteral("fresh_bytes"), QStringLiteral("png"), + UrlFactory::getProviderTypeFromQtMapId(mapId)); + tile->etag = QByteArrayLiteral("\"etag\""); + tile->expiresAt = 4102444800; // 2100-01-01 + + QVERIFY(QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, tile))); + + QVERIFY(reply.isFinished()); + QVERIFY(reply.isCached()); + QCOMPARE(reply.mapImageData(), QByteArrayLiteral("fresh_bytes")); + + const TileFetchStats after = TileFetchMetrics::instance().snapshot(); + QCOMPARE(after.cacheHits, before.cacheHits + 1); +} + +void QGeoTiledMapReplyQGCTest::_testMustRevalidateCacheTileNotServedDirectly() +{ + if (!QGCNetworkHelper::isInternetAvailable()) { + QSKIP("Revalidation path requires network availability"); + } + + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 7, 8, 4); + QNetworkAccessManager nam; + QGeoTiledMapReplyQGC reply(&nam, QNetworkRequest(QUrl(QStringLiteral("http://127.0.0.1:9/t.png"))), spec); + + const TileFetchStats before = TileFetchMetrics::instance().snapshot(); + + // Fresh expiry but must-revalidate set => treated as needing revalidation, + // so it must NOT be served as a direct cache hit. + auto tile = QSharedPointer::create(QStringLiteral("mr_hash"), QByteArrayLiteral("mr_bytes"), QStringLiteral("png"), + UrlFactory::getProviderTypeFromQtMapId(mapId)); + tile->etag = QByteArrayLiteral("\"etag\""); + tile->expiresAt = 4102444800; // 2100-01-01 + tile->mustRevalidate = true; + + QVERIFY(QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, tile))); + + const TileFetchStats after = TileFetchMetrics::instance().snapshot(); + QCOMPARE(after.cacheHits, before.cacheHits); + QVERIFY(!reply.isCached()); +} + +void QGeoTiledMapReplyQGCTest::_testExpiredRevalidatableTileServedStale() +{ + const int mapId = validMapId(); + QVERIFY(mapId > 0); + + const QGeoTileSpec spec = tileSpec(mapId, 9, 10, 4); + // No fetcher and no QNAM => the background revalidator no-ops, isolating the + // stale-serve decision. EXPIRED + validators + NOT must-revalidate must still be + // served from cache (stale-while-revalidate offline, or direct serve when no net). + QGeoTiledMapReplyQGC reply(nullptr, QNetworkRequest(QUrl(QStringLiteral("https://example.com/t.png"))), spec); + + const TileFetchStats before = TileFetchMetrics::instance().snapshot(); + + auto tile = QSharedPointer::create(QStringLiteral("swr_hash"), QByteArrayLiteral("swr_bytes"), QStringLiteral("png"), + UrlFactory::getProviderTypeFromQtMapId(mapId)); + tile->etag = QByteArrayLiteral("\"etag\""); + tile->expiresAt = 1000; // 1970 => expired + tile->mustRevalidate = false; // plain expiry => stale-while-revalidate eligible + + QVERIFY(QMetaObject::invokeMethod(&reply, "_cacheReply", Qt::DirectConnection, Q_ARG(QSharedPointer, tile))); + + QVERIFY(reply.isFinished()); + QVERIFY(reply.isCached()); + QCOMPARE(reply.mapImageData(), QByteArrayLiteral("swr_bytes")); + + const TileFetchStats after = TileFetchMetrics::instance().snapshot(); + QCOMPARE(after.cacheHits, before.cacheHits + 1); +} + +UT_REGISTER_TEST(QGeoTiledMapReplyQGCTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.h b/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.h new file mode 100644 index 000000000000..ea544aa75f10 --- /dev/null +++ b/test/QtLocationPlugin/QGeoTiledMapReplyQGCTest.h @@ -0,0 +1,19 @@ +#pragma once + +#include "QtLocationTestBase.h" + +class QGeoTiledMapReplyQGCTest : public QtLocationTestBase +{ + Q_OBJECT + +private slots: + void _testCacheReplyMarksCachedAndFinished(); + void _testCacheReplyNullTileSetsError(); + void _testNonExpiredCacheServedWithoutNetworkManager(); + void _testHandleNetworkResultSuccessCachesAndFinishes(); + void _testHandleNetworkResultHttpErrorSetsError(); + void _testHandleNetworkResult304WithoutCacheSetsError(); + void _testFreshCacheServedWithoutRevalidation(); + void _testMustRevalidateCacheTileNotServedDirectly(); + void _testExpiredRevalidatableTileServedStale(); +}; diff --git a/test/QtLocationPlugin/QtLocationTestBase.h b/test/QtLocationPlugin/QtLocationTestBase.h new file mode 100644 index 000000000000..a42488fe5918 --- /dev/null +++ b/test/QtLocationPlugin/QtLocationTestBase.h @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "QGCMapUrlEngine.h" +#include "UnitTest.h" + +class QtLocationTestBase : public UnitTest +{ +protected: + static int validMapId() + { + const QStringList providerTypes = UrlFactory::getProviderTypes(); + if (providerTypes.isEmpty()) { + return -1; + } + return UrlFactory::getQtMapIdFromProviderType(providerTypes.first()); + } + + static QGeoTileSpec tileSpec(int mapId, int x, int y, int zoom) + { + QGeoTileSpec spec; + spec.setMapId(mapId); + spec.setX(x); + spec.setY(y); + spec.setZoom(zoom); + return spec; + } +}; diff --git a/test/QtLocationPlugin/TileFetchMetricsTest.cc b/test/QtLocationPlugin/TileFetchMetricsTest.cc new file mode 100644 index 000000000000..dc702c074e84 --- /dev/null +++ b/test/QtLocationPlugin/TileFetchMetricsTest.cc @@ -0,0 +1,79 @@ +#include "TileFetchMetricsTest.h" + +#include "TileFetchMetrics.h" + +void TileFetchMetricsTest::init() +{ + UnitTest::init(); + // Process-wide singleton: reset so each test starts from a known baseline. + TileFetchMetrics::instance().reset(); +} + +void TileFetchMetricsTest::_testCacheHitMissCounters() +{ + TileFetchMetrics &m = TileFetchMetrics::instance(); + const TileFetchStats before = m.snapshot(); + + m.recordCacheHit(); + m.recordCacheHit(); + m.recordCacheMiss(); + + const TileFetchStats after = m.snapshot(); + QCOMPARE(after.cacheHits, before.cacheHits + 2); + QCOMPARE(after.cacheMisses, before.cacheMisses + 1); +} + +void TileFetchMetricsTest::_testNetworkSuccessBytesAndLatency() +{ + TileFetchMetrics &m = TileFetchMetrics::instance(); + const TileFetchStats before = m.snapshot(); + + m.recordNetworkSuccess(2048, 120); + + const TileFetchStats after = m.snapshot(); + QCOMPARE(after.networkSuccess, before.networkSuccess + 1); + QCOMPARE(after.bytesDownloaded, before.bytesDownloaded + 2048); + QCOMPARE(after.totalLatencyMs, before.totalLatencyMs + 120); + QCOMPARE(after.latencySamples, before.latencySamples + 1); +} + +void TileFetchMetricsTest::_testNotModifiedCounter() +{ + TileFetchMetrics &m = TileFetchMetrics::instance(); + const TileFetchStats before = m.snapshot(); + + m.recordNotModified(30); + + const TileFetchStats after = m.snapshot(); + QCOMPARE(after.notModified, before.notModified + 1); + QCOMPARE(after.latencySamples, before.latencySamples + 1); +} + +void TileFetchMetricsTest::_testNetworkErrorCounter() +{ + TileFetchMetrics &m = TileFetchMetrics::instance(); + const TileFetchStats before = m.snapshot(); + + m.recordNetworkError(50); + + const TileFetchStats after = m.snapshot(); + QCOMPARE(after.networkErrors, before.networkErrors + 1); +} + +void TileFetchMetricsTest::_testReset() +{ + TileFetchMetrics &m = TileFetchMetrics::instance(); + m.recordCacheHit(); + m.recordNetworkSuccess(100, 10); + + m.reset(); + + const TileFetchStats after = m.snapshot(); + QCOMPARE(after.cacheHits, static_cast(0)); + QCOMPARE(after.networkSuccess, static_cast(0)); + QCOMPARE(after.bytesDownloaded, static_cast(0)); + QCOMPARE(after.totalFetches(), static_cast(0)); + QCOMPARE(after.averageLatencyMs(), 0.0); +} + +UT_REGISTER_TEST(TileFetchMetricsTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/TileFetchMetricsTest.h b/test/QtLocationPlugin/TileFetchMetricsTest.h new file mode 100644 index 000000000000..7e012115d6b6 --- /dev/null +++ b/test/QtLocationPlugin/TileFetchMetricsTest.h @@ -0,0 +1,16 @@ +#pragma once + +#include "UnitTest.h" + +class TileFetchMetricsTest : public UnitTest +{ + Q_OBJECT + +private slots: + void init() override; + void _testCacheHitMissCounters(); + void _testNetworkSuccessBytesAndLatency(); + void _testNotModifiedCounter(); + void _testNetworkErrorCounter(); + void _testReset(); +}; diff --git a/test/QtLocationPlugin/UrlFactoryTest.cc b/test/QtLocationPlugin/UrlFactoryTest.cc index 3797f7b7ef44..751f2bd08ec4 100644 --- a/test/QtLocationPlugin/UrlFactoryTest.cc +++ b/test/QtLocationPlugin/UrlFactoryTest.cc @@ -1,5 +1,6 @@ #include "UrlFactoryTest.h" +#include "BingMapProvider.h" #include "MapProvider.h" #include "QGCMapUrlEngine.h" #include "QGCTileSet.h" @@ -71,7 +72,8 @@ void UrlFactoryTest::_testMapIdFromInvalidType() void UrlFactoryTest::_testProviderTypeFromInvalidId() { QVERIFY(UrlFactory::getProviderTypeFromQtMapId(-1).isEmpty()); - expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("map id not found:")); + expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, + QRegularExpression("provider not found from id:")); QVERIFY(UrlFactory::getProviderTypeFromQtMapId(99999).isEmpty()); verifyExpectedLogMessage(); } @@ -98,18 +100,52 @@ void UrlFactoryTest::_testProviderTypeFromHashRoundtrip() void UrlFactoryTest::_testHashFromInvalidType() { - expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("provider not found for type:")); QCOMPARE(UrlFactory::hashFromProviderType(QString()), -1); + expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("type not found:")); + QCOMPARE(UrlFactory::hashFromProviderType(QStringLiteral("NoSuchProvider")), -1); verifyExpectedLogMessage(); } void UrlFactoryTest::_testProviderTypeFromInvalidHash() { - expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("provider not found from hash:")); + expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, + QRegularExpression("provider not found from hash:")); QVERIFY(UrlFactory::providerTypeFromHash(0).isEmpty()); verifyExpectedLogMessage(); } +void UrlFactoryTest::_testRetinaTileHashDisjointFrom1x() +{ + // In a headless unit test useRetinaTiles() is false (no QGuiApplication DPR > 1), + // so getTileHash() yields the 1x hash. The @2x cache key reserves a disjoint + // hash space by adding kRetinaHashOffset (1e9) to the provider-hash field, per + // QGCMapUrlEngine.cpp. Reproduce that here to assert the spaces never collide and + // both decode back to the same provider. + constexpr int kRetinaHashOffset = 1000000000; + + const int x = 100; + const int y = 200; + const int z = 5; + + const QString hash1x = UrlFactory::getTileHash(kBingRoad, x, y, z); + QCOMPARE(hash1x.length(), 29); + + const int baseHash = UrlFactory::hashFromProviderType(kBingRoad); + QVERIFY(baseHash > 0); + const QString hash2x = QString::asprintf("%010d%08d%08d%03d", baseHash + kRetinaHashOffset, x, y, z); + + // Disjoint: a 1x tile and its @2x counterpart never share a cache key. + QVERIFY(hash1x != hash2x); + + // Both keys decode back to the same provider (offset is stripped on lookup). + QCOMPARE(UrlFactory::tileHashToType(hash1x), kBingRoad); + QCOMPARE(UrlFactory::tileHashToType(hash2x), kBingRoad); + + // The retina key sorts above the entire 1x mapId range, guaranteeing no overlap + // with any other provider's 1x hash field. + QVERIFY(hash2x.left(10).toInt() >= kRetinaHashOffset); +} + // --- Tile hash encode/decode --- void UrlFactoryTest::_testGetTileHashFormat() @@ -131,8 +167,10 @@ void UrlFactoryTest::_testTileHashToTypeRoundtrip() void UrlFactoryTest::_testTileHashToTypeInvalid() { - expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("provider not found from hash:")); QVERIFY(UrlFactory::tileHashToType(QStringLiteral("garbage")).isEmpty()); + expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, + QRegularExpression("provider not found from hash:")); + QVERIFY(UrlFactory::tileHashToType(QStringLiteral("0000000000garbagehash")).isEmpty()); verifyExpectedLogMessage(); } @@ -169,7 +207,7 @@ void UrlFactoryTest::_testGetTileURLByType() const QUrl url = UrlFactory::getTileURL(kBingRoad, 301, 385, 10); QVERIFY(url.isValid()); QVERIFY(!url.isEmpty()); - QVERIFY(url.scheme().startsWith(QStringLiteral("http"))); + QCOMPARE(url.scheme(), QStringLiteral("https")); } void UrlFactoryTest::_testGetTileURLByMapId() @@ -179,7 +217,23 @@ void UrlFactoryTest::_testGetTileURLByMapId() const QUrl url = UrlFactory::getTileURL(id, 301, 385, 10); QVERIFY(url.isValid()); QVERIFY(!url.isEmpty()); - QVERIFY(url.scheme().startsWith(QStringLiteral("http"))); + QCOMPARE(url.scheme(), QStringLiteral("https")); +} + +void UrlFactoryTest::_testGetTileURLUsesLayerSlugNotName() +{ + // Regression: MapQuest/VWorld must interpolate the layer slug (_mapTypeId), + // not the display name (_mapName) which contains spaces. + const QUrl mapQuest = UrlFactory::getTileURL(QStringLiteral("MapQuest Map"), 5, 6, 4); + QVERIFY(!mapQuest.isEmpty()); + QVERIFY(mapQuest.path().contains(QStringLiteral("/map/"))); + QVERIFY(!mapQuest.toString().contains(QStringLiteral("MapQuest"))); + + // VWorld serves only Korea; x/y must fall in its valid range at this zoom. + const QUrl vworld = UrlFactory::getTileURL(QStringLiteral("VWorld Street Map"), 860, 400, 10); + QVERIFY(!vworld.isEmpty()); + QVERIFY(vworld.toString().contains(QStringLiteral("/Base/"))); + QVERIFY(!vworld.toString().contains(QStringLiteral("Street Map"))); } void UrlFactoryTest::_testGetTileURLInvalidInputs() @@ -200,7 +254,7 @@ void UrlFactoryTest::_testAverageSizeForKnownProviders() void UrlFactoryTest::_testAverageSizeForInvalidType() { expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("type not found:")); - QCOMPARE(UrlFactory::averageSizeForType(QStringLiteral("Nonexistent")), QGC_AVERAGE_TILE_SIZE); + QCOMPARE(UrlFactory::averageSizeForType(QStringLiteral("NonexistentForAvgSize")), QGC_AVERAGE_TILE_SIZE); verifyExpectedLogMessage(); } @@ -306,7 +360,7 @@ void UrlFactoryTest::_testGetMapProviderValid() auto provider = UrlFactory::getMapProviderFromQtMapId(id); QVERIFY(provider != nullptr); QCOMPARE(provider->getMapName(), kBingRoad); - QVERIFY(provider->isBingProvider()); + QVERIFY(std::dynamic_pointer_cast(provider) != nullptr); auto byType = UrlFactory::getMapProviderFromProviderType(kBingRoad); QVERIFY(byType != nullptr); @@ -318,8 +372,49 @@ void UrlFactoryTest::_testGetMapProviderInvalid() QVERIFY(UrlFactory::getMapProviderFromQtMapId(-1) == nullptr); QVERIFY(UrlFactory::getMapProviderFromProviderType(QString()) == nullptr); expectLogMessage("QtLocationPlugin.QGCMapUrlEngine", QtWarningMsg, QRegularExpression("type not found:")); - QVERIFY(UrlFactory::getMapProviderFromProviderType(QStringLiteral("Nonexistent")) == nullptr); + QVERIFY(UrlFactory::getMapProviderFromProviderType(QStringLiteral("NonexistentForProviderLookup")) == nullptr); verifyExpectedLogMessage(); } +void UrlFactoryTest::_testGetTileURLGoldenStrings() +{ + struct Golden + { + QString type; + QString expected; + }; + + const QList cases = { + {QStringLiteral("Street Map"), QStringLiteral("https://tile.openstreetmap.org/10/301/385.png")}, + {QStringLiteral("Statkart Topo"), + QStringLiteral("https://cache.kartverket.no/v1/wmts/1.0.0/topo4/default/webmercator/10/385/301.png")}, + {QStringLiteral("Statkart Basemap"), + QStringLiteral( + "https://cache.kartverket.no/v1/wmts/1.0.0/norgeskart_bakgrunn/default/webmercator/10/385/301.png")}, + {QStringLiteral("Svalbard Topo"), + QStringLiteral("https://geodata.npolar.no/arcgis/rest/services/Basisdata/NP_Basiskart_Svalbard_WMTS_3857/" + "MapServer/WMTS/tile/1.0.0/Basisdata_NP_Basiskart_Svalbard_WMTS_3857/default/default028mm/" + "10/385/301")}, + {QStringLiteral("Eniro Topo"), + QStringLiteral("https://map.eniro.com/geowebcache/service/tms1.0.0/map/10/301/638.png")}, + {QStringLiteral("MapQuest Map"), QStringLiteral("https://otile3.mqcdn.com/tiles/1.0.0/map/10/301/385.jpg")}, + {QStringLiteral("MapQuest Sat"), QStringLiteral("https://otile3.mqcdn.com/tiles/1.0.0/sat/10/301/385.jpg")}, + {QStringLiteral("Japan-GSI Contour"), + QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/std/10/301/385.png")}, + {QStringLiteral("Japan-GSI Seamless"), + QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/seamlessphoto/10/301/385.jpg")}, + {QStringLiteral("Japan-GSI Anaglyph"), + QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/anaglyphmap_color/10/301/385.png")}, + {QStringLiteral("Japan-GSI Slope"), + QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/slopemap/10/301/385.png")}, + {QStringLiteral("Japan-GSI Relief"), + QStringLiteral("https://cyberjapandata.gsi.go.jp/xyz/relief/10/301/385.png")}, + }; + + for (const auto& c : cases) { + const QString actual = UrlFactory::getTileURL(c.type, 301, 385, 10).toString(); + QCOMPARE(actual, c.expected); + } +} + UT_REGISTER_TEST(UrlFactoryTest, TestLabel::Unit) diff --git a/test/QtLocationPlugin/UrlFactoryTest.h b/test/QtLocationPlugin/UrlFactoryTest.h index bcf1f7351642..9930b3ca3b28 100644 --- a/test/QtLocationPlugin/UrlFactoryTest.h +++ b/test/QtLocationPlugin/UrlFactoryTest.h @@ -21,13 +21,16 @@ private slots: void _testProviderTypeFromInvalidHash(); void _testGetTileHashFormat(); void _testTileHashToTypeRoundtrip(); + void _testRetinaTileHashDisjointFrom1x(); void _testTileHashToTypeInvalid(); void _testGetImageFormatByType(); void _testGetImageFormatByMapId(); void _testGetImageFormatInvalidInputs(); void _testGetTileURLByType(); void _testGetTileURLByMapId(); + void _testGetTileURLUsesLayerSlugNotName(); void _testGetTileURLInvalidInputs(); + void _testGetTileURLGoldenStrings(); void _testAverageSizeForKnownProviders(); void _testAverageSizeForInvalidType(); void _testIsElevationTrue(); diff --git a/test/UnitTestFramework/Fixtures/RAIIFixtures.cc b/test/UnitTestFramework/Fixtures/RAIIFixtures.cc index 3f0ea8c82930..13e93091eec4 100644 --- a/test/UnitTestFramework/Fixtures/RAIIFixtures.cc +++ b/test/UnitTestFramework/Fixtures/RAIIFixtures.cc @@ -96,6 +96,11 @@ void NetworkReplyFixture::setHttpStatus(int statusCode) setAttribute(QNetworkRequest::HttpStatusCodeAttribute, statusCode); } +void NetworkReplyFixture::setRawHeader(const QByteArray& name, const QByteArray& value) +{ + QNetworkReply::setRawHeader(name, value); +} + void NetworkReplyFixture::setRedirectTarget(const QUrl& target) { setAttribute(QNetworkRequest::RedirectionTargetAttribute, target); diff --git a/test/UnitTestFramework/Fixtures/RAIIFixtures.h b/test/UnitTestFramework/Fixtures/RAIIFixtures.h index a585c55bb3f4..0478df3511e3 100644 --- a/test/UnitTestFramework/Fixtures/RAIIFixtures.h +++ b/test/UnitTestFramework/Fixtures/RAIIFixtures.h @@ -76,6 +76,7 @@ class NetworkReplyFixture final : public QNetworkReply ~NetworkReplyFixture() override; void setHttpStatus(int statusCode); + void setRawHeader(const QByteArray& name, const QByteArray& value); void setRedirectTarget(const QUrl& target); void setNetworkError(QNetworkReply::NetworkError errorCode, const QString& message); void setContentType(const QString& contentType); diff --git a/test/Utilities/Database/QGCSqlHelperTest.cc b/test/Utilities/Database/QGCSqlHelperTest.cc index 087d9008736f..bbe2f34f8c8c 100644 --- a/test/Utilities/Database/QGCSqlHelperTest.cc +++ b/test/Utilities/Database/QGCSqlHelperTest.cc @@ -147,4 +147,41 @@ void QGCSqlHelperTest::_applySqlitePragmas() QCOMPARE(q.value(0).toInt(), 1); } +void QGCSqlHelperTest::_execPreparedSuccess() +{ + QGCSqlHelper::ScopedConnection conn(QStringLiteral(":memory:")); + QVERIFY(conn.isValid()); + QSqlQuery setup(conn.database()); + QVERIFY(setup.exec(QStringLiteral("CREATE TABLE t (id INTEGER PRIMARY KEY, name TEXT)"))); + + QSqlQuery insert(conn.database()); + QVERIFY(QGCSqlHelper::execPrepared(insert, QStringLiteral("INSERT INTO t(id, name) VALUES(?, ?)"), 1, QStringLiteral("alpha"))); + + QSqlQuery select(conn.database()); + QVERIFY(QGCSqlHelper::execPrepared(select, QStringLiteral("SELECT name FROM t WHERE id = ?"), 1)); + QVERIFY(select.next()); + QCOMPARE(select.value(0).toString(), QStringLiteral("alpha")); +} + +void QGCSqlHelperTest::_execPreparedPrepareFailure() +{ + QGCSqlHelper::ScopedConnection conn(QStringLiteral(":memory:")); + QVERIFY(conn.isValid()); + + QSqlQuery query(conn.database()); + QVERIFY(!QGCSqlHelper::execPrepared(query, QStringLiteral("SELECT * FROM nonexistent WHERE id = ?"), 1)); +} + +void QGCSqlHelperTest::_execPreparedExecFailure() +{ + QGCSqlHelper::ScopedConnection conn(QStringLiteral(":memory:")); + QVERIFY(conn.isValid()); + QSqlQuery setup(conn.database()); + QVERIFY(setup.exec(QStringLiteral("CREATE TABLE t (id INTEGER PRIMARY KEY)"))); + + QSqlQuery insert(conn.database()); + QVERIFY(QGCSqlHelper::execPrepared(insert, QStringLiteral("INSERT INTO t(id) VALUES(?)"), 1)); + QVERIFY(!QGCSqlHelper::execPrepared(insert, QStringLiteral("INSERT INTO t(id) VALUES(?)"), 1)); +} + UT_REGISTER_TEST(QGCSqlHelperTest, TestLabel::Unit, TestLabel::Utilities) diff --git a/test/Utilities/Database/QGCSqlHelperTest.h b/test/Utilities/Database/QGCSqlHelperTest.h index d76b52d3496e..891794dfab67 100644 --- a/test/Utilities/Database/QGCSqlHelperTest.h +++ b/test/Utilities/Database/QGCSqlHelperTest.h @@ -14,4 +14,7 @@ private slots: void _scopedConnectionUniqueNames(); void _scopedConnectionCleanup(); void _applySqlitePragmas(); + void _execPreparedSuccess(); + void _execPreparedPrepareFailure(); + void _execPreparedExecFailure(); }; diff --git a/test/Utilities/Network/QGCNetworkHelperTest.cc b/test/Utilities/Network/QGCNetworkHelperTest.cc index 957cf799ccab..e1cbce451686 100644 --- a/test/Utilities/Network/QGCNetworkHelperTest.cc +++ b/test/Utilities/Network/QGCNetworkHelperTest.cc @@ -493,6 +493,85 @@ void QGCNetworkHelperTest::_testReplyHelpersNullReply() QVERIFY(!QGCNetworkHelper::isJsonResponse(nullptr)); } +// ============================================================================ +// Retry / Transient Error Helpers Tests +// ============================================================================ +void QGCNetworkHelperTest::_testIsTransientErrorClassifiesStatusAndErrorCodes() +{ + using NE = QNetworkReply::NetworkError; + + QVERIFY(QGCNetworkHelper::isTransientError(NE::NoError, 429)); + QVERIFY(QGCNetworkHelper::isTransientError(NE::NoError, 500)); + QVERIFY(QGCNetworkHelper::isTransientError(NE::NoError, 503)); + QVERIFY(QGCNetworkHelper::isTransientError(NE::TimeoutError, 0)); + QVERIFY(QGCNetworkHelper::isTransientError(NE::ServiceUnavailableError, 0)); + QVERIFY(QGCNetworkHelper::isTransientError(NE::RemoteHostClosedError, 0)); + + QVERIFY(!QGCNetworkHelper::isTransientError(NE::NoError, 200)); + QVERIFY(!QGCNetworkHelper::isTransientError(NE::NoError, 404)); + QVERIFY(!QGCNetworkHelper::isTransientError(NE::ContentNotFoundError, 404)); + QVERIFY(!QGCNetworkHelper::isTransientError(NE::AuthenticationRequiredError, 401)); +} + +void QGCNetworkHelperTest::_testRetryAfterFromReplyNull() +{ + QVERIFY(!QGCNetworkHelper::retryAfterFromReply(nullptr).has_value()); +} + +void QGCNetworkHelperTest::_testRetryAfterFromReplyParsesSeconds() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com"))); + reply.setRawHeader("Retry-After", "3"); + + const auto delay = QGCNetworkHelper::retryAfterFromReply(&reply); + QVERIFY(delay.has_value()); + QCOMPARE(delay->count(), 3000); +} + +void QGCNetworkHelperTest::_testRetryAfterFromReplyClampsHugeValue() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com"))); + reply.setRawHeader("Retry-After", "120"); + + const auto delay = QGCNetworkHelper::retryAfterFromReply(&reply); + QVERIFY(delay.has_value()); + QCOMPARE(delay->count(), QGCNetworkHelper::kMaxRateLimitDelay.count()); +} + +// Regression: a parseable-but-huge delta-seconds must not overflow qint64 in the *1000. +void QGCNetworkHelperTest::_testRetryAfterFromReplyClampsOverflowValue() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com"))); + reply.setRawHeader("Retry-After", "9999999999999999"); + + const auto delay = QGCNetworkHelper::retryAfterFromReply(&reply); + QVERIFY(delay.has_value()); + QCOMPARE(delay->count(), QGCNetworkHelper::kMaxRateLimitDelay.count()); +} + +void QGCNetworkHelperTest::_testRetryAfterFromReplyHttpDate() +{ + TestFixtures::NetworkReplyFixture futureReply(QUrl(QStringLiteral("https://example.com"))); + futureReply.setRawHeader("Retry-After", "Sun, 06 Nov 2050 08:49:37 +0000"); + const auto futureDelay = QGCNetworkHelper::retryAfterFromReply(&futureReply); + QVERIFY(futureDelay.has_value()); + QCOMPARE(futureDelay->count(), QGCNetworkHelper::kMaxRateLimitDelay.count()); + + TestFixtures::NetworkReplyFixture pastReply(QUrl(QStringLiteral("https://example.com"))); + pastReply.setRawHeader("Retry-After", "Sun, 06 Nov 1994 08:49:37 +0000"); + const auto pastDelay = QGCNetworkHelper::retryAfterFromReply(&pastReply); + QVERIFY(pastDelay.has_value()); + QCOMPARE(pastDelay->count(), 0); +} + +void QGCNetworkHelperTest::_testRetryAfterFromReplyUnparseable() +{ + TestFixtures::NetworkReplyFixture reply(QUrl(QStringLiteral("https://example.com"))); + reply.setRawHeader("Retry-After", "not-a-date"); + + QVERIFY(!QGCNetworkHelper::retryAfterFromReply(&reply).has_value()); +} + // ============================================================================ // Network Availability Tests // ============================================================================ diff --git a/test/Utilities/Network/QGCNetworkHelperTest.h b/test/Utilities/Network/QGCNetworkHelperTest.h index f47676c56390..19b54ccc7883 100644 --- a/test/Utilities/Network/QGCNetworkHelperTest.h +++ b/test/Utilities/Network/QGCNetworkHelperTest.h @@ -57,6 +57,15 @@ private slots: void _testParseJsonReplyNull(); void _testReplyHelpersNullReply(); + // Retry / transient error helpers tests + void _testIsTransientErrorClassifiesStatusAndErrorCodes(); + void _testRetryAfterFromReplyNull(); + void _testRetryAfterFromReplyParsesSeconds(); + void _testRetryAfterFromReplyClampsHugeValue(); + void _testRetryAfterFromReplyClampsOverflowValue(); + void _testRetryAfterFromReplyHttpDate(); + void _testRetryAfterFromReplyUnparseable(); + // Network availability tests void _testIsNetworkAvailable(); void _testConnectionTypeName();