From bafa5f056e9dc40a9c09049937f456010c42783a Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 30 Apr 2025 18:36:32 +0100 Subject: [PATCH 01/49] Remove 'vector_math' dependency --- example/pubspec.lock | 2 +- example/pubspec.yaml | 1 - lib/src/geo/latlng_bounds.dart | 2 +- lib/src/gestures/map_interactive_viewer.dart | 2 +- lib/src/map/camera/camera.dart | 2 +- lib/src/map/controller/map_controller_impl.dart | 2 +- lib/src/misc/deg_rad_conversions.dart | 7 +++++++ pubspec.yaml | 1 - 8 files changed, 12 insertions(+), 7 deletions(-) create mode 100644 lib/src/misc/deg_rad_conversions.dart diff --git a/example/pubspec.lock b/example/pubspec.lock index 029357d84..0daa550df 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -501,7 +501,7 @@ packages: source: hosted version: "3.1.4" vector_math: - dependency: "direct main" + dependency: transitive description: name: vector_math sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" diff --git a/example/pubspec.yaml b/example/pubspec.yaml index bc2452a77..2b72737c9 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -19,7 +19,6 @@ dependencies: proj4dart: ^2.1.0 shared_preferences: ^2.3.4 url_launcher: ^6.3.1 - vector_math: ^2.1.4 dependency_overrides: flutter_map: diff --git a/lib/src/geo/latlng_bounds.dart b/lib/src/geo/latlng_bounds.dart index 62724972f..0cc465609 100644 --- a/lib/src/geo/latlng_bounds.dart +++ b/lib/src/geo/latlng_bounds.dart @@ -1,7 +1,7 @@ import 'dart:math'; +import 'package:flutter_map/src/misc/deg_rad_conversions.dart'; import 'package:latlong2/latlong.dart'; -import 'package:vector_math/vector_math_64.dart'; /// Data structure representing rectangular bounding box constrained by its /// northwest and southeast corners diff --git a/lib/src/gestures/map_interactive_viewer.dart b/lib/src/gestures/map_interactive_viewer.dart index 124172ef6..1363712f3 100644 --- a/lib/src/gestures/map_interactive_viewer.dart +++ b/lib/src/gestures/map_interactive_viewer.dart @@ -5,9 +5,9 @@ import 'package:flutter/gestures.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/misc/deg_rad_conversions.dart'; import 'package:flutter_map/src/misc/extensions.dart'; import 'package:latlong2/latlong.dart'; -import 'package:vector_math/vector_math_64.dart'; part 'package:flutter_map/src/gestures/compound_animations.dart'; diff --git a/lib/src/map/camera/camera.dart b/lib/src/map/camera/camera.dart index 5d046e8ec..f58e6a10f 100644 --- a/lib/src/map/camera/camera.dart +++ b/lib/src/map/camera/camera.dart @@ -3,9 +3,9 @@ import 'dart:math' as math; import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/map/inherited_model.dart'; +import 'package:flutter_map/src/misc/deg_rad_conversions.dart'; import 'package:flutter_map/src/misc/extensions.dart'; import 'package:latlong2/latlong.dart'; -import 'package:vector_math/vector_math_64.dart'; /// Describes the view of a map. This includes the size/zoom/position/crs as /// well as the minimum/maximum zoom. This class is mostly immutable but has diff --git a/lib/src/map/controller/map_controller_impl.dart b/lib/src/map/controller/map_controller_impl.dart index d36c605aa..0ed0ed9b9 100644 --- a/lib/src/map/controller/map_controller_impl.dart +++ b/lib/src/map/controller/map_controller_impl.dart @@ -3,10 +3,10 @@ import 'dart:async'; import 'package:flutter/widgets.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/gestures/map_interactive_viewer.dart'; +import 'package:flutter_map/src/misc/deg_rad_conversions.dart'; import 'package:flutter_map/src/misc/extensions.dart'; import 'package:flutter_map/src/misc/move_and_rotate_result.dart'; import 'package:latlong2/latlong.dart'; -import 'package:vector_math/vector_math_64.dart'; /// Implements [MapController] whilst exposing methods for internal use which /// should not be visible to the user (e.g. for setting the current camera). diff --git a/lib/src/misc/deg_rad_conversions.dart b/lib/src/misc/deg_rad_conversions.dart new file mode 100644 index 000000000..c31c7ea7c --- /dev/null +++ b/lib/src/misc/deg_rad_conversions.dart @@ -0,0 +1,7 @@ +import 'dart:math'; + +/// Constant factor to convert and angle from degrees to radians. +const double degrees2Radians = pi / 180.0; + +/// Constant factor to convert and angle from radians to degrees. +const double radians2Degrees = 180.0 / pi; diff --git a/pubspec.yaml b/pubspec.yaml index 65019902f..e9c935ce0 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -37,7 +37,6 @@ dependencies: meta: ^1.11.0 polylabel: ^1.0.1 proj4dart: ^2.1.0 - vector_math: ^2.1.4 dev_dependencies: flutter_lints: ">=4.0.0 <6.0.0" From cfd187b402aba6eaca11a30794df6affb2318318 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sat, 3 May 2025 13:04:05 +0100 Subject: [PATCH 02/49] Initial caching implementation --- example/lib/misc/tile_providers.dart | 5 +- example/pubspec.lock | 56 ++++ lib/flutter_map.dart | 8 +- .../provider.dart} | 0 .../io_impl_provider.dart} | 0 .../stub_provider.dart} | 0 .../network/native/caching/manager.dart | 250 ++++++++++++++++ .../caching/persistent_registry_workers.dart | 77 +++++ .../native/caching/tile_information.dart | 66 +++++ .../network/native/image_provider_v2.dart | 276 ++++++++++++++++++ .../{ => network}/network_image_provider.dart | 0 .../{ => network}/network_tile_provider.dart | 2 +- pubspec.yaml | 3 + test/full_coverage_test.dart | 10 +- .../network_image_provider_test.dart | 2 +- 15 files changed, 742 insertions(+), 13 deletions(-) rename lib/src/layer/tile_layer/tile_provider/{asset_tile_provider.dart => asset/provider.dart} (100%) rename lib/src/layer/tile_layer/tile_provider/{file_providers/tile_provider_io.dart => file/io_impl_provider.dart} (100%) rename lib/src/layer/tile_layer/tile_provider/{file_providers/tile_provider_stub.dart => file/stub_provider.dart} (100%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart rename lib/src/layer/tile_layer/tile_provider/{ => network}/network_image_provider.dart (100%) rename lib/src/layer/tile_layer/tile_provider/{ => network}/network_tile_provider.dart (98%) diff --git a/example/lib/misc/tile_providers.dart b/example/lib/misc/tile_providers.dart index 8c6db0b27..3ca29a147 100644 --- a/example/lib/misc/tile_providers.dart +++ b/example/lib/misc/tile_providers.dart @@ -1,10 +1,11 @@ import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; +//import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; TileLayer get openStreetMapTileLayer => TileLayer( urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', // Use the recommended flutter_map_cancellable_tile_provider package to // support the cancellation of loading tiles. - tileProvider: CancellableNetworkTileProvider(), + // TODO: change + //tileProvider: CancellableNetworkTileProvider(), ); diff --git a/example/pubspec.lock b/example/pubspec.lock index 0daa550df..79656615d 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -41,6 +41,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.19.1" + crypto: + dependency: transitive + description: + name: crypto + sha256: "1e445881f28f22d6140f181e07737b22f1e099a5e1ff94b0af2f9e4a463f4855" + url: "https://pub.dev" + source: hosted + version: "3.0.6" dart_earcut: dependency: transitive description: @@ -89,6 +97,14 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.1" + fixnum: + dependency: transitive + description: + name: fixnum + sha256: b6dc7065e46c974bc7c5f143080a6764ec7a4be6da1285ececdc37be96de53be + url: "https://pub.dev" + source: hosted + version: "1.1.1" flutter: dependency: "direct main" description: flutter @@ -255,6 +271,30 @@ packages: url: "https://pub.dev" source: hosted version: "1.9.1" + path_provider: + dependency: transitive + description: + name: path_provider + sha256: "50c5dd5b6e1aaf6fb3a78b33f6aa3afca52bf903a8a5298f53101fdaee55bbcd" + url: "https://pub.dev" + source: hosted + version: "2.1.5" + path_provider_android: + dependency: transitive + description: + name: path_provider_android + sha256: d0d310befe2c8ab9e7f393288ccbb11b60c019c6b5afc21973eeee4dda2b35e9 + url: "https://pub.dev" + source: hosted + version: "2.2.17" + path_provider_foundation: + dependency: transitive + description: + name: path_provider_foundation + sha256: "4843174df4d288f5e29185bd6e72a6fbdf5a4a4602717eed565497429f179942" + url: "https://pub.dev" + source: hosted + version: "2.4.1" path_provider_linux: dependency: transitive description: @@ -380,6 +420,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.10.1" + sprintf: + dependency: transitive + description: + name: sprintf + sha256: "1fc9ffe69d4df602376b52949af107d8f5703b77cda567c4d7d86a0693120f23" + url: "https://pub.dev" + source: hosted + version: "7.0.0" stack_trace: dependency: transitive description: @@ -500,6 +548,14 @@ packages: url: "https://pub.dev" source: hosted version: "3.1.4" + uuid: + dependency: transitive + description: + name: uuid + sha256: a5be9ef6618a7ac1e964353ef476418026db906c4facdedaa299b7a2e71690ff + url: "https://pub.dev" + source: hosted + version: "4.5.1" vector_math: dependency: transitive description: diff --git a/lib/flutter_map.dart b/lib/flutter_map.dart index acb68995f..cbedb2c2b 100644 --- a/lib/flutter_map.dart +++ b/lib/flutter_map.dart @@ -43,11 +43,11 @@ export 'package:flutter_map/src/layer/tile_layer/tile_coordinates.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_display.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_image.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_layer.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/asset_tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/file_providers/tile_provider_stub.dart' - if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file_providers/tile_provider_io.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network_tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_provider.dart' + if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_event.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_transformer.dart'; export 'package:flutter_map/src/map/camera/camera.dart'; diff --git a/lib/src/layer/tile_layer/tile_provider/asset_tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/asset/provider.dart similarity index 100% rename from lib/src/layer/tile_layer/tile_provider/asset_tile_provider.dart rename to lib/src/layer/tile_layer/tile_provider/asset/provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/file_providers/tile_provider_io.dart b/lib/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart similarity index 100% rename from lib/src/layer/tile_layer/tile_provider/file_providers/tile_provider_io.dart rename to lib/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/file_providers/tile_provider_stub.dart b/lib/src/layer/tile_layer/tile_provider/file/stub_provider.dart similarity index 100% rename from lib/src/layer/tile_layer/tile_provider/file_providers/tile_provider_stub.dart rename to lib/src/layer/tile_layer/tile_provider/file/stub_provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart new file mode 100644 index 000000000..cb38fcb4b --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -0,0 +1,250 @@ +import 'dart:async'; +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:flutter/foundation.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; + +part 'persistent_registry_workers.dart'; +part 'tile_information.dart'; + +/// Singleton class which manages built-in tile caching on native platforms +/// +/// Built-in tile caching is simple and based on the tile's HTTP headers. +/// +/// > [!IMPORTANT] +/// > Built-in tile caching is not a replacement for caching which can better +/// > guarantee resilience. It should not solely be used where not having +/// > cached tiles may lead to a dangerous situation - for example, offline +/// > mapping. This provides no guarantees as to the safety of cached tiles. +/// +/// By default, caching is performed in a caching directory set by the OS, which +/// may be cleared at any time. +/// +/// The registry used to manage tiles is in JSON. There is no guarantee that the +/// registry will remain valid (not corrupt). A corrupt registry will result in +/// all cached tiles being lost. +/// +/// Tile server URLs which use (for example) API keys create tiles with UUIDs +/// including the volatile part of the URL. If this part of the URL is changed, +/// all tiles previously stored will become in-accessible. +/// +/// The cache does not peristently monitor usage (eg. hits) of the cache. +/// +/// The primary purpose of this caching is to reduce the number of requests +/// to tile servers. +/// +/// --- +/// +/// The singleton is not disposed of. The only time it would make sense to +/// close the singleton is when either: +/// * there are no more tile providers/layers using it +/// * the app is destroyed +/// +/// However, it is more difficult to track the first condition, allbeit possible. +/// More importantly, creating and opening a new instance takes some time, so +/// this should be minimized. +/// +/// It is not possible to detect reliably when the process is stopped*. However, +/// we should ideally close any open file handles after we no longer need them +/// (when the singleton is disposed). Additionally, for performance, opening +/// the persistent registry file once as a [RandomAccessFile] and synchonously +/// writing to it is preferred to repeatedly opening it for async writing. +/// Therefore, a long-living isolate is used. +/// +/// The isolate maintains its own in-memory registry (just as this class does +/// directly). The isolate registry and main registry should remain in sync. +/// The isolate registry is used only for writing to, which then writes +/// to the persistent registry synchronously using an open [RandomAccessFile]. +/// The main registry is used only for reading and writing. The main and isolate +/// registries are populated from the persistent registry when an instance is +/// created. +/// +/// When the program is terminated (or hot-reloaded in Dart), the isolate is +/// usually terminated. This usually results in the file handle being closed as +/// well. Closure of the persistent registry file is important before another +/// manager instance is created - otherwise the lock obtained will prevent +/// the new instance from working correctly. It is assumed the OS closes the +/// open file handles, but that does mean every write to the persistent +/// registry must be flushed. +// TODO: Expose for other providers? +@immutable +class TileCachingManager { + const TileCachingManager._({ + required String cacheDirectory, + required void Function(String uuid, CachedTileInformation? tileInfo) + persistentRegistryWriter, + required Map registry, + }) : _cacheDirectory = cacheDirectory, + _writeToPersistentRegistry = persistentRegistryWriter, + _registry = registry; + + /// Current instance of singleton + /// + /// Completer pattern used to obtain a lock - the first tile loaded takes + /// slightly longer, as it has to create the first instance, so other tiles + /// loaded simultaneously must wait instead of all attempting to create + /// multiple singletons. + static Completer? _instance; + + final String _cacheDirectory; + final void Function(String uuid, CachedTileInformation? tileInfo) + _writeToPersistentRegistry; + final Map _registry; + + /// Returns the current caching instance if one is already available, + /// otherwise create and open a new instance + /// + /// If an instance is already being created, this will wait until that + /// instance is available instead of creating a new one. + /// + /// Returns `null` if an instance does not exist and one could not be created. + static Future getInstanceOrCreate({ + String? cacheDirectory, + }) async { + if (_instance != null) return await _instance!.future; + + _instance = Completer(); + + final Directory resolvedCacheDirectory; + try { + resolvedCacheDirectory = Directory( + p.join( + cacheDirectory ?? + (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ), + ); + } on MissingPlatformDirectoryException { + return null; + } + + try { + await resolvedCacheDirectory.create(recursive: true); + } on FileSystemException { + return null; + } + + final persistentRegistryFilePath = + p.join(resolvedCacheDirectory.absolute.path, 'manager.json'); + final persistentRegistryFile = File(persistentRegistryFilePath); + + final Map registry; + try { + if (await persistentRegistryFile.exists()) { + final parsedCacheManager = await compute( + _parsePersistentRegistryWorker, + persistentRegistryFilePath, + ); + if (parsedCacheManager == null) { + await resolvedCacheDirectory.delete(recursive: true); + await resolvedCacheDirectory.create(recursive: true); + await persistentRegistryFile.create(recursive: true); + registry = HashMap(); + } else { + registry = parsedCacheManager; + } + + //for (final MapEntry(key: uuid, value: tileInfo) in registry.entries) {} + } else { + await persistentRegistryFile.create(recursive: true); + registry = HashMap(); + } + } on FileSystemException { + return null; + } + + final receivePort = ReceivePort(); + try { + await Isolate.spawn( + _persistentRegistryWorkerIsolate, + ( + port: receivePort.sendPort, + persistentRegistryFilePath: persistentRegistryFilePath, + initialRegistry: registry, + ), + ); + } catch (e) { + return null; + } + final workerSendPort = await receivePort.first as SendPort; + + final instance = TileCachingManager._( + cacheDirectory: resolvedCacheDirectory.absolute.path, + persistentRegistryWriter: (uuid, tileInfo) => + workerSendPort.send((uuid: uuid, tileInfo: tileInfo)), + registry: registry, + ); + + _instance!.complete(instance); + return instance; + } + + /// Retrieve a tile from the cache, if it exists + Future< + ({ + Uint8List bytes, + CachedTileInformation tileInfo, + })?> getTile( + String uuid, + ) async { + if (!_registry.containsKey(uuid)) { + unawaited(removeTile(uuid)); + return null; + } + + final tileFile = File(p.join(_cacheDirectory, uuid)); + + try { + return (bytes: await tileFile.readAsBytes(), tileInfo: _registry[uuid]!); + } on FileSystemException { + unawaited(removeTile(uuid)); + return null; + } + } + + /// Add or update a tile in the cache + /// + /// [bytes] is required if the tile is not already cached. + Future putTile( + String uuid, + CachedTileInformation tileInfo, [ + Uint8List? bytes, + ]) async { + if (_registry[uuid] case final existingTileInfo? + when tileInfo == existingTileInfo) { + return; + } + + final tileFile = File(p.join(_cacheDirectory, uuid)); + + if (bytes == null && !await tileFile.exists()) { + // more expensive condition last + throw ArgumentError.notNull('bytes'); + } + + if (bytes != null) { + try { + await tileFile.create(recursive: true); + await tileFile.writeAsBytes(bytes); + } on FileSystemException { + return; + } + } + + _registry[uuid] = tileInfo; + _writeToPersistentRegistry(uuid, tileInfo); + } + + /// Remove a tile from the cache + Future removeTile(String uuid) async { + final tileFile = File(p.join(_cacheDirectory, uuid)); + if (await tileFile.exists()) await tileFile.delete(); + + if (_registry.remove(uuid) == null) return; + _writeToPersistentRegistry(uuid, null); + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart new file mode 100644 index 000000000..f7c69fa44 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart @@ -0,0 +1,77 @@ +part of 'manager.dart'; + +/// Isolate worker which maintains its own registry and sequences writes to +/// the persistent registry +/// +/// See documentation on [TileCachingManager] for more info. +Future _persistentRegistryWorkerIsolate( + ({ + SendPort port, + String persistentRegistryFilePath, + Map initialRegistry, + }) input, +) async { + final registry = input.initialRegistry; + + final writer = + File(input.persistentRegistryFilePath).openSync(mode: FileMode.writeOnly); + + final receivePort = ReceivePort(); + final incomingRegistryUpdates = StreamIterator( + receivePort.map((val) { + final (:uuid, :tileInfo) = + val as ({String uuid, CachedTileInformation? tileInfo}); + + if (tileInfo == null) { + registry.remove(uuid); + return null; + } + registry[uuid] = tileInfo; + return null; + }), + ); + + input.port.send(receivePort.sendPort); + + while (await incomingRegistryUpdates.moveNext()) { + final encoded = jsonEncode(registry); + writer.setPositionSync(0); + writer.writeStringSync(encoded); + writer.flushSync(); + } + + writer.closeSync(); +} + +/// Decode the JSON within the persistent registry into a mapping of tile +/// UUIDs to their [CachedTileInformation]s +/// +/// Should be used within an isolate/[compute]r. +/// +/// If the JSON is invalid or the file cannot be read, this returns null. +HashMap? _parsePersistentRegistryWorker( + String persistentRegistryFilePath, +) { + final String json; + try { + json = File(persistentRegistryFilePath).readAsStringSync(); + } on FileSystemException { + return null; + } + + final Map parsed; + try { + parsed = jsonDecode(json) as Map; + } on FormatException { + return null; + } + + return HashMap.from( + parsed.map( + (key, value) => MapEntry( + key, + CachedTileInformation.fromJson(value as Map), + ), + ), + ); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart new file mode 100644 index 000000000..87c309542 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart @@ -0,0 +1,66 @@ +part of 'manager.dart'; + +/// Metadata about a tile cached with the [TileCachingManager] +@immutable +class CachedTileInformation { + /// Create a new metadata container + /// + /// [lastModifiedLocally] should be set to [DateTime.timestamp]. Other + /// properties should be set based on the tile's HTTP response headers. + const CachedTileInformation({ + required this.lastModifiedLocally, + required this.staleAt, + required this.lastModified, + required this.etag, + }); + + /// Decode metadata from JSON + CachedTileInformation.fromJson(Map json) + : lastModifiedLocally = + HttpDate.parse(json['lastModifiedLocally'] as String), + staleAt = HttpDate.parse(json['staleAt'] as String), + lastModified = json['lastModified'] as String == '' + ? null + : HttpDate.parse(json['lastModified'] as String), + etag = json['etag'] as String == '' ? null : json['etag'] as String; + + /// Used to efficiently allow updates to already cached tiles + /// + /// Must be set to [DateTime.timestamp] when a new tile is cached or a tile + /// is updated. + final DateTime lastModifiedLocally; + + /// The date/time at which the tile becomes stale according to the HTTP spec + final DateTime staleAt; + + /// The tile's [HttpHeaders.lastModifiedHeader] + final DateTime? lastModified; + + /// The tile's [HttpHeaders.etagHeader] + final String? etag; + + /// Whether the tile is currently stale + bool get isStale => DateTime.timestamp().isAfter(staleAt); + + /// Convert the metadata to JSON + Map toJson() => { + 'lastModifiedLocally': HttpDate.format(lastModifiedLocally), + 'staleAt': HttpDate.format(staleAt), + 'lastModified': + lastModified == null ? '' : HttpDate.format(lastModified!), + 'etag': etag == null ? '' : etag!, + }; + + @override + int get hashCode => + Object.hash(lastModifiedLocally, staleAt, lastModified, etag); + + @override + bool operator ==(Object other) => + identical(this, other) || + (other is CachedTileInformation && + lastModifiedLocally == other.lastModifiedLocally && + staleAt == other.staleAt && + lastModified == other.lastModified && + etag == other.etag); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart b/lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart new file mode 100644 index 000000000..48f5ca848 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart @@ -0,0 +1,276 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:math'; +import 'dart:ui'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/painting.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; +import 'package:http/http.dart'; +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; +import 'package:uuid/uuid.dart'; + +/// Dedicated [ImageProvider] to fetch tiles from the network +/// +/// Supports falling back to a secondary URL, if the primary URL fetch fails. +/// Note that specifying a [fallbackUrl] will prevent this image provider from +/// being cached. +@immutable +class MapNetworkImageProviderv2 + extends ImageProvider { + /// The URL to fetch the tile from (GET request) + final String url; + + /// The URL to fetch the tile from (GET request), in the event the original + /// [url] request fails + /// + /// If this is non-null, [operator==] will always return `false` (except if + /// the two objects are [identical]). Therefore, if this is non-null, this + /// image provider will not be cached in memory. + final String? fallbackUrl; + + /// The headers to include with the tile fetch request + /// + /// Not included in [operator==]. + final Map headers; + + /// The HTTP client to use to make network requests + /// + /// Not included in [operator==]. + final Client httpClient; + + /// Whether to ignore exceptions and errors that occur whilst fetching tiles + /// over the network, and just return a transparent tile + final bool silenceExceptions; + + /// Function invoked when the image starts loading (not from cache) + /// + /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only + /// after all tiles have loaded. + final void Function() startedLoading; + + /// Function invoked when the image completes loading bytes from the network + /// + /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only + /// after all tiles have loaded. + final void Function() finishedLoadingBytes; + + /// Create a dedicated [ImageProvider] to fetch tiles from the network + /// + /// Supports falling back to a secondary URL, if the primary URL fetch fails. + /// Note that specifying a [fallbackUrl] will prevent this image provider from + /// being cached. + const MapNetworkImageProviderv2({ + required this.url, + required this.fallbackUrl, + required this.headers, + required this.httpClient, + required this.silenceExceptions, + required this.startedLoading, + required this.finishedLoadingBytes, + }); + + static final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + + @override + ImageStreamCompleter loadImage( + MapNetworkImageProviderv2 key, + ImageDecoderCallback decode, + ) => + MultiFrameImageStreamCompleter( + codec: _load(key, decode), + scale: 1, + debugLabel: url, + informationCollector: () => [ + DiagnosticsProperty('URL', url), + DiagnosticsProperty('Fallback URL', fallbackUrl), + DiagnosticsProperty('Current provider', key), + ], + ); + + Future _load( + MapNetworkImageProviderv2 key, + ImageDecoderCallback decode, { + bool useFallback = false, + }) async { + startedLoading(); + + final resolvedUrl = useFallback ? fallbackUrl ?? '' : url; + final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); + + // TODO: Allow disabling caching + final cachingManager = (await TileCachingManager.getInstanceOrCreate())!; + // TODO: Remove force null check, then fallback to non-caching + + final cachedTile = await cachingManager.getTile(uuid); + + Future handleOk(Response response) async { + final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; + final etag = response.headers[HttpHeaders.etagHeader]; + + cachingManager.putTile( + uuid, + CachedTileInformation( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt(response), + lastModified: + lastModified != null ? HttpDate.parse(lastModified) : null, + etag: etag, + ), + response.bodyBytes, + ); + + finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); + } + + Future handleNotOk(Response response) async { + // Optimistically try to decode the response anyway + try { + finishedLoadingBytes(); + return await decode( + await ImmutableBuffer.fromUint8List(response.bodyBytes), + ); + } catch (err) { + // Otherwise fallback to a cached tile if we have one + if (cachedTile != null) { + finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + // Otherwise fallback to the fallback URL + if (!useFallback && fallbackUrl != null) { + finishedLoadingBytes(); + return _load(key, decode, useFallback: true); + } + + // Otherwise throw an exception/silently fail + if (!silenceExceptions) { + finishedLoadingBytes(); + throw HttpException( + 'Recieved ${response.statusCode}, and body was not a decodable image', + uri: Uri.parse(resolvedUrl), + ); + } + + finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) + .then(decode); + } finally { + scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); + } + } + + if (cachedTile != null) { + // If we have a cached tile that's not stale, return it + if (!cachedTile.tileInfo.isStale) { + finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + // Otherwise, ask the server what's going on - supply any details we have + final response = await httpClient.get( + Uri.parse(resolvedUrl), + headers: { + ...headers, + if (cachedTile.tileInfo.lastModified case final lastModified?) + HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), + if (cachedTile.tileInfo.etag case final etag?) + HttpHeaders.ifNoneMatchHeader: etag, + }, + ); + + // Server says nothing's changed - but might return new useful headers + if (response.statusCode == HttpStatus.notModified) { + final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; + final etag = response.headers[HttpHeaders.etagHeader]; + + cachingManager.putTile( + uuid, + CachedTileInformation( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt(response), + lastModified: lastModified != null + ? HttpDate.parse(lastModified) + : cachedTile.tileInfo.lastModified, + etag: etag ?? cachedTile.tileInfo.etag, + ), + ); + + finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + if (response.statusCode == HttpStatus.ok) { + return await handleOk(response); + } + return await handleNotOk(response); + } + + final response = await httpClient.get( + Uri.parse(resolvedUrl), + headers: headers, + ); + + if (response.statusCode == HttpStatus.ok) { + return await handleOk(response); + } + return await handleNotOk(response); + } + + static DateTime _calculateStaleAt(Response response) { + final addToNow = DateTime.timestamp().add; + + if (response.headers[HttpHeaders.cacheControlHeader]?.toLowerCase() + case final cacheControl?) { + final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; + + if (maxAge == null) { + if (response.headers[HttpHeaders.expiresHeader]?.toLowerCase() + case final expires?) { + return HttpDate.parse(expires); + } + return addToNow(const Duration(days: 7)); + } else { + if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { + return addToNow( + Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), + ); + } + + final estimatedAge = max( + 0, + DateTime.timestamp() + .difference( + HttpDate.parse(response.headers[HttpHeaders.dateHeader]!), + ) + .inSeconds, + ); + return addToNow( + Duration(seconds: int.parse(maxAge) - estimatedAge), + ); + } + } else { + return addToNow(const Duration(days: 7)); + } + } + + @override + SynchronousFuture obtainKey( + ImageConfiguration configuration, + ) => + SynchronousFuture(this); + + @override + bool operator ==(Object other) => + identical(this, other) || + (other is MapNetworkImageProviderv2 && + fallbackUrl == null && + url == other.url); + + @override + int get hashCode => + Object.hashAll([url, if (fallbackUrl != null) fallbackUrl]); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network_image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/network_image_provider.dart similarity index 100% rename from lib/src/layer/tile_layer/tile_provider/network_image_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/network_image_provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/network_tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart similarity index 98% rename from lib/src/layer/tile_layer/tile_provider/network_tile_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart index 8b9fd8ba8..5e2a9c6c7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network_tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart @@ -3,7 +3,7 @@ import 'dart:collection'; import 'package:flutter/rendering.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network_image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; import 'package:http/http.dart'; import 'package:http/retry.dart'; diff --git a/pubspec.yaml b/pubspec.yaml index e9c935ce0..c6c187e8f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -35,8 +35,11 @@ dependencies: latlong2: ^0.9.1 logger: ^2.0.0 meta: ^1.11.0 + path: ^1.9.1 + path_provider: ^2.1.5 polylabel: ^1.0.1 proj4dart: ^2.1.0 + uuid: ^4.5.1 dev_dependencies: flutter_lints: ">=4.0.0 <6.0.0" diff --git a/test/full_coverage_test.dart b/test/full_coverage_test.dart index 3e1f687bd..9ae6f6090 100644 --- a/test/full_coverage_test.dart +++ b/test/full_coverage_test.dart @@ -30,12 +30,12 @@ import 'package:flutter_map/src/layer/tile_layer/tile_image.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_image_manager.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_image_view.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_layer.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/asset_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/file_providers/tile_provider_io.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/file_providers/tile_provider_stub.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network_image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range_calculator.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_scale_calculator.dart'; diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index 50a50779f..b125fcc6e 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -4,7 +4,7 @@ import 'dart:typed_data'; import 'package:flutter/painting.dart'; import 'package:flutter/widgets.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network_image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:http/http.dart'; import 'package:mocktail/mocktail.dart'; From 6007304f386e7050ce6b6295873b0735682ada31 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 4 May 2025 13:01:32 +0100 Subject: [PATCH 03/49] Improved performance of persistent registry writer & format efficiency Fixed bugs in persistent registry writer Added cache size limiter Added `MapCachingOptions` to configure caching externally Minor other renaming & refactoring --- .../network/native/caching/manager.dart | 52 +++++-- .../network/native/caching/options.dart | 42 ++++++ .../caching/persistent_registry_workers.dart | 130 ++++++++++++++---- .../native/caching/tile_information.dart | 20 +-- ...e_provider_v2.dart => image_provider.dart} | 28 ++-- .../network/native/network_tile_provider.dart | 106 ++++++++++++++ 6 files changed, 324 insertions(+), 54 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart rename lib/src/layer/tile_layer/tile_provider/network/native/{image_provider_v2.dart => image_provider.dart} (91%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index cb38fcb4b..495a2661b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -5,6 +5,8 @@ import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -72,8 +74,9 @@ part 'tile_information.dart'; /// registry must be flushed. // TODO: Expose for other providers? @immutable -class TileCachingManager { - const TileCachingManager._({ +@internal +class MapTileCachingManager { + const MapTileCachingManager._({ required String cacheDirectory, required void Function(String uuid, CachedTileInformation? tileInfo) persistentRegistryWriter, @@ -82,13 +85,15 @@ class TileCachingManager { _writeToPersistentRegistry = persistentRegistryWriter, _registry = registry; + static const _persistentRegistryFileName = 'manager.json'; + /// Current instance of singleton /// /// Completer pattern used to obtain a lock - the first tile loaded takes /// slightly longer, as it has to create the first instance, so other tiles /// loaded simultaneously must wait instead of all attempting to create /// multiple singletons. - static Completer? _instance; + static Completer? _instance; final String _cacheDirectory; final void Function(String uuid, CachedTileInformation? tileInfo) @@ -101,19 +106,24 @@ class TileCachingManager { /// If an instance is already being created, this will wait until that /// instance is available instead of creating a new one. /// + /// [options] is only used to configure a new instance. If [options] changes, + /// a new instance will not be created. + /// /// Returns `null` if an instance does not exist and one could not be created. - static Future getInstanceOrCreate({ - String? cacheDirectory, + static Future getInstanceOrCreate({ + MapCachingOptions? options, }) async { if (_instance != null) return await _instance!.future; _instance = Completer(); + options ??= const MapCachingOptions(); + final Directory resolvedCacheDirectory; try { resolvedCacheDirectory = Directory( p.join( - cacheDirectory ?? + options.cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, 'fm_cache', ), @@ -128,8 +138,10 @@ class TileCachingManager { return null; } - final persistentRegistryFilePath = - p.join(resolvedCacheDirectory.absolute.path, 'manager.json'); + final persistentRegistryFilePath = p.join( + resolvedCacheDirectory.absolute.path, + _persistentRegistryFileName, + ); final persistentRegistryFile = File(persistentRegistryFilePath); final Map registry; @@ -146,9 +158,21 @@ class TileCachingManager { registry = HashMap(); } else { registry = parsedCacheManager; - } - //for (final MapEntry(key: uuid, value: tileInfo) in registry.entries) {} + if (options.maxCacheSize case final sizeLimit?) { + // This can cause some delay when creating + // But it's much better than lagging or inconsistent registries + (await compute( + _limitCacheSizeWorker, + ( + cacheDirectoryPath: resolvedCacheDirectory.absolute.path, + persistentRegistryFileName: _persistentRegistryFileName, + sizeLimit: sizeLimit, + ), + )) + .forEach(registry.remove); + } + } } else { await persistentRegistryFile.create(recursive: true); registry = HashMap(); @@ -172,7 +196,7 @@ class TileCachingManager { } final workerSendPort = await receivePort.first as SendPort; - final instance = TileCachingManager._( + final instance = MapTileCachingManager._( cacheDirectory: resolvedCacheDirectory.absolute.path, persistentRegistryWriter: (uuid, tileInfo) => workerSendPort.send((uuid: uuid, tileInfo: tileInfo)), @@ -192,7 +216,7 @@ class TileCachingManager { String uuid, ) async { if (!_registry.containsKey(uuid)) { - unawaited(removeTile(uuid)); + unawaited(_removeTile(uuid)); return null; } @@ -201,7 +225,7 @@ class TileCachingManager { try { return (bytes: await tileFile.readAsBytes(), tileInfo: _registry[uuid]!); } on FileSystemException { - unawaited(removeTile(uuid)); + unawaited(_removeTile(uuid)); return null; } } @@ -240,7 +264,7 @@ class TileCachingManager { } /// Remove a tile from the cache - Future removeTile(String uuid) async { + Future _removeTile(String uuid) async { final tileFile = File(p.join(_cacheDirectory, uuid)); if (await tileFile.exists()) await tileFile.delete(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart new file mode 100644 index 000000000..7c8fce261 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart @@ -0,0 +1,42 @@ +import 'package:meta/meta.dart'; + +/// Configuration of the built-in caching +@immutable +class MapCachingOptions { + /// Path to the caching directory to use + /// + /// This must be accessible to the program. + /// + /// Defaults to a platform provided temporary directory. + final String? cacheDirectory; + + /// Preferred maximum size (in bytes) of the cache + /// + /// This is applied when the internal caching mechanism is created (on the + /// first tile load in the main memory space for the app). It is not an + /// absolute limit. + /// + /// This may cause some slight delay to the loading of the first tiles, + /// especially if the size is large and the cache does exceed the size. If + /// the visible delay becomes too large, disable this and manage the cache + /// size manually if necessary. + /// + /// Defaults to 1GB. Set to `null` to disable. + final int? maxCacheSize; + + /// Create a configuration for caching + const MapCachingOptions({ + this.cacheDirectory, + this.maxCacheSize = 1000000000, + }); + + @override + int get hashCode => Object.hash(cacheDirectory, maxCacheSize); + + @override + bool operator ==(Object other) => + identical(this, other) || + (other is MapCachingOptions && + cacheDirectory == other.cacheDirectory && + maxCacheSize == other.maxCacheSize); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart index f7c69fa44..cb13b2a6b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart @@ -3,7 +3,17 @@ part of 'manager.dart'; /// Isolate worker which maintains its own registry and sequences writes to /// the persistent registry /// -/// See documentation on [TileCachingManager] for more info. +/// We cannot use [IOSink] from [File.openWrite], since we need to overwrite the +/// entire file on every write. [RandomAccessFile] allows this, and may also be +/// faster (especially for sync operations). However, it does not sequence +/// writes as [IOSink] does: attempting multiple writes at the same time throws +/// errors. If we use sync operations on every incoming update, this shouldn't +/// be an issue - instead, we use a debouncer (at 50ms, which is small enough +/// that the user should not usually terminate the isolate very close to loading +/// tiles, but also small enough to group adjacent tile loads), so manual +/// sequencing and locking is required. +/// +/// See documentation on [MapTileCachingManager] for more info. Future _persistentRegistryWorkerIsolate( ({ SendPort port, @@ -12,35 +22,48 @@ Future _persistentRegistryWorkerIsolate( }) input, ) async { final registry = input.initialRegistry; + final writer = File(input.persistentRegistryFilePath) + .openSync(mode: FileMode.writeOnlyAppend); - final writer = - File(input.persistentRegistryFilePath).openSync(mode: FileMode.writeOnly); + var writeLocker = Completer()..complete(); + var alreadyWaitingToWrite = false; + Future write() async { + if (alreadyWaitingToWrite) return; + alreadyWaitingToWrite = true; + await writeLocker.future; + writeLocker = Completer(); + alreadyWaitingToWrite = false; - final receivePort = ReceivePort(); - final incomingRegistryUpdates = StreamIterator( - receivePort.map((val) { - final (:uuid, :tileInfo) = - val as ({String uuid, CachedTileInformation? tileInfo}); - - if (tileInfo == null) { - registry.remove(uuid); - return null; - } - registry[uuid] = tileInfo; - return null; - }), - ); + final encoded = jsonEncode(registry); + writer + ..setPositionSync(0) + ..writeStringSync(encoded) + ..truncateSync(writer.positionSync()) + ..flushSync(); + + writeLocker.complete(); + } + + Timer createWriteDebouncer() => + Timer(const Duration(milliseconds: 50), write); + Timer? writeDebouncer; + final receivePort = ReceivePort(); input.port.send(receivePort.sendPort); - while (await incomingRegistryUpdates.moveNext()) { - final encoded = jsonEncode(registry); - writer.setPositionSync(0); - writer.writeStringSync(encoded); - writer.flushSync(); - } + await for (final val in receivePort) { + final (:uuid, :tileInfo) = + val as ({String uuid, CachedTileInformation? tileInfo}); - writer.closeSync(); + if (tileInfo == null) { + registry.remove(uuid); + } else { + registry[uuid] = tileInfo; + } + + writeDebouncer?.cancel(); + writeDebouncer = createWriteDebouncer(); + } } /// Decode the JSON within the persistent registry into a mapping of tile @@ -75,3 +98,62 @@ HashMap? _parsePersistentRegistryWorker( ), ); } + +/// Remove tile files from the cache directory, 'first'-modified and largest +/// first, until the total size is below the set limit +/// +/// Returns removed tile UUIDs. +/// +/// This does not alter any registries in memory. +Future> _limitCacheSizeWorker( + ({ + String cacheDirectoryPath, + String persistentRegistryFileName, + int sizeLimit + }) input, +) async { + final cacheDirectory = Directory(input.cacheDirectoryPath); + + final currentCacheSize = await cacheDirectory + .list() + .fold(0, (sum, file) => sum + file.statSync().size); + if (currentCacheSize <= input.sizeLimit) return []; + + final mapping = + SplayTreeMap>(); + bool foundManager = false; + await for (final file in cacheDirectory.list()) { + if (file is! File) continue; + if (!foundManager && + p.basename(file.absolute.path) == input.persistentRegistryFileName) { + foundManager = true; + continue; + } + + final FileStat stat; + try { + stat = file.statSync(); + } on FileSystemException { + return []; + } + + (mapping[stat.modified] ??= []) // `stat.accessed` is unreliable + .add((file: file, uuid: p.basename(file.path), size: stat.size)); + } + + // Delete largest oldest files first + int collectedSize = 0; + final collectedUuids = []; + outer: + for (final MapEntry(key: _, value: files) in mapping.entries) { + files.sort((a, b) => b.size.compareTo(a.size)); + for (final (:file, :uuid, :size) in files) { + collectedUuids.add(uuid); + collectedSize += size; + file.deleteSync(); + if (currentCacheSize - collectedSize <= input.sizeLimit) break outer; + } + } + + return collectedUuids; +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart index 87c309542..43ed69785 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart @@ -1,7 +1,8 @@ part of 'manager.dart'; -/// Metadata about a tile cached with the [TileCachingManager] +/// Metadata about a tile cached with the [MapTileCachingManager] @immutable +@internal class CachedTileInformation { /// Create a new metadata container /// @@ -19,10 +20,13 @@ class CachedTileInformation { : lastModifiedLocally = HttpDate.parse(json['lastModifiedLocally'] as String), staleAt = HttpDate.parse(json['staleAt'] as String), - lastModified = json['lastModified'] as String == '' - ? null - : HttpDate.parse(json['lastModified'] as String), - etag = json['etag'] as String == '' ? null : json['etag'] as String; + lastModified = json.containsKey('lastModified') && + (json['lastModified'] as String).isNotEmpty + ? HttpDate.parse(json['lastModified'] as String) + : null, + etag = json.containsKey('etag') && (json['etag'] as String).isNotEmpty + ? json['etag'] as String + : null; /// Used to efficiently allow updates to already cached tiles /// @@ -46,9 +50,9 @@ class CachedTileInformation { Map toJson() => { 'lastModifiedLocally': HttpDate.format(lastModifiedLocally), 'staleAt': HttpDate.format(staleAt), - 'lastModified': - lastModified == null ? '' : HttpDate.format(lastModified!), - 'etag': etag == null ? '' : etag!, + if (lastModified != null) + 'lastModified': HttpDate.format(lastModified!), + if (etag != null) 'etag': etag!, }; @override diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart b/lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart similarity index 91% rename from lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart rename to lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart index 48f5ca848..29ff64555 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/image_provider_v2.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart @@ -7,6 +7,7 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; import 'package:http/http.dart'; import 'package:uuid/data.dart'; import 'package:uuid/rng.dart'; @@ -18,8 +19,8 @@ import 'package:uuid/uuid.dart'; /// Note that specifying a [fallbackUrl] will prevent this image provider from /// being cached. @immutable -class MapNetworkImageProviderv2 - extends ImageProvider { +class CachingNetworkTileImageProvider + extends ImageProvider { /// The URL to fetch the tile from (GET request) final String url; @@ -45,6 +46,13 @@ class MapNetworkImageProviderv2 /// over the network, and just return a transparent tile final bool silenceExceptions; + /// Configuration of built-in caching + /// + /// See online documentation for more information about built-in caching. + /// + /// Set to `null` to disable. See [MapCachingOptions] for defaults. + final MapCachingOptions? cachingOptions; + /// Function invoked when the image starts loading (not from cache) /// /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only @@ -62,12 +70,13 @@ class MapNetworkImageProviderv2 /// Supports falling back to a secondary URL, if the primary URL fetch fails. /// Note that specifying a [fallbackUrl] will prevent this image provider from /// being cached. - const MapNetworkImageProviderv2({ + const CachingNetworkTileImageProvider({ required this.url, required this.fallbackUrl, required this.headers, required this.httpClient, required this.silenceExceptions, + required this.cachingOptions, required this.startedLoading, required this.finishedLoadingBytes, }); @@ -76,7 +85,7 @@ class MapNetworkImageProviderv2 @override ImageStreamCompleter loadImage( - MapNetworkImageProviderv2 key, + CachingNetworkTileImageProvider key, ImageDecoderCallback decode, ) => MultiFrameImageStreamCompleter( @@ -91,7 +100,7 @@ class MapNetworkImageProviderv2 ); Future _load( - MapNetworkImageProviderv2 key, + CachingNetworkTileImageProvider key, ImageDecoderCallback decode, { bool useFallback = false, }) async { @@ -101,7 +110,9 @@ class MapNetworkImageProviderv2 final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); // TODO: Allow disabling caching - final cachingManager = (await TileCachingManager.getInstanceOrCreate())!; + final cachingManager = (await MapTileCachingManager.getInstanceOrCreate( + options: cachingOptions, + ))!; // TODO: Remove force null check, then fallback to non-caching final cachedTile = await cachingManager.getTile(uuid); @@ -166,6 +177,7 @@ class MapNetworkImageProviderv2 if (cachedTile != null) { // If we have a cached tile that's not stale, return it if (!cachedTile.tileInfo.isStale) { + print('from ache'); finishedLoadingBytes(); return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); } @@ -258,7 +270,7 @@ class MapNetworkImageProviderv2 } @override - SynchronousFuture obtainKey( + SynchronousFuture obtainKey( ImageConfiguration configuration, ) => SynchronousFuture(this); @@ -266,7 +278,7 @@ class MapNetworkImageProviderv2 @override bool operator ==(Object other) => identical(this, other) || - (other is MapNetworkImageProviderv2 && + (other is CachingNetworkTileImageProvider && fallbackUrl == null && url == other.url); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart new file mode 100644 index 000000000..d7091acc2 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart @@ -0,0 +1,106 @@ +import 'dart:async'; +import 'dart:collection'; + +import 'package:flutter/rendering.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; +import 'package:http/http.dart'; +import 'package:http/retry.dart'; + +/// [TileProvider] to fetch tiles from the network +/// +/// By default, a [RetryClient] is used to retry failed requests. 'dart:http' +/// or 'dart:io' might be needed to override this. +/// +/// On the web, the 'User-Agent' header cannot be changed as specified in +/// [TileLayer.tileProvider]'s documentation, due to a Dart/browser limitation. +/// +/// Does not support cancellation of tile loading via +/// [TileProvider.getImageWithCancelLoadingSupport], as abortion of in-flight +/// HTTP requests on the web is +/// [not yet supported in Dart](https://github.com/dart-lang/http/issues/424). +class NetworkTileProvider extends TileProvider { + /// [TileProvider] to fetch tiles from the network + /// + /// By default, a [RetryClient] is used to retry failed requests. 'dart:http' + /// or 'dart:io' might be needed to override this. + /// + /// On the web, the 'User-Agent' header cannot be changed, as specified in + /// [TileLayer.tileProvider]'s documentation, due to a Dart/browser limitation. + /// + /// Does not support cancellation of tile loading via + /// [TileProvider.getImageWithCancelLoadingSupport], as abortion of in-flight + /// HTTP requests on the web is + /// [not yet supported in Dart](https://github.com/dart-lang/http/issues/424). + NetworkTileProvider({ + super.headers, + Client? httpClient, + this.silenceExceptions = false, + this.cachingOptions, + }) : _isInternallyCreatedClient = httpClient == null, + _httpClient = httpClient ?? RetryClient(Client()); + + /// Whether to ignore exceptions and errors that occur whilst fetching tiles + /// over the network, and just return a transparent tile + final bool silenceExceptions; + + /// Configuration of built-in caching + /// + /// See online documentation for more information about built-in caching. + /// + /// Changing the configuration whilst caching is already working (after the + /// first tile has been loaded) is not supported. Changing the configuration + /// whilst a cache already exists is supported, but changing the + /// [MapCachingOptions.cacheDirectory] will not remove the old directory. + /// + /// Set to `null` to disable. See [MapCachingOptions] for defaults. + final MapCachingOptions? cachingOptions; + + /// Long living client used to make all tile requests by + /// [MapNetworkImageProvider] for the duration that this provider is + /// alive + /// + /// Not automatically closed if created externally and passed as an argument + /// during construction. + final Client _httpClient; + + /// Whether [_httpClient] was created on construction (and not passed in) + final bool _isInternallyCreatedClient; + + /// Each [Completer] is completed once the corresponding tile has finished + /// loading + /// + /// Used to avoid disposing of [_httpClient] whilst HTTP requests are still + /// underway. + /// + /// Does not include tiles loaded from session cache. + final _tilesInProgress = HashMap>(); + + @override + ImageProvider getImage(TileCoordinates coordinates, TileLayer options) => + CachingNetworkTileImageProvider( + url: getTileUrl(coordinates, options), + fallbackUrl: getTileFallbackUrl(coordinates, options), + headers: headers, + httpClient: _httpClient, + silenceExceptions: silenceExceptions, + cachingOptions: cachingOptions, + startedLoading: () => _tilesInProgress[coordinates] = Completer(), + finishedLoadingBytes: () { + _tilesInProgress[coordinates]?.complete(); + _tilesInProgress.remove(coordinates); + }, + ); + + @override + Future dispose() async { + if (_tilesInProgress.isNotEmpty) { + await Future.wait(_tilesInProgress.values.map((c) => c.future)); + } + if (_isInternallyCreatedClient) _httpClient.close(); + + super.dispose(); + } +} From 6fe737ca0a593fbcda153a71591d19582b499f50 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 4 May 2025 17:02:48 +0100 Subject: [PATCH 04/49] Refactored to re-enable web support Added `MapCachingOptions.overrideFreshAge` --- example/lib/misc/tile_providers.dart | 6 +- lib/flutter_map.dart | 7 +- ...rovider.dart => native_tile_provider.dart} | 0 ..._provider.dart => stub_tile_provider.dart} | 4 +- .../caching/options.dart | 18 +- .../image_provider.dart} | 51 ++-- .../network/independent/tile_loader.dart | 14 + .../tile_provider.dart} | 15 +- .../network/native/caching/manager.dart | 16 +- .../native/caching/tile_information.dart | 1 - .../network/native/image_provider.dart | 288 ------------------ .../network/native/network_tile_provider.dart | 106 ------- .../network/native/tile_loader.dart | 211 +++++++++++++ .../network/web/tile_loader.dart | 49 +++ test/full_coverage_test.dart | 8 +- .../network_image_provider_test.dart | 41 ++- 16 files changed, 375 insertions(+), 460 deletions(-) rename lib/src/layer/tile_layer/tile_provider/file/{io_impl_provider.dart => native_tile_provider.dart} (100%) rename lib/src/layer/tile_layer/tile_provider/file/{stub_provider.dart => stub_tile_provider.dart} (85%) rename lib/src/layer/tile_layer/tile_provider/network/{native => independent}/caching/options.dart (66%) rename lib/src/layer/tile_layer/tile_provider/network/{network_image_provider.dart => independent/image_provider.dart} (73%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart rename lib/src/layer/tile_layer/tile_provider/network/{network_tile_provider.dart => independent/tile_provider.dart} (87%) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart diff --git a/example/lib/misc/tile_providers.dart b/example/lib/misc/tile_providers.dart index 3ca29a147..e0ea30153 100644 --- a/example/lib/misc/tile_providers.dart +++ b/example/lib/misc/tile_providers.dart @@ -1,11 +1,15 @@ import 'package:flutter_map/flutter_map.dart'; +import 'package:http/http.dart'; +import 'package:http/retry.dart'; //import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; +final httpClient = RetryClient(Client()); + TileLayer get openStreetMapTileLayer => TileLayer( urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', // Use the recommended flutter_map_cancellable_tile_provider package to // support the cancellation of loading tiles. // TODO: change - //tileProvider: CancellableNetworkTileProvider(), + tileProvider: NetworkTileProvider(httpClient: httpClient), ); diff --git a/lib/flutter_map.dart b/lib/flutter_map.dart index cbedb2c2b..f6e81fe4e 100644 --- a/lib/flutter_map.dart +++ b/lib/flutter_map.dart @@ -45,9 +45,10 @@ export 'package:flutter_map/src/layer/tile_layer/tile_image.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_layer.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_provider.dart' - if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart' + if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_event.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_transformer.dart'; export 'package:flutter_map/src/map/camera/camera.dart'; diff --git a/lib/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart b/lib/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart similarity index 100% rename from lib/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart rename to lib/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart diff --git a/lib/src/layer/tile_layer/tile_provider/file/stub_provider.dart b/lib/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart similarity index 85% rename from lib/src/layer/tile_layer/tile_provider/file/stub_provider.dart rename to lib/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart index d6d92121b..9d25e3310 100644 --- a/lib/src/layer/tile_layer/tile_provider/file/stub_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart @@ -21,5 +21,7 @@ class FileTileProvider extends TileProvider { @override ImageProvider getImage(TileCoordinates coordinates, TileLayer options) => throw UnsupportedError( - 'The current platform does not have access to IO (the local filesystem), and therefore does not support `FileTileProvider`'); + 'The current platform does not have access to IO (the local ' + 'filesystem), and therefore does not support `FileTileProvider`', + ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart similarity index 66% rename from lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart rename to lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart index 7c8fce261..9c66362c3 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/options.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart @@ -24,11 +24,24 @@ class MapCachingOptions { /// Defaults to 1GB. Set to `null` to disable. final int? maxCacheSize; + /// Override the duration of time a tile is considered fresh for + /// + /// Defaults to `null`: use duration calculated from each tile's HTTP headers. + final Duration? overrideFreshAge; + /// Create a configuration for caching const MapCachingOptions({ this.cacheDirectory, this.maxCacheSize = 1000000000, - }); + this.overrideFreshAge, + }) : assert( + maxCacheSize == null || maxCacheSize > 0, + '`maxCacheSize` must be greater than 0 or disabled', + ), + assert( + overrideFreshAge == null || overrideFreshAge > Duration.zero, + '`overrideFreshAge` must be greater than 0 or disabled', + ); @override int get hashCode => Object.hash(cacheDirectory, maxCacheSize); @@ -38,5 +51,6 @@ class MapCachingOptions { identical(this, other) || (other is MapCachingOptions && cacheDirectory == other.cacheDirectory && - maxCacheSize == other.maxCacheSize); + maxCacheSize == other.maxCacheSize && + overrideFreshAge == other.overrideFreshAge); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/network_image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart similarity index 73% rename from lib/src/layer/tile_layer/tile_provider/network/network_image_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart index 0a2a3d02c..46a750d48 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/network_image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart @@ -4,6 +4,9 @@ import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart' + if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart' + if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; import 'package:http/http.dart'; /// Dedicated [ImageProvider] to fetch tiles from the network @@ -12,7 +15,7 @@ import 'package:http/http.dart'; /// Note that specifying a [fallbackUrl] will prevent this image provider from /// being cached. @immutable -class MapNetworkImageProvider extends ImageProvider { +class NetworkTileImageProvider extends ImageProvider { /// The URL to fetch the tile from (GET request) final String url; @@ -36,8 +39,20 @@ class MapNetworkImageProvider extends ImageProvider { /// Whether to ignore exceptions and errors that occur whilst fetching tiles /// over the network, and just return a transparent tile + /// + /// Not included in [operator==]. final bool silenceExceptions; + /// Configuration of built-in caching on native platforms + /// + /// See online documentation for more information about built-in caching. + /// + /// Set to `null` to disable. See [MapCachingOptions] for defaults. Caching + /// is always disabled on the web. + /// + /// Not included in [operator==]. + final MapCachingOptions? cachingOptions; + /// Function invoked when the image starts loading (not from cache) /// /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only @@ -55,19 +70,20 @@ class MapNetworkImageProvider extends ImageProvider { /// Supports falling back to a secondary URL, if the primary URL fetch fails. /// Note that specifying a [fallbackUrl] will prevent this image provider from /// being cached. - const MapNetworkImageProvider({ + const NetworkTileImageProvider({ required this.url, required this.fallbackUrl, required this.headers, required this.httpClient, required this.silenceExceptions, + required this.cachingOptions, required this.startedLoading, required this.finishedLoadingBytes, }); @override ImageStreamCompleter loadImage( - MapNetworkImageProvider key, + NetworkTileImageProvider key, ImageDecoderCallback decode, ) => MultiFrameImageStreamCompleter( @@ -82,33 +98,14 @@ class MapNetworkImageProvider extends ImageProvider { ); Future _load( - MapNetworkImageProvider key, + NetworkTileImageProvider key, ImageDecoderCallback decode, { bool useFallback = false, - }) { - startedLoading(); - - return httpClient - .readBytes( - Uri.parse(useFallback ? fallbackUrl ?? '' : url), - headers: headers, - ) - .whenComplete(finishedLoadingBytes) - .then(ImmutableBuffer.fromUint8List) - .then(decode) - .onError((err, stack) { - scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); - if (useFallback || fallbackUrl == null) { - if (!silenceExceptions) throw err; - return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) - .then(decode); - } - return _load(key, decode, useFallback: true); - }); - } + }) => + loadTileImage(key, decode, useFallback: useFallback); @override - SynchronousFuture obtainKey( + SynchronousFuture obtainKey( ImageConfiguration configuration, ) => SynchronousFuture(this); @@ -116,7 +113,7 @@ class MapNetworkImageProvider extends ImageProvider { @override bool operator ==(Object other) => identical(this, other) || - (other is MapNetworkImageProvider && + (other is NetworkTileImageProvider && fallbackUrl == null && url == other.url); diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart new file mode 100644 index 000000000..d25b6f86b --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart @@ -0,0 +1,14 @@ +import 'dart:ui'; + +import 'package:flutter/painting.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; +import 'package:meta/meta.dart'; + +@internal +Future loadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) => + simpleLoadTileImage(key, decode, useFallback: useFallback); diff --git a/lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart similarity index 87% rename from lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart index 5e2a9c6c7..1a98f555c 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart @@ -3,7 +3,7 @@ import 'dart:collection'; import 'package:flutter/rendering.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; import 'package:http/http.dart'; import 'package:http/retry.dart'; @@ -36,6 +36,7 @@ class NetworkTileProvider extends TileProvider { super.headers, Client? httpClient, this.silenceExceptions = false, + this.cachingOptions = const MapCachingOptions(), }) : _isInternallyCreatedClient = httpClient == null, _httpClient = httpClient ?? RetryClient(Client()); @@ -43,8 +44,15 @@ class NetworkTileProvider extends TileProvider { /// over the network, and just return a transparent tile final bool silenceExceptions; + /// Configuration of built-in caching + /// + /// See online documentation for more information about built-in caching. + /// + /// Set to `null` to disable. See [MapCachingOptions] for defaults. + final MapCachingOptions? cachingOptions; + /// Long living client used to make all tile requests by - /// [MapNetworkImageProvider] for the duration that this provider is + /// [NetworkTileImageProvider] for the duration that this provider is /// alive /// /// Not automatically closed if created externally and passed as an argument @@ -65,12 +73,13 @@ class NetworkTileProvider extends TileProvider { @override ImageProvider getImage(TileCoordinates coordinates, TileLayer options) => - MapNetworkImageProvider( + NetworkTileImageProvider( url: getTileUrl(coordinates, options), fallbackUrl: getTileFallbackUrl(coordinates, options), headers: headers, httpClient: _httpClient, silenceExceptions: silenceExceptions, + cachingOptions: cachingOptions, startedLoading: () => _tilesInProgress[coordinates] = Completer(), finishedLoadingBytes: () { _tilesInProgress[coordinates]?.complete(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index 495a2661b..1fc30c7c8 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -5,8 +5,7 @@ import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; -import 'package:meta/meta.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -72,17 +71,16 @@ part 'tile_information.dart'; /// the new instance from working correctly. It is assumed the OS closes the /// open file handles, but that does mean every write to the persistent /// registry must be flushed. -// TODO: Expose for other providers? +// TODO: Expose for other providers? How to expose without breaking IO boundary? @immutable -@internal class MapTileCachingManager { const MapTileCachingManager._({ required String cacheDirectory, required void Function(String uuid, CachedTileInformation? tileInfo) - persistentRegistryWriter, + writeToPersistentRegistry, required Map registry, }) : _cacheDirectory = cacheDirectory, - _writeToPersistentRegistry = persistentRegistryWriter, + _writeToPersistentRegistry = writeToPersistentRegistry, _registry = registry; static const _persistentRegistryFileName = 'manager.json'; @@ -111,14 +109,12 @@ class MapTileCachingManager { /// /// Returns `null` if an instance does not exist and one could not be created. static Future getInstanceOrCreate({ - MapCachingOptions? options, + required MapCachingOptions options, }) async { if (_instance != null) return await _instance!.future; _instance = Completer(); - options ??= const MapCachingOptions(); - final Directory resolvedCacheDirectory; try { resolvedCacheDirectory = Directory( @@ -198,7 +194,7 @@ class MapTileCachingManager { final instance = MapTileCachingManager._( cacheDirectory: resolvedCacheDirectory.absolute.path, - persistentRegistryWriter: (uuid, tileInfo) => + writeToPersistentRegistry: (uuid, tileInfo) => workerSendPort.send((uuid: uuid, tileInfo: tileInfo)), registry: registry, ); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart index 43ed69785..ffa55b88b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart @@ -2,7 +2,6 @@ part of 'manager.dart'; /// Metadata about a tile cached with the [MapTileCachingManager] @immutable -@internal class CachedTileInformation { /// Create a new metadata container /// diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart deleted file mode 100644 index 29ff64555..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/native/image_provider.dart +++ /dev/null @@ -1,288 +0,0 @@ -import 'dart:async'; -import 'dart:io'; -import 'dart:math'; -import 'dart:ui'; - -import 'package:flutter/foundation.dart'; -import 'package:flutter/painting.dart'; -import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; -import 'package:http/http.dart'; -import 'package:uuid/data.dart'; -import 'package:uuid/rng.dart'; -import 'package:uuid/uuid.dart'; - -/// Dedicated [ImageProvider] to fetch tiles from the network -/// -/// Supports falling back to a secondary URL, if the primary URL fetch fails. -/// Note that specifying a [fallbackUrl] will prevent this image provider from -/// being cached. -@immutable -class CachingNetworkTileImageProvider - extends ImageProvider { - /// The URL to fetch the tile from (GET request) - final String url; - - /// The URL to fetch the tile from (GET request), in the event the original - /// [url] request fails - /// - /// If this is non-null, [operator==] will always return `false` (except if - /// the two objects are [identical]). Therefore, if this is non-null, this - /// image provider will not be cached in memory. - final String? fallbackUrl; - - /// The headers to include with the tile fetch request - /// - /// Not included in [operator==]. - final Map headers; - - /// The HTTP client to use to make network requests - /// - /// Not included in [operator==]. - final Client httpClient; - - /// Whether to ignore exceptions and errors that occur whilst fetching tiles - /// over the network, and just return a transparent tile - final bool silenceExceptions; - - /// Configuration of built-in caching - /// - /// See online documentation for more information about built-in caching. - /// - /// Set to `null` to disable. See [MapCachingOptions] for defaults. - final MapCachingOptions? cachingOptions; - - /// Function invoked when the image starts loading (not from cache) - /// - /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only - /// after all tiles have loaded. - final void Function() startedLoading; - - /// Function invoked when the image completes loading bytes from the network - /// - /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only - /// after all tiles have loaded. - final void Function() finishedLoadingBytes; - - /// Create a dedicated [ImageProvider] to fetch tiles from the network - /// - /// Supports falling back to a secondary URL, if the primary URL fetch fails. - /// Note that specifying a [fallbackUrl] will prevent this image provider from - /// being cached. - const CachingNetworkTileImageProvider({ - required this.url, - required this.fallbackUrl, - required this.headers, - required this.httpClient, - required this.silenceExceptions, - required this.cachingOptions, - required this.startedLoading, - required this.finishedLoadingBytes, - }); - - static final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); - - @override - ImageStreamCompleter loadImage( - CachingNetworkTileImageProvider key, - ImageDecoderCallback decode, - ) => - MultiFrameImageStreamCompleter( - codec: _load(key, decode), - scale: 1, - debugLabel: url, - informationCollector: () => [ - DiagnosticsProperty('URL', url), - DiagnosticsProperty('Fallback URL', fallbackUrl), - DiagnosticsProperty('Current provider', key), - ], - ); - - Future _load( - CachingNetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, - }) async { - startedLoading(); - - final resolvedUrl = useFallback ? fallbackUrl ?? '' : url; - final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); - - // TODO: Allow disabling caching - final cachingManager = (await MapTileCachingManager.getInstanceOrCreate( - options: cachingOptions, - ))!; - // TODO: Remove force null check, then fallback to non-caching - - final cachedTile = await cachingManager.getTile(uuid); - - Future handleOk(Response response) async { - final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; - final etag = response.headers[HttpHeaders.etagHeader]; - - cachingManager.putTile( - uuid, - CachedTileInformation( - lastModifiedLocally: DateTime.timestamp(), - staleAt: _calculateStaleAt(response), - lastModified: - lastModified != null ? HttpDate.parse(lastModified) : null, - etag: etag, - ), - response.bodyBytes, - ); - - finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); - } - - Future handleNotOk(Response response) async { - // Optimistically try to decode the response anyway - try { - finishedLoadingBytes(); - return await decode( - await ImmutableBuffer.fromUint8List(response.bodyBytes), - ); - } catch (err) { - // Otherwise fallback to a cached tile if we have one - if (cachedTile != null) { - finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - // Otherwise fallback to the fallback URL - if (!useFallback && fallbackUrl != null) { - finishedLoadingBytes(); - return _load(key, decode, useFallback: true); - } - - // Otherwise throw an exception/silently fail - if (!silenceExceptions) { - finishedLoadingBytes(); - throw HttpException( - 'Recieved ${response.statusCode}, and body was not a decodable image', - uri: Uri.parse(resolvedUrl), - ); - } - - finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) - .then(decode); - } finally { - scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); - } - } - - if (cachedTile != null) { - // If we have a cached tile that's not stale, return it - if (!cachedTile.tileInfo.isStale) { - print('from ache'); - finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - // Otherwise, ask the server what's going on - supply any details we have - final response = await httpClient.get( - Uri.parse(resolvedUrl), - headers: { - ...headers, - if (cachedTile.tileInfo.lastModified case final lastModified?) - HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), - if (cachedTile.tileInfo.etag case final etag?) - HttpHeaders.ifNoneMatchHeader: etag, - }, - ); - - // Server says nothing's changed - but might return new useful headers - if (response.statusCode == HttpStatus.notModified) { - final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; - final etag = response.headers[HttpHeaders.etagHeader]; - - cachingManager.putTile( - uuid, - CachedTileInformation( - lastModifiedLocally: DateTime.timestamp(), - staleAt: _calculateStaleAt(response), - lastModified: lastModified != null - ? HttpDate.parse(lastModified) - : cachedTile.tileInfo.lastModified, - etag: etag ?? cachedTile.tileInfo.etag, - ), - ); - - finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - if (response.statusCode == HttpStatus.ok) { - return await handleOk(response); - } - return await handleNotOk(response); - } - - final response = await httpClient.get( - Uri.parse(resolvedUrl), - headers: headers, - ); - - if (response.statusCode == HttpStatus.ok) { - return await handleOk(response); - } - return await handleNotOk(response); - } - - static DateTime _calculateStaleAt(Response response) { - final addToNow = DateTime.timestamp().add; - - if (response.headers[HttpHeaders.cacheControlHeader]?.toLowerCase() - case final cacheControl?) { - final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; - - if (maxAge == null) { - if (response.headers[HttpHeaders.expiresHeader]?.toLowerCase() - case final expires?) { - return HttpDate.parse(expires); - } - return addToNow(const Duration(days: 7)); - } else { - if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { - return addToNow( - Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), - ); - } - - final estimatedAge = max( - 0, - DateTime.timestamp() - .difference( - HttpDate.parse(response.headers[HttpHeaders.dateHeader]!), - ) - .inSeconds, - ); - return addToNow( - Duration(seconds: int.parse(maxAge) - estimatedAge), - ); - } - } else { - return addToNow(const Duration(days: 7)); - } - } - - @override - SynchronousFuture obtainKey( - ImageConfiguration configuration, - ) => - SynchronousFuture(this); - - @override - bool operator ==(Object other) => - identical(this, other) || - (other is CachingNetworkTileImageProvider && - fallbackUrl == null && - url == other.url); - - @override - int get hashCode => - Object.hashAll([url, if (fallbackUrl != null) fallbackUrl]); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart deleted file mode 100644 index d7091acc2..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/native/network_tile_provider.dart +++ /dev/null @@ -1,106 +0,0 @@ -import 'dart:async'; -import 'dart:collection'; - -import 'package:flutter/rendering.dart'; -import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/options.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; -import 'package:http/http.dart'; -import 'package:http/retry.dart'; - -/// [TileProvider] to fetch tiles from the network -/// -/// By default, a [RetryClient] is used to retry failed requests. 'dart:http' -/// or 'dart:io' might be needed to override this. -/// -/// On the web, the 'User-Agent' header cannot be changed as specified in -/// [TileLayer.tileProvider]'s documentation, due to a Dart/browser limitation. -/// -/// Does not support cancellation of tile loading via -/// [TileProvider.getImageWithCancelLoadingSupport], as abortion of in-flight -/// HTTP requests on the web is -/// [not yet supported in Dart](https://github.com/dart-lang/http/issues/424). -class NetworkTileProvider extends TileProvider { - /// [TileProvider] to fetch tiles from the network - /// - /// By default, a [RetryClient] is used to retry failed requests. 'dart:http' - /// or 'dart:io' might be needed to override this. - /// - /// On the web, the 'User-Agent' header cannot be changed, as specified in - /// [TileLayer.tileProvider]'s documentation, due to a Dart/browser limitation. - /// - /// Does not support cancellation of tile loading via - /// [TileProvider.getImageWithCancelLoadingSupport], as abortion of in-flight - /// HTTP requests on the web is - /// [not yet supported in Dart](https://github.com/dart-lang/http/issues/424). - NetworkTileProvider({ - super.headers, - Client? httpClient, - this.silenceExceptions = false, - this.cachingOptions, - }) : _isInternallyCreatedClient = httpClient == null, - _httpClient = httpClient ?? RetryClient(Client()); - - /// Whether to ignore exceptions and errors that occur whilst fetching tiles - /// over the network, and just return a transparent tile - final bool silenceExceptions; - - /// Configuration of built-in caching - /// - /// See online documentation for more information about built-in caching. - /// - /// Changing the configuration whilst caching is already working (after the - /// first tile has been loaded) is not supported. Changing the configuration - /// whilst a cache already exists is supported, but changing the - /// [MapCachingOptions.cacheDirectory] will not remove the old directory. - /// - /// Set to `null` to disable. See [MapCachingOptions] for defaults. - final MapCachingOptions? cachingOptions; - - /// Long living client used to make all tile requests by - /// [MapNetworkImageProvider] for the duration that this provider is - /// alive - /// - /// Not automatically closed if created externally and passed as an argument - /// during construction. - final Client _httpClient; - - /// Whether [_httpClient] was created on construction (and not passed in) - final bool _isInternallyCreatedClient; - - /// Each [Completer] is completed once the corresponding tile has finished - /// loading - /// - /// Used to avoid disposing of [_httpClient] whilst HTTP requests are still - /// underway. - /// - /// Does not include tiles loaded from session cache. - final _tilesInProgress = HashMap>(); - - @override - ImageProvider getImage(TileCoordinates coordinates, TileLayer options) => - CachingNetworkTileImageProvider( - url: getTileUrl(coordinates, options), - fallbackUrl: getTileFallbackUrl(coordinates, options), - headers: headers, - httpClient: _httpClient, - silenceExceptions: silenceExceptions, - cachingOptions: cachingOptions, - startedLoading: () => _tilesInProgress[coordinates] = Completer(), - finishedLoadingBytes: () { - _tilesInProgress[coordinates]?.complete(); - _tilesInProgress.remove(coordinates); - }, - ); - - @override - Future dispose() async { - if (_tilesInProgress.isNotEmpty) { - await Future.wait(_tilesInProgress.values.map((c) => c.future)); - } - if (_isInternallyCreatedClient) _httpClient.close(); - - super.dispose(); - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart new file mode 100644 index 000000000..e3e11ccf3 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart @@ -0,0 +1,211 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:math'; +import 'dart:ui'; + +import 'package:flutter/painting.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; +import 'package:http/http.dart'; +import 'package:meta/meta.dart'; +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; +import 'package:uuid/uuid.dart'; + +final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + +@internal +Future loadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) => + _ioLoadTileImage(key, decode, useFallback: useFallback); + +Future _ioLoadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) async { + key.startedLoading(); + + final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; + final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); + + if (key.cachingOptions == null) { + return simpleLoadTileImage(key, decode, useFallback: useFallback); + } + final cachingManager = await MapTileCachingManager.getInstanceOrCreate( + options: key.cachingOptions!, + ); + if (cachingManager == null) { + return simpleLoadTileImage(key, decode, useFallback: useFallback); + } + + final cachedTile = await cachingManager.getTile(uuid); + + Future handleOk(Response response) async { + final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; + final etag = response.headers[HttpHeaders.etagHeader]; + + unawaited(cachingManager.putTile( + uuid, + CachedTileInformation( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt( + response, + overrideFreshAge: key.cachingOptions!.overrideFreshAge, + ), + lastModified: + lastModified != null ? HttpDate.parse(lastModified) : null, + etag: etag, + ), + response.bodyBytes, + )); + + key.finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); + } + + Future handleNotOk(Response response) async { + // Optimistically try to decode the response anyway + try { + key.finishedLoadingBytes(); + return await decode( + await ImmutableBuffer.fromUint8List(response.bodyBytes), + ); + } catch (err) { + // Otherwise fallback to a cached tile if we have one + if (cachedTile != null) { + key.finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + // Otherwise fallback to the fallback URL + if (!useFallback && key.fallbackUrl != null) { + key.finishedLoadingBytes(); + return _ioLoadTileImage(key, decode, useFallback: true); + } + + // Otherwise throw an exception/silently fail + if (!key.silenceExceptions) { + key.finishedLoadingBytes(); + throw HttpException( + 'Recieved ${response.statusCode}, and body was not a decodable image', + uri: Uri.parse(resolvedUrl), + ); + } + + key.finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) + .then(decode); + } finally { + scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); + } + } + + if (cachedTile != null) { + // If we have a cached tile that's not stale, return it + if (!cachedTile.tileInfo.isStale) { + key.finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + // Otherwise, ask the server what's going on - supply any details we have + final response = await key.httpClient.get( + Uri.parse(resolvedUrl), + headers: { + ...key.headers, + if (cachedTile.tileInfo.lastModified case final lastModified?) + HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), + if (cachedTile.tileInfo.etag case final etag?) + HttpHeaders.ifNoneMatchHeader: etag, + }, + ); + + // Server says nothing's changed - but might return new useful headers + if (response.statusCode == HttpStatus.notModified) { + final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; + final etag = response.headers[HttpHeaders.etagHeader]; + + unawaited(cachingManager.putTile( + uuid, + CachedTileInformation( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt( + response, + overrideFreshAge: key.cachingOptions!.overrideFreshAge, + ), + lastModified: lastModified != null + ? HttpDate.parse(lastModified) + : cachedTile.tileInfo.lastModified, + etag: etag ?? cachedTile.tileInfo.etag, + ), + )); + + key.finishedLoadingBytes(); + return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); + } + + if (response.statusCode == HttpStatus.ok) { + return await handleOk(response); + } + return await handleNotOk(response); + } + + final response = await key.httpClient.get( + Uri.parse(resolvedUrl), + headers: key.headers, + ); + + if (response.statusCode == HttpStatus.ok) { + return await handleOk(response); + } + return await handleNotOk(response); +} + +DateTime _calculateStaleAt( + Response response, { + required Duration? overrideFreshAge, +}) { + final addToNow = DateTime.timestamp().add; + + if (overrideFreshAge case final overrideFreshAge?) { + return addToNow(overrideFreshAge); + } + + if (response.headers[HttpHeaders.cacheControlHeader]?.toLowerCase() + case final cacheControl?) { + final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; + + if (maxAge == null) { + if (response.headers[HttpHeaders.expiresHeader]?.toLowerCase() + case final expires?) { + return HttpDate.parse(expires); + } + return addToNow(const Duration(days: 7)); + } else { + if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { + return addToNow( + Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), + ); + } + + final estimatedAge = max( + 0, + DateTime.timestamp() + .difference( + HttpDate.parse(response.headers[HttpHeaders.dateHeader]!), + ) + .inSeconds, + ); + return addToNow( + Duration(seconds: int.parse(maxAge) - estimatedAge), + ); + } + } else { + return addToNow(const Duration(days: 7)); + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart new file mode 100644 index 000000000..611f0e24c --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart @@ -0,0 +1,49 @@ +import 'dart:async'; +import 'dart:ui'; + +import 'package:flutter/painting.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:meta/meta.dart'; + +@internal +Future loadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) => + _webLoadTileImage(key, decode, useFallback: useFallback); + +@internal +Future simpleLoadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) => + _webLoadTileImage(key, decode, useFallback: useFallback); + +Future _webLoadTileImage( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) { + key.startedLoading(); + + return key.httpClient + .readBytes( + Uri.parse(useFallback ? key.fallbackUrl ?? '' : key.url), + headers: key.headers, + ) + .whenComplete(key.finishedLoadingBytes) + .then(ImmutableBuffer.fromUint8List) + .then(decode) + .onError((err, stack) { + scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); + if (useFallback || key.fallbackUrl == null) { + if (!key.silenceExceptions) throw err; + return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) + .then(decode); + } + return _webLoadTileImage(key, decode, useFallback: true); + }); +} diff --git a/test/full_coverage_test.dart b/test/full_coverage_test.dart index 9ae6f6090..d158d5894 100644 --- a/test/full_coverage_test.dart +++ b/test/full_coverage_test.dart @@ -32,10 +32,10 @@ import 'package:flutter_map/src/layer/tile_layer/tile_image_view.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_layer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/io_impl_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range_calculator.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_scale_calculator.dart'; diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index b125fcc6e..9766a7eb2 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -4,7 +4,7 @@ import 'dart:typed_data'; import 'package:flutter/painting.dart'; import 'package:flutter/widgets.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/network_image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:http/http.dart'; import 'package:mocktail/mocktail.dart'; @@ -66,12 +66,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: null, headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -105,12 +106,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: null, headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -140,12 +142,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: null, headers: headers, httpClient: mockClient, silenceExceptions: true, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -183,12 +186,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -229,12 +233,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: true, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -267,12 +272,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -305,12 +311,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: true, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -345,12 +352,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: null, headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -388,12 +396,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: null, headers: headers, httpClient: mockClient, silenceExceptions: true, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -433,12 +442,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -482,12 +492,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -523,12 +534,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: false, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -569,12 +581,13 @@ void main() { bool startedLoadingTriggered = false; bool finishedLoadingTriggered = false; - final provider = MapNetworkImageProvider( + final provider = NetworkTileImageProvider( url: url.toString(), fallbackUrl: fallbackUrl.toString(), headers: headers, httpClient: mockClient, silenceExceptions: true, + cachingOptions: null, startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); From 25ebb25d22af2c4edb07bff6b85ce6188fef3a15 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 4 May 2025 18:10:46 +0100 Subject: [PATCH 05/49] Improved startup performance by not waiting for caching instance Improved performance by moving tile file writing into long-lived isolate worker to reduce overheads --- .../network/native/caching/manager.dart | 222 +++++++++--------- .../caching/persistent_registry_workers.dart | 12 + .../network/native/tile_loader.dart | 48 ++-- 3 files changed, 148 insertions(+), 134 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index 1fc30c7c8..c2f28534d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -71,6 +71,9 @@ part 'tile_information.dart'; /// the new instance from working correctly. It is assumed the OS closes the /// open file handles, but that does mean every write to the persistent /// registry must be flushed. +/// +/// A long-lasting isolate is also for writing tile files to reduce the +/// overheads of async file operations. // TODO: Expose for other providers? How to expose without breaking IO boundary? @immutable class MapTileCachingManager { @@ -78,129 +81,147 @@ class MapTileCachingManager { required String cacheDirectory, required void Function(String uuid, CachedTileInformation? tileInfo) writeToPersistentRegistry, + required void Function(String tileFilePath, Uint8List bytes) writeTileFile, required Map registry, }) : _cacheDirectory = cacheDirectory, _writeToPersistentRegistry = writeToPersistentRegistry, + _writeTileFile = writeTileFile, _registry = registry; static const _persistentRegistryFileName = 'manager.json'; - /// Current instance of singleton - /// - /// Completer pattern used to obtain a lock - the first tile loaded takes - /// slightly longer, as it has to create the first instance, so other tiles - /// loaded simultaneously must wait instead of all attempting to create - /// multiple singletons. - static Completer? _instance; + static MapTileCachingManager? _instance; + static bool _instanceBeingCreated = false; final String _cacheDirectory; final void Function(String uuid, CachedTileInformation? tileInfo) _writeToPersistentRegistry; + final void Function(String tileFilePath, Uint8List bytes) _writeTileFile; final Map _registry; - /// Returns the current caching instance if one is already available, - /// otherwise create and open a new instance + /// Returns the current instance if one is already available; otherwise, start + /// creating a new instance in the background and return `null` /// - /// If an instance is already being created, this will wait until that - /// instance is available instead of creating a new one. + /// This will also return null if an instance is already being created in the + /// background but is not ready yet, or if one could not be created (for + /// example due to an error when attempting to create the instance). /// /// [options] is only used to configure a new instance. If [options] changes, /// a new instance will not be created. - /// - /// Returns `null` if an instance does not exist and one could not be created. - static Future getInstanceOrCreate({ + static MapTileCachingManager? getInstanceOrCreate({ required MapCachingOptions options, - }) async { - if (_instance != null) return await _instance!.future; + }) { + if (_instance != null) return _instance!; - _instance = Completer(); + if (_instanceBeingCreated) return null; + _instanceBeingCreated = true; - final Directory resolvedCacheDirectory; - try { - resolvedCacheDirectory = Directory( - p.join( - options.cacheDirectory ?? - (await getApplicationCacheDirectory()).absolute.path, - 'fm_cache', - ), - ); - } on MissingPlatformDirectoryException { - return null; - } + () async { + final Directory resolvedCacheDirectory; + try { + resolvedCacheDirectory = Directory( + p.join( + options.cacheDirectory ?? + (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ), + ); + } on MissingPlatformDirectoryException { + return null; + } - try { - await resolvedCacheDirectory.create(recursive: true); - } on FileSystemException { - return null; - } + try { + await resolvedCacheDirectory.create(recursive: true); + } on FileSystemException { + return null; + } - final persistentRegistryFilePath = p.join( - resolvedCacheDirectory.absolute.path, - _persistentRegistryFileName, - ); - final persistentRegistryFile = File(persistentRegistryFilePath); + final persistentRegistryFilePath = p.join( + resolvedCacheDirectory.absolute.path, + _persistentRegistryFileName, + ); + final persistentRegistryFile = File(persistentRegistryFilePath); - final Map registry; - try { - if (await persistentRegistryFile.exists()) { - final parsedCacheManager = await compute( - _parsePersistentRegistryWorker, - persistentRegistryFilePath, - ); - if (parsedCacheManager == null) { - await resolvedCacheDirectory.delete(recursive: true); - await resolvedCacheDirectory.create(recursive: true); - await persistentRegistryFile.create(recursive: true); - registry = HashMap(); - } else { - registry = parsedCacheManager; + final Map registry; + try { + if (await persistentRegistryFile.exists()) { + final parsedCacheManager = await compute( + _parsePersistentRegistryWorker, + persistentRegistryFilePath, + debugLabel: '[flutter_map: cache] Persistent Registry Parser', + ); + if (parsedCacheManager == null) { + await resolvedCacheDirectory.delete(recursive: true); + await resolvedCacheDirectory.create(recursive: true); + await persistentRegistryFile.create(recursive: true); + registry = HashMap(); + } else { + registry = parsedCacheManager; - if (options.maxCacheSize case final sizeLimit?) { - // This can cause some delay when creating - // But it's much better than lagging or inconsistent registries - (await compute( - _limitCacheSizeWorker, - ( - cacheDirectoryPath: resolvedCacheDirectory.absolute.path, - persistentRegistryFileName: _persistentRegistryFileName, - sizeLimit: sizeLimit, - ), - )) - .forEach(registry.remove); + if (options.maxCacheSize case final sizeLimit?) { + // This can cause some delay when creating + // But it's much better than lagging or inconsistent registries + (await compute( + _limitCacheSizeWorker, + ( + cacheDirectoryPath: resolvedCacheDirectory.absolute.path, + persistentRegistryFileName: _persistentRegistryFileName, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Limiter', + )) + .forEach(registry.remove); + } } + } else { + await persistentRegistryFile.create(recursive: true); + registry = HashMap(); } - } else { - await persistentRegistryFile.create(recursive: true); - registry = HashMap(); + } on FileSystemException { + return null; } - } on FileSystemException { - return null; - } - final receivePort = ReceivePort(); - try { - await Isolate.spawn( - _persistentRegistryWorkerIsolate, - ( - port: receivePort.sendPort, - persistentRegistryFilePath: persistentRegistryFilePath, - initialRegistry: registry, - ), - ); - } catch (e) { - return null; - } - final workerSendPort = await receivePort.first as SendPort; + final registryWorkerReceivePort = ReceivePort(); + try { + await Isolate.spawn( + _persistentRegistryWorkerIsolate, + ( + port: registryWorkerReceivePort.sendPort, + persistentRegistryFilePath: persistentRegistryFilePath, + initialRegistry: registry, + ), + debugName: '[flutter_map: cache] Persistent Registry Worker', + ); + } catch (e) { + return null; + } + final registryWorkerSendPort = + await registryWorkerReceivePort.first as SendPort; - final instance = MapTileCachingManager._( - cacheDirectory: resolvedCacheDirectory.absolute.path, - writeToPersistentRegistry: (uuid, tileInfo) => - workerSendPort.send((uuid: uuid, tileInfo: tileInfo)), - registry: registry, - ); + final tileFileWriterWorkerReceivePort = ReceivePort(); + try { + await Isolate.spawn( + _tileFileWriterWorkerIsolate, + tileFileWriterWorkerReceivePort.sendPort, + debugName: '[flutter_map: cache] Tile File Writer', + ); + } catch (e) { + return null; + } + final tileFileWriterWorkerSendPort = + await tileFileWriterWorkerReceivePort.first as SendPort; - _instance!.complete(instance); - return instance; + _instance = MapTileCachingManager._( + cacheDirectory: resolvedCacheDirectory.absolute.path, + writeToPersistentRegistry: (uuid, tileInfo) => + registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)), + writeTileFile: (tileFilePath, bytes) => tileFileWriterWorkerSendPort + .send((tileFilePath: tileFilePath, bytes: bytes)), + registry: registry, + ); + }(); + + return null; } /// Retrieve a tile from the cache, if it exists @@ -239,20 +260,9 @@ class MapTileCachingManager { return; } - final tileFile = File(p.join(_cacheDirectory, uuid)); - - if (bytes == null && !await tileFile.exists()) { - // more expensive condition last - throw ArgumentError.notNull('bytes'); - } - if (bytes != null) { - try { - await tileFile.create(recursive: true); - await tileFile.writeAsBytes(bytes); - } on FileSystemException { - return; - } + final tileFilePath = p.join(_cacheDirectory, uuid); + _writeTileFile(tileFilePath, bytes); } _registry[uuid] = tileInfo; diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart index cb13b2a6b..95e95dd92 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart @@ -66,6 +66,18 @@ Future _persistentRegistryWorkerIsolate( } } +/// Isolate worker which writes bytes to files synchronously +Future _tileFileWriterWorkerIsolate(SendPort port) async { + final receivePort = ReceivePort(); + port.send(receivePort.sendPort); + + await for (final val in receivePort) { + final (:tileFilePath, :bytes) = + val as ({String tileFilePath, Uint8List bytes}); + File(tileFilePath).writeAsBytesSync(bytes); + } +} + /// Decode the JSON within the persistent registry into a mapping of tile /// UUIDs to their [CachedTileInformation]s /// diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart index e3e11ccf3..985f6f3cb 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart @@ -31,19 +31,19 @@ Future _ioLoadTileImage( }) async { key.startedLoading(); - final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; - final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); - if (key.cachingOptions == null) { return simpleLoadTileImage(key, decode, useFallback: useFallback); } - final cachingManager = await MapTileCachingManager.getInstanceOrCreate( + final cachingManager = MapTileCachingManager.getInstanceOrCreate( options: key.cachingOptions!, ); if (cachingManager == null) { return simpleLoadTileImage(key, decode, useFallback: useFallback); } + final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; + final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); + final cachedTile = await cachingManager.getTile(uuid); Future handleOk(Response response) async { @@ -65,40 +65,34 @@ Future _ioLoadTileImage( response.bodyBytes, )); - key.finishedLoadingBytes(); return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); } Future handleNotOk(Response response) async { // Optimistically try to decode the response anyway try { - key.finishedLoadingBytes(); return await decode( await ImmutableBuffer.fromUint8List(response.bodyBytes), ); } catch (err) { // Otherwise fallback to a cached tile if we have one if (cachedTile != null) { - key.finishedLoadingBytes(); return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); } // Otherwise fallback to the fallback URL if (!useFallback && key.fallbackUrl != null) { - key.finishedLoadingBytes(); return _ioLoadTileImage(key, decode, useFallback: true); } // Otherwise throw an exception/silently fail if (!key.silenceExceptions) { - key.finishedLoadingBytes(); throw HttpException( 'Recieved ${response.statusCode}, and body was not a decodable image', uri: Uri.parse(resolvedUrl), ); } - key.finishedLoadingBytes(); return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) .then(decode); } finally { @@ -124,6 +118,7 @@ Future _ioLoadTileImage( HttpHeaders.ifNoneMatchHeader: etag, }, ); + key.finishedLoadingBytes(); // Server says nothing's changed - but might return new useful headers if (response.statusCode == HttpStatus.notModified) { @@ -145,7 +140,6 @@ Future _ioLoadTileImage( ), )); - key.finishedLoadingBytes(); return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); } @@ -159,6 +153,7 @@ Future _ioLoadTileImage( Uri.parse(resolvedUrl), headers: key.headers, ); + key.finishedLoadingBytes(); if (response.statusCode == HttpStatus.ok) { return await handleOk(response); @@ -185,27 +180,24 @@ DateTime _calculateStaleAt( case final expires?) { return HttpDate.parse(expires); } + return addToNow(const Duration(days: 7)); - } else { - if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { - return addToNow( - Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), - ); - } + } - final estimatedAge = max( - 0, - DateTime.timestamp() - .difference( - HttpDate.parse(response.headers[HttpHeaders.dateHeader]!), - ) - .inSeconds, - ); + if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { return addToNow( - Duration(seconds: int.parse(maxAge) - estimatedAge), + Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), ); } - } else { - return addToNow(const Duration(days: 7)); + + final estimatedAge = max( + 0, + DateTime.timestamp() + .difference(HttpDate.parse(response.headers[HttpHeaders.dateHeader]!)) + .inSeconds, + ); + return addToNow(Duration(seconds: int.parse(maxAge) - estimatedAge)); } + + return addToNow(const Duration(days: 7)); } From b79cb070d7cd2557f6409599e33bfff37a771b23 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 4 May 2025 18:40:04 +0100 Subject: [PATCH 06/49] Exposed `MapTileCachingManager` & `CachedMapTileMetadata` Renamed `CachedTileInformation` to `CachedMapTileMetadata` Improved documentation --- lib/flutter_map.dart | 3 + .../network/independent/caching/manager.dart | 68 +++++++++++++++++++ .../network/independent/caching/options.dart | 3 +- .../caching/tile_metadata.dart} | 29 +++++--- .../network/native/caching/manager.dart | 45 +++--------- .../caching/persistent_registry_workers.dart | 10 +-- .../network/native/tile_loader.dart | 5 +- 7 files changed, 108 insertions(+), 55 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart rename lib/src/layer/tile_layer/tile_provider/network/{native/caching/tile_information.dart => independent/caching/tile_metadata.dart} (72%) diff --git a/lib/flutter_map.dart b/lib/flutter_map.dart index f6e81fe4e..6f7da9682 100644 --- a/lib/flutter_map.dart +++ b/lib/flutter_map.dart @@ -47,7 +47,10 @@ export 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.da export 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart' if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart' + if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_event.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_transformer.dart'; diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart new file mode 100644 index 000000000..c703d3988 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart @@ -0,0 +1,68 @@ +import 'dart:typed_data'; + +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; + +/// {@template fm.mtcm} +/// Singleton class which manages simple built-in tile caching based HTTP +/// headers for native (I/O) platforms only +/// +/// > [!IMPORTANT] +/// > Built-in tile caching is not a replacement for caching which can better +/// > guarantee resilience. It should not solely be used where not having +/// > cached tiles may lead to a dangerous situation - for example, offline +/// > mapping. This provides no guarantees as to the safety of cached tiles. +/// +/// For more information, see the online documentation. +/// +/// --- +/// +/// Direct usage of this class is not usually necessary. It is visible so other +/// tile providers may make use of it. +/// +/// --- +/// {@endtemplate} +@immutable +class MapTileCachingManager { + const MapTileCachingManager._(); + + /// Returns the current instance if one is already available; otherwise, start + /// creating a new instance in the background and return `null` + /// + /// This will also return null if an instance is already being created in the + /// background but is not ready yet, or if one could not be created (for + /// example due to an error when attempting to create the instance). + /// + /// [options] is only used to configure a new instance. If [options] changes, + /// a new instance will not be created. + static MapTileCachingManager? getInstanceOrCreate({ + required MapCachingOptions options, + }) => + throw UnsupportedError( + '`MapTileCachingManager` is only implemented on I/O platforms', + ); + + /// Retrieve a tile from the cache, if it exists + Future< + ({ + Uint8List bytes, + CachedMapTileMetadata tileInfo, + })?> getTile( + String uuid, + ) => + throw UnsupportedError( + '`MapTileCachingManager` is only implemented on I/O platforms', + ); + + /// Add or update a tile in the cache + /// + /// [bytes] is required if the tile is not already cached. + Future putTile( + String uuid, + CachedMapTileMetadata tileInfo, [ + Uint8List? bytes, + ]) => + throw UnsupportedError( + '`MapTileCachingManager` is only implemented on I/O platforms', + ); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart index 9c66362c3..99cc2ec9b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart @@ -7,7 +7,8 @@ class MapCachingOptions { /// /// This must be accessible to the program. /// - /// Defaults to a platform provided temporary directory. + /// Defaults to a platform provided cache directory, which may be cleared by + /// the OS at any time. final String? cacheDirectory; /// Preferred maximum size (in bytes) of the cache diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart similarity index 72% rename from lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart rename to lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart index ffa55b88b..d5ced5cc5 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/tile_information.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart @@ -1,13 +1,20 @@ -part of 'manager.dart'; +import 'dart:io' show HttpDate; // this is web safe! + +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; /// Metadata about a tile cached with the [MapTileCachingManager] +/// +/// Direct usage of this class is not usually necessary. It is visible so other +/// tile providers may make use of it. @immutable -class CachedTileInformation { - /// Create a new metadata container +class CachedMapTileMetadata { + /// Create new metadata /// - /// [lastModifiedLocally] should be set to [DateTime.timestamp]. Other - /// properties should be set based on the tile's HTTP response headers. - const CachedTileInformation({ + /// [lastModifiedLocally] must be set to [DateTime.timestamp]. Other + /// properties should usually be set based on the tile's HTTP response + /// headers. + const CachedMapTileMetadata({ required this.lastModifiedLocally, required this.staleAt, required this.lastModified, @@ -15,7 +22,7 @@ class CachedTileInformation { }); /// Decode metadata from JSON - CachedTileInformation.fromJson(Map json) + CachedMapTileMetadata.fromJson(Map json) : lastModifiedLocally = HttpDate.parse(json['lastModifiedLocally'] as String), staleAt = HttpDate.parse(json['staleAt'] as String), @@ -36,16 +43,16 @@ class CachedTileInformation { /// The date/time at which the tile becomes stale according to the HTTP spec final DateTime staleAt; - /// The tile's [HttpHeaders.lastModifiedHeader] + /// The tile's last modified HTTP header final DateTime? lastModified; - /// The tile's [HttpHeaders.etagHeader] + /// The tile's etag HTTP header final String? etag; /// Whether the tile is currently stale bool get isStale => DateTime.timestamp().isAfter(staleAt); - /// Convert the metadata to JSON + /// Encode the metadata to JSON Map toJson() => { 'lastModifiedLocally': HttpDate.format(lastModifiedLocally), 'staleAt': HttpDate.format(staleAt), @@ -61,7 +68,7 @@ class CachedTileInformation { @override bool operator ==(Object other) => identical(this, other) || - (other is CachedTileInformation && + (other is CachedMapTileMetadata && lastModifiedLocally == other.lastModifiedLocally && staleAt == other.staleAt && lastModified == other.lastModified && diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index c2f28534d..8d2e52522 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -6,39 +6,13 @@ import 'dart:isolate'; import 'package:flutter/foundation.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; part 'persistent_registry_workers.dart'; -part 'tile_information.dart'; -/// Singleton class which manages built-in tile caching on native platforms -/// -/// Built-in tile caching is simple and based on the tile's HTTP headers. -/// -/// > [!IMPORTANT] -/// > Built-in tile caching is not a replacement for caching which can better -/// > guarantee resilience. It should not solely be used where not having -/// > cached tiles may lead to a dangerous situation - for example, offline -/// > mapping. This provides no guarantees as to the safety of cached tiles. -/// -/// By default, caching is performed in a caching directory set by the OS, which -/// may be cleared at any time. -/// -/// The registry used to manage tiles is in JSON. There is no guarantee that the -/// registry will remain valid (not corrupt). A corrupt registry will result in -/// all cached tiles being lost. -/// -/// Tile server URLs which use (for example) API keys create tiles with UUIDs -/// including the volatile part of the URL. If this part of the URL is changed, -/// all tiles previously stored will become in-accessible. -/// -/// The cache does not peristently monitor usage (eg. hits) of the cache. -/// -/// The primary purpose of this caching is to reduce the number of requests -/// to tile servers. -/// -/// --- +/// {@macro fm.mtcm} /// /// The singleton is not disposed of. The only time it would make sense to /// close the singleton is when either: @@ -74,15 +48,14 @@ part 'tile_information.dart'; /// /// A long-lasting isolate is also for writing tile files to reduce the /// overheads of async file operations. -// TODO: Expose for other providers? How to expose without breaking IO boundary? @immutable class MapTileCachingManager { const MapTileCachingManager._({ required String cacheDirectory, - required void Function(String uuid, CachedTileInformation? tileInfo) + required void Function(String uuid, CachedMapTileMetadata? tileInfo) writeToPersistentRegistry, required void Function(String tileFilePath, Uint8List bytes) writeTileFile, - required Map registry, + required Map registry, }) : _cacheDirectory = cacheDirectory, _writeToPersistentRegistry = writeToPersistentRegistry, _writeTileFile = writeTileFile, @@ -94,10 +67,10 @@ class MapTileCachingManager { static bool _instanceBeingCreated = false; final String _cacheDirectory; - final void Function(String uuid, CachedTileInformation? tileInfo) + final void Function(String uuid, CachedMapTileMetadata? tileInfo) _writeToPersistentRegistry; final void Function(String tileFilePath, Uint8List bytes) _writeTileFile; - final Map _registry; + final Map _registry; /// Returns the current instance if one is already available; otherwise, start /// creating a new instance in the background and return `null` @@ -142,7 +115,7 @@ class MapTileCachingManager { ); final persistentRegistryFile = File(persistentRegistryFilePath); - final Map registry; + final Map registry; try { if (await persistentRegistryFile.exists()) { final parsedCacheManager = await compute( @@ -228,7 +201,7 @@ class MapTileCachingManager { Future< ({ Uint8List bytes, - CachedTileInformation tileInfo, + CachedMapTileMetadata tileInfo, })?> getTile( String uuid, ) async { @@ -252,7 +225,7 @@ class MapTileCachingManager { /// [bytes] is required if the tile is not already cached. Future putTile( String uuid, - CachedTileInformation tileInfo, [ + CachedMapTileMetadata tileInfo, [ Uint8List? bytes, ]) async { if (_registry[uuid] case final existingTileInfo? diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart index 95e95dd92..eb8dfe102 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart @@ -18,7 +18,7 @@ Future _persistentRegistryWorkerIsolate( ({ SendPort port, String persistentRegistryFilePath, - Map initialRegistry, + Map initialRegistry, }) input, ) async { final registry = input.initialRegistry; @@ -53,7 +53,7 @@ Future _persistentRegistryWorkerIsolate( await for (final val in receivePort) { final (:uuid, :tileInfo) = - val as ({String uuid, CachedTileInformation? tileInfo}); + val as ({String uuid, CachedMapTileMetadata? tileInfo}); if (tileInfo == null) { registry.remove(uuid); @@ -79,12 +79,12 @@ Future _tileFileWriterWorkerIsolate(SendPort port) async { } /// Decode the JSON within the persistent registry into a mapping of tile -/// UUIDs to their [CachedTileInformation]s +/// UUIDs to their [CachedMapTileMetadata]s /// /// Should be used within an isolate/[compute]r. /// /// If the JSON is invalid or the file cannot be read, this returns null. -HashMap? _parsePersistentRegistryWorker( +HashMap? _parsePersistentRegistryWorker( String persistentRegistryFilePath, ) { final String json; @@ -105,7 +105,7 @@ HashMap? _parsePersistentRegistryWorker( parsed.map( (key, value) => MapEntry( key, - CachedTileInformation.fromJson(value as Map), + CachedMapTileMetadata.fromJson(value as Map), ), ), ); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart index 985f6f3cb..4fa507908 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart @@ -5,6 +5,7 @@ import 'dart:ui'; import 'package:flutter/painting.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; @@ -52,7 +53,7 @@ Future _ioLoadTileImage( unawaited(cachingManager.putTile( uuid, - CachedTileInformation( + CachedMapTileMetadata( lastModifiedLocally: DateTime.timestamp(), staleAt: _calculateStaleAt( response, @@ -127,7 +128,7 @@ Future _ioLoadTileImage( unawaited(cachingManager.putTile( uuid, - CachedTileInformation( + CachedMapTileMetadata( lastModifiedLocally: DateTime.timestamp(), staleAt: _calculateStaleAt( response, From 92c37cb2e69ea51332c4daa6399061fdeb8ed7f6 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 4 May 2025 22:55:16 +0100 Subject: [PATCH 07/49] Added `MapCachingOptions.cacheKeyGenerator` --- .../network/independent/caching/options.dart | 21 +++++++++++++++++-- .../network/native/tile_loader.dart | 3 ++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart index 99cc2ec9b..aa7c66e80 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart @@ -30,11 +30,22 @@ class MapCachingOptions { /// Defaults to `null`: use duration calculated from each tile's HTTP headers. final Duration? overrideFreshAge; + /// Function to convert a tile URL to a key used in the cache + /// + /// This may be useful where parts of the URL are volatile or do not represent + /// the tile image, for example, API keys contained with the query parameters. + /// + /// The resulting key should be unique to that tile URL. + /// + /// Defaults to generating a UUID from the entire URL string. + final String Function(String url)? cacheKeyGenerator; + /// Create a configuration for caching const MapCachingOptions({ this.cacheDirectory, this.maxCacheSize = 1000000000, this.overrideFreshAge, + this.cacheKeyGenerator, }) : assert( maxCacheSize == null || maxCacheSize > 0, '`maxCacheSize` must be greater than 0 or disabled', @@ -45,7 +56,12 @@ class MapCachingOptions { ); @override - int get hashCode => Object.hash(cacheDirectory, maxCacheSize); + int get hashCode => Object.hash( + cacheDirectory, + maxCacheSize, + overrideFreshAge, + cacheKeyGenerator, + ); @override bool operator ==(Object other) => @@ -53,5 +69,6 @@ class MapCachingOptions { (other is MapCachingOptions && cacheDirectory == other.cacheDirectory && maxCacheSize == other.maxCacheSize && - overrideFreshAge == other.overrideFreshAge); + overrideFreshAge == other.overrideFreshAge && + cacheKeyGenerator == other.cacheKeyGenerator); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart index 4fa507908..09da80a2f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart @@ -43,7 +43,8 @@ Future _ioLoadTileImage( } final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; - final uuid = _uuid.v5(Namespace.url.value, resolvedUrl); + final uuid = key.cachingOptions!.cacheKeyGenerator?.call(resolvedUrl) ?? + _uuid.v5(Namespace.url.value, resolvedUrl); final cachedTile = await cachingManager.getTile(uuid); From c1a8bb69a289ec81189ea6799fe6ef1c771bdea5 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sat, 10 May 2025 12:39:29 +0100 Subject: [PATCH 08/49] Reverted to waiting for manager instance to be created before returning from `getInstance` --- .../network/independent/caching/manager.dart | 20 +- .../network/native/caching/manager.dart | 186 ++++++++---------- .../network/native/tile_loader.dart | 10 +- 3 files changed, 99 insertions(+), 117 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart index c703d3988..48f2cca69 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart @@ -26,17 +26,19 @@ import 'package:meta/meta.dart'; class MapTileCachingManager { const MapTileCachingManager._(); - /// Returns the current instance if one is already available; otherwise, start - /// creating a new instance in the background and return `null` + /// Returns an existing instance if available, else creates one and returns + /// once ready /// - /// This will also return null if an instance is already being created in the - /// background but is not ready yet, or if one could not be created (for - /// example due to an error when attempting to create the instance). + /// If this is called multiple times simultanously, they will lock so that + /// only one instance is created. /// - /// [options] is only used to configure a new instance. If [options] changes, - /// a new instance will not be created. - static MapTileCachingManager? getInstanceOrCreate({ - required MapCachingOptions options, + /// Throws if an instance did not exist and could not be created. In this + /// case, tile provider users should fallback to a non-caching implementation. + /// + /// If an instance is not already available, [options] is used to configure + /// the new instance. + static Future getInstance({ + MapCachingOptions options = const MapCachingOptions(), }) => throw UnsupportedError( '`MapTileCachingManager` is only implemented on I/O platforms', diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index 8d2e52522..9a122efda 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -63,8 +63,7 @@ class MapTileCachingManager { static const _persistentRegistryFileName = 'manager.json'; - static MapTileCachingManager? _instance; - static bool _instanceBeingCreated = false; + static Completer? _instance; final String _cacheDirectory; final void Function(String uuid, CachedMapTileMetadata? tileInfo) @@ -72,42 +71,34 @@ class MapTileCachingManager { final void Function(String tileFilePath, Uint8List bytes) _writeTileFile; final Map _registry; - /// Returns the current instance if one is already available; otherwise, start - /// creating a new instance in the background and return `null` + /// Returns an existing instance if available, else creates one and returns + /// once ready /// - /// This will also return null if an instance is already being created in the - /// background but is not ready yet, or if one could not be created (for - /// example due to an error when attempting to create the instance). + /// If this is called multiple times simultanously, they will lock so that + /// only one instance is created. /// - /// [options] is only used to configure a new instance. If [options] changes, - /// a new instance will not be created. - static MapTileCachingManager? getInstanceOrCreate({ - required MapCachingOptions options, - }) { - if (_instance != null) return _instance!; - - if (_instanceBeingCreated) return null; - _instanceBeingCreated = true; - - () async { - final Directory resolvedCacheDirectory; - try { - resolvedCacheDirectory = Directory( - p.join( - options.cacheDirectory ?? - (await getApplicationCacheDirectory()).absolute.path, - 'fm_cache', - ), - ); - } on MissingPlatformDirectoryException { - return null; - } + /// Throws if an instance did not exist and could not be created. In this + /// case, tile provider users should fallback to a non-caching implementation. + /// + /// If an instance is not already available, [options] is used to configure + /// the new instance. + static Future getInstance({ + MapCachingOptions options = const MapCachingOptions(), + }) async { + if (_instance != null) return await _instance!.future; - try { - await resolvedCacheDirectory.create(recursive: true); - } on FileSystemException { - return null; - } + _instance = Completer(); + final MapTileCachingManager instance; + + try { + final resolvedCacheDirectory = Directory( + p.join( + options.cacheDirectory ?? + (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ), + ); + await resolvedCacheDirectory.create(recursive: true); final persistentRegistryFilePath = p.join( resolvedCacheDirectory.absolute.path, @@ -116,75 +107,63 @@ class MapTileCachingManager { final persistentRegistryFile = File(persistentRegistryFilePath); final Map registry; - try { - if (await persistentRegistryFile.exists()) { - final parsedCacheManager = await compute( - _parsePersistentRegistryWorker, - persistentRegistryFilePath, - debugLabel: '[flutter_map: cache] Persistent Registry Parser', - ); - if (parsedCacheManager == null) { - await resolvedCacheDirectory.delete(recursive: true); - await resolvedCacheDirectory.create(recursive: true); - await persistentRegistryFile.create(recursive: true); - registry = HashMap(); - } else { - registry = parsedCacheManager; - - if (options.maxCacheSize case final sizeLimit?) { - // This can cause some delay when creating - // But it's much better than lagging or inconsistent registries - (await compute( - _limitCacheSizeWorker, - ( - cacheDirectoryPath: resolvedCacheDirectory.absolute.path, - persistentRegistryFileName: _persistentRegistryFileName, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Limiter', - )) - .forEach(registry.remove); - } - } - } else { + if (await persistentRegistryFile.exists()) { + final parsedCacheManager = await compute( + _parsePersistentRegistryWorker, + persistentRegistryFilePath, + debugLabel: '[flutter_map: cache] Persistent Registry Parser', + ); + if (parsedCacheManager == null) { + await resolvedCacheDirectory.delete(recursive: true); + await resolvedCacheDirectory.create(recursive: true); await persistentRegistryFile.create(recursive: true); registry = HashMap(); + } else { + registry = parsedCacheManager; + + if (options.maxCacheSize case final sizeLimit?) { + // This can cause some delay when creating + // But it's much better than lagging or inconsistent registries + (await compute( + _limitCacheSizeWorker, + ( + cacheDirectoryPath: resolvedCacheDirectory.absolute.path, + persistentRegistryFileName: _persistentRegistryFileName, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Limiter', + )) + .forEach(registry.remove); + } } - } on FileSystemException { - return null; + } else { + await persistentRegistryFile.create(recursive: true); + registry = HashMap(); } final registryWorkerReceivePort = ReceivePort(); - try { - await Isolate.spawn( - _persistentRegistryWorkerIsolate, - ( - port: registryWorkerReceivePort.sendPort, - persistentRegistryFilePath: persistentRegistryFilePath, - initialRegistry: registry, - ), - debugName: '[flutter_map: cache] Persistent Registry Worker', - ); - } catch (e) { - return null; - } + await Isolate.spawn( + _persistentRegistryWorkerIsolate, + ( + port: registryWorkerReceivePort.sendPort, + persistentRegistryFilePath: persistentRegistryFilePath, + initialRegistry: registry, + ), + debugName: '[flutter_map: cache] Persistent Registry Worker', + ); final registryWorkerSendPort = await registryWorkerReceivePort.first as SendPort; final tileFileWriterWorkerReceivePort = ReceivePort(); - try { - await Isolate.spawn( - _tileFileWriterWorkerIsolate, - tileFileWriterWorkerReceivePort.sendPort, - debugName: '[flutter_map: cache] Tile File Writer', - ); - } catch (e) { - return null; - } + await Isolate.spawn( + _tileFileWriterWorkerIsolate, + tileFileWriterWorkerReceivePort.sendPort, + debugName: '[flutter_map: cache] Tile File Writer', + ); final tileFileWriterWorkerSendPort = await tileFileWriterWorkerReceivePort.first as SendPort; - _instance = MapTileCachingManager._( + instance = MapTileCachingManager._( cacheDirectory: resolvedCacheDirectory.absolute.path, writeToPersistentRegistry: (uuid, tileInfo) => registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)), @@ -192,9 +171,13 @@ class MapTileCachingManager { .send((tileFilePath: tileFilePath, bytes: bytes)), registry: registry, ); - }(); + } catch (error, stackTrace) { + _instance!.completeError(error, stackTrace); + rethrow; + } - return null; + _instance!.complete(instance); + return instance; } /// Retrieve a tile from the cache, if it exists @@ -205,19 +188,14 @@ class MapTileCachingManager { })?> getTile( String uuid, ) async { - if (!_registry.containsKey(uuid)) { - unawaited(_removeTile(uuid)); - return null; - } - final tileFile = File(p.join(_cacheDirectory, uuid)); - try { - return (bytes: await tileFile.readAsBytes(), tileInfo: _registry[uuid]!); - } on FileSystemException { - unawaited(_removeTile(uuid)); - return null; + if (_registry[uuid] case final tileInfo? when await tileFile.exists()) { + return (bytes: await tileFile.readAsBytes(), tileInfo: tileInfo); } + + unawaited(removeTile(uuid)); + return null; } /// Add or update a tile in the cache @@ -243,7 +221,7 @@ class MapTileCachingManager { } /// Remove a tile from the cache - Future _removeTile(String uuid) async { + Future removeTile(String uuid) async { final tileFile = File(p.join(_cacheDirectory, uuid)); if (await tileFile.exists()) await tileFile.delete(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart index 09da80a2f..50c0f173d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart @@ -35,10 +35,12 @@ Future _ioLoadTileImage( if (key.cachingOptions == null) { return simpleLoadTileImage(key, decode, useFallback: useFallback); } - final cachingManager = MapTileCachingManager.getInstanceOrCreate( - options: key.cachingOptions!, - ); - if (cachingManager == null) { + final MapTileCachingManager cachingManager; + try { + cachingManager = await MapTileCachingManager.getInstance( + options: key.cachingOptions!, + ); + } catch (_) { return simpleLoadTileImage(key, decode, useFallback: useFallback); } From bd5dfe7e924d812135d7b80f1403bbf514aa7741 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 14 May 2025 20:12:52 +0100 Subject: [PATCH 09/49] Added size monitor --- .../network/native/caching/manager.dart | 30 ++++++++----- .../caching/persistent_registry_workers.dart | 45 ++++++++++++++++--- 2 files changed, 59 insertions(+), 16 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart index 9a122efda..d08aa28be 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart @@ -54,22 +54,23 @@ class MapTileCachingManager { required String cacheDirectory, required void Function(String uuid, CachedMapTileMetadata? tileInfo) writeToPersistentRegistry, - required void Function(String tileFilePath, Uint8List bytes) writeTileFile, - required Map registry, + required void Function(String tileFilePath, Uint8List? bytes) writeTileFile, + required HashMap registry, }) : _cacheDirectory = cacheDirectory, _writeToPersistentRegistry = writeToPersistentRegistry, _writeTileFile = writeTileFile, _registry = registry; - static const _persistentRegistryFileName = 'manager.json'; + static const _persistentRegistryFileName = 'registry.json'; + static const _sizeMonitorFileName = 'sizeMonitor'; static Completer? _instance; final String _cacheDirectory; final void Function(String uuid, CachedMapTileMetadata? tileInfo) _writeToPersistentRegistry; - final void Function(String tileFilePath, Uint8List bytes) _writeTileFile; - final Map _registry; + final void Function(String tileFilePath, Uint8List? bytes) _writeTileFile; + final HashMap _registry; /// Returns an existing instance if available, else creates one and returns /// once ready @@ -106,17 +107,22 @@ class MapTileCachingManager { ); final persistentRegistryFile = File(persistentRegistryFilePath); - final Map registry; + final sizeMonitorFilePath = p.join( + resolvedCacheDirectory.absolute.path, + _sizeMonitorFileName, + ); + + final HashMap registry; if (await persistentRegistryFile.exists()) { final parsedCacheManager = await compute( _parsePersistentRegistryWorker, persistentRegistryFilePath, debugLabel: '[flutter_map: cache] Persistent Registry Parser', ); + if (parsedCacheManager == null) { await resolvedCacheDirectory.delete(recursive: true); await resolvedCacheDirectory.create(recursive: true); - await persistentRegistryFile.create(recursive: true); registry = HashMap(); } else { registry = parsedCacheManager; @@ -137,7 +143,6 @@ class MapTileCachingManager { } } } else { - await persistentRegistryFile.create(recursive: true); registry = HashMap(); } @@ -157,7 +162,10 @@ class MapTileCachingManager { final tileFileWriterWorkerReceivePort = ReceivePort(); await Isolate.spawn( _tileFileWriterWorkerIsolate, - tileFileWriterWorkerReceivePort.sendPort, + ( + port: tileFileWriterWorkerReceivePort.sendPort, + sizeMonitorFilePath: sizeMonitorFilePath, + ), debugName: '[flutter_map: cache] Tile File Writer', ); final tileFileWriterWorkerSendPort = @@ -222,8 +230,8 @@ class MapTileCachingManager { /// Remove a tile from the cache Future removeTile(String uuid) async { - final tileFile = File(p.join(_cacheDirectory, uuid)); - if (await tileFile.exists()) await tileFile.delete(); + final tileFilePath = p.join(_cacheDirectory, uuid); + _writeTileFile(tileFilePath, null); if (_registry.remove(uuid) == null) return; _writeToPersistentRegistry(uuid, null); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart index eb8dfe102..6da10514a 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart @@ -66,15 +66,50 @@ Future _persistentRegistryWorkerIsolate( } } -/// Isolate worker which writes bytes to files synchronously -Future _tileFileWriterWorkerIsolate(SendPort port) async { +/// Isolate worker which writes & deletes tile files, and updates the size +/// monitor, synchronously +Future _tileFileWriterWorkerIsolate( + ({ + SendPort port, + String sizeMonitorFilePath, + }) input, +) async { + final sizeMonitorWriter = File(input.sizeMonitorFilePath) + .openSync(mode: FileMode.append) + ..setPositionSync(0); + int currentSize = + sizeMonitorWriter.readSync(8).buffer.asInt64List().elementAtOrNull(0) ?? + 0; + final allocatedWriteBinBuffer = Uint8List(8); + final receivePort = ReceivePort(); - port.send(receivePort.sendPort); + input.port.send(receivePort.sendPort); await for (final val in receivePort) { final (:tileFilePath, :bytes) = - val as ({String tileFilePath, Uint8List bytes}); - File(tileFilePath).writeAsBytesSync(bytes); + val as ({String tileFilePath, Uint8List? bytes}); + + final tileFile = File(tileFilePath); + final tileFileExists = tileFile.existsSync(); + + final existingTileSize = tileFileExists ? tileFile.lengthSync() : 0; + final newTileSize = bytes?.lengthInBytes ?? 0; + if (newTileSize - existingTileSize case final deltaSize + when deltaSize != 0) { + currentSize += deltaSize; + sizeMonitorWriter + ..setPositionSync(0) + ..writeFromSync( + allocatedWriteBinBuffer..buffer.asInt64List()[0] = currentSize, + ) + ..flushSync(); + } + + if (bytes != null) { + tileFile.writeAsBytesSync(bytes); + } else if (tileFileExists) { + tileFile.deleteSync(); + } } } From 2c38eb82d3d8d8d80a4c75529c388934206f1653 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 15 May 2025 12:31:01 +0100 Subject: [PATCH 10/49] Major refactoring & renaming Allowed `NetworkTileProvider` to accept any `MapCachingProvider` implementation, defaulting to `BuiltInMapCachingProvider` Used size monitor to check cache size before applying limit if necessary --- example/lib/main.dart | 4 +- lib/flutter_map.dart | 10 +- .../built_in/built_in_caching_provider.dart | 94 +++++++ .../caching/built_in/impl/native/native.dart | 205 +++++++++++++++ .../native}/persistent_registry_workers.dart | 22 +- .../network/caching/built_in/impl/stub.dart | 42 +++ .../caching/built_in/impl/web/web.dart | 41 +++ .../network/caching/caching_provider.dart | 43 ++++ .../disabled/disabled_caching_provider.dart | 23 ++ .../caching/tile_metadata.dart | 9 +- .../image_provider.dart | 21 +- .../image_provider/tile_loader_simple.dart | 27 ++ .../tile_loader_with_caching.dart} | 110 +++----- .../network/independent/caching/manager.dart | 70 ----- .../network/independent/caching/options.dart | 74 ------ .../network/independent/tile_loader.dart | 14 - .../network/native/caching/manager.dart | 239 ------------------ .../{independent => }/tile_provider.dart | 13 +- .../network/web/tile_loader.dart | 49 ---- test/full_coverage_test.dart | 4 +- .../network_image_provider_test.dart | 29 ++- 21 files changed, 578 insertions(+), 565 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart rename lib/src/layer/tile_layer/tile_provider/network/{native/caching => caching/built_in/impl/native}/persistent_registry_workers.dart (91%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart rename lib/src/layer/tile_layer/tile_provider/network/{independent => }/caching/tile_metadata.dart (89%) rename lib/src/layer/tile_layer/tile_provider/network/{independent => image_provider}/image_provider.dart (84%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart rename lib/src/layer/tile_layer/tile_provider/network/{native/tile_loader.dart => image_provider/tile_loader_with_caching.dart} (60%) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart rename lib/src/layer/tile_layer/tile_provider/network/{independent => }/tile_provider.dart (92%) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index 05d447137..6817c5b6c 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,4 +1,5 @@ import 'package:flutter/material.dart'; +import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; @@ -34,8 +35,9 @@ import 'package:flutter_map_example/pages/tile_loading_error_handle.dart'; import 'package:flutter_map_example/pages/wms_tile_layer.dart'; import 'package:flutter_web_plugins/url_strategy.dart'; -void main() { +Future main() async { usePathUrlStrategy(); + await BuiltInMapCachingProvider.getOrCreateInstance().isInitialised; runApp(const MyApp()); } diff --git a/lib/flutter_map.dart b/lib/flutter_map.dart index 6f7da9682..d82163df9 100644 --- a/lib/flutter_map.dart +++ b/lib/flutter_map.dart @@ -47,11 +47,11 @@ export 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.da export 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart' if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart' - if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; -export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_event.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_transformer.dart'; export 'package:flutter_map/src/map/camera/camera.dart'; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart new file mode 100644 index 000000000..258ee1865 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -0,0 +1,94 @@ +import 'package:flutter_map/flutter_map.dart'; +// TODO: On Dart 3.8 min, update to remove `@internal`s, switch to privates and conditional parts +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart' + if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart' + if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart'; + +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; +import 'package:uuid/uuid.dart'; + +/// Simple built-in map caching respecting HTTP headers using the filesystem +/// and a JSON registry, on native (non-web) platforms only +/// +/// This is enabled by default. For more information, see the online +/// documentation. +abstract interface class BuiltInMapCachingProvider + implements MapCachingProvider { + /// if a singleton instance exists, return it, otherwise create a new + /// singleton instance (and start asynchronously initialising it) + /// + /// If an instance already exists, the provided configuration will be ignored. + /// + /// See individual properties for more information about configuration. + factory BuiltInMapCachingProvider.getOrCreateInstance({ + /// Path to the caching directory to use + /// + /// This must be accessible to the program. + /// + /// Defaults to a platform provided cache directory, which may be cleared by + /// the OS at any time. + String? cacheDirectory, + + /// Preferred maximum size (in bytes) of the cache + /// + /// This is applied when the internal caching mechanism is created (on the + /// first tile load in the main memory space for the app). It is not an + /// absolute limit. + /// + /// This may cause some slight delay to the loading of the first tiles, + /// especially if the size is large and the cache does exceed the size. If + /// the visible delay becomes too large, disable this and manage the cache + /// size manually if necessary. + /// + /// Defaults to 1GB. Set to `null` to disable. + int? maxCacheSize = 1000000000, + + /// Override the duration of time a tile is considered fresh for + /// + /// Defaults to `null`: use duration calculated from each tile's HTTP headers. + Duration? overrideFreshAge, + + /// Function to convert a tile URL to a key used in the cache + /// + /// This may be useful where parts of the URL are volatile or do not represent + /// the tile image, for example, API keys contained with the query parameters. + /// + /// The resulting key should be unique to that tile URL. + /// + /// Defaults to generating a UUID from the entire URL string. + String Function(String url)? cacheKeyGenerator, + }) { + assert( + maxCacheSize == null || maxCacheSize > 0, + '`maxCacheSize` must be greater than 0 or disabled', + ); + assert( + overrideFreshAge == null || overrideFreshAge > Duration.zero, + '`overrideFreshAge` must be greater than 0 or disabled', + ); + return _instance ??= BuiltInMapCachingProviderImpl.createAndInitialise( + cacheDirectory: cacheDirectory, + maxCacheSize: maxCacheSize, + overrideFreshAge: overrideFreshAge, + cacheKeyGenerator: + cacheKeyGenerator ?? (url) => _uuid.v5(Namespace.url.value, url), + ); + } + + static BuiltInMapCachingProviderImpl? _instance; + + static final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + + /// Completes when the current instance has initialised and is ready to load + /// and write tiles + /// + /// See online documentation to see how to use this to preload caching to + /// remove the initial delay before loading tiles. + /// + /// May complete with an error if initialisation failed. + /// + /// On the web, this will always complete successfully immediately in the same + /// event loop. Caching will not be available. + Future get isInitialised; +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart new file mode 100644 index 000000000..c3db4f11c --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -0,0 +1,205 @@ +import 'dart:async'; +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; +import 'package:path_provider/path_provider.dart'; + +part 'persistent_registry_workers.dart'; + +@internal +class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { + final String? cacheDirectory; + final int? maxCacheSize; + final Duration? overrideFreshAge; + final String Function(String url) cacheKeyGenerator; + + @internal + BuiltInMapCachingProviderImpl.createAndInitialise({ + required this.cacheDirectory, + required this.maxCacheSize, + required this.overrideFreshAge, + required this.cacheKeyGenerator, + }) { + _initialise(); + } + + static const _persistentRegistryFileName = 'registry.json'; + static const _sizeMonitorFileName = 'sizeMonitor'; + + late final String _cacheDirectory; + late final void Function(String uuid, CachedMapTileMetadata? tileInfo) + _writeToPersistentRegistry; + late final void Function(String tileFilePath, Uint8List? bytes) + _writeTileFile; + late final HashMap _registry; + + Completer? _isInitialised; + + @override + bool get isSupported => true; + + @override + Future get isInitialised => _isInitialised!.future; + + Future _initialise() async { + if (_isInitialised != null) return await _isInitialised!.future; + + _isInitialised = Completer(); + + try { + _cacheDirectory = p.join( + cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ); + final cacheDirectoryIO = Directory(_cacheDirectory); + await cacheDirectoryIO.create(recursive: true); + + final persistentRegistryFilePath = p.join( + _cacheDirectory, + _persistentRegistryFileName, + ); + final persistentRegistryFile = File(persistentRegistryFilePath); + + final sizeMonitorFilePath = p.join( + _cacheDirectory, + _sizeMonitorFileName, + ); + + if (await persistentRegistryFile.exists()) { + final parsedCacheManager = await compute( + _parsePersistentRegistryWorker, + persistentRegistryFilePath, + debugLabel: '[flutter_map: cache] Persistent Registry Parser', + ); + + if (parsedCacheManager == null) { + await cacheDirectoryIO.delete(recursive: true); + await cacheDirectoryIO.create(recursive: true); + _registry = HashMap(); + } else { + _registry = parsedCacheManager; + + if (maxCacheSize case final sizeLimit?) { + // This can cause some delay when creating + // But it's much better than lagging or inconsistent registries + (await compute( + _limitCacheSizeWorker, + ( + cacheDirectoryPath: _cacheDirectory, + persistentRegistryFileName: _persistentRegistryFileName, + sizeMonitorFilePath: sizeMonitorFilePath, + sizeMonitorFileName: _sizeMonitorFileName, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Limiter', + )) + .forEach(_registry.remove); + } + } + } else { + _registry = HashMap(); + } + + final registryWorkerReceivePort = ReceivePort(); + await Isolate.spawn( + _persistentRegistryWorkerIsolate, + ( + port: registryWorkerReceivePort.sendPort, + persistentRegistryFilePath: persistentRegistryFilePath, + initialRegistry: _registry, + ), + debugName: '[flutter_map: cache] Persistent Registry Worker', + ); + final registryWorkerSendPort = + await registryWorkerReceivePort.first as SendPort; + + final tileFileWriterWorkerReceivePort = ReceivePort(); + await Isolate.spawn( + _tileFileWriterWorkerIsolate, + ( + port: tileFileWriterWorkerReceivePort.sendPort, + sizeMonitorFilePath: sizeMonitorFilePath, + ), + debugName: '[flutter_map: cache] Tile File Writer', + ); + final tileFileWriterWorkerSendPort = + await tileFileWriterWorkerReceivePort.first as SendPort; + + _writeToPersistentRegistry = (uuid, tileInfo) => + registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)); + _writeTileFile = (tileFilePath, bytes) => tileFileWriterWorkerSendPort + .send((tileFilePath: tileFilePath, bytes: bytes)); + } catch (error, stackTrace) { + _isInitialised!.completeError(error, stackTrace); + rethrow; + } + + _isInitialised!.complete(); + } + + @override + Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + String url, + ) async { + await isInitialised; + + final uuid = cacheKeyGenerator(url); + + final tileFile = File(p.join(_cacheDirectory, uuid)); + + if (_registry[uuid] case final tileInfo? when await tileFile.exists()) { + return (bytes: await tileFile.readAsBytes(), tileInfo: tileInfo); + } + + unawaited(_removeTile(uuid)); + return null; + } + + @override + Future putTile({ + required String url, + required CachedMapTileMetadata tileInfo, + Uint8List? bytes, + }) async { + await isInitialised; + + final uuid = cacheKeyGenerator(url); + final resolvedTileInfo = overrideFreshAge != null + ? CachedMapTileMetadata( + lastModifiedLocally: tileInfo.lastModifiedLocally, + staleAt: DateTime.timestamp().add(overrideFreshAge!), + lastModified: tileInfo.lastModified, + etag: tileInfo.etag, + ) + : tileInfo; + + if (_registry[uuid] case final existingTileInfo? + when resolvedTileInfo == existingTileInfo) { + return; + } + + if (bytes != null) { + final tileFilePath = p.join(_cacheDirectory, uuid); + _writeTileFile(tileFilePath, bytes); + } + + _registry[uuid] = resolvedTileInfo; + _writeToPersistentRegistry(uuid, resolvedTileInfo); + } + + Future _removeTile(String uuid) async { + await isInitialised; + + final tileFilePath = p.join(_cacheDirectory, uuid); + _writeTileFile(tileFilePath, null); + + if (_registry.remove(uuid) == null) return; + _writeToPersistentRegistry(uuid, null); + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart similarity index 91% rename from lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart rename to lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart index 6da10514a..fb5e4306f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/persistent_registry_workers.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart @@ -1,4 +1,4 @@ -part of 'manager.dart'; +part of 'native.dart'; /// Isolate worker which maintains its own registry and sequences writes to /// the persistent registry @@ -12,8 +12,6 @@ part of 'manager.dart'; /// that the user should not usually terminate the isolate very close to loading /// tiles, but also small enough to group adjacent tile loads), so manual /// sequencing and locking is required. -/// -/// See documentation on [MapTileCachingManager] for more info. Future _persistentRegistryWorkerIsolate( ({ SendPort port, @@ -156,19 +154,26 @@ Future> _limitCacheSizeWorker( ({ String cacheDirectoryPath, String persistentRegistryFileName, + String sizeMonitorFilePath, + String sizeMonitorFileName, int sizeLimit }) input, ) async { final cacheDirectory = Directory(input.cacheDirectoryPath); - final currentCacheSize = await cacheDirectory - .list() - .fold(0, (sum, file) => sum + file.statSync().size); + final sizeMonitorReader = File(input.sizeMonitorFilePath).openSync() + ..setPositionSync(0); + final currentCacheSize = + sizeMonitorReader.readSync(8).buffer.asInt64List().elementAtOrNull(0) ?? + 0; + sizeMonitorReader.closeSync(); + if (currentCacheSize <= input.sizeLimit) return []; final mapping = SplayTreeMap>(); bool foundManager = false; + bool foundSizeMonitor = false; await for (final file in cacheDirectory.list()) { if (file is! File) continue; if (!foundManager && @@ -176,6 +181,11 @@ Future> _limitCacheSizeWorker( foundManager = true; continue; } + if (!foundSizeMonitor && + p.basename(file.absolute.path) == input.sizeMonitorFileName) { + foundSizeMonitor = true; + continue; + } final FileStat stat; try { diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart new file mode 100644 index 000000000..3c569a34a --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -0,0 +1,42 @@ +import 'dart:typed_data'; + +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; + +/// Internal stub implementation of [BuiltInMapCachingProvider] +/// +/// Implemented based on platform in `native/` and `web/`. These must follow +/// the same structure as this stub. +@internal +class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { + final String? cacheDirectory; + final int? maxCacheSize; + final Duration? overrideFreshAge; + final String Function(String url) cacheKeyGenerator; + + @internal + const BuiltInMapCachingProviderImpl.createAndInitialise({ + required this.cacheDirectory, + required this.maxCacheSize, + required this.overrideFreshAge, + required this.cacheKeyGenerator, + }); + + @override + external Future get isInitialised; + + @override + external bool get isSupported; + + @override + external Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + String url, + ); + + @override + external Future putTile({ + required String url, + required CachedMapTileMetadata tileInfo, + Uint8List? bytes, + }); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart new file mode 100644 index 000000000..5ec042691 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -0,0 +1,41 @@ +import 'dart:typed_data'; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; + +@internal +class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { + final String? cacheDirectory; + final int? maxCacheSize; + final Duration? overrideFreshAge; + final String Function(String url) cacheKeyGenerator; + + @internal + const BuiltInMapCachingProviderImpl.createAndInitialise({ + required this.cacheDirectory, + required this.maxCacheSize, + required this.overrideFreshAge, + required this.cacheKeyGenerator, + }); + + @override + bool get isSupported => false; + + @override + Future get isInitialised => SynchronousFuture(null); + + @override + Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + String url, + ) => + throw UnsupportedError('Built-in map caching is not supported on web'); + + @override + Future putTile({ + required String url, + required CachedMapTileMetadata tileInfo, + Uint8List? bytes, + }) => + throw UnsupportedError('Built-in map caching is not supported on web'); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart new file mode 100644 index 000000000..b5935e304 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -0,0 +1,43 @@ +import 'dart:typed_data'; + +import 'package:flutter_map/flutter_map.dart'; + +/// Provides tile caching facilities based on HTTP headers (see +/// [CachedMapTileMetadata]) to [TileProvider]s +/// +/// Some caching plugins may choose instead to provide a dedicated +/// [TileProvider], in which case the flutter_map-provided caching facilities +/// are irrelevant. +abstract interface class MapCachingProvider { + /// Whether this caching provider is "currently supported" + /// + /// This can mean multiple things depending on the implementation's choice. + /// However, it is used the same in the [NetworkTileProvider] implementaiton. + /// + /// In some implementations, such as [BuiltInMapCachingProvider], this is set + /// constantly to indicate whether the implementation supports the current + /// platform. [getTile] and other implementation specific methods are used + /// to automatically wait for the internal initialisation to be complete + /// before returning a tile. In this case, the provider delays the loading of + /// tiles until initialisation is complete. + /// + /// In other implementations, [isSupported] may be set to indicate the + /// internal initialisation status. In this case, the provider does not delay + /// loading of tiles until initialisation is complete, and instead + /// automatically switches to using cached tiles once ready. + bool get isSupported; + + /// Retrieve a tile from the cache, if it exists + Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + String url, + ); + + /// Add or update a tile in the cache + /// + /// [bytes] is required if the tile is not already cached. + Future putTile({ + required String url, + required CachedMapTileMetadata tileInfo, + Uint8List? bytes, + }); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart new file mode 100644 index 000000000..2fd40ac1f --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart @@ -0,0 +1,23 @@ +import 'dart:typed_data'; + +import 'package:flutter_map/flutter_map.dart'; + +/// Map caching provider which disables caching +class DisabledMapCachingProvider implements MapCachingProvider { + /// Disable map caching through the [NetworkTileProvider.cachingProvider] + const DisabledMapCachingProvider(); + + @override + bool get isSupported => false; + + @override + Never getTile(String url) => throw StateError('Caching should be disabled'); + + @override + Never putTile({ + required String url, + required CachedMapTileMetadata tileInfo, + Uint8List? bytes, + }) => + throw StateError('Caching should be disabled'); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart similarity index 89% rename from lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart rename to lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index d5ced5cc5..d7be4e615 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -3,10 +3,13 @@ import 'dart:io' show HttpDate; // this is web safe! import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; -/// Metadata about a tile cached with the [MapTileCachingManager] +/// Metadata about a tile cached with a [MapCachingProvider] /// -/// Direct usage of this class is not usually necessary. It is visible so other -/// tile providers may make use of it. +/// Map caching is based on HTTP headers, which this class contains and can +/// encode/decode to/from JSON. +/// +/// External usage of this class is not usually necessary. It is visible so +/// other tile providers may make use of it. @immutable class CachedMapTileMetadata { /// Create new metadata diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart similarity index 84% rename from lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 46a750d48..ec472830b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -1,14 +1,17 @@ import 'dart:async'; +import 'dart:io' + show HttpHeaders, HttpDate, HttpException, HttpStatus; // this is web safe! +import 'dart:math'; import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart' - if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart' - if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; import 'package:http/http.dart'; +part 'tile_loader_simple.dart'; +part 'tile_loader_with_caching.dart'; + /// Dedicated [ImageProvider] to fetch tiles from the network /// /// Supports falling back to a secondary URL, if the primary URL fetch fails. @@ -43,15 +46,15 @@ class NetworkTileImageProvider extends ImageProvider { /// Not included in [operator==]. final bool silenceExceptions; - /// Configuration of built-in caching on native platforms + /// Caching provider used to get cached tiles /// /// See online documentation for more information about built-in caching. /// - /// Set to `null` to disable. See [MapCachingOptions] for defaults. Caching - /// is always disabled on the web. + /// Defaults to [BuiltInMapCachingProvider]. Set to + /// [DisabledMapCachingProvider] to disable. /// /// Not included in [operator==]. - final MapCachingOptions? cachingOptions; + final MapCachingProvider? cachingProvider; /// Function invoked when the image starts loading (not from cache) /// @@ -76,7 +79,7 @@ class NetworkTileImageProvider extends ImageProvider { required this.headers, required this.httpClient, required this.silenceExceptions, - required this.cachingOptions, + required this.cachingProvider, required this.startedLoading, required this.finishedLoadingBytes, }); @@ -102,7 +105,7 @@ class NetworkTileImageProvider extends ImageProvider { ImageDecoderCallback decode, { bool useFallback = false, }) => - loadTileImage(key, decode, useFallback: useFallback); + _loadTileImageWithCaching(key, decode, useFallback: useFallback); @override SynchronousFuture obtainKey( diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart new file mode 100644 index 000000000..8e945c06b --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart @@ -0,0 +1,27 @@ +part of 'image_provider.dart'; + +Future _loadTileImageSimple( + NetworkTileImageProvider key, + ImageDecoderCallback decode, { + bool useFallback = false, +}) { + key.startedLoading(); + + return key.httpClient + .readBytes( + Uri.parse(useFallback ? key.fallbackUrl ?? '' : key.url), + headers: key.headers, + ) + .whenComplete(key.finishedLoadingBytes) + .then(ImmutableBuffer.fromUint8List) + .then(decode) + .onError((err, stack) { + scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); + if (useFallback || key.fallbackUrl == null) { + if (!key.silenceExceptions) throw err; + return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) + .then(decode); + } + return _loadTileImageSimple(key, decode, useFallback: true); + }); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart similarity index 60% rename from lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart rename to lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart index 50c0f173d..b18cf4c4d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/native/tile_loader.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart @@ -1,73 +1,45 @@ -import 'dart:async'; -import 'dart:io'; -import 'dart:math'; -import 'dart:ui'; - -import 'package:flutter/painting.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; -import 'package:http/http.dart'; -import 'package:meta/meta.dart'; -import 'package:uuid/data.dart'; -import 'package:uuid/rng.dart'; -import 'package:uuid/uuid.dart'; - -final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); - -@internal -Future loadTileImage( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) => - _ioLoadTileImage(key, decode, useFallback: useFallback); +part of 'image_provider.dart'; -Future _ioLoadTileImage( +Future _loadTileImageWithCaching( NetworkTileImageProvider key, ImageDecoderCallback decode, { bool useFallback = false, }) async { key.startedLoading(); - if (key.cachingOptions == null) { - return simpleLoadTileImage(key, decode, useFallback: useFallback); + final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; + + final cachingProvider = + key.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); + + if (!cachingProvider.isSupported) { + return _loadTileImageSimple(key, decode, useFallback: useFallback); } - final MapTileCachingManager cachingManager; + + final ({Uint8List bytes, CachedMapTileMetadata tileInfo})? cachedTile; try { - cachingManager = await MapTileCachingManager.getInstance( - options: key.cachingOptions!, - ); + cachedTile = await cachingProvider.getTile(resolvedUrl); } catch (_) { - return simpleLoadTileImage(key, decode, useFallback: useFallback); + return _loadTileImageSimple(key, decode, useFallback: useFallback); } - final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; - final uuid = key.cachingOptions!.cacheKeyGenerator?.call(resolvedUrl) ?? - _uuid.v5(Namespace.url.value, resolvedUrl); - - final cachedTile = await cachingManager.getTile(uuid); - Future handleOk(Response response) async { final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; final etag = response.headers[HttpHeaders.etagHeader]; - unawaited(cachingManager.putTile( - uuid, - CachedMapTileMetadata( - lastModifiedLocally: DateTime.timestamp(), - staleAt: _calculateStaleAt( - response, - overrideFreshAge: key.cachingOptions!.overrideFreshAge, + unawaited( + cachingProvider.putTile( + url: resolvedUrl, + tileInfo: CachedMapTileMetadata( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt(response), + lastModified: + lastModified != null ? HttpDate.parse(lastModified) : null, + etag: etag, ), - lastModified: - lastModified != null ? HttpDate.parse(lastModified) : null, - etag: etag, + bytes: response.bodyBytes, ), - response.bodyBytes, - )); + ); return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); } @@ -86,7 +58,7 @@ Future _ioLoadTileImage( // Otherwise fallback to the fallback URL if (!useFallback && key.fallbackUrl != null) { - return _ioLoadTileImage(key, decode, useFallback: true); + return _loadTileImageWithCaching(key, decode, useFallback: true); } // Otherwise throw an exception/silently fail @@ -129,20 +101,19 @@ Future _ioLoadTileImage( final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; final etag = response.headers[HttpHeaders.etagHeader]; - unawaited(cachingManager.putTile( - uuid, - CachedMapTileMetadata( - lastModifiedLocally: DateTime.timestamp(), - staleAt: _calculateStaleAt( - response, - overrideFreshAge: key.cachingOptions!.overrideFreshAge, + unawaited( + cachingProvider.putTile( + url: resolvedUrl, + tileInfo: CachedMapTileMetadata( + lastModifiedLocally: DateTime.timestamp(), + staleAt: _calculateStaleAt(response), + lastModified: lastModified != null + ? HttpDate.parse(lastModified) + : cachedTile.tileInfo.lastModified, + etag: etag ?? cachedTile.tileInfo.etag, ), - lastModified: lastModified != null - ? HttpDate.parse(lastModified) - : cachedTile.tileInfo.lastModified, - etag: etag ?? cachedTile.tileInfo.etag, ), - )); + ); return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); } @@ -165,16 +136,9 @@ Future _ioLoadTileImage( return await handleNotOk(response); } -DateTime _calculateStaleAt( - Response response, { - required Duration? overrideFreshAge, -}) { +DateTime _calculateStaleAt(Response response) { final addToNow = DateTime.timestamp().add; - if (overrideFreshAge case final overrideFreshAge?) { - return addToNow(overrideFreshAge); - } - if (response.headers[HttpHeaders.cacheControlHeader]?.toLowerCase() case final cacheControl?) { final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart deleted file mode 100644 index 48f2cca69..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/manager.dart +++ /dev/null @@ -1,70 +0,0 @@ -import 'dart:typed_data'; - -import 'package:flutter_map/flutter_map.dart'; -import 'package:meta/meta.dart'; - -/// {@template fm.mtcm} -/// Singleton class which manages simple built-in tile caching based HTTP -/// headers for native (I/O) platforms only -/// -/// > [!IMPORTANT] -/// > Built-in tile caching is not a replacement for caching which can better -/// > guarantee resilience. It should not solely be used where not having -/// > cached tiles may lead to a dangerous situation - for example, offline -/// > mapping. This provides no guarantees as to the safety of cached tiles. -/// -/// For more information, see the online documentation. -/// -/// --- -/// -/// Direct usage of this class is not usually necessary. It is visible so other -/// tile providers may make use of it. -/// -/// --- -/// {@endtemplate} -@immutable -class MapTileCachingManager { - const MapTileCachingManager._(); - - /// Returns an existing instance if available, else creates one and returns - /// once ready - /// - /// If this is called multiple times simultanously, they will lock so that - /// only one instance is created. - /// - /// Throws if an instance did not exist and could not be created. In this - /// case, tile provider users should fallback to a non-caching implementation. - /// - /// If an instance is not already available, [options] is used to configure - /// the new instance. - static Future getInstance({ - MapCachingOptions options = const MapCachingOptions(), - }) => - throw UnsupportedError( - '`MapTileCachingManager` is only implemented on I/O platforms', - ); - - /// Retrieve a tile from the cache, if it exists - Future< - ({ - Uint8List bytes, - CachedMapTileMetadata tileInfo, - })?> getTile( - String uuid, - ) => - throw UnsupportedError( - '`MapTileCachingManager` is only implemented on I/O platforms', - ); - - /// Add or update a tile in the cache - /// - /// [bytes] is required if the tile is not already cached. - Future putTile( - String uuid, - CachedMapTileMetadata tileInfo, [ - Uint8List? bytes, - ]) => - throw UnsupportedError( - '`MapTileCachingManager` is only implemented on I/O platforms', - ); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart deleted file mode 100644 index aa7c66e80..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart +++ /dev/null @@ -1,74 +0,0 @@ -import 'package:meta/meta.dart'; - -/// Configuration of the built-in caching -@immutable -class MapCachingOptions { - /// Path to the caching directory to use - /// - /// This must be accessible to the program. - /// - /// Defaults to a platform provided cache directory, which may be cleared by - /// the OS at any time. - final String? cacheDirectory; - - /// Preferred maximum size (in bytes) of the cache - /// - /// This is applied when the internal caching mechanism is created (on the - /// first tile load in the main memory space for the app). It is not an - /// absolute limit. - /// - /// This may cause some slight delay to the loading of the first tiles, - /// especially if the size is large and the cache does exceed the size. If - /// the visible delay becomes too large, disable this and manage the cache - /// size manually if necessary. - /// - /// Defaults to 1GB. Set to `null` to disable. - final int? maxCacheSize; - - /// Override the duration of time a tile is considered fresh for - /// - /// Defaults to `null`: use duration calculated from each tile's HTTP headers. - final Duration? overrideFreshAge; - - /// Function to convert a tile URL to a key used in the cache - /// - /// This may be useful where parts of the URL are volatile or do not represent - /// the tile image, for example, API keys contained with the query parameters. - /// - /// The resulting key should be unique to that tile URL. - /// - /// Defaults to generating a UUID from the entire URL string. - final String Function(String url)? cacheKeyGenerator; - - /// Create a configuration for caching - const MapCachingOptions({ - this.cacheDirectory, - this.maxCacheSize = 1000000000, - this.overrideFreshAge, - this.cacheKeyGenerator, - }) : assert( - maxCacheSize == null || maxCacheSize > 0, - '`maxCacheSize` must be greater than 0 or disabled', - ), - assert( - overrideFreshAge == null || overrideFreshAge > Duration.zero, - '`overrideFreshAge` must be greater than 0 or disabled', - ); - - @override - int get hashCode => Object.hash( - cacheDirectory, - maxCacheSize, - overrideFreshAge, - cacheKeyGenerator, - ); - - @override - bool operator ==(Object other) => - identical(this, other) || - (other is MapCachingOptions && - cacheDirectory == other.cacheDirectory && - maxCacheSize == other.maxCacheSize && - overrideFreshAge == other.overrideFreshAge && - cacheKeyGenerator == other.cacheKeyGenerator); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart deleted file mode 100644 index d25b6f86b..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/tile_loader.dart +++ /dev/null @@ -1,14 +0,0 @@ -import 'dart:ui'; - -import 'package:flutter/painting.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart'; -import 'package:meta/meta.dart'; - -@internal -Future loadTileImage( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) => - simpleLoadTileImage(key, decode, useFallback: useFallback); diff --git a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart b/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart deleted file mode 100644 index d08aa28be..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/native/caching/manager.dart +++ /dev/null @@ -1,239 +0,0 @@ -import 'dart:async'; -import 'dart:collection'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:isolate'; - -import 'package:flutter/foundation.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/options.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/caching/tile_metadata.dart'; -import 'package:path/path.dart' as p; -import 'package:path_provider/path_provider.dart'; - -part 'persistent_registry_workers.dart'; - -/// {@macro fm.mtcm} -/// -/// The singleton is not disposed of. The only time it would make sense to -/// close the singleton is when either: -/// * there are no more tile providers/layers using it -/// * the app is destroyed -/// -/// However, it is more difficult to track the first condition, allbeit possible. -/// More importantly, creating and opening a new instance takes some time, so -/// this should be minimized. -/// -/// It is not possible to detect reliably when the process is stopped*. However, -/// we should ideally close any open file handles after we no longer need them -/// (when the singleton is disposed). Additionally, for performance, opening -/// the persistent registry file once as a [RandomAccessFile] and synchonously -/// writing to it is preferred to repeatedly opening it for async writing. -/// Therefore, a long-living isolate is used. -/// -/// The isolate maintains its own in-memory registry (just as this class does -/// directly). The isolate registry and main registry should remain in sync. -/// The isolate registry is used only for writing to, which then writes -/// to the persistent registry synchronously using an open [RandomAccessFile]. -/// The main registry is used only for reading and writing. The main and isolate -/// registries are populated from the persistent registry when an instance is -/// created. -/// -/// When the program is terminated (or hot-reloaded in Dart), the isolate is -/// usually terminated. This usually results in the file handle being closed as -/// well. Closure of the persistent registry file is important before another -/// manager instance is created - otherwise the lock obtained will prevent -/// the new instance from working correctly. It is assumed the OS closes the -/// open file handles, but that does mean every write to the persistent -/// registry must be flushed. -/// -/// A long-lasting isolate is also for writing tile files to reduce the -/// overheads of async file operations. -@immutable -class MapTileCachingManager { - const MapTileCachingManager._({ - required String cacheDirectory, - required void Function(String uuid, CachedMapTileMetadata? tileInfo) - writeToPersistentRegistry, - required void Function(String tileFilePath, Uint8List? bytes) writeTileFile, - required HashMap registry, - }) : _cacheDirectory = cacheDirectory, - _writeToPersistentRegistry = writeToPersistentRegistry, - _writeTileFile = writeTileFile, - _registry = registry; - - static const _persistentRegistryFileName = 'registry.json'; - static const _sizeMonitorFileName = 'sizeMonitor'; - - static Completer? _instance; - - final String _cacheDirectory; - final void Function(String uuid, CachedMapTileMetadata? tileInfo) - _writeToPersistentRegistry; - final void Function(String tileFilePath, Uint8List? bytes) _writeTileFile; - final HashMap _registry; - - /// Returns an existing instance if available, else creates one and returns - /// once ready - /// - /// If this is called multiple times simultanously, they will lock so that - /// only one instance is created. - /// - /// Throws if an instance did not exist and could not be created. In this - /// case, tile provider users should fallback to a non-caching implementation. - /// - /// If an instance is not already available, [options] is used to configure - /// the new instance. - static Future getInstance({ - MapCachingOptions options = const MapCachingOptions(), - }) async { - if (_instance != null) return await _instance!.future; - - _instance = Completer(); - final MapTileCachingManager instance; - - try { - final resolvedCacheDirectory = Directory( - p.join( - options.cacheDirectory ?? - (await getApplicationCacheDirectory()).absolute.path, - 'fm_cache', - ), - ); - await resolvedCacheDirectory.create(recursive: true); - - final persistentRegistryFilePath = p.join( - resolvedCacheDirectory.absolute.path, - _persistentRegistryFileName, - ); - final persistentRegistryFile = File(persistentRegistryFilePath); - - final sizeMonitorFilePath = p.join( - resolvedCacheDirectory.absolute.path, - _sizeMonitorFileName, - ); - - final HashMap registry; - if (await persistentRegistryFile.exists()) { - final parsedCacheManager = await compute( - _parsePersistentRegistryWorker, - persistentRegistryFilePath, - debugLabel: '[flutter_map: cache] Persistent Registry Parser', - ); - - if (parsedCacheManager == null) { - await resolvedCacheDirectory.delete(recursive: true); - await resolvedCacheDirectory.create(recursive: true); - registry = HashMap(); - } else { - registry = parsedCacheManager; - - if (options.maxCacheSize case final sizeLimit?) { - // This can cause some delay when creating - // But it's much better than lagging or inconsistent registries - (await compute( - _limitCacheSizeWorker, - ( - cacheDirectoryPath: resolvedCacheDirectory.absolute.path, - persistentRegistryFileName: _persistentRegistryFileName, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Limiter', - )) - .forEach(registry.remove); - } - } - } else { - registry = HashMap(); - } - - final registryWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - _persistentRegistryWorkerIsolate, - ( - port: registryWorkerReceivePort.sendPort, - persistentRegistryFilePath: persistentRegistryFilePath, - initialRegistry: registry, - ), - debugName: '[flutter_map: cache] Persistent Registry Worker', - ); - final registryWorkerSendPort = - await registryWorkerReceivePort.first as SendPort; - - final tileFileWriterWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - _tileFileWriterWorkerIsolate, - ( - port: tileFileWriterWorkerReceivePort.sendPort, - sizeMonitorFilePath: sizeMonitorFilePath, - ), - debugName: '[flutter_map: cache] Tile File Writer', - ); - final tileFileWriterWorkerSendPort = - await tileFileWriterWorkerReceivePort.first as SendPort; - - instance = MapTileCachingManager._( - cacheDirectory: resolvedCacheDirectory.absolute.path, - writeToPersistentRegistry: (uuid, tileInfo) => - registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)), - writeTileFile: (tileFilePath, bytes) => tileFileWriterWorkerSendPort - .send((tileFilePath: tileFilePath, bytes: bytes)), - registry: registry, - ); - } catch (error, stackTrace) { - _instance!.completeError(error, stackTrace); - rethrow; - } - - _instance!.complete(instance); - return instance; - } - - /// Retrieve a tile from the cache, if it exists - Future< - ({ - Uint8List bytes, - CachedMapTileMetadata tileInfo, - })?> getTile( - String uuid, - ) async { - final tileFile = File(p.join(_cacheDirectory, uuid)); - - if (_registry[uuid] case final tileInfo? when await tileFile.exists()) { - return (bytes: await tileFile.readAsBytes(), tileInfo: tileInfo); - } - - unawaited(removeTile(uuid)); - return null; - } - - /// Add or update a tile in the cache - /// - /// [bytes] is required if the tile is not already cached. - Future putTile( - String uuid, - CachedMapTileMetadata tileInfo, [ - Uint8List? bytes, - ]) async { - if (_registry[uuid] case final existingTileInfo? - when tileInfo == existingTileInfo) { - return; - } - - if (bytes != null) { - final tileFilePath = p.join(_cacheDirectory, uuid); - _writeTileFile(tileFilePath, bytes); - } - - _registry[uuid] = tileInfo; - _writeToPersistentRegistry(uuid, tileInfo); - } - - /// Remove a tile from the cache - Future removeTile(String uuid) async { - final tileFilePath = p.join(_cacheDirectory, uuid); - _writeTileFile(tileFilePath, null); - - if (_registry.remove(uuid) == null) return; - _writeToPersistentRegistry(uuid, null); - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart similarity index 92% rename from lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart rename to lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index 1a98f555c..a31b18eb0 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -3,7 +3,7 @@ import 'dart:collection'; import 'package:flutter/rendering.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart'; import 'package:http/http.dart'; import 'package:http/retry.dart'; @@ -36,7 +36,7 @@ class NetworkTileProvider extends TileProvider { super.headers, Client? httpClient, this.silenceExceptions = false, - this.cachingOptions = const MapCachingOptions(), + this.cachingProvider, }) : _isInternallyCreatedClient = httpClient == null, _httpClient = httpClient ?? RetryClient(Client()); @@ -44,12 +44,13 @@ class NetworkTileProvider extends TileProvider { /// over the network, and just return a transparent tile final bool silenceExceptions; - /// Configuration of built-in caching + /// Caching provider used to get cached tiles /// /// See online documentation for more information about built-in caching. /// - /// Set to `null` to disable. See [MapCachingOptions] for defaults. - final MapCachingOptions? cachingOptions; + /// Defaults to [BuiltInMapCachingProvider]. Set to + /// [DisabledMapCachingProvider] to disable. + final MapCachingProvider? cachingProvider; /// Long living client used to make all tile requests by /// [NetworkTileImageProvider] for the duration that this provider is @@ -79,7 +80,7 @@ class NetworkTileProvider extends TileProvider { headers: headers, httpClient: _httpClient, silenceExceptions: silenceExceptions, - cachingOptions: cachingOptions, + cachingProvider: cachingProvider, startedLoading: () => _tilesInProgress[coordinates] = Completer(), finishedLoadingBytes: () { _tilesInProgress[coordinates]?.complete(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart b/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart deleted file mode 100644 index 611f0e24c..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/web/tile_loader.dart +++ /dev/null @@ -1,49 +0,0 @@ -import 'dart:async'; -import 'dart:ui'; - -import 'package:flutter/painting.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; -import 'package:meta/meta.dart'; - -@internal -Future loadTileImage( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) => - _webLoadTileImage(key, decode, useFallback: useFallback); - -@internal -Future simpleLoadTileImage( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) => - _webLoadTileImage(key, decode, useFallback: useFallback); - -Future _webLoadTileImage( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) { - key.startedLoading(); - - return key.httpClient - .readBytes( - Uri.parse(useFallback ? key.fallbackUrl ?? '' : key.url), - headers: key.headers, - ) - .whenComplete(key.finishedLoadingBytes) - .then(ImmutableBuffer.fromUint8List) - .then(decode) - .onError((err, stack) { - scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); - if (useFallback || key.fallbackUrl == null) { - if (!key.silenceExceptions) throw err; - return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) - .then(decode); - } - return _webLoadTileImage(key, decode, useFallback: true); - }); -} diff --git a/test/full_coverage_test.dart b/test/full_coverage_test.dart index d158d5894..1f422fc07 100644 --- a/test/full_coverage_test.dart +++ b/test/full_coverage_test.dart @@ -34,8 +34,8 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/asset/provider.da import 'package:flutter_map/src/layer/tile_layer/tile_provider/base_tile_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/native_tile_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/file/stub_tile_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/tile_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/tile_provider.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_range_calculator.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_scale_calculator.dart'; diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index 9766a7eb2..91d5c3cbc 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -4,7 +4,8 @@ import 'dart:typed_data'; import 'package:flutter/painting.dart'; import 'package:flutter/widgets.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/independent/image_provider.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:http/http.dart'; import 'package:mocktail/mocktail.dart'; @@ -72,7 +73,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -112,7 +113,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -148,7 +149,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: true, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -192,7 +193,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -239,7 +240,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: true, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -278,7 +279,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -317,7 +318,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: true, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -358,7 +359,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -402,7 +403,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: true, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -448,7 +449,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -498,7 +499,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -540,7 +541,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: false, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); @@ -587,7 +588,7 @@ void main() { headers: headers, httpClient: mockClient, silenceExceptions: true, - cachingOptions: null, + cachingProvider: const DisabledMapCachingProvider(), startedLoading: () => startedLoadingTriggered = true, finishedLoadingBytes: () => finishedLoadingTriggered = true, ); From e9c4465c55fef691564ab8b75c489c3d94493844 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 15 May 2025 12:35:00 +0100 Subject: [PATCH 11/49] Fixed linting issue --- .../tile_provider/network/caching/built_in/impl/web/web.dart | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 5ec042691..93cdb10aa 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -1,5 +1,3 @@ -import 'dart:typed_data'; - import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; From e67f7e4c43bca4f2b14d7327744c4173a75a6414 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 15 May 2025 22:55:57 +0100 Subject: [PATCH 12/49] Improved speed and efficiency of size limiter Refactored workers into independent source files Added size monitor auto-repair Added read-only option --- lib/src/layer/tile_layer/tile_layer.dart | 9 +- .../built_in/built_in_caching_provider.dart | 17 +- .../caching/built_in/impl/native/native.dart | 26 ++- .../native/persistent_registry_workers.dart | 216 ------------------ .../workers/persistent_registry_parser.dart | 38 +++ .../workers/persistent_registry_writer.dart | 80 +++++++ .../impl/native/workers/size_limiter.dart | 86 +++++++ .../workers/tile_writer_size_monitor.dart | 60 +++++ .../workers/utils/size_monitor_opener.dart | 93 ++++++++ .../network/caching/built_in/impl/stub.dart | 2 + .../caching/built_in/impl/web/web.dart | 2 + 11 files changed, 397 insertions(+), 232 deletions(-) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart diff --git a/lib/src/layer/tile_layer/tile_layer.dart b/lib/src/layer/tile_layer/tile_layer.dart index ca73239eb..2caecec42 100644 --- a/lib/src/layer/tile_layer/tile_layer.dart +++ b/lib/src/layer/tile_layer/tile_layer.dart @@ -362,20 +362,17 @@ class _TileLayerState extends State with TickerProviderStateMixin { false && (kReleaseMode || kProfileMode) && !_unblockOpenStreetMapUrl; void _warnOpenStreetMapUrl() { if (!_isOpenStreetMapUrl || !kDebugMode || _unblockOpenStreetMapUrl) return; - Logger(printer: PrettyPrinter(methodCount: 0)).e( + Logger(printer: PrettyPrinter(methodCount: 0)).w( '''\x1B[1m\x1B[3mflutter_map\x1B[0m flutter_map wants to help keep map data available for everyone. We use the public OpenStreetMap tile servers in our code examples & demo app, but they are NOT free to use by everyone. -In an upcoming non-major release, requests to 'tile.openstreetmap.org' or -'tile.osm.org' will be blocked by default in release mode. Please review https://operations.osmfoundation.org/policies/tiles/ to see if your project is compliant with their Tile Usage Policy. For more information, see https://docs.fleaflet.dev/tile-servers/using-openstreetmap-direct. It describes in additional detail why we feel it is important to do this, how -you can unblock the tile servers if your use-case is acceptable, the timeframes -for this new policy, and how we're working to reduce requests without any extra -work from you.''', +you can disable this warning, the timeframes for this new policy, and how we're +working to reduce requests without any extra work from you.''', ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 258ee1865..5aeb9fe65 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -46,18 +46,28 @@ abstract interface class BuiltInMapCachingProvider /// Override the duration of time a tile is considered fresh for /// - /// Defaults to `null`: use duration calculated from each tile's HTTP headers. + /// Defaults to `null`: use duration calculated from each tile's HTTP + /// headers. Duration? overrideFreshAge, /// Function to convert a tile URL to a key used in the cache /// - /// This may be useful where parts of the URL are volatile or do not represent - /// the tile image, for example, API keys contained with the query parameters. + /// This may be useful where parts of the URL are volatile or do not + /// represent the tile image, for example, API keys contained with the query + /// parameters. /// /// The resulting key should be unique to that tile URL. /// /// Defaults to generating a UUID from the entire URL string. String Function(String url)? cacheKeyGenerator, + + /// Prevent any tiles from being added or updated + /// + /// Does not disable the size limiter if the cache size is larger than + /// `maxCacheSize`. + /// + /// Defaults to `false`. + bool readOnly = false, }) { assert( maxCacheSize == null || maxCacheSize > 0, @@ -73,6 +83,7 @@ abstract interface class BuiltInMapCachingProvider overrideFreshAge: overrideFreshAge, cacheKeyGenerator: cacheKeyGenerator ?? (url) => _uuid.v5(Namespace.url.value, url), + readOnly: readOnly, ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index c3db4f11c..72e1ba9cd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -1,23 +1,25 @@ import 'dart:async'; import 'dart:collection'; -import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; -part 'persistent_registry_workers.dart'; - @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; final String Function(String url) cacheKeyGenerator; + final bool readOnly; @internal BuiltInMapCachingProviderImpl.createAndInitialise({ @@ -25,6 +27,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.maxCacheSize, required this.overrideFreshAge, required this.cacheKeyGenerator, + required this.readOnly, }) { _initialise(); } @@ -50,6 +53,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future _initialise() async { if (_isInitialised != null) return await _isInitialised!.future; + final stopwatch = Stopwatch()..start(); _isInitialised = Completer(); try { @@ -73,7 +77,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { if (await persistentRegistryFile.exists()) { final parsedCacheManager = await compute( - _parsePersistentRegistryWorker, + persistentRegistryParserWorker, persistentRegistryFilePath, debugLabel: '[flutter_map: cache] Persistent Registry Parser', ); @@ -89,7 +93,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { // This can cause some delay when creating // But it's much better than lagging or inconsistent registries (await compute( - _limitCacheSizeWorker, + sizeLimiterWorker, ( cacheDirectoryPath: _cacheDirectory, persistentRegistryFileName: _persistentRegistryFileName, @@ -108,7 +112,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final registryWorkerReceivePort = ReceivePort(); await Isolate.spawn( - _persistentRegistryWorkerIsolate, + persistentRegistryWriterWorker, ( port: registryWorkerReceivePort.sendPort, persistentRegistryFilePath: persistentRegistryFilePath, @@ -121,10 +125,13 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final tileFileWriterWorkerReceivePort = ReceivePort(); await Isolate.spawn( - _tileFileWriterWorkerIsolate, + tileWriterSizeMonitorWorker, ( port: tileFileWriterWorkerReceivePort.sendPort, + cacheDirectoryPath: _cacheDirectory, + persistentRegistryFileName: _persistentRegistryFileName, sizeMonitorFilePath: sizeMonitorFilePath, + sizeMonitorFileName: _sizeMonitorFileName, ), debugName: '[flutter_map: cache] Tile File Writer', ); @@ -140,6 +147,9 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { rethrow; } + stopwatch.stop(); + print(stopwatch.elapsedMilliseconds); + _isInitialised!.complete(); } @@ -167,6 +177,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required CachedMapTileMetadata tileInfo, Uint8List? bytes, }) async { + if (readOnly) return; + await isInitialised; final uuid = cacheKeyGenerator(url); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart deleted file mode 100644 index fb5e4306f..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/persistent_registry_workers.dart +++ /dev/null @@ -1,216 +0,0 @@ -part of 'native.dart'; - -/// Isolate worker which maintains its own registry and sequences writes to -/// the persistent registry -/// -/// We cannot use [IOSink] from [File.openWrite], since we need to overwrite the -/// entire file on every write. [RandomAccessFile] allows this, and may also be -/// faster (especially for sync operations). However, it does not sequence -/// writes as [IOSink] does: attempting multiple writes at the same time throws -/// errors. If we use sync operations on every incoming update, this shouldn't -/// be an issue - instead, we use a debouncer (at 50ms, which is small enough -/// that the user should not usually terminate the isolate very close to loading -/// tiles, but also small enough to group adjacent tile loads), so manual -/// sequencing and locking is required. -Future _persistentRegistryWorkerIsolate( - ({ - SendPort port, - String persistentRegistryFilePath, - Map initialRegistry, - }) input, -) async { - final registry = input.initialRegistry; - final writer = File(input.persistentRegistryFilePath) - .openSync(mode: FileMode.writeOnlyAppend); - - var writeLocker = Completer()..complete(); - var alreadyWaitingToWrite = false; - Future write() async { - if (alreadyWaitingToWrite) return; - alreadyWaitingToWrite = true; - await writeLocker.future; - writeLocker = Completer(); - alreadyWaitingToWrite = false; - - final encoded = jsonEncode(registry); - writer - ..setPositionSync(0) - ..writeStringSync(encoded) - ..truncateSync(writer.positionSync()) - ..flushSync(); - - writeLocker.complete(); - } - - Timer createWriteDebouncer() => - Timer(const Duration(milliseconds: 50), write); - Timer? writeDebouncer; - - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - - await for (final val in receivePort) { - final (:uuid, :tileInfo) = - val as ({String uuid, CachedMapTileMetadata? tileInfo}); - - if (tileInfo == null) { - registry.remove(uuid); - } else { - registry[uuid] = tileInfo; - } - - writeDebouncer?.cancel(); - writeDebouncer = createWriteDebouncer(); - } -} - -/// Isolate worker which writes & deletes tile files, and updates the size -/// monitor, synchronously -Future _tileFileWriterWorkerIsolate( - ({ - SendPort port, - String sizeMonitorFilePath, - }) input, -) async { - final sizeMonitorWriter = File(input.sizeMonitorFilePath) - .openSync(mode: FileMode.append) - ..setPositionSync(0); - int currentSize = - sizeMonitorWriter.readSync(8).buffer.asInt64List().elementAtOrNull(0) ?? - 0; - final allocatedWriteBinBuffer = Uint8List(8); - - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - - await for (final val in receivePort) { - final (:tileFilePath, :bytes) = - val as ({String tileFilePath, Uint8List? bytes}); - - final tileFile = File(tileFilePath); - final tileFileExists = tileFile.existsSync(); - - final existingTileSize = tileFileExists ? tileFile.lengthSync() : 0; - final newTileSize = bytes?.lengthInBytes ?? 0; - if (newTileSize - existingTileSize case final deltaSize - when deltaSize != 0) { - currentSize += deltaSize; - sizeMonitorWriter - ..setPositionSync(0) - ..writeFromSync( - allocatedWriteBinBuffer..buffer.asInt64List()[0] = currentSize, - ) - ..flushSync(); - } - - if (bytes != null) { - tileFile.writeAsBytesSync(bytes); - } else if (tileFileExists) { - tileFile.deleteSync(); - } - } -} - -/// Decode the JSON within the persistent registry into a mapping of tile -/// UUIDs to their [CachedMapTileMetadata]s -/// -/// Should be used within an isolate/[compute]r. -/// -/// If the JSON is invalid or the file cannot be read, this returns null. -HashMap? _parsePersistentRegistryWorker( - String persistentRegistryFilePath, -) { - final String json; - try { - json = File(persistentRegistryFilePath).readAsStringSync(); - } on FileSystemException { - return null; - } - - final Map parsed; - try { - parsed = jsonDecode(json) as Map; - } on FormatException { - return null; - } - - return HashMap.from( - parsed.map( - (key, value) => MapEntry( - key, - CachedMapTileMetadata.fromJson(value as Map), - ), - ), - ); -} - -/// Remove tile files from the cache directory, 'first'-modified and largest -/// first, until the total size is below the set limit -/// -/// Returns removed tile UUIDs. -/// -/// This does not alter any registries in memory. -Future> _limitCacheSizeWorker( - ({ - String cacheDirectoryPath, - String persistentRegistryFileName, - String sizeMonitorFilePath, - String sizeMonitorFileName, - int sizeLimit - }) input, -) async { - final cacheDirectory = Directory(input.cacheDirectoryPath); - - final sizeMonitorReader = File(input.sizeMonitorFilePath).openSync() - ..setPositionSync(0); - final currentCacheSize = - sizeMonitorReader.readSync(8).buffer.asInt64List().elementAtOrNull(0) ?? - 0; - sizeMonitorReader.closeSync(); - - if (currentCacheSize <= input.sizeLimit) return []; - - final mapping = - SplayTreeMap>(); - bool foundManager = false; - bool foundSizeMonitor = false; - await for (final file in cacheDirectory.list()) { - if (file is! File) continue; - if (!foundManager && - p.basename(file.absolute.path) == input.persistentRegistryFileName) { - foundManager = true; - continue; - } - if (!foundSizeMonitor && - p.basename(file.absolute.path) == input.sizeMonitorFileName) { - foundSizeMonitor = true; - continue; - } - - final FileStat stat; - try { - stat = file.statSync(); - } on FileSystemException { - return []; - } - - (mapping[stat.modified] ??= []) // `stat.accessed` is unreliable - .add((file: file, uuid: p.basename(file.path), size: stat.size)); - } - - // Delete largest oldest files first - int collectedSize = 0; - final collectedUuids = []; - outer: - for (final MapEntry(key: _, value: files) in mapping.entries) { - files.sort((a, b) => b.size.compareTo(a.size)); - for (final (:file, :uuid, :size) in files) { - collectedUuids.add(uuid); - collectedSize += size; - file.deleteSync(); - if (currentCacheSize - collectedSize <= input.sizeLimit) break outer; - } - } - - return collectedUuids; -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart new file mode 100644 index 000000000..89a0be4d6 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart @@ -0,0 +1,38 @@ +import 'dart:collection'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; + +/// Decode the JSON within the persistent registry into a mapping of tile +/// UUIDs to their [CachedMapTileMetadata]s +/// +/// If the JSON is invalid or the file cannot be read, this returns null. +@internal +HashMap? persistentRegistryParserWorker( + String persistentRegistryFilePath, +) { + final String json; + try { + json = File(persistentRegistryFilePath).readAsStringSync(); + } on FileSystemException { + return null; + } + + final Map parsed; + try { + parsed = jsonDecode(json) as Map; + } on FormatException { + return null; + } + + return HashMap.from( + parsed.map( + (key, value) => MapEntry( + key, + CachedMapTileMetadata.fromJson(value as Map), + ), + ), + ); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart new file mode 100644 index 000000000..8adde042e --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart @@ -0,0 +1,80 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:flutter_map/flutter_map.dart'; +import 'package:meta/meta.dart'; + +/// Isolate worker which maintains its own registry and sequences writes to +/// the persistent registry +/// +/// We cannot use [IOSink] from [File.openWrite], since we need to overwrite the +/// entire file on every write. [RandomAccessFile] allows this, and may also be +/// faster (especially for sync operations). However, it does not sequence +/// writes as [IOSink] does: attempting multiple writes at the same time throws +/// errors. If we use sync operations on every incoming update, this shouldn't +/// be an issue - instead, we use a debouncer (at 50ms, which is small enough +/// that the user should not usually terminate the isolate very close to loading +/// tiles, but also small enough to group adjacent tile loads), so manual +/// sequencing and locking is required. +@internal +Future persistentRegistryWriterWorker( + ({ + SendPort port, + String persistentRegistryFilePath, + Map initialRegistry, + }) input, +) async { + final registry = input.initialRegistry; + final writer = + File(input.persistentRegistryFilePath).openSync(mode: FileMode.writeOnly); + + // We rewrite the registry from the initial state in-case it was size limited, + // for example + final encoded = jsonEncode(registry); + writer + ..writeStringSync(encoded) + ..truncateSync(writer.positionSync()) + ..flushSync(); + + var writeLocker = Completer()..complete(); + var alreadyWaitingToWrite = false; + Future write() async { + if (alreadyWaitingToWrite) return; + alreadyWaitingToWrite = true; + await writeLocker.future; + writeLocker = Completer(); + alreadyWaitingToWrite = false; + + final encoded = jsonEncode(registry); + writer + ..setPositionSync(0) + ..writeStringSync(encoded) + ..truncateSync(writer.positionSync()) + ..flushSync(); + + writeLocker.complete(); + } + + Timer createWriteDebouncer() => + Timer(const Duration(milliseconds: 50), write); + Timer? writeDebouncer; + + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + + await for (final val in receivePort) { + final (:uuid, :tileInfo) = + val as ({String uuid, CachedMapTileMetadata? tileInfo}); + + if (tileInfo == null) { + registry.remove(uuid); + } else { + registry[uuid] = tileInfo; + } + + writeDebouncer?.cancel(); + writeDebouncer = createWriteDebouncer(); + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart new file mode 100644 index 000000000..81a9befe5 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart @@ -0,0 +1,86 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:collection/collection.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +typedef _SizeLimiterTile = ({String path, int size, DateTime sortKey}); + +/// Remove tile files from the cache directory until the total size is below the +/// set limit +/// +/// Removes the least recently accessed tiles first. Tries to remove as few +/// tiles as possible (largest first if last accessed at same time). +/// +/// Returns removed tile UUIDs. +/// +/// This does not alter any registries in memory. +@internal +Future> sizeLimiterWorker( + ({ + String cacheDirectoryPath, + String persistentRegistryFileName, + String sizeMonitorFilePath, + String sizeMonitorFileName, + int sizeLimit + }) input, +) async { + final cacheDirectory = Directory(input.cacheDirectoryPath); + + final (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( + cacheDirectoryPath: input.cacheDirectoryPath, + persistentRegistryFileName: input.persistentRegistryFileName, + sizeMonitorFileName: input.sizeMonitorFileName, + sizeMonitorFilePath: input.sizeMonitorFilePath, + ); + + if (currentSize <= input.sizeLimit) { + sizeMonitor.closeSync(); + return []; + } + + final tiles = await Future.wait<_SizeLimiterTile>( + cacheDirectory.listSync().whereType().where((f) { + final uuid = p.basename(f.absolute.path); + return uuid != input.persistentRegistryFileName && + uuid != input.sizeMonitorFileName; + }).map((f) async { + final stat = await f.stat(); + // `stat.accessed` may be unstable on some OSs, but seems to work enough? + return (path: f.absolute.path, size: stat.size, sortKey: stat.accessed); + }), + ); + + int compareSortKeys(_SizeLimiterTile a, _SizeLimiterTile b) => + a.sortKey.compareTo(b.sortKey); + int compareInverseSizes(_SizeLimiterTile a, _SizeLimiterTile b) => + b.size.compareTo(a.size); + tiles.sort(compareSortKeys.then(compareInverseSizes)); + + int i = 0; + int deletedSize = 0; + final deletedTiles = () sync* { + while (currentSize - deletedSize > input.sizeLimit && i < tiles.length) { + final tile = tiles[i++]; + final uuid = p.basename(tile.path); + + deletedSize += tile.size; + yield uuid; + yield File(tile.path).delete(); + } + }(); + + sizeMonitor + ..setPositionSync(0) + ..writeFromSync( + Uint8List(8)..buffer.asInt64List()[0] = currentSize - deletedSize, + ) + ..flushSync() + ..closeSync(); + + await Future.wait(deletedTiles.whereType>()); + + return deletedTiles.whereType().toList(growable: false); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart new file mode 100644 index 000000000..cd3e3205d --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart @@ -0,0 +1,60 @@ +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; +import 'package:meta/meta.dart'; + +/// Isolate worker which writes & deletes tile files, and updates the size +/// monitor, synchronously +@internal +Future tileWriterSizeMonitorWorker( + ({ + SendPort port, + String cacheDirectoryPath, + String persistentRegistryFileName, + String sizeMonitorFilePath, + String sizeMonitorFileName, + }) input, +) async { + int currentSize; + final RandomAccessFile sizeMonitor; + (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( + cacheDirectoryPath: input.cacheDirectoryPath, + persistentRegistryFileName: input.persistentRegistryFileName, + sizeMonitorFileName: input.sizeMonitorFileName, + sizeMonitorFilePath: input.sizeMonitorFilePath, + ); + + final allocatedWriteBinBuffer = Uint8List(8); + + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + + await for (final val in receivePort) { + final (:tileFilePath, :bytes) = + val as ({String tileFilePath, Uint8List? bytes}); + + final tileFile = File(tileFilePath); + final tileFileExists = tileFile.existsSync(); + + final existingTileSize = tileFileExists ? tileFile.lengthSync() : 0; + final newTileSize = bytes?.lengthInBytes ?? 0; + if (newTileSize - existingTileSize case final deltaSize + when deltaSize != 0) { + currentSize += deltaSize; + sizeMonitor + ..setPositionSync(0) + ..writeFromSync( + allocatedWriteBinBuffer..buffer.asInt64List()[0] = currentSize, + ) + ..flushSync(); + } + + if (bytes != null) { + tileFile.writeAsBytesSync(bytes); + } else if (tileFileExists) { + tileFile.deleteSync(); + } + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart new file mode 100644 index 000000000..af79deac0 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart @@ -0,0 +1,93 @@ +import 'dart:async'; +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; + +/// Opens and reads the existing size monitor if available +/// +/// If one does not exist, it calculates the current cache size and writes it +/// to a new size monitor. +/// +/// The returned [RandomAccessFile] is open - closure is the responsibility of +/// the caller. +@internal +Future<({int currentSize, RandomAccessFile sizeMonitor})> + getOrCreateSizeMonitor({ + required String cacheDirectoryPath, + required String persistentRegistryFileName, + required String sizeMonitorFilePath, + required String sizeMonitorFileName, +}) async { + final sizeMonitorFile = File(sizeMonitorFilePath); + + final sizeMonitor = sizeMonitorFile.openSync(mode: FileMode.append) + ..setPositionSync(0); + final bytes = sizeMonitor.readSync(8); + + if (bytes.length == 8) { + return ( + currentSize: bytes.buffer.asInt64List()[0], + sizeMonitor: sizeMonitor, + ); + } + + final calculatedCurrentSize = await Directory(cacheDirectoryPath) + .listSync() + .whereType() + .where( + (f) { + final uuid = p.basename(f.absolute.path); + return uuid != persistentRegistryFileName && + uuid != sizeMonitorFileName; + }, + ) + .map((f) => f.length()) + .asyncFold(0, (v, l) => v + l); + + sizeMonitor + ..setPositionSync(0) + ..writeFromSync( + Uint8List(8)..buffer.asInt64List()[0] = calculatedCurrentSize, + ) + ..flushSync(); + + return (currentSize: calculatedCurrentSize, sizeMonitor: sizeMonitor); +} + +extension _AsyncFold on Iterable> { + /// Reduces a collection of [Future]s to a single value by iteratively + /// combining each element of the collection when it completes with an + /// existing value + /// + /// The result must not depend on the order of completetion and [combine] + /// calls. + Future asyncFold( + T initialValue, + T Function(T previousValue, E element) combine, + ) async { + var value = initialValue; + + bool hasFinishedIterating = false; + int waiting = 0; + final completer = Completer(); + + for (final element in this) { + waiting++; + unawaited( + element.then((result) { + value = combine(value, result); + waiting--; + if (hasFinishedIterating && waiting == 0) completer.complete(); + }), + ); + } + + if (waiting == 0) return value; + + hasFinishedIterating = true; + await completer.future; + return value; + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 3c569a34a..1b9eaa89f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -13,6 +13,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final int? maxCacheSize; final Duration? overrideFreshAge; final String Function(String url) cacheKeyGenerator; + final bool readOnly; @internal const BuiltInMapCachingProviderImpl.createAndInitialise({ @@ -20,6 +21,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.maxCacheSize, required this.overrideFreshAge, required this.cacheKeyGenerator, + required this.readOnly, }); @override diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 93cdb10aa..44bdecaa5 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -8,6 +8,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final int? maxCacheSize; final Duration? overrideFreshAge; final String Function(String url) cacheKeyGenerator; + final bool readOnly; @internal const BuiltInMapCachingProviderImpl.createAndInitialise({ @@ -15,6 +16,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.maxCacheSize, required this.overrideFreshAge, required this.cacheKeyGenerator, + required this.readOnly, }); @override From 36d7268a46d1fea661d6239922e8954407d13d5b Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 15 May 2025 22:57:19 +0100 Subject: [PATCH 13/49] Removed leftover debugging tools --- .../network/caching/built_in/impl/native/native.dart | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 72e1ba9cd..d463bfe22 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -53,7 +53,6 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future _initialise() async { if (_isInitialised != null) return await _isInitialised!.future; - final stopwatch = Stopwatch()..start(); _isInitialised = Completer(); try { @@ -147,9 +146,6 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { rethrow; } - stopwatch.stop(); - print(stopwatch.elapsedMilliseconds); - _isInitialised!.complete(); } From 5fe9133eb1e2b04a192dd483aba2e142655a1e32 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sat, 17 May 2025 17:37:33 +0100 Subject: [PATCH 14/49] Improved documentation --- .../caching/built_in/built_in_caching_provider.dart | 13 +++++++++---- .../network/caching/caching_provider.dart | 7 +++++-- .../caching/disabled/disabled_caching_provider.dart | 4 ++-- .../network/caching/tile_metadata.dart | 5 +++-- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 5aeb9fe65..8decfd9da 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -8,11 +8,16 @@ import 'package:uuid/data.dart'; import 'package:uuid/rng.dart'; import 'package:uuid/uuid.dart'; -/// Simple built-in map caching respecting HTTP headers using the filesystem -/// and a JSON registry, on native (non-web) platforms only +/// Simple built-in map caching using a JSON + I/O storage mechanism, on native +/// (non-web) platforms only /// -/// This is enabled by default. For more information, see the online -/// documentation. +/// Usually uses HTTP headers to determine tile freshness, although +/// `overrideFreshAge` can override this. +/// +/// This is enabled by default in flutter_map, when using the +/// [NetworkTileProvider] (or cancellable version). +/// +/// For more information, see the online documentation. abstract interface class BuiltInMapCachingProvider implements MapCachingProvider { /// if a singleton instance exists, return it, otherwise create a new diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index b5935e304..1e0b8f314 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -2,12 +2,15 @@ import 'dart:typed_data'; import 'package:flutter_map/flutter_map.dart'; -/// Provides tile caching facilities based on HTTP headers (see -/// [CachedMapTileMetadata]) to [TileProvider]s +/// Provides tile caching facilities to [TileProvider]s /// /// Some caching plugins may choose instead to provide a dedicated /// [TileProvider], in which case the flutter_map-provided caching facilities /// are irrelevant. +/// +/// The [CachedMapTileMetadata] object is used to store metadata alongside +/// cached tiles. Its intended purpose is primarily for caching based on HTTP +/// headers - however, this is not a requirement. abstract interface class MapCachingProvider { /// Whether this caching provider is "currently supported" /// diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart index 2fd40ac1f..2bd2650a3 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart @@ -2,9 +2,9 @@ import 'dart:typed_data'; import 'package:flutter_map/flutter_map.dart'; -/// Map caching provider which disables caching +/// Map caching provider which disables built-in caching class DisabledMapCachingProvider implements MapCachingProvider { - /// Disable map caching through the [NetworkTileProvider.cachingProvider] + /// Disable map caching through [NetworkTileProvider.cachingProvider] const DisabledMapCachingProvider(); @override diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index d7be4e615..3fd2139a0 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -5,8 +5,9 @@ import 'package:meta/meta.dart'; /// Metadata about a tile cached with a [MapCachingProvider] /// -/// Map caching is based on HTTP headers, which this class contains and can -/// encode/decode to/from JSON. +/// Caching is usually determined with HTTP headers. However, if a specific +/// implementation chooses to, it can solely use [staleAt] and set the other +/// properties to a dummy value. /// /// External usage of this class is not usually necessary. It is visible so /// other tile providers may make use of it. From 192860cf2f1a2e9eddecbb7a92b20a45a286fd9b Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 18 May 2025 15:05:12 +0100 Subject: [PATCH 15/49] Switch to FlatBuffers instead of JSON Other performance improvements to initialisation --- example/pubspec.lock | 8 + .../built_in/built_in_caching_provider.dart | 6 +- .../impl/native/flatbufs/registry.fbs | 19 ++ .../impl/native/flatbufs/registry.g.dart | 272 ++++++++++++++++++ .../caching/built_in/impl/native/native.dart | 38 +-- .../workers/persistent_registry_parser.dart | 46 +-- .../workers/persistent_registry_writer.dart | 79 +++-- .../workers/tile_writer_size_monitor.dart | 6 +- .../workers/utils/size_monitor_opener.dart | 16 ++ .../network/caching/tile_metadata.dart | 24 -- pubspec.yaml | 1 + 11 files changed, 436 insertions(+), 79 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart diff --git a/example/pubspec.lock b/example/pubspec.lock index 79656615d..6378d2e06 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -105,6 +105,14 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.1" + flat_buffers: + dependency: transitive + description: + name: flat_buffers + sha256: "380bdcba5664a718bfd4ea20a45d39e13684f5318fcd8883066a55e21f37f4c3" + url: "https://pub.dev" + source: hosted + version: "23.5.26" flutter: dependency: "direct main" description: flutter diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 8decfd9da..b684aa5a2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -8,9 +8,13 @@ import 'package:uuid/data.dart'; import 'package:uuid/rng.dart'; import 'package:uuid/uuid.dart'; -/// Simple built-in map caching using a JSON + I/O storage mechanism, on native +/// Simple built-in map caching using an I/O storage mechanism, for native /// (non-web) platforms only /// +/// Uses FlatBuffers to store a centralised registry which is operated on as +/// a map in memory to maximise performance, with tile blobs stored raw as +/// files and a second file used to track the size of the cache. +/// /// Usually uses HTTP headers to determine tile freshness, although /// `overrideFreshAge` can override this. /// diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs new file mode 100644 index 000000000..d9e784f70 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs @@ -0,0 +1,19 @@ +namespace FMCache; + +table TileMetadata { + last_modified_locally: int64; + stale_at: int64; + last_modified: int64; + etag: string; +} + +table TileMetadataEntry { + id: string; + metadata: TileMetadata; +} + +table TileMetadataMap { + entries: [TileMetadataEntry]; +} + +root_type TileMetadataMap; \ No newline at end of file diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart new file mode 100644 index 000000000..11f553289 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart @@ -0,0 +1,272 @@ +// automatically generated by the FlatBuffers compiler, do not modify +// ignore_for_file: unused_import, unused_field, unused_element, unused_local_variable, constant_identifier_names, public_member_api_docs + +import 'dart:typed_data' show Uint8List; +import 'package:flat_buffers/flat_buffers.dart' as fb; + +class TileMetadata { + factory TileMetadata(List bytes) { + final rootRef = fb.BufferContext.fromBytes(bytes); + return reader.read(rootRef, 0); + } + TileMetadata._(this._bc, this._bcOffset); + + static const fb.Reader reader = _TileMetadataReader(); + + final fb.BufferContext _bc; + final int _bcOffset; + + int get lastModifiedLocally => + const fb.Int64Reader().vTableGet(_bc, _bcOffset, 4, 0); + int get staleAt => const fb.Int64Reader().vTableGet(_bc, _bcOffset, 6, 0); + int get lastModified => + const fb.Int64Reader().vTableGet(_bc, _bcOffset, 8, 0); + String? get etag => + const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10); + + @override + String toString() { + return 'TileMetadata{lastModifiedLocally: $lastModifiedLocally, staleAt: $staleAt, lastModified: $lastModified, etag: $etag}'; + } +} + +class _TileMetadataReader extends fb.TableReader { + const _TileMetadataReader(); + + @override + TileMetadata createObject(fb.BufferContext bc, int offset) => + TileMetadata._(bc, offset); +} + +class TileMetadataBuilder { + TileMetadataBuilder(this.fbBuilder); + + final fb.Builder fbBuilder; + + void begin() { + fbBuilder.startTable(4); + } + + int addLastModifiedLocally(int? lastModifiedLocally) { + fbBuilder.addInt64(0, lastModifiedLocally); + return fbBuilder.offset; + } + + int addStaleAt(int? staleAt) { + fbBuilder.addInt64(1, staleAt); + return fbBuilder.offset; + } + + int addLastModified(int? lastModified) { + fbBuilder.addInt64(2, lastModified); + return fbBuilder.offset; + } + + int addEtagOffset(int? offset) { + fbBuilder.addOffset(3, offset); + return fbBuilder.offset; + } + + int finish() { + return fbBuilder.endTable(); + } +} + +class TileMetadataObjectBuilder extends fb.ObjectBuilder { + final int? _lastModifiedLocally; + final int? _staleAt; + final int? _lastModified; + final String? _etag; + + TileMetadataObjectBuilder({ + int? lastModifiedLocally, + int? staleAt, + int? lastModified, + String? etag, + }) : _lastModifiedLocally = lastModifiedLocally, + _staleAt = staleAt, + _lastModified = lastModified, + _etag = etag; + + /// Finish building, and store into the [fbBuilder]. + @override + int finish(fb.Builder fbBuilder) { + final int? etagOffset = _etag == null ? null : fbBuilder.writeString(_etag); + fbBuilder.startTable(4); + fbBuilder.addInt64(0, _lastModifiedLocally); + fbBuilder.addInt64(1, _staleAt); + fbBuilder.addInt64(2, _lastModified); + fbBuilder.addOffset(3, etagOffset); + return fbBuilder.endTable(); + } + + /// Convenience method to serialize to byte list. + @override + Uint8List toBytes([String? fileIdentifier]) { + final fbBuilder = fb.Builder(deduplicateTables: false); + fbBuilder.finish(finish(fbBuilder), fileIdentifier); + return fbBuilder.buffer; + } +} + +class TileMetadataEntry { + factory TileMetadataEntry(List bytes) { + final rootRef = fb.BufferContext.fromBytes(bytes); + return reader.read(rootRef, 0); + } + TileMetadataEntry._(this._bc, this._bcOffset); + + static const fb.Reader reader = _TileMetadataEntryReader(); + + final fb.BufferContext _bc; + final int _bcOffset; + + String? get id => + const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 4); + TileMetadata? get metadata => + TileMetadata.reader.vTableGetNullable(_bc, _bcOffset, 6); + + @override + String toString() { + return 'TileMetadataEntry{id: $id, metadata: $metadata}'; + } +} + +class _TileMetadataEntryReader extends fb.TableReader { + const _TileMetadataEntryReader(); + + @override + TileMetadataEntry createObject(fb.BufferContext bc, int offset) => + TileMetadataEntry._(bc, offset); +} + +class TileMetadataEntryBuilder { + TileMetadataEntryBuilder(this.fbBuilder); + + final fb.Builder fbBuilder; + + void begin() { + fbBuilder.startTable(2); + } + + int addIdOffset(int? offset) { + fbBuilder.addOffset(0, offset); + return fbBuilder.offset; + } + + int addMetadataOffset(int? offset) { + fbBuilder.addOffset(1, offset); + return fbBuilder.offset; + } + + int finish() { + return fbBuilder.endTable(); + } +} + +class TileMetadataEntryObjectBuilder extends fb.ObjectBuilder { + final String? _id; + final TileMetadataObjectBuilder? _metadata; + + TileMetadataEntryObjectBuilder({ + String? id, + TileMetadataObjectBuilder? metadata, + }) : _id = id, + _metadata = metadata; + + /// Finish building, and store into the [fbBuilder]. + @override + int finish(fb.Builder fbBuilder) { + final int? idOffset = _id == null ? null : fbBuilder.writeString(_id); + final int? metadataOffset = _metadata?.getOrCreateOffset(fbBuilder); + fbBuilder.startTable(2); + fbBuilder.addOffset(0, idOffset); + fbBuilder.addOffset(1, metadataOffset); + return fbBuilder.endTable(); + } + + /// Convenience method to serialize to byte list. + @override + Uint8List toBytes([String? fileIdentifier]) { + final fbBuilder = fb.Builder(deduplicateTables: false); + fbBuilder.finish(finish(fbBuilder), fileIdentifier); + return fbBuilder.buffer; + } +} + +class TileMetadataMap { + factory TileMetadataMap(List bytes) { + final rootRef = fb.BufferContext.fromBytes(bytes); + return reader.read(rootRef, 0); + } + TileMetadataMap._(this._bc, this._bcOffset); + + static const fb.Reader reader = _TileMetadataMapReader(); + + final fb.BufferContext _bc; + final int _bcOffset; + + List? get entries => + const fb.ListReader(TileMetadataEntry.reader) + .vTableGetNullable(_bc, _bcOffset, 4); + + @override + String toString() { + return 'TileMetadataMap{entries: $entries}'; + } +} + +class _TileMetadataMapReader extends fb.TableReader { + const _TileMetadataMapReader(); + + @override + TileMetadataMap createObject(fb.BufferContext bc, int offset) => + TileMetadataMap._(bc, offset); +} + +class TileMetadataMapBuilder { + TileMetadataMapBuilder(this.fbBuilder); + + final fb.Builder fbBuilder; + + void begin() { + fbBuilder.startTable(1); + } + + int addEntriesOffset(int? offset) { + fbBuilder.addOffset(0, offset); + return fbBuilder.offset; + } + + int finish() { + return fbBuilder.endTable(); + } +} + +class TileMetadataMapObjectBuilder extends fb.ObjectBuilder { + final List? _entries; + + TileMetadataMapObjectBuilder({ + List? entries, + }) : _entries = entries; + + /// Finish building, and store into the [fbBuilder]. + @override + int finish(fb.Builder fbBuilder) { + final int? entriesOffset = _entries == null + ? null + : fbBuilder.writeList( + _entries.map((b) => b.getOrCreateOffset(fbBuilder)).toList()); + fbBuilder.startTable(1); + fbBuilder.addOffset(0, entriesOffset); + return fbBuilder.endTable(); + } + + /// Convenience method to serialize to byte list. + @override + Uint8List toBytes([String? fileIdentifier]) { + final fbBuilder = fb.Builder(deduplicateTables: false); + fbBuilder.finish(finish(fbBuilder), fileIdentifier); + return fbBuilder.buffer; + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index d463bfe22..99d3abc69 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -9,6 +9,7 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -32,8 +33,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _initialise(); } - static const _persistentRegistryFileName = 'registry.json'; - static const _sizeMonitorFileName = 'sizeMonitor'; + static const _persistentRegistryFileName = 'registry.bin'; + static const _sizeMonitorFileName = 'sizeMonitor.bin'; late final String _cacheDirectory; late final void Function(String uuid, CachedMapTileMetadata? tileInfo) @@ -89,20 +90,25 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _registry = parsedCacheManager; if (maxCacheSize case final sizeLimit?) { - // This can cause some delay when creating - // But it's much better than lagging or inconsistent registries - (await compute( - sizeLimiterWorker, - ( - cacheDirectoryPath: _cacheDirectory, - persistentRegistryFileName: _persistentRegistryFileName, - sizeMonitorFilePath: sizeMonitorFilePath, - sizeMonitorFileName: _sizeMonitorFileName, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Limiter', - )) - .forEach(_registry.remove); + final currentSize = + await asyncGetOnlySizeMonitor(sizeMonitorFilePath); + + if (currentSize == null || currentSize > sizeLimit) { + // This can cause some delay when creating + // But it's much better than lagging or inconsistent registries + (await compute( + sizeLimiterWorker, + ( + cacheDirectoryPath: _cacheDirectory, + persistentRegistryFileName: _persistentRegistryFileName, + sizeMonitorFilePath: sizeMonitorFilePath, + sizeMonitorFileName: _sizeMonitorFileName, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Limiter', + )) + .forEach(_registry.remove); + } } } } else { diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart index 89a0be4d6..86473570f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart @@ -1,38 +1,48 @@ import 'dart:collection'; -import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart'; import 'package:meta/meta.dart'; -/// Decode the JSON within the persistent registry into a mapping of tile -/// UUIDs to their [CachedMapTileMetadata]s +/// Unpack the FlatBuffer registry into a mapping of tile UUIDs to their +/// [CachedMapTileMetadata]s /// -/// If the JSON is invalid or the file cannot be read, this returns null. +/// If the FlatBuffer file is invalid or the file cannot be read, this returns +/// null. @internal HashMap? persistentRegistryParserWorker( String persistentRegistryFilePath, ) { - final String json; + final Uint8List bin; try { - json = File(persistentRegistryFilePath).readAsStringSync(); + bin = File(persistentRegistryFilePath).readAsBytesSync(); } on FileSystemException { return null; } - final Map parsed; + final tileMetadataMap = TileMetadataMap(bin); + if (tileMetadataMap.entries == null) return null; + try { - parsed = jsonDecode(json) as Map; - } on FormatException { + return HashMap.fromIterable( + tileMetadataMap.entries!, + key: (e) => (e as TileMetadataEntry).id!, + value: (e) { + final metadata = (e as TileMetadataEntry).metadata!; + return CachedMapTileMetadata( + lastModifiedLocally: + DateTime.fromMillisecondsSinceEpoch(metadata.lastModifiedLocally), + staleAt: DateTime.fromMillisecondsSinceEpoch(metadata.staleAt), + lastModified: metadata.lastModified == 0 + ? null + : DateTime.fromMillisecondsSinceEpoch(metadata.lastModified), + etag: metadata.etag, + ); + }, + ); + } catch (_) { return null; } - - return HashMap.from( - parsed.map( - (key, value) => MapEntry( - key, - CachedMapTileMetadata.fromJson(value as Map), - ), - ), - ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart index 8adde042e..277baa9a6 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart @@ -1,9 +1,10 @@ import 'dart:async'; -import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; +import 'package:flat_buffers/flat_buffers.dart' as fb; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart'; import 'package:meta/meta.dart'; /// Isolate worker which maintains its own registry and sequences writes to @@ -26,18 +27,13 @@ Future persistentRegistryWriterWorker( Map initialRegistry, }) input, ) async { + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + final registry = input.initialRegistry; final writer = File(input.persistentRegistryFilePath).openSync(mode: FileMode.writeOnly); - // We rewrite the registry from the initial state in-case it was size limited, - // for example - final encoded = jsonEncode(registry); - writer - ..writeStringSync(encoded) - ..truncateSync(writer.positionSync()) - ..flushSync(); - var writeLocker = Completer()..complete(); var alreadyWaitingToWrite = false; Future write() async { @@ -47,12 +43,7 @@ Future persistentRegistryWriterWorker( writeLocker = Completer(); alreadyWaitingToWrite = false; - final encoded = jsonEncode(registry); - writer - ..setPositionSync(0) - ..writeStringSync(encoded) - ..truncateSync(writer.positionSync()) - ..flushSync(); + _writeFlatbuf(registry, writer); writeLocker.complete(); } @@ -61,8 +52,7 @@ Future persistentRegistryWriterWorker( Timer(const Duration(milliseconds: 50), write); Timer? writeDebouncer; - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); + write(); await for (final val in receivePort) { final (:uuid, :tileInfo) = @@ -78,3 +68,58 @@ Future persistentRegistryWriterWorker( writeDebouncer = createWriteDebouncer(); } } + +void _writeFlatbuf( + Map registry, + RandomAccessFile fileWriter, +) { + final registryIds = registry.keys.toList(growable: false); + final registryMetadatas = registry.values.toList(growable: false); + + final builder = fb.Builder(initialSize: 1048576); + + final entriesOffset = builder.writeList( + List.generate( + registry.length, + (i) { + final id = registryIds[i]; + final metadata = registryMetadatas[i]; + + final fbId = builder.writeString(id, asciiOptimization: true); + final fbEtag = metadata.etag == null + ? null + : builder.writeString(metadata.etag!, asciiOptimization: true); + + final fbTileMetadata = (TileMetadataBuilder(builder) + ..begin() + ..addLastModifiedLocally( + metadata.lastModifiedLocally.millisecondsSinceEpoch, + ) + ..addStaleAt(metadata.staleAt.millisecondsSinceEpoch) + ..addEtagOffset(fbEtag) + ..addLastModified(metadata.lastModified?.millisecondsSinceEpoch)) + .finish(); + + return (TileMetadataEntryBuilder(builder) + ..begin() + ..addIdOffset(fbId) + ..addMetadataOffset(fbTileMetadata)) + .finish(); + }, + growable: false, + ), + ); + + final metadataMapOffset = (TileMetadataMapBuilder(builder) + ..begin() + ..addEntriesOffset(entriesOffset)) + .finish(); + + builder.finish(metadataMapOffset); + + fileWriter + ..setPositionSync(0) + ..writeFromSync(builder.buffer) + ..truncateSync(fileWriter.positionSync()) + ..flushSync(); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart index cd3e3205d..ec56d2007 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart @@ -17,6 +17,9 @@ Future tileWriterSizeMonitorWorker( String sizeMonitorFileName, }) input, ) async { + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + int currentSize; final RandomAccessFile sizeMonitor; (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( @@ -28,9 +31,6 @@ Future tileWriterSizeMonitorWorker( final allocatedWriteBinBuffer = Uint8List(8); - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - await for (final val in receivePort) { final (:tileFilePath, :bytes) = val as ({String tileFilePath, Uint8List? bytes}); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart index af79deac0..28432474f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart @@ -5,6 +5,22 @@ import 'dart:typed_data'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; +/// Asynchronously read the existing size monitor if available, +/// returning `null` if unavailable +@internal +Future asyncGetOnlySizeMonitor(String sizeMonitorFilePath) async { + final sizeMonitorFile = File(sizeMonitorFilePath); + + final sizeMonitor = await sizeMonitorFile.open(mode: FileMode.append); + await sizeMonitor.setPosition(0); + final bytes = await sizeMonitor.read(8); + + await sizeMonitor.close(); + + if (bytes.length == 8) return bytes.buffer.asInt64List()[0]; + return null; +} + /// Opens and reads the existing size monitor if available /// /// If one does not exist, it calculates the current cache size and writes it diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index 3fd2139a0..89e318b78 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -1,5 +1,3 @@ -import 'dart:io' show HttpDate; // this is web safe! - import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; @@ -25,19 +23,6 @@ class CachedMapTileMetadata { required this.etag, }); - /// Decode metadata from JSON - CachedMapTileMetadata.fromJson(Map json) - : lastModifiedLocally = - HttpDate.parse(json['lastModifiedLocally'] as String), - staleAt = HttpDate.parse(json['staleAt'] as String), - lastModified = json.containsKey('lastModified') && - (json['lastModified'] as String).isNotEmpty - ? HttpDate.parse(json['lastModified'] as String) - : null, - etag = json.containsKey('etag') && (json['etag'] as String).isNotEmpty - ? json['etag'] as String - : null; - /// Used to efficiently allow updates to already cached tiles /// /// Must be set to [DateTime.timestamp] when a new tile is cached or a tile @@ -56,15 +41,6 @@ class CachedMapTileMetadata { /// Whether the tile is currently stale bool get isStale => DateTime.timestamp().isAfter(staleAt); - /// Encode the metadata to JSON - Map toJson() => { - 'lastModifiedLocally': HttpDate.format(lastModifiedLocally), - 'staleAt': HttpDate.format(staleAt), - if (lastModified != null) - 'lastModified': HttpDate.format(lastModified!), - if (etag != null) 'etag': etag!, - }; - @override int get hashCode => Object.hash(lastModifiedLocally, staleAt, lastModified, etag); diff --git a/pubspec.yaml b/pubspec.yaml index c6c187e8f..6925994ae 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -29,6 +29,7 @@ dependencies: async: ^2.11.0 collection: ^1.18.0 dart_earcut: ^1.1.0 + flat_buffers: ^23.5.26 flutter: sdk: flutter http: ^1.2.1 From 9321d0c3d50353a4668e612e1b0f7c7436274635 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 May 2025 14:40:18 +0100 Subject: [PATCH 16/49] Return number of tiles loaded from `isInitialised` Integrate basic stats and loading into example app --- example/lib/main.dart | 23 ++++++-- example/lib/misc/timed_future.dart | 27 ++++++++++ example/lib/pages/home.dart | 31 ++++++++++- example/lib/widgets/drawer/menu_drawer.dart | 53 +++++++++++++++++-- .../built_in/built_in_caching_provider.dart | 11 ++-- .../impl/native/flatbufs/registry.fbs | 4 +- .../caching/built_in/impl/native/native.dart | 17 +++--- ...dart => persistent_registry_unpacker.dart} | 8 +-- .../workers/persistent_registry_writer.dart | 4 +- .../network/caching/built_in/impl/stub.dart | 2 +- .../caching/built_in/impl/web/web.dart | 3 +- 11 files changed, 151 insertions(+), 32 deletions(-) create mode 100644 example/lib/misc/timed_future.dart rename lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/{persistent_registry_parser.dart => persistent_registry_unpacker.dart} (88%) diff --git a/example/lib/main.dart b/example/lib/main.dart index 6817c5b6c..c2770e435 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,5 +1,6 @@ import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; @@ -33,16 +34,28 @@ import 'package:flutter_map_example/pages/sliding_map.dart'; import 'package:flutter_map_example/pages/tile_builder.dart'; import 'package:flutter_map_example/pages/tile_loading_error_handle.dart'; import 'package:flutter_map_example/pages/wms_tile_layer.dart'; +import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_web_plugins/url_strategy.dart'; Future main() async { usePathUrlStrategy(); - await BuiltInMapCachingProvider.getOrCreateInstance().isInitialised; - runApp(const MyApp()); + + final cachingInstance = BuiltInMapCachingProvider.getOrCreateInstance(); + final cacheInitComplete = cachingInstance.isSupported + ? cachingInstance.isInitialised.timed() + : null; + MenuDrawer.cacheInitComplete.value = cacheInitComplete; + + runApp(MyApp(cacheInitComplete: cacheInitComplete)); } class MyApp extends StatelessWidget { - const MyApp({super.key}); + const MyApp({ + super.key, + required this.cacheInitComplete, + }); + + final TimedFuture? cacheInitComplete; @override Widget build(BuildContext context) { @@ -52,7 +65,9 @@ class MyApp extends StatelessWidget { useMaterial3: true, colorSchemeSeed: const Color(0xFF8dea88), ), - home: const HomePage(), + home: HomePage( + cacheInitComplete: cacheInitComplete, + ), routes: { CancellableTileProviderPage.route: (context) => const CancellableTileProviderPage(), diff --git a/example/lib/misc/timed_future.dart b/example/lib/misc/timed_future.dart new file mode 100644 index 000000000..f6a984561 --- /dev/null +++ b/example/lib/misc/timed_future.dart @@ -0,0 +1,27 @@ +import 'dart:async'; + +typedef TimedFuture = ({ + Future result, + Future duration, + Future<({E result, Duration duration})> future, +}); + +extension CreateTimedFuture on Future { + /// Augments the future with the length of time it took to complete + /// + /// Note that the time is started on invocation of this method, not the actual + /// duration of the future from when it was created. + TimedFuture timed() { + final timer = Stopwatch()..start(); + final duration = Completer(); + final future = Completer<({E result, Duration duration})>(); + whenComplete(() => duration.complete((timer..stop()).elapsed)); + then( + (result) => duration.future.then( + (duration) => future.complete((result: result, duration: duration)), + ), + onError: future.completeError, + ); + return (result: this, duration: duration.future, future: future.future); + } +} diff --git a/example/lib/pages/home.dart b/example/lib/pages/home.dart index 28eec1058..4e8222d48 100644 --- a/example/lib/pages/home.dart +++ b/example/lib/pages/home.dart @@ -3,6 +3,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map_example/misc/tile_providers.dart'; +import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/widgets/drawer/floating_menu_button.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_map_example/widgets/first_start_dialog.dart'; @@ -13,7 +14,12 @@ import 'package:url_launcher/url_launcher.dart'; class HomePage extends StatefulWidget { static const String route = '/'; - const HomePage({super.key}); + const HomePage({ + super.key, + required this.cacheInitComplete, + }); + + final TimedFuture? cacheInitComplete; @override State createState() => _HomePageState(); @@ -32,10 +38,33 @@ class _HomePageState extends State { drawer: const MenuDrawer(HomePage.route), body: Stack( children: [ + if (widget.cacheInitComplete case final cacheInitComplete?) + FutureBuilder( + future: cacheInitComplete.result, + builder: (context, snapshot) { + if (snapshot.connectionState == ConnectionState.done) { + return const ColoredBox(color: Color(0xFFE0E0E0)); + } + return const Center( + child: Row( + mainAxisSize: MainAxisSize.min, + spacing: 16, + children: [ + SizedBox.square( + dimension: 24, + child: CircularProgressIndicator.adaptive(), + ), + Text('Awaiting cache initialisation'), + ], + ), + ); + }, + ), FlutterMap( options: const MapOptions( initialCenter: LatLng(51.5, -0.09), initialZoom: 5, + backgroundColor: Colors.transparent, ), children: [ openStreetMapTileLayer, diff --git a/example/lib/widgets/drawer/menu_drawer.dart b/example/lib/widgets/drawer/menu_drawer.dart index f48e7dc59..dfe7a82ba 100644 --- a/example/lib/widgets/drawer/menu_drawer.dart +++ b/example/lib/widgets/drawer/menu_drawer.dart @@ -1,5 +1,6 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; @@ -42,12 +43,21 @@ class MenuDrawer extends StatelessWidget { const MenuDrawer(this.currentRoute, {super.key}); + static final ValueNotifier?> cacheInitComplete = + ValueNotifier(null); + @override Widget build(BuildContext context) { return Drawer( child: ListView( children: [ - DrawerHeader( + Container( + padding: const EdgeInsets.fromLTRB(16, 32, 16, 16) + .add(EdgeInsets.only(top: MediaQuery.paddingOf(context).top)), + margin: const EdgeInsets.only(bottom: 8), + decoration: BoxDecoration( + border: Border(bottom: Divider.createBorderSide(context)), + ), child: Column( mainAxisAlignment: MainAxisAlignment.center, children: [ @@ -65,12 +75,49 @@ class MenuDrawer extends StatelessWidget { textAlign: TextAlign.center, style: TextStyle(fontSize: 14), ), + const SizedBox(height: 8), if (kIsWeb) - const Text( + Text( _isWASM ? 'Running with WASM' : 'Running without WASM', textAlign: TextAlign.center, - style: TextStyle(fontSize: 14), + style: Theme.of(context).textTheme.bodySmall, ), + ValueListenableBuilder( + valueListenable: cacheInitComplete, + builder: (context, value, _) { + if (value == null) { + return Text( + 'No map cache in use', + style: Theme.of(context).textTheme.bodySmall, + ); + } + return FutureBuilder( + future: value.future, + builder: (context, snapshot) { + if (snapshot.hasError) { + return Text( + 'Failed to load or recover map cache', + style: Theme.of(context).textTheme.bodySmall, + ); + } + if (snapshot.hasData) { + return Text( + 'Loaded map cache of ' + '${snapshot.requireData.result!} tiles in ' + '${snapshot.requireData.duration.inMilliseconds}' + '\u00a0ms${kDebugMode ? ' (debug\u00a0mode)' : ''}', + textAlign: TextAlign.center, + style: Theme.of(context).textTheme.bodySmall, + ); + } + return Text( + 'Loading map cache...', + style: Theme.of(context).textTheme.bodySmall, + ); + }, + ); + }, + ), ], ), ), diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index b684aa5a2..141de2007 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -106,9 +106,10 @@ abstract interface class BuiltInMapCachingProvider /// See online documentation to see how to use this to preload caching to /// remove the initial delay before loading tiles. /// - /// May complete with an error if initialisation failed. - /// - /// On the web, this will always complete successfully immediately in the same - /// event loop. Caching will not be available. - Future get isInitialised; + /// Completes with: + /// * on native platforms, the number of cached tiles (at initialisation) + /// * or an error if initialisation fails and could not be recovered + /// * on web platforms, `null` (synchronously & immediately), and caching + /// will not be available + Future get isInitialised; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs index d9e784f70..c9373980d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs @@ -1,5 +1,3 @@ -namespace FMCache; - table TileMetadata { last_modified_locally: int64; stale_at: int64; @@ -8,7 +6,7 @@ table TileMetadata { } table TileMetadataEntry { - id: string; + id: string (key); metadata: TileMetadata; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 99d3abc69..3c4de2380 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -5,7 +5,7 @@ import 'dart:isolate'; import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart'; @@ -43,18 +43,18 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _writeTileFile; late final HashMap _registry; - Completer? _isInitialised; + Completer? _isInitialised; @override bool get isSupported => true; @override - Future get isInitialised => _isInitialised!.future; + Future get isInitialised => _isInitialised!.future; - Future _initialise() async { - if (_isInitialised != null) return await _isInitialised!.future; + Future _initialise() async { + if (_isInitialised != null) return isInitialised; - _isInitialised = Completer(); + _isInitialised = Completer(); try { _cacheDirectory = p.join( @@ -77,7 +77,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { if (await persistentRegistryFile.exists()) { final parsedCacheManager = await compute( - persistentRegistryParserWorker, + persistentRegistryUnpackerWorker, persistentRegistryFilePath, debugLabel: '[flutter_map: cache] Persistent Registry Parser', ); @@ -152,7 +152,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { rethrow; } - _isInitialised!.complete(); + _isInitialised!.complete(_registry.length); + return _registry.length; } @override diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart similarity index 88% rename from lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart rename to lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart index 86473570f..4311f6ba5 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_parser.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart @@ -12,7 +12,7 @@ import 'package:meta/meta.dart'; /// If the FlatBuffer file is invalid or the file cannot be read, this returns /// null. @internal -HashMap? persistentRegistryParserWorker( +HashMap? persistentRegistryUnpackerWorker( String persistentRegistryFilePath, ) { final Uint8List bin; @@ -22,10 +22,10 @@ HashMap? persistentRegistryParserWorker( return null; } - final tileMetadataMap = TileMetadataMap(bin); - if (tileMetadataMap.entries == null) return null; - try { + final tileMetadataMap = TileMetadataMap(bin); + if (tileMetadataMap.entries == null) return null; + return HashMap.fromIterable( tileMetadataMap.entries!, key: (e) => (e as TileMetadataEntry).id!, diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart index 277baa9a6..f896873c5 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart @@ -43,7 +43,7 @@ Future persistentRegistryWriterWorker( writeLocker = Completer(); alreadyWaitingToWrite = false; - _writeFlatbuf(registry, writer); + _writeFlatbuffer(registry, writer); writeLocker.complete(); } @@ -69,7 +69,7 @@ Future persistentRegistryWriterWorker( } } -void _writeFlatbuf( +void _writeFlatbuffer( Map registry, RandomAccessFile fileWriter, ) { diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 1b9eaa89f..87f6978ad 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -25,7 +25,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { }); @override - external Future get isInitialised; + external Future get isInitialised; @override external bool get isSupported; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 44bdecaa5..28c199e1e 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -23,7 +23,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { bool get isSupported => false; @override - Future get isInitialised => SynchronousFuture(null); + // ignore: prefer_void_to_null + Future get isInitialised => SynchronousFuture(null); @override Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( From fd7a79b727dcfc53696decfb25d0d34669ae5dbc Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 May 2025 16:01:02 +0100 Subject: [PATCH 17/49] Added call to `WidgetsFlutterBinding.ensureInitialized` internally --- .../network/caching/built_in/impl/native/native.dart | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 3c4de2380..b76b7e23a 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -4,6 +4,7 @@ import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; +import 'package:flutter/widgets.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; @@ -54,6 +55,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future _initialise() async { if (_isInitialised != null) return isInitialised; + WidgetsFlutterBinding.ensureInitialized(); _isInitialised = Completer(); try { From 0997c706638d0c1c05899e9c6440220ff80fcaca Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 May 2025 16:43:18 +0100 Subject: [PATCH 18/49] Drop default cache size limit to 800 MB --- .../network/caching/built_in/built_in_caching_provider.dart | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 141de2007..bc2ddeddd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -50,8 +50,8 @@ abstract interface class BuiltInMapCachingProvider /// the visible delay becomes too large, disable this and manage the cache /// size manually if necessary. /// - /// Defaults to 1GB. Set to `null` to disable. - int? maxCacheSize = 1000000000, + /// Defaults to 800 MB. Set to `null` to disable. + int? maxCacheSize = 800000000, /// Override the duration of time a tile is considered fresh for /// From 55471461e9a9143d8b49997d4366d258492df79f Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 May 2025 17:57:26 +0100 Subject: [PATCH 19/49] Minor general improvements Use `DisabledMapCachingProvider` as a mixin (for built-in caching provider on web) Removed `WidgetsFlutterBinding.ensureInitialized()` from internals Improved documentation --- example/lib/main.dart | 2 + .../built_in/built_in_caching_provider.dart | 3 +- .../caching/built_in/impl/native/native.dart | 218 ++++++++---------- .../impl/native/workers/size_limiter.dart | 11 +- .../workers/tile_writer_size_monitor.dart | 4 - .../workers/utils/size_monitor_opener.dart | 8 +- .../caching/built_in/impl/web/web.dart | 22 +- .../network/caching/caching_provider.dart | 28 ++- .../disabled/disabled_caching_provider.dart | 11 +- 9 files changed, 133 insertions(+), 174 deletions(-) diff --git a/example/lib/main.dart b/example/lib/main.dart index c2770e435..a612eaa2f 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -38,6 +38,8 @@ import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_web_plugins/url_strategy.dart'; Future main() async { + WidgetsFlutterBinding.ensureInitialized(); + usePathUrlStrategy(); final cachingInstance = BuiltInMapCachingProvider.getOrCreateInstance(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index bc2ddeddd..046b81394 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -1,5 +1,4 @@ import 'package:flutter_map/flutter_map.dart'; -// TODO: On Dart 3.8 min, update to remove `@internal`s, switch to privates and conditional parts import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart' if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart' if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart'; @@ -111,5 +110,7 @@ abstract interface class BuiltInMapCachingProvider /// * or an error if initialisation fails and could not be recovered /// * on web platforms, `null` (synchronously & immediately), and caching /// will not be available + /// + /// [isSupported] will be set to determine the current platform's support. Future get isInitialised; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index b76b7e23a..b3f691708 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -4,7 +4,6 @@ import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; -import 'package:flutter/widgets.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; @@ -17,6 +16,9 @@ import 'package:path_provider/path_provider.dart'; @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { + static const persistentRegistryFileName = 'registry.bin'; + static const sizeMonitorFileName = 'sizeMonitor.bin'; + final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; @@ -31,132 +33,109 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.cacheKeyGenerator, required this.readOnly, }) { - _initialise(); - } + // This should only be called/constructed once + _isInitialised.complete( + () async { + _cacheDirectoryPath = p.join( + this.cacheDirectory ?? + (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ); + final cacheDirectory = Directory(_cacheDirectoryPath); + await cacheDirectory.create(recursive: true); + + final persistentRegistryFilePath = + p.join(_cacheDirectoryPath, persistentRegistryFileName); + final persistentRegistryFile = File(persistentRegistryFilePath); + + final sizeMonitorFilePath = + p.join(_cacheDirectoryPath, sizeMonitorFileName); + + if (await persistentRegistryFile.exists()) { + final parsedCacheManager = await compute( + persistentRegistryUnpackerWorker, + persistentRegistryFilePath, + debugLabel: '[flutter_map: cache] Persistent Registry Unpacker', + ); + + if (parsedCacheManager == null) { + await cacheDirectory.delete(recursive: true); + await cacheDirectory.create(recursive: true); + _registry = HashMap(); + } else { + _registry = parsedCacheManager; + + if (maxCacheSize case final sizeLimit?) { + final currentSize = + await asyncGetOnlySizeMonitor(sizeMonitorFilePath); + + if (currentSize == null || currentSize > sizeLimit) { + (await compute( + sizeLimiterWorker, + ( + cacheDirectoryPath: _cacheDirectoryPath, + sizeMonitorFilePath: sizeMonitorFilePath, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Limiter', + )) + .forEach(_registry.remove); + } + } + } + } else { + _registry = HashMap(); + } + + final registryWorkerReceivePort = ReceivePort(); + await Isolate.spawn( + persistentRegistryWriterWorker, + ( + port: registryWorkerReceivePort.sendPort, + persistentRegistryFilePath: persistentRegistryFilePath, + initialRegistry: _registry, + ), + debugName: '[flutter_map: cache] Persistent Registry Writer', + ); + final registryWorkerSendPort = + await registryWorkerReceivePort.first as SendPort; + + final tileFileWriterWorkerReceivePort = ReceivePort(); + await Isolate.spawn( + tileWriterSizeMonitorWorker, + ( + port: tileFileWriterWorkerReceivePort.sendPort, + cacheDirectoryPath: _cacheDirectoryPath, + sizeMonitorFilePath: sizeMonitorFilePath, + ), + debugName: '[flutter_map: cache] Tile File & Size Monitor Writer', + ); + final tileFileWriterWorkerSendPort = + await tileFileWriterWorkerReceivePort.first as SendPort; - static const _persistentRegistryFileName = 'registry.bin'; - static const _sizeMonitorFileName = 'sizeMonitor.bin'; + _writeToPersistentRegistry = (uuid, tileInfo) => + registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)); + _writeTileFile = (tileFilePath, bytes) => tileFileWriterWorkerSendPort + .send((tileFilePath: tileFilePath, bytes: bytes)); - late final String _cacheDirectory; + return _registry.length; + }(), + ); + } + + late final String _cacheDirectoryPath; late final void Function(String uuid, CachedMapTileMetadata? tileInfo) _writeToPersistentRegistry; late final void Function(String tileFilePath, Uint8List? bytes) _writeTileFile; late final HashMap _registry; - Completer? _isInitialised; - + final _isInitialised = Completer(); @override - bool get isSupported => true; + Future get isInitialised => _isInitialised.future; @override - Future get isInitialised => _isInitialised!.future; - - Future _initialise() async { - if (_isInitialised != null) return isInitialised; - - WidgetsFlutterBinding.ensureInitialized(); - _isInitialised = Completer(); - - try { - _cacheDirectory = p.join( - cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, - 'fm_cache', - ); - final cacheDirectoryIO = Directory(_cacheDirectory); - await cacheDirectoryIO.create(recursive: true); - - final persistentRegistryFilePath = p.join( - _cacheDirectory, - _persistentRegistryFileName, - ); - final persistentRegistryFile = File(persistentRegistryFilePath); - - final sizeMonitorFilePath = p.join( - _cacheDirectory, - _sizeMonitorFileName, - ); - - if (await persistentRegistryFile.exists()) { - final parsedCacheManager = await compute( - persistentRegistryUnpackerWorker, - persistentRegistryFilePath, - debugLabel: '[flutter_map: cache] Persistent Registry Parser', - ); - - if (parsedCacheManager == null) { - await cacheDirectoryIO.delete(recursive: true); - await cacheDirectoryIO.create(recursive: true); - _registry = HashMap(); - } else { - _registry = parsedCacheManager; - - if (maxCacheSize case final sizeLimit?) { - final currentSize = - await asyncGetOnlySizeMonitor(sizeMonitorFilePath); - - if (currentSize == null || currentSize > sizeLimit) { - // This can cause some delay when creating - // But it's much better than lagging or inconsistent registries - (await compute( - sizeLimiterWorker, - ( - cacheDirectoryPath: _cacheDirectory, - persistentRegistryFileName: _persistentRegistryFileName, - sizeMonitorFilePath: sizeMonitorFilePath, - sizeMonitorFileName: _sizeMonitorFileName, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Limiter', - )) - .forEach(_registry.remove); - } - } - } - } else { - _registry = HashMap(); - } - - final registryWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - persistentRegistryWriterWorker, - ( - port: registryWorkerReceivePort.sendPort, - persistentRegistryFilePath: persistentRegistryFilePath, - initialRegistry: _registry, - ), - debugName: '[flutter_map: cache] Persistent Registry Worker', - ); - final registryWorkerSendPort = - await registryWorkerReceivePort.first as SendPort; - - final tileFileWriterWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - tileWriterSizeMonitorWorker, - ( - port: tileFileWriterWorkerReceivePort.sendPort, - cacheDirectoryPath: _cacheDirectory, - persistentRegistryFileName: _persistentRegistryFileName, - sizeMonitorFilePath: sizeMonitorFilePath, - sizeMonitorFileName: _sizeMonitorFileName, - ), - debugName: '[flutter_map: cache] Tile File Writer', - ); - final tileFileWriterWorkerSendPort = - await tileFileWriterWorkerReceivePort.first as SendPort; - - _writeToPersistentRegistry = (uuid, tileInfo) => - registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)); - _writeTileFile = (tileFilePath, bytes) => tileFileWriterWorkerSendPort - .send((tileFilePath: tileFilePath, bytes: bytes)); - } catch (error, stackTrace) { - _isInitialised!.completeError(error, stackTrace); - rethrow; - } - - _isInitialised!.complete(_registry.length); - return _registry.length; - } + bool get isSupported => true; @override Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( @@ -165,8 +144,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { await isInitialised; final uuid = cacheKeyGenerator(url); - - final tileFile = File(p.join(_cacheDirectory, uuid)); + final tileFile = File(p.join(_cacheDirectoryPath, uuid)); if (_registry[uuid] case final tileInfo? when await tileFile.exists()) { return (bytes: await tileFile.readAsBytes(), tileInfo: tileInfo); @@ -202,7 +180,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { } if (bytes != null) { - final tileFilePath = p.join(_cacheDirectory, uuid); + final tileFilePath = p.join(_cacheDirectoryPath, uuid); _writeTileFile(tileFilePath, bytes); } @@ -213,7 +191,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future _removeTile(String uuid) async { await isInitialised; - final tileFilePath = p.join(_cacheDirectory, uuid); + final tileFilePath = p.join(_cacheDirectoryPath, uuid); _writeTileFile(tileFilePath, null); if (_registry.remove(uuid) == null) return; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart index 81a9befe5..22e5f5227 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart @@ -2,6 +2,7 @@ import 'dart:io'; import 'dart:typed_data'; import 'package:collection/collection.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; @@ -21,18 +22,14 @@ typedef _SizeLimiterTile = ({String path, int size, DateTime sortKey}); Future> sizeLimiterWorker( ({ String cacheDirectoryPath, - String persistentRegistryFileName, String sizeMonitorFilePath, - String sizeMonitorFileName, - int sizeLimit + int sizeLimit, }) input, ) async { final cacheDirectory = Directory(input.cacheDirectoryPath); final (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( cacheDirectoryPath: input.cacheDirectoryPath, - persistentRegistryFileName: input.persistentRegistryFileName, - sizeMonitorFileName: input.sizeMonitorFileName, sizeMonitorFilePath: input.sizeMonitorFilePath, ); @@ -44,8 +41,8 @@ Future> sizeLimiterWorker( final tiles = await Future.wait<_SizeLimiterTile>( cacheDirectory.listSync().whereType().where((f) { final uuid = p.basename(f.absolute.path); - return uuid != input.persistentRegistryFileName && - uuid != input.sizeMonitorFileName; + return uuid != BuiltInMapCachingProviderImpl.persistentRegistryFileName && + uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; }).map((f) async { final stat = await f.stat(); // `stat.accessed` may be unstable on some OSs, but seems to work enough? diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart index ec56d2007..e990b8da2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart @@ -12,9 +12,7 @@ Future tileWriterSizeMonitorWorker( ({ SendPort port, String cacheDirectoryPath, - String persistentRegistryFileName, String sizeMonitorFilePath, - String sizeMonitorFileName, }) input, ) async { final receivePort = ReceivePort(); @@ -24,8 +22,6 @@ Future tileWriterSizeMonitorWorker( final RandomAccessFile sizeMonitor; (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( cacheDirectoryPath: input.cacheDirectoryPath, - persistentRegistryFileName: input.persistentRegistryFileName, - sizeMonitorFileName: input.sizeMonitorFileName, sizeMonitorFilePath: input.sizeMonitorFilePath, ); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart index 28432474f..da142cc02 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart @@ -2,6 +2,7 @@ import 'dart:async'; import 'dart:io'; import 'dart:typed_data'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; @@ -32,9 +33,7 @@ Future asyncGetOnlySizeMonitor(String sizeMonitorFilePath) async { Future<({int currentSize, RandomAccessFile sizeMonitor})> getOrCreateSizeMonitor({ required String cacheDirectoryPath, - required String persistentRegistryFileName, required String sizeMonitorFilePath, - required String sizeMonitorFileName, }) async { final sizeMonitorFile = File(sizeMonitorFilePath); @@ -55,8 +54,9 @@ Future<({int currentSize, RandomAccessFile sizeMonitor})> .where( (f) { final uuid = p.basename(f.absolute.path); - return uuid != persistentRegistryFileName && - uuid != sizeMonitorFileName; + return uuid != + BuiltInMapCachingProviderImpl.persistentRegistryFileName && + uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; }, ) .map((f) => f.length()) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 28c199e1e..6b424b02e 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -3,7 +3,9 @@ import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; @internal -class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { +class BuiltInMapCachingProviderImpl + with DisabledMapCachingProvider + implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; @@ -20,23 +22,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { }); @override - bool get isSupported => false; - - @override + // False positive lint // ignore: prefer_void_to_null Future get isInitialised => SynchronousFuture(null); - - @override - Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( - String url, - ) => - throw UnsupportedError('Built-in map caching is not supported on web'); - - @override - Future putTile({ - required String url, - required CachedMapTileMetadata tileInfo, - Uint8List? bytes, - }) => - throw UnsupportedError('Built-in map caching is not supported on web'); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index 1e0b8f314..3f62a62e9 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -12,32 +12,30 @@ import 'package:flutter_map/flutter_map.dart'; /// cached tiles. Its intended purpose is primarily for caching based on HTTP /// headers - however, this is not a requirement. abstract interface class MapCachingProvider { - /// Whether this caching provider is "currently supported" + /// Whether this caching provider is "currently supported": whether the + /// tile provider should attempt to use it, or fallback to a non-caching + /// alternative /// - /// This can mean multiple things depending on the implementation's choice. - /// However, it is used the same in the [NetworkTileProvider] implementaiton. + /// Tile providers must not call [getTile] or [putTile] if this is `false`. + /// [getTile] and [putTile] should gracefully throw if this is `false`. + /// This should not throw. /// - /// In some implementations, such as [BuiltInMapCachingProvider], this is set - /// constantly to indicate whether the implementation supports the current - /// platform. [getTile] and other implementation specific methods are used - /// to automatically wait for the internal initialisation to be complete - /// before returning a tile. In this case, the provider delays the loading of - /// tiles until initialisation is complete. - /// - /// In other implementations, [isSupported] may be set to indicate the - /// internal initialisation status. In this case, the provider does not delay - /// loading of tiles until initialisation is complete, and instead - /// automatically switches to using cached tiles once ready. + /// If this is always `false`, consider mixing in or using + /// [DisabledMapCachingProvider] directly. bool get isSupported; /// Retrieve a tile from the cache, if it exists + /// + /// This may throw. Tile providers should anticipate this and fallback to a + /// non-caching alternative. Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( String url, ); /// Add or update a tile in the cache /// - /// [bytes] is required if the tile is not already cached. + /// [bytes] is required if the tile is not already cached. The behaviour is + /// implementation specific if bytes are not supplied when required. Future putTile({ required String url, required CachedMapTileMetadata tileInfo, diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart index 2bd2650a3..8feb5b439 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart @@ -2,16 +2,17 @@ import 'dart:typed_data'; import 'package:flutter_map/flutter_map.dart'; -/// Map caching provider which disables built-in caching -class DisabledMapCachingProvider implements MapCachingProvider { - /// Disable map caching through [NetworkTileProvider.cachingProvider] +/// Caching provider which disables built-in caching +mixin class DisabledMapCachingProvider implements MapCachingProvider { + /// Disable built-in map caching const DisabledMapCachingProvider(); @override bool get isSupported => false; @override - Never getTile(String url) => throw StateError('Caching should be disabled'); + Never getTile(String url) => + throw UnsupportedError('Must not be called if `isSupported` is `false`'); @override Never putTile({ @@ -19,5 +20,5 @@ class DisabledMapCachingProvider implements MapCachingProvider { required CachedMapTileMetadata tileInfo, Uint8List? bytes, }) => - throw StateError('Caching should be disabled'); + throw UnsupportedError('Must not be called if `isSupported` is `false`'); } From db508517ed5baa1e81838f65cf10a31f01101b2e Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 19 May 2025 19:27:02 +0100 Subject: [PATCH 20/49] Replace `catch (_)` with `on Exception` --- .../impl/native/workers/persistent_registry_unpacker.dart | 2 +- .../network/image_provider/tile_loader_with_caching.dart | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart index 4311f6ba5..f63706b4a 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart @@ -42,7 +42,7 @@ HashMap? persistentRegistryUnpackerWorker( ); }, ); - } catch (_) { + } on Exception { return null; } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart index b18cf4c4d..6ba598b4c 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart @@ -19,7 +19,7 @@ Future _loadTileImageWithCaching( final ({Uint8List bytes, CachedMapTileMetadata tileInfo})? cachedTile; try { cachedTile = await cachingProvider.getTile(resolvedUrl); - } catch (_) { + } on Exception { return _loadTileImageSimple(key, decode, useFallback: useFallback); } @@ -50,7 +50,7 @@ Future _loadTileImageWithCaching( return await decode( await ImmutableBuffer.fromUint8List(response.bodyBytes), ); - } catch (err) { + } on Exception { // Otherwise fallback to a cached tile if we have one if (cachedTile != null) { return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); From 92452f63767af17bf0159d8838a697815c62d5ca Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 20 May 2025 12:02:11 +0100 Subject: [PATCH 21/49] Switched back to JSON from FlatBuffers Removed `CachedMapTileMetadata.lastModifiedLocally` Store `CachedMapTileMetadata.staleAt` & `.lastModified` as milliseconds since epoch to improve performance/efficiency Compress/obfuscate `CachedMapTileMetadata` JSON representation --- example/pubspec.lock | 8 - .../built_in/built_in_caching_provider.dart | 6 +- .../impl/native/flatbufs/registry.fbs | 17 -- .../impl/native/flatbufs/registry.g.dart | 272 ------------------ .../caching/built_in/impl/native/native.dart | 8 +- .../workers/persistent_registry_unpacker.dart | 39 +-- .../workers/persistent_registry_writer.dart | 69 +---- .../network/caching/tile_metadata.dart | 62 ++-- .../tile_loader_with_caching.dart | 2 - pubspec.yaml | 1 - 10 files changed, 63 insertions(+), 421 deletions(-) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart diff --git a/example/pubspec.lock b/example/pubspec.lock index 6378d2e06..79656615d 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -105,14 +105,6 @@ packages: url: "https://pub.dev" source: hosted version: "1.1.1" - flat_buffers: - dependency: transitive - description: - name: flat_buffers - sha256: "380bdcba5664a718bfd4ea20a45d39e13684f5318fcd8883066a55e21f37f4c3" - url: "https://pub.dev" - source: hosted - version: "23.5.26" flutter: dependency: "direct main" description: flutter diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 046b81394..0b09f1211 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -10,9 +10,9 @@ import 'package:uuid/uuid.dart'; /// Simple built-in map caching using an I/O storage mechanism, for native /// (non-web) platforms only /// -/// Uses FlatBuffers to store a centralised registry which is operated on as -/// a map in memory to maximise performance, with tile blobs stored raw as -/// files and a second file used to track the size of the cache. +/// Uses JSON to store a centralised registry which is operated on as a map in +/// memory to maximise performance, with tile blobs stored raw as files and a +/// second file used to track the size of the cache. /// /// Usually uses HTTP headers to determine tile freshness, although /// `overrideFreshAge` can override this. diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs deleted file mode 100644 index c9373980d..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.fbs +++ /dev/null @@ -1,17 +0,0 @@ -table TileMetadata { - last_modified_locally: int64; - stale_at: int64; - last_modified: int64; - etag: string; -} - -table TileMetadataEntry { - id: string (key); - metadata: TileMetadata; -} - -table TileMetadataMap { - entries: [TileMetadataEntry]; -} - -root_type TileMetadataMap; \ No newline at end of file diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart deleted file mode 100644 index 11f553289..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart +++ /dev/null @@ -1,272 +0,0 @@ -// automatically generated by the FlatBuffers compiler, do not modify -// ignore_for_file: unused_import, unused_field, unused_element, unused_local_variable, constant_identifier_names, public_member_api_docs - -import 'dart:typed_data' show Uint8List; -import 'package:flat_buffers/flat_buffers.dart' as fb; - -class TileMetadata { - factory TileMetadata(List bytes) { - final rootRef = fb.BufferContext.fromBytes(bytes); - return reader.read(rootRef, 0); - } - TileMetadata._(this._bc, this._bcOffset); - - static const fb.Reader reader = _TileMetadataReader(); - - final fb.BufferContext _bc; - final int _bcOffset; - - int get lastModifiedLocally => - const fb.Int64Reader().vTableGet(_bc, _bcOffset, 4, 0); - int get staleAt => const fb.Int64Reader().vTableGet(_bc, _bcOffset, 6, 0); - int get lastModified => - const fb.Int64Reader().vTableGet(_bc, _bcOffset, 8, 0); - String? get etag => - const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 10); - - @override - String toString() { - return 'TileMetadata{lastModifiedLocally: $lastModifiedLocally, staleAt: $staleAt, lastModified: $lastModified, etag: $etag}'; - } -} - -class _TileMetadataReader extends fb.TableReader { - const _TileMetadataReader(); - - @override - TileMetadata createObject(fb.BufferContext bc, int offset) => - TileMetadata._(bc, offset); -} - -class TileMetadataBuilder { - TileMetadataBuilder(this.fbBuilder); - - final fb.Builder fbBuilder; - - void begin() { - fbBuilder.startTable(4); - } - - int addLastModifiedLocally(int? lastModifiedLocally) { - fbBuilder.addInt64(0, lastModifiedLocally); - return fbBuilder.offset; - } - - int addStaleAt(int? staleAt) { - fbBuilder.addInt64(1, staleAt); - return fbBuilder.offset; - } - - int addLastModified(int? lastModified) { - fbBuilder.addInt64(2, lastModified); - return fbBuilder.offset; - } - - int addEtagOffset(int? offset) { - fbBuilder.addOffset(3, offset); - return fbBuilder.offset; - } - - int finish() { - return fbBuilder.endTable(); - } -} - -class TileMetadataObjectBuilder extends fb.ObjectBuilder { - final int? _lastModifiedLocally; - final int? _staleAt; - final int? _lastModified; - final String? _etag; - - TileMetadataObjectBuilder({ - int? lastModifiedLocally, - int? staleAt, - int? lastModified, - String? etag, - }) : _lastModifiedLocally = lastModifiedLocally, - _staleAt = staleAt, - _lastModified = lastModified, - _etag = etag; - - /// Finish building, and store into the [fbBuilder]. - @override - int finish(fb.Builder fbBuilder) { - final int? etagOffset = _etag == null ? null : fbBuilder.writeString(_etag); - fbBuilder.startTable(4); - fbBuilder.addInt64(0, _lastModifiedLocally); - fbBuilder.addInt64(1, _staleAt); - fbBuilder.addInt64(2, _lastModified); - fbBuilder.addOffset(3, etagOffset); - return fbBuilder.endTable(); - } - - /// Convenience method to serialize to byte list. - @override - Uint8List toBytes([String? fileIdentifier]) { - final fbBuilder = fb.Builder(deduplicateTables: false); - fbBuilder.finish(finish(fbBuilder), fileIdentifier); - return fbBuilder.buffer; - } -} - -class TileMetadataEntry { - factory TileMetadataEntry(List bytes) { - final rootRef = fb.BufferContext.fromBytes(bytes); - return reader.read(rootRef, 0); - } - TileMetadataEntry._(this._bc, this._bcOffset); - - static const fb.Reader reader = _TileMetadataEntryReader(); - - final fb.BufferContext _bc; - final int _bcOffset; - - String? get id => - const fb.StringReader().vTableGetNullable(_bc, _bcOffset, 4); - TileMetadata? get metadata => - TileMetadata.reader.vTableGetNullable(_bc, _bcOffset, 6); - - @override - String toString() { - return 'TileMetadataEntry{id: $id, metadata: $metadata}'; - } -} - -class _TileMetadataEntryReader extends fb.TableReader { - const _TileMetadataEntryReader(); - - @override - TileMetadataEntry createObject(fb.BufferContext bc, int offset) => - TileMetadataEntry._(bc, offset); -} - -class TileMetadataEntryBuilder { - TileMetadataEntryBuilder(this.fbBuilder); - - final fb.Builder fbBuilder; - - void begin() { - fbBuilder.startTable(2); - } - - int addIdOffset(int? offset) { - fbBuilder.addOffset(0, offset); - return fbBuilder.offset; - } - - int addMetadataOffset(int? offset) { - fbBuilder.addOffset(1, offset); - return fbBuilder.offset; - } - - int finish() { - return fbBuilder.endTable(); - } -} - -class TileMetadataEntryObjectBuilder extends fb.ObjectBuilder { - final String? _id; - final TileMetadataObjectBuilder? _metadata; - - TileMetadataEntryObjectBuilder({ - String? id, - TileMetadataObjectBuilder? metadata, - }) : _id = id, - _metadata = metadata; - - /// Finish building, and store into the [fbBuilder]. - @override - int finish(fb.Builder fbBuilder) { - final int? idOffset = _id == null ? null : fbBuilder.writeString(_id); - final int? metadataOffset = _metadata?.getOrCreateOffset(fbBuilder); - fbBuilder.startTable(2); - fbBuilder.addOffset(0, idOffset); - fbBuilder.addOffset(1, metadataOffset); - return fbBuilder.endTable(); - } - - /// Convenience method to serialize to byte list. - @override - Uint8List toBytes([String? fileIdentifier]) { - final fbBuilder = fb.Builder(deduplicateTables: false); - fbBuilder.finish(finish(fbBuilder), fileIdentifier); - return fbBuilder.buffer; - } -} - -class TileMetadataMap { - factory TileMetadataMap(List bytes) { - final rootRef = fb.BufferContext.fromBytes(bytes); - return reader.read(rootRef, 0); - } - TileMetadataMap._(this._bc, this._bcOffset); - - static const fb.Reader reader = _TileMetadataMapReader(); - - final fb.BufferContext _bc; - final int _bcOffset; - - List? get entries => - const fb.ListReader(TileMetadataEntry.reader) - .vTableGetNullable(_bc, _bcOffset, 4); - - @override - String toString() { - return 'TileMetadataMap{entries: $entries}'; - } -} - -class _TileMetadataMapReader extends fb.TableReader { - const _TileMetadataMapReader(); - - @override - TileMetadataMap createObject(fb.BufferContext bc, int offset) => - TileMetadataMap._(bc, offset); -} - -class TileMetadataMapBuilder { - TileMetadataMapBuilder(this.fbBuilder); - - final fb.Builder fbBuilder; - - void begin() { - fbBuilder.startTable(1); - } - - int addEntriesOffset(int? offset) { - fbBuilder.addOffset(0, offset); - return fbBuilder.offset; - } - - int finish() { - return fbBuilder.endTable(); - } -} - -class TileMetadataMapObjectBuilder extends fb.ObjectBuilder { - final List? _entries; - - TileMetadataMapObjectBuilder({ - List? entries, - }) : _entries = entries; - - /// Finish building, and store into the [fbBuilder]. - @override - int finish(fb.Builder fbBuilder) { - final int? entriesOffset = _entries == null - ? null - : fbBuilder.writeList( - _entries.map((b) => b.getOrCreateOffset(fbBuilder)).toList()); - fbBuilder.startTable(1); - fbBuilder.addOffset(0, entriesOffset); - return fbBuilder.endTable(); - } - - /// Convenience method to serialize to byte list. - @override - Uint8List toBytes([String? fileIdentifier]) { - final fbBuilder = fb.Builder(deduplicateTables: false); - fbBuilder.finish(finish(fbBuilder), fileIdentifier); - return fbBuilder.buffer; - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index b3f691708..881d09a3e 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -16,7 +16,7 @@ import 'package:path_provider/path_provider.dart'; @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { - static const persistentRegistryFileName = 'registry.bin'; + static const persistentRegistryFileName = 'registry.json'; static const sizeMonitorFileName = 'sizeMonitor.bin'; final String? cacheDirectory; @@ -167,18 +167,12 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final uuid = cacheKeyGenerator(url); final resolvedTileInfo = overrideFreshAge != null ? CachedMapTileMetadata( - lastModifiedLocally: tileInfo.lastModifiedLocally, staleAt: DateTime.timestamp().add(overrideFreshAge!), lastModified: tileInfo.lastModified, etag: tileInfo.etag, ) : tileInfo; - if (_registry[uuid] case final existingTileInfo? - when resolvedTileInfo == existingTileInfo) { - return; - } - if (bytes != null) { final tileFilePath = p.join(_cacheDirectoryPath, uuid); _writeTileFile(tileFilePath, bytes); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart index f63706b4a..7b602d575 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart @@ -1,9 +1,8 @@ import 'dart:collection'; +import 'dart:convert'; import 'dart:io'; -import 'dart:typed_data'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart'; import 'package:meta/meta.dart'; /// Unpack the FlatBuffer registry into a mapping of tile UUIDs to their @@ -15,34 +14,26 @@ import 'package:meta/meta.dart'; HashMap? persistentRegistryUnpackerWorker( String persistentRegistryFilePath, ) { - final Uint8List bin; + final String json; try { - bin = File(persistentRegistryFilePath).readAsBytesSync(); + json = File(persistentRegistryFilePath).readAsStringSync(); } on FileSystemException { return null; } + final Map parsed; try { - final tileMetadataMap = TileMetadataMap(bin); - if (tileMetadataMap.entries == null) return null; - - return HashMap.fromIterable( - tileMetadataMap.entries!, - key: (e) => (e as TileMetadataEntry).id!, - value: (e) { - final metadata = (e as TileMetadataEntry).metadata!; - return CachedMapTileMetadata( - lastModifiedLocally: - DateTime.fromMillisecondsSinceEpoch(metadata.lastModifiedLocally), - staleAt: DateTime.fromMillisecondsSinceEpoch(metadata.staleAt), - lastModified: metadata.lastModified == 0 - ? null - : DateTime.fromMillisecondsSinceEpoch(metadata.lastModified), - etag: metadata.etag, - ); - }, - ); - } on Exception { + parsed = jsonDecode(json) as Map; + } on FormatException { return null; } + + return HashMap.from( + parsed.map( + (key, value) => MapEntry( + key, + CachedMapTileMetadata.fromJson(value as Map), + ), + ), + ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart index f896873c5..57aee9162 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart @@ -1,10 +1,9 @@ import 'dart:async'; +import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; -import 'package:flat_buffers/flat_buffers.dart' as fb; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/flatbufs/registry.g.dart'; import 'package:meta/meta.dart'; /// Isolate worker which maintains its own registry and sequences writes to @@ -43,17 +42,22 @@ Future persistentRegistryWriterWorker( writeLocker = Completer(); alreadyWaitingToWrite = false; - _writeFlatbuffer(registry, writer); + final encoded = jsonEncode(registry); + writer + ..setPositionSync(0) + ..writeStringSync(encoded) + ..truncateSync(writer.positionSync()) + ..flushSync(); writeLocker.complete(); } + write(); + Timer createWriteDebouncer() => Timer(const Duration(milliseconds: 50), write); Timer? writeDebouncer; - write(); - await for (final val in receivePort) { final (:uuid, :tileInfo) = val as ({String uuid, CachedMapTileMetadata? tileInfo}); @@ -68,58 +72,3 @@ Future persistentRegistryWriterWorker( writeDebouncer = createWriteDebouncer(); } } - -void _writeFlatbuffer( - Map registry, - RandomAccessFile fileWriter, -) { - final registryIds = registry.keys.toList(growable: false); - final registryMetadatas = registry.values.toList(growable: false); - - final builder = fb.Builder(initialSize: 1048576); - - final entriesOffset = builder.writeList( - List.generate( - registry.length, - (i) { - final id = registryIds[i]; - final metadata = registryMetadatas[i]; - - final fbId = builder.writeString(id, asciiOptimization: true); - final fbEtag = metadata.etag == null - ? null - : builder.writeString(metadata.etag!, asciiOptimization: true); - - final fbTileMetadata = (TileMetadataBuilder(builder) - ..begin() - ..addLastModifiedLocally( - metadata.lastModifiedLocally.millisecondsSinceEpoch, - ) - ..addStaleAt(metadata.staleAt.millisecondsSinceEpoch) - ..addEtagOffset(fbEtag) - ..addLastModified(metadata.lastModified?.millisecondsSinceEpoch)) - .finish(); - - return (TileMetadataEntryBuilder(builder) - ..begin() - ..addIdOffset(fbId) - ..addMetadataOffset(fbTileMetadata)) - .finish(); - }, - growable: false, - ), - ); - - final metadataMapOffset = (TileMetadataMapBuilder(builder) - ..begin() - ..addEntriesOffset(entriesOffset)) - .finish(); - - builder.finish(metadataMapOffset); - - fileWriter - ..setPositionSync(0) - ..writeFromSync(builder.buffer) - ..truncateSync(fileWriter.positionSync()) - ..flushSync(); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index 89e318b78..31619a983 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -1,56 +1,64 @@ +import 'dart:io' show HttpHeaders; // web safe! + import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; /// Metadata about a tile cached with a [MapCachingProvider] /// /// Caching is usually determined with HTTP headers. However, if a specific -/// implementation chooses to, it can solely use [staleAt] and set the other -/// properties to a dummy value. +/// implementation chooses to, it can solely use [isStale] and set the other +/// properties to `null`. /// /// External usage of this class is not usually necessary. It is visible so /// other tile providers may make use of it. @immutable class CachedMapTileMetadata { /// Create new metadata - /// - /// [lastModifiedLocally] must be set to [DateTime.timestamp]. Other - /// properties should usually be set based on the tile's HTTP response - /// headers. - const CachedMapTileMetadata({ - required this.lastModifiedLocally, - required this.staleAt, - required this.lastModified, + CachedMapTileMetadata({ + required DateTime staleAt, + required DateTime? lastModified, required this.etag, - }); + }) : _staleAt = staleAt.millisecondsSinceEpoch, + _lastModified = lastModified?.millisecondsSinceEpoch; - /// Used to efficiently allow updates to already cached tiles - /// - /// Must be set to [DateTime.timestamp] when a new tile is cached or a tile - /// is updated. - final DateTime lastModifiedLocally; + /// Decode metadata from JSON + CachedMapTileMetadata.fromJson(Map json) + : _staleAt = json['a'] as int, + _lastModified = json.containsKey('b') ? json['b'] as int : null, + etag = json.containsKey('c') ? json['c'] as String : null; - /// The date/time at which the tile becomes stale according to the HTTP spec - final DateTime staleAt; + final int _staleAt; - /// The tile's last modified HTTP header - final DateTime? lastModified; + /// If available, the value in [HttpHeaders.lastModifiedHeader] + DateTime? get lastModified => _lastModified == null + ? null + : DateTime.fromMillisecondsSinceEpoch(_lastModified); + final int? _lastModified; - /// The tile's etag HTTP header + /// If available, the value in [HttpHeaders.etagHeader] final String? etag; - /// Whether the tile is currently stale - bool get isStale => DateTime.timestamp().isAfter(staleAt); + /// Whether this tile should be considered stale + /// + /// Usually this is implemented by storing the timestamp at which the tile + /// becomes stale, and comparing that to the current timestamp. + bool get isStale => DateTime.timestamp().millisecondsSinceEpoch > _staleAt; + + /// Encode the metadata to JSON + Map toJson() => { + 'a': _staleAt, + if (_lastModified != null) 'b': _lastModified, + if (etag != null) 'c': etag, + }; @override - int get hashCode => - Object.hash(lastModifiedLocally, staleAt, lastModified, etag); + int get hashCode => Object.hash(_staleAt, lastModified, etag); @override bool operator ==(Object other) => identical(this, other) || (other is CachedMapTileMetadata && - lastModifiedLocally == other.lastModifiedLocally && - staleAt == other.staleAt && + _staleAt == other._staleAt && lastModified == other.lastModified && etag == other.etag); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart index 6ba598b4c..650ff0cb0 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart @@ -31,7 +31,6 @@ Future _loadTileImageWithCaching( cachingProvider.putTile( url: resolvedUrl, tileInfo: CachedMapTileMetadata( - lastModifiedLocally: DateTime.timestamp(), staleAt: _calculateStaleAt(response), lastModified: lastModified != null ? HttpDate.parse(lastModified) : null, @@ -105,7 +104,6 @@ Future _loadTileImageWithCaching( cachingProvider.putTile( url: resolvedUrl, tileInfo: CachedMapTileMetadata( - lastModifiedLocally: DateTime.timestamp(), staleAt: _calculateStaleAt(response), lastModified: lastModified != null ? HttpDate.parse(lastModified) diff --git a/pubspec.yaml b/pubspec.yaml index 6925994ae..c6c187e8f 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -29,7 +29,6 @@ dependencies: async: ^2.11.0 collection: ^1.18.0 dart_earcut: ^1.1.0 - flat_buffers: ^23.5.26 flutter: sdk: flutter http: ^1.2.1 From 92a60b1d0bbeba8c7d18afc13b49a574acc4c7bb Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 20 May 2025 12:43:29 +0100 Subject: [PATCH 22/49] Fixed size limiter --- .../built_in/impl/native/workers/size_limiter.dart | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart index 22e5f5227..f9a2591ae 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart @@ -58,16 +58,18 @@ Future> sizeLimiterWorker( int i = 0; int deletedSize = 0; - final deletedTiles = () sync* { + final deletedFiles = >[]; + final deletedUuids = () sync* { while (currentSize - deletedSize > input.sizeLimit && i < tiles.length) { final tile = tiles[i++]; final uuid = p.basename(tile.path); deletedSize += tile.size; + deletedFiles.add(File(tile.path).delete()); yield uuid; - yield File(tile.path).delete(); } - }(); + }() + .toList(growable: false); sizeMonitor ..setPositionSync(0) @@ -77,7 +79,7 @@ Future> sizeLimiterWorker( ..flushSync() ..closeSync(); - await Future.wait(deletedTiles.whereType>()); + await Future.wait(deletedFiles); - return deletedTiles.whereType().toList(growable: false); + return deletedUuids; } From 3fe5adcd6afbef2c94d6b2f85745620209a3480b Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 20 May 2025 13:56:08 +0100 Subject: [PATCH 23/49] Minor improvement --- .../network/caching/built_in/built_in_caching_provider.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 0b09f1211..b3dffa7e6 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -50,7 +50,7 @@ abstract interface class BuiltInMapCachingProvider /// size manually if necessary. /// /// Defaults to 800 MB. Set to `null` to disable. - int? maxCacheSize = 800000000, + int? maxCacheSize = 800_000_000, /// Override the duration of time a tile is considered fresh for /// From 6078fe296896e306033768c1f1aa7219486992c9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 22 May 2025 23:43:07 +0100 Subject: [PATCH 24/49] Fixed minor bug in example app --- example/lib/pages/home.dart | 42 +++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/example/lib/pages/home.dart b/example/lib/pages/home.dart index 4e8222d48..1ecf42107 100644 --- a/example/lib/pages/home.dart +++ b/example/lib/pages/home.dart @@ -39,26 +39,28 @@ class _HomePageState extends State { body: Stack( children: [ if (widget.cacheInitComplete case final cacheInitComplete?) - FutureBuilder( - future: cacheInitComplete.result, - builder: (context, snapshot) { - if (snapshot.connectionState == ConnectionState.done) { - return const ColoredBox(color: Color(0xFFE0E0E0)); - } - return const Center( - child: Row( - mainAxisSize: MainAxisSize.min, - spacing: 16, - children: [ - SizedBox.square( - dimension: 24, - child: CircularProgressIndicator.adaptive(), - ), - Text('Awaiting cache initialisation'), - ], - ), - ); - }, + Positioned.fill( + child: FutureBuilder( + future: cacheInitComplete.result, + builder: (context, snapshot) { + if (snapshot.connectionState == ConnectionState.done) { + return const ColoredBox(color: Color(0xFFE0E0E0)); + } + return const Center( + child: Row( + mainAxisSize: MainAxisSize.min, + spacing: 16, + children: [ + SizedBox.square( + dimension: 24, + child: CircularProgressIndicator.adaptive(), + ), + Text('Awaiting cache initialisation'), + ], + ), + ); + }, + ), ), FlutterMap( options: const MapOptions( From 37da3d45d2d7c2ce8b39ebeb033d9eb0b8f02eed Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Fri, 23 May 2025 13:35:10 +0100 Subject: [PATCH 25/49] Improved network tile image provider Prepared NTIP for abortable HTTP requests Merged caching & non-caching NTIPs Improved NTIP tests --- .../image_provider/consolidate_response.dart | 83 ++ .../image_provider/image_provider.dart | 285 +++++- .../image_provider/tile_loader_simple.dart | 27 - .../tile_loader_with_caching.dart | 169 ---- .../tile_provider/network/tile_provider.dart | 57 +- .../network_image_provider_test.dart | 824 +++++++++++------- 6 files changed, 876 insertions(+), 569 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/image_provider/consolidate_response.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/consolidate_response.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/consolidate_response.dart new file mode 100644 index 000000000..eacf725b4 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/consolidate_response.dart @@ -0,0 +1,83 @@ +// Adapted from Flutter (c 2014 BSD The Flutter Authors) method to work without +// `dart:io` using a `StreamedResponse` + +import 'dart:async'; +import 'dart:convert'; + +import 'package:flutter/foundation.dart'; +import 'package:http/http.dart'; +import 'package:meta/meta.dart'; + +/// Efficiently converts the response body of an [Response] into a +/// [Uint8List]. +/// +/// Assumes response has been uncompressed automatically. +/// +/// See [consolidateHttpClientResponseBytes] for more info. +@internal +Future consolidateStreamedResponseBytes( + StreamedResponse response, { + BytesReceivedCallback? onBytesReceived, +}) { + final completer = Completer.sync(); + final output = _OutputBuffer(); + + int? expectedContentLength = response.contentLength; + if (expectedContentLength == -1) expectedContentLength = null; + + int bytesReceived = 0; + late final StreamSubscription> subscription; + subscription = response.stream.listen( + (chunk) { + output.add(chunk); + if (onBytesReceived != null) { + bytesReceived += chunk.length; + try { + onBytesReceived(bytesReceived, expectedContentLength); + } catch (error, stackTrace) { + completer.completeError(error, stackTrace); + subscription.cancel(); + return; + } + } + }, + onDone: () { + output.close(); + completer.complete(output.bytes); + }, + onError: completer.completeError, + cancelOnError: true, + ); + + return completer.future; +} + +class _OutputBuffer extends ByteConversionSinkBase { + List>? _chunks = >[]; + int _contentLength = 0; + Uint8List? _bytes; + + @override + void add(List chunk) { + assert(_bytes == null, '`_bytes` must be `null`'); + _chunks!.add(chunk); + _contentLength += chunk.length; + } + + @override + void close() { + if (_bytes != null) { + // We've already been closed; this is a no-op + return; + } + _bytes = Uint8List(_contentLength); + int offset = 0; + for (final List chunk in _chunks!) { + _bytes!.setRange(offset, offset + chunk.length, chunk); + offset += chunk.length; + } + _chunks = null; + } + + Uint8List get bytes => _bytes!; +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index ec472830b..eacaf5f93 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -1,16 +1,14 @@ import 'dart:async'; -import 'dart:io' - show HttpHeaders, HttpDate, HttpException, HttpStatus; // this is web safe! +import 'dart:io' show HttpHeaders, HttpDate, HttpStatus; // this is web safe! import 'dart:math'; import 'dart:ui'; import 'package:flutter/foundation.dart'; import 'package:flutter/painting.dart'; import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/image_provider/consolidate_response.dart'; import 'package:http/http.dart'; - -part 'tile_loader_simple.dart'; -part 'tile_loader_with_caching.dart'; +import 'package:meta/meta.dart'; /// Dedicated [ImageProvider] to fetch tiles from the network /// @@ -18,6 +16,7 @@ part 'tile_loader_with_caching.dart'; /// Note that specifying a [fallbackUrl] will prevent this image provider from /// being cached. @immutable +@internal class NetworkTileImageProvider extends ImageProvider { /// The URL to fetch the tile from (GET request) final String url; @@ -40,12 +39,23 @@ class NetworkTileImageProvider extends ImageProvider { /// Not included in [operator==]. final Client httpClient; + /// Completes when the tile request should be aborted + /// + /// Not included in [operator==]. + final Future? abortTrigger; + /// Whether to ignore exceptions and errors that occur whilst fetching tiles /// over the network, and just return a transparent tile /// /// Not included in [operator==]. final bool silenceExceptions; + /// Whether to optimistically attempt to decode HTTP responses that have a + /// non-successful status code as an image + /// + /// Not included in [operator==]. + final bool attemptDecodeOfHttpErrorResponses; + /// Caching provider used to get cached tiles /// /// See online documentation for more information about built-in caching. @@ -56,18 +66,6 @@ class NetworkTileImageProvider extends ImageProvider { /// Not included in [operator==]. final MapCachingProvider? cachingProvider; - /// Function invoked when the image starts loading (not from cache) - /// - /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only - /// after all tiles have loaded. - final void Function() startedLoading; - - /// Function invoked when the image completes loading bytes from the network - /// - /// Used with [finishedLoadingBytes] to safely dispose of the [httpClient] only - /// after all tiles have loaded. - final void Function() finishedLoadingBytes; - /// Create a dedicated [ImageProvider] to fetch tiles from the network /// /// Supports falling back to a secondary URL, if the primary URL fetch fails. @@ -78,34 +76,254 @@ class NetworkTileImageProvider extends ImageProvider { required this.fallbackUrl, required this.headers, required this.httpClient, + required this.abortTrigger, required this.silenceExceptions, + required this.attemptDecodeOfHttpErrorResponses, required this.cachingProvider, - required this.startedLoading, - required this.finishedLoadingBytes, }); @override ImageStreamCompleter loadImage( NetworkTileImageProvider key, ImageDecoderCallback decode, - ) => - MultiFrameImageStreamCompleter( - codec: _load(key, decode), - scale: 1, - debugLabel: url, - informationCollector: () => [ - DiagnosticsProperty('URL', url), - DiagnosticsProperty('Fallback URL', fallbackUrl), - DiagnosticsProperty('Current provider', key), - ], - ); + ) { + final chunkEvents = StreamController(); + + return MultiFrameImageStreamCompleter( + codec: _loadImage(key, chunkEvents, decode), + chunkEvents: chunkEvents.stream, + scale: 1, + debugLabel: key.url, + informationCollector: () => [ + DiagnosticsProperty('URL', url), + DiagnosticsProperty('Fallback URL', fallbackUrl), + DiagnosticsProperty('Current provider', key), + ], + ); + } - Future _load( + Future _loadImage( NetworkTileImageProvider key, + StreamController chunkEvents, ImageDecoderCallback decode, { bool useFallback = false, - }) => - _loadTileImageWithCaching(key, decode, useFallback: useFallback); + }) async { + // Create utility methods + void evict() => + scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); + Future decodeBytes(Uint8List bytes) => + ImmutableBuffer.fromUint8List(bytes).then(decode); + + // Resolve URIs + final resolvedUrl = useFallback ? fallbackUrl ?? '' : url; + final Uri uri; + try { + uri = Uri.parse(resolvedUrl); + } on FormatException { + evict(); + chunkEvents.close(); + rethrow; + } + + // Prepare caching provider & load cached tile if available + ({Uint8List bytes, CachedMapTileMetadata tileInfo})? cachedTile; + final cachingProvider = + this.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); + if (cachingProvider.isSupported) { + try { + cachedTile = await cachingProvider.getTile(resolvedUrl); + } on Exception { + cachedTile = null; + } + } + + // Create method to get bytes from server + Future<({Uint8List bytes, StreamedResponse response})> get({ + Map? additionalHeaders, + }) async { + // TODO: Support cancellation + // final request = AbortableRequest('GET', uri, abortTrigger: abortTrigger); + final request = Request('GET', uri); + + request.headers.addAll(headers); + if (additionalHeaders != null) request.headers.addAll(additionalHeaders); + + final response = await httpClient.send(request); + + final bytes = await consolidateStreamedResponseBytes( + response, + onBytesReceived: (cumulative, total) => chunkEvents.add( + ImageChunkEvent( + cumulativeBytesLoaded: cumulative, + expectedTotalBytes: total, + ), + ), + ); + + return (bytes: bytes, response: response); + } + + // Create method to interact with cache + void cachePut({ + required Uint8List? bytes, + required Map headers, + }) { + if (!cachingProvider.isSupported) return; + + final lastModified = headers[HttpHeaders.lastModifiedHeader]; + final etag = headers[HttpHeaders.etagHeader]; + + DateTime calculateStaleAt() { + final addToNow = DateTime.timestamp().add; + + if (headers[HttpHeaders.cacheControlHeader]?.toLowerCase() + case final cacheControl?) { + final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; + + if (maxAge == null) { + if (headers[HttpHeaders.expiresHeader]?.toLowerCase() + case final expires?) { + return HttpDate.parse(expires); + } + + return addToNow(const Duration(days: 7)); + } + + if (headers[HttpHeaders.ageHeader] case final currentAge?) { + return addToNow( + Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), + ); + } + + final estimatedAge = max( + 0, + DateTime.timestamp() + .difference(HttpDate.parse(headers[HttpHeaders.dateHeader]!)) + .inSeconds, + ); + return addToNow(Duration(seconds: int.parse(maxAge) - estimatedAge)); + } + + return addToNow(const Duration(days: 7)); + } + + cachingProvider.putTile( + url: resolvedUrl, + tileInfo: CachedMapTileMetadata( + staleAt: calculateStaleAt(), + lastModified: + lastModified != null ? HttpDate.parse(lastModified) : null, + etag: etag, + ), + bytes: bytes, + ); + } + + // Main logic + // All `decodeBytes` calls should be awaited so errors may be handled + try { + if (cachedTile != null && !cachedTile.tileInfo.isStale) { + // If we have a cached tile that's not stale, return it + return await decodeBytes(cachedTile.bytes); + } + + // Otherwise, ask the server what's going on - supply any details we have + final (:bytes, :response) = await get( + additionalHeaders: { + if (cachedTile?.tileInfo.lastModified case final lastModified?) + HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), + if (cachedTile?.tileInfo.etag case final etag?) + HttpHeaders.ifNoneMatchHeader: etag, + }, + ); + + // Server says nothing's changed - but might return new useful headers + if (cachedTile != null && response.statusCode == HttpStatus.notModified) { + cachePut(bytes: null, headers: response.headers); + return await decodeBytes(cachedTile.bytes); + } + + // Server says the image has changed - store it new + if (response.statusCode == HttpStatus.ok) { + cachePut(bytes: bytes, headers: response.headers); + return await decodeBytes(bytes); + } + + // It's likely an error at this point + // If the user has disabled attempted-decode, we just throw and catch + // below + // Otherwise we try to decode it anyway, without memory caching + if (!attemptDecodeOfHttpErrorResponses) { + throw NetworkImageLoadException( + statusCode: response.statusCode, + uri: uri, + ); + } + evict(); + try { + return await decodeBytes(bytes); + } catch (err, stackTrace) { + // If it throws, we don't want to throw the decode error, as that's not + // useful for users + // Instead, we throw an exception reporting the failed HTTP request, + // which is caught by the non-specific catch block below to initiate the + // retry/silence mechanisms if applicable + // We do retain the stack trace, so that it might be clear we attempted + // to decode it + // We piggyback off of an error meant for `NetworkImage` - it's the same + // as we need + Error.throwWithStackTrace( + NetworkImageLoadException( + statusCode: response.statusCode, + uri: uri, + ), + stackTrace, + ); + } + } + // TODO: Support cancellation + /* on AbortedRequest { + // This is a planned exception, we just quit silently + + evict(); + chunkEvents.close(); + return await decodeBytes(TileProvider.transparentImage); + } */ + on ClientException catch (e) { + // This could be a wide range of issues, potentially ours, potentially + // network, etc. + + evict(); + + // Try to detect errors thrown from requests being aborted due to the + // client being closed + // This can occur when the map/tile layer is disposed early - in older + // versions, we used manual tracking to avoid disposing too early, but now + // we just attempt to catch (it's cleaner & easier) + if (e.message.contains('closed') || e.message.contains('cancel')) { + return await decodeBytes(TileProvider.transparentImage); + } + + if (useFallback || fallbackUrl == null) { + chunkEvents.close(); + if (!silenceExceptions) rethrow; + return await decodeBytes(TileProvider.transparentImage); + } + return _loadImage(key, chunkEvents, decode, useFallback: true); + } catch (e) { + // Non-specific catch to catch decoding errors, the manually thrown HTTP + // exception, etc. + + evict(); + + if (useFallback || fallbackUrl == null) { + chunkEvents.close(); + if (!silenceExceptions) rethrow; + return await decodeBytes(TileProvider.transparentImage); + } + return _loadImage(key, chunkEvents, decode, useFallback: true); + } + } @override SynchronousFuture obtainKey( @@ -118,6 +336,7 @@ class NetworkTileImageProvider extends ImageProvider { identical(this, other) || (other is NetworkTileImageProvider && fallbackUrl == null && + other.fallbackUrl == null && url == other.url); @override diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart deleted file mode 100644 index 8e945c06b..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_simple.dart +++ /dev/null @@ -1,27 +0,0 @@ -part of 'image_provider.dart'; - -Future _loadTileImageSimple( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) { - key.startedLoading(); - - return key.httpClient - .readBytes( - Uri.parse(useFallback ? key.fallbackUrl ?? '' : key.url), - headers: key.headers, - ) - .whenComplete(key.finishedLoadingBytes) - .then(ImmutableBuffer.fromUint8List) - .then(decode) - .onError((err, stack) { - scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); - if (useFallback || key.fallbackUrl == null) { - if (!key.silenceExceptions) throw err; - return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) - .then(decode); - } - return _loadTileImageSimple(key, decode, useFallback: true); - }); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart deleted file mode 100644 index 650ff0cb0..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/tile_loader_with_caching.dart +++ /dev/null @@ -1,169 +0,0 @@ -part of 'image_provider.dart'; - -Future _loadTileImageWithCaching( - NetworkTileImageProvider key, - ImageDecoderCallback decode, { - bool useFallback = false, -}) async { - key.startedLoading(); - - final resolvedUrl = useFallback ? key.fallbackUrl ?? '' : key.url; - - final cachingProvider = - key.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); - - if (!cachingProvider.isSupported) { - return _loadTileImageSimple(key, decode, useFallback: useFallback); - } - - final ({Uint8List bytes, CachedMapTileMetadata tileInfo})? cachedTile; - try { - cachedTile = await cachingProvider.getTile(resolvedUrl); - } on Exception { - return _loadTileImageSimple(key, decode, useFallback: useFallback); - } - - Future handleOk(Response response) async { - final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; - final etag = response.headers[HttpHeaders.etagHeader]; - - unawaited( - cachingProvider.putTile( - url: resolvedUrl, - tileInfo: CachedMapTileMetadata( - staleAt: _calculateStaleAt(response), - lastModified: - lastModified != null ? HttpDate.parse(lastModified) : null, - etag: etag, - ), - bytes: response.bodyBytes, - ), - ); - - return ImmutableBuffer.fromUint8List(response.bodyBytes).then(decode); - } - - Future handleNotOk(Response response) async { - // Optimistically try to decode the response anyway - try { - return await decode( - await ImmutableBuffer.fromUint8List(response.bodyBytes), - ); - } on Exception { - // Otherwise fallback to a cached tile if we have one - if (cachedTile != null) { - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - // Otherwise fallback to the fallback URL - if (!useFallback && key.fallbackUrl != null) { - return _loadTileImageWithCaching(key, decode, useFallback: true); - } - - // Otherwise throw an exception/silently fail - if (!key.silenceExceptions) { - throw HttpException( - 'Recieved ${response.statusCode}, and body was not a decodable image', - uri: Uri.parse(resolvedUrl), - ); - } - - return ImmutableBuffer.fromUint8List(TileProvider.transparentImage) - .then(decode); - } finally { - scheduleMicrotask(() => PaintingBinding.instance.imageCache.evict(key)); - } - } - - if (cachedTile != null) { - // If we have a cached tile that's not stale, return it - if (!cachedTile.tileInfo.isStale) { - key.finishedLoadingBytes(); - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - // Otherwise, ask the server what's going on - supply any details we have - final response = await key.httpClient.get( - Uri.parse(resolvedUrl), - headers: { - ...key.headers, - if (cachedTile.tileInfo.lastModified case final lastModified?) - HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), - if (cachedTile.tileInfo.etag case final etag?) - HttpHeaders.ifNoneMatchHeader: etag, - }, - ); - key.finishedLoadingBytes(); - - // Server says nothing's changed - but might return new useful headers - if (response.statusCode == HttpStatus.notModified) { - final lastModified = response.headers[HttpHeaders.lastModifiedHeader]; - final etag = response.headers[HttpHeaders.etagHeader]; - - unawaited( - cachingProvider.putTile( - url: resolvedUrl, - tileInfo: CachedMapTileMetadata( - staleAt: _calculateStaleAt(response), - lastModified: lastModified != null - ? HttpDate.parse(lastModified) - : cachedTile.tileInfo.lastModified, - etag: etag ?? cachedTile.tileInfo.etag, - ), - ), - ); - - return ImmutableBuffer.fromUint8List(cachedTile.bytes).then(decode); - } - - if (response.statusCode == HttpStatus.ok) { - return await handleOk(response); - } - return await handleNotOk(response); - } - - final response = await key.httpClient.get( - Uri.parse(resolvedUrl), - headers: key.headers, - ); - key.finishedLoadingBytes(); - - if (response.statusCode == HttpStatus.ok) { - return await handleOk(response); - } - return await handleNotOk(response); -} - -DateTime _calculateStaleAt(Response response) { - final addToNow = DateTime.timestamp().add; - - if (response.headers[HttpHeaders.cacheControlHeader]?.toLowerCase() - case final cacheControl?) { - final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; - - if (maxAge == null) { - if (response.headers[HttpHeaders.expiresHeader]?.toLowerCase() - case final expires?) { - return HttpDate.parse(expires); - } - - return addToNow(const Duration(days: 7)); - } - - if (response.headers[HttpHeaders.ageHeader] case final currentAge?) { - return addToNow( - Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), - ); - } - - final estimatedAge = max( - 0, - DateTime.timestamp() - .difference(HttpDate.parse(response.headers[HttpHeaders.dateHeader]!)) - .inSeconds, - ); - return addToNow(Duration(seconds: int.parse(maxAge) - estimatedAge)); - } - - return addToNow(const Duration(days: 7)); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index a31b18eb0..e7896b890 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:collection'; import 'package:flutter/rendering.dart'; import 'package:flutter_map/flutter_map.dart'; @@ -36,14 +35,26 @@ class NetworkTileProvider extends TileProvider { super.headers, Client? httpClient, this.silenceExceptions = false, + this.attemptDecodeOfHttpErrorResponses = true, this.cachingProvider, }) : _isInternallyCreatedClient = httpClient == null, _httpClient = httpClient ?? RetryClient(Client()); /// Whether to ignore exceptions and errors that occur whilst fetching tiles /// over the network, and just return a transparent tile + /// + /// Defaults to `false`. final bool silenceExceptions; + /// Whether to optimistically attempt to decode HTTP responses that have a + /// non-successful status code as an image + /// + /// If the decode is unnsuccessful, the behaviour depends on + /// [silenceExceptions]. + /// + /// Defaults to `true`. + final bool attemptDecodeOfHttpErrorResponses; + /// Caching provider used to get cached tiles /// /// See online documentation for more information about built-in caching. @@ -63,38 +74,46 @@ class NetworkTileProvider extends TileProvider { /// Whether [_httpClient] was created on construction (and not passed in) final bool _isInternallyCreatedClient; - /// Each [Completer] is completed once the corresponding tile has finished - /// loading - /// - /// Used to avoid disposing of [_httpClient] whilst HTTP requests are still - /// underway. - /// - /// Does not include tiles loaded from session cache. - final _tilesInProgress = HashMap>(); + @override + // TODO: True when abortable + bool get supportsCancelLoading => false; @override - ImageProvider getImage(TileCoordinates coordinates, TileLayer options) => + ImageProvider getImage( + TileCoordinates coordinates, + TileLayer options, + ) => NetworkTileImageProvider( url: getTileUrl(coordinates, options), fallbackUrl: getTileFallbackUrl(coordinates, options), headers: headers, httpClient: _httpClient, + abortTrigger: null, silenceExceptions: silenceExceptions, + attemptDecodeOfHttpErrorResponses: attemptDecodeOfHttpErrorResponses, cachingProvider: cachingProvider, - startedLoading: () => _tilesInProgress[coordinates] = Completer(), - finishedLoadingBytes: () { - _tilesInProgress[coordinates]?.complete(); - _tilesInProgress.remove(coordinates); - }, ); + /*@override + ImageProvider getImageWithCancelLoadingSupport( + TileCoordinates coordinates, + TileLayer options, + Future cancelLoading, + ) => + NetworkTileImageProvider( + url: getTileUrl(coordinates, options), + fallbackUrl: getTileFallbackUrl(coordinates, options), + headers: headers, + httpClient: _httpClient, + abortTrigger: cancelLoading, + silenceExceptions: silenceExceptions, + attemptDecodeOfHttpErrorResponses: attemptDecodeOfHttpErrorResponses, + cachingProvider: cachingProvider, + );*/ + @override Future dispose() async { - if (_tilesInProgress.isNotEmpty) { - await Future.wait(_tilesInProgress.values.map((c) => c.future)); - } if (_isInternallyCreatedClient) _httpClient.close(); - super.dispose(); } } diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index 91d5c3cbc..c8c6a32db 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -12,7 +12,7 @@ import 'package:mocktail/mocktail.dart'; import '../../../test_utils/test_tile_image.dart'; -class MockHttpClient extends Mock implements BaseClient {} +class MockHttpClient extends Mock implements Client {} // Helper function to resolve the ImageInfo from the ImageProvider. Future getImageInfo(ImageProvider provider) { @@ -51,46 +51,56 @@ void main() { final mockClient = MockHttpClient(); + NetworkTileImageProvider createDefaultImageProvider( + Uri url, { + Uri? fallbackUrl, + bool silenceExceptions = false, + }) => + NetworkTileImageProvider( + url: url.toString(), + fallbackUrl: fallbackUrl?.toString(), + headers: headers, + httpClient: mockClient, + abortTrigger: null, + silenceExceptions: silenceExceptions, + attemptDecodeOfHttpErrorResponses: true, + cachingProvider: const DisabledMapCachingProvider(), + ); + setUpAll(() { // Ensure the Mock library has example values for Uri. registerFallbackValue(Uri()); + registerFallbackValue(Request('GET', Uri())); // TODO: Abortable? }); // We expect a request to be made to the correct URL with the appropriate headers. testWidgets( 'Valid/expected response', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) - .thenAnswer((_) async => testWhiteTileBytes); - - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: null, - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, + when(() => mockClient.send(any())).thenAnswer( + (_) async => StreamedResponse(Stream.value(testWhiteTileBytes), 200), ); - expect(startedLoadingTriggered, false); + final url = randomUrl(); + final provider = createDefaultImageProvider(url); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(img!.image.width, equals(256)); expect(img.image.height, equals(256)); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); @@ -98,425 +108,592 @@ void main() { // We expect the request to be made, and a HTTP ClientException to be bubbled // up to the caller. testWidgets( - 'Server failure - no fallback, exceptions enabled', + 'ClientException - no fallback, exceptions enabled', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) + when(() => mockClient.send(any())) .thenAnswer((_) async => throw ClientException('Server error')); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: null, - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, - ); - - expect(startedLoadingTriggered, false); + final url = randomUrl(); + final provider = createDefaultImageProvider(url); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNull); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Server failure - no fallback, exceptions silenced', + 'ClientException - no fallback, exceptions silenced', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) + when(() => mockClient.send(any())) .thenAnswer((_) async => throw ClientException('Server error')); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: null, - headers: headers, - httpClient: mockClient, - silenceExceptions: true, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, - ); - - expect(startedLoadingTriggered, false); + final url = randomUrl(); + final provider = createDefaultImageProvider(url, silenceExceptions: true); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); // We expect the regular URL to be called once, then the fallback URL. testWidgets( - 'Server failure - successful fallback, exceptions enabled', + 'ClientException - successful fallback', (tester) async { final url = randomUrl(); - when(() => mockClient.readBytes(url, headers: any(named: 'headers'))) - .thenAnswer((_) async => throw ClientException('Server error')); + when( + () => mockClient.send( + any(that: isA().having((r) => r.url, 'URL', url))), + ).thenAnswer((_) async => throw ClientException('Server error')); final fallbackUrl = randomUrl(fallback: true); - when(() => - mockClient.readBytes(fallbackUrl, headers: any(named: 'headers'))) - .thenAnswer((_) async { - return testWhiteTileBytes; - }); - - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, + when( + () => mockClient.send(any( + that: isA().having((r) => r.url, 'URL', fallbackUrl))), + ).thenAnswer( + (_) async => StreamedResponse(Stream.value(testWhiteTileBytes), 200), ); - expect(startedLoadingTriggered, false); + final provider = + createDefaultImageProvider(url, fallbackUrl: fallbackUrl); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(img!.image.width, equals(256)); expect(img.image.height, equals(256)); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Server failure - successful fallback, exceptions silenced', + 'ClientException - failed fallback, exceptions enabled', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(url, headers: any(named: 'headers'))) + when(() => mockClient.send(any())) .thenAnswer((_) async => throw ClientException('Server error')); + final url = randomUrl(); final fallbackUrl = randomUrl(fallback: true); - when(() => - mockClient.readBytes(fallbackUrl, headers: any(named: 'headers'))) - .thenAnswer((_) async { - return testWhiteTileBytes; - }); + final provider = + createDefaultImageProvider(url, fallbackUrl: fallbackUrl); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; + final img = await tester.runAsync(() => getImageInfo(provider)); - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, + expect(img, isNull); + expect(tester.takeException(), isInstanceOf()); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + }, + timeout: defaultTimeout, + ); + + testWidgets( + 'ClientException - failed fallback, exceptions silenced', + (tester) async { + when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) + .thenAnswer((_) async => throw ClientException('Server error')); + + final url = randomUrl(); + final fallbackUrl = randomUrl(fallback: true); + final provider = createDefaultImageProvider( + url, + fallbackUrl: fallbackUrl, silenceExceptions: true, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, ); - expect(startedLoadingTriggered, false); - final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Server failure - failed fallback, exceptions enabled', + 'HTTP errstatus - no fallback, exceptions enabled', (tester) async { + when(() => mockClient.send(any())) + .thenAnswer((_) async => StreamedResponse(const Stream.empty(), 400)); + final url = randomUrl(); - final fallbackUrl = randomUrl(fallback: true); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) - .thenAnswer((_) async => throw ClientException('Server error')); + final provider = createDefaultImageProvider(url); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; + final img = await tester.runAsync(() => getImageInfo(provider)); - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, - ); + expect(img, isNull); + expect(tester.takeException(), isInstanceOf()); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + }, + timeout: defaultTimeout, + ); + testWidgets( + 'HTTP errstatus - no fallback, exceptions silenced', + (tester) async { + when(() => mockClient.send(any())) + .thenAnswer((_) async => StreamedResponse(const Stream.empty(), 400)); - expect(startedLoadingTriggered, false); + final url = randomUrl(); + final provider = createDefaultImageProvider(url, silenceExceptions: true); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - - expect(img, isNull); - expect(tester.takeException(), isInstanceOf()); + expect(img, isNotNull); + expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Server failure - failed fallback, exceptions silenced', + 'HTTP errstatus - successful fallback', (tester) async { final url = randomUrl(); - final fallbackUrl = randomUrl(fallback: true); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) - .thenAnswer((_) async => throw ClientException('Server error')); - - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; + when( + () => mockClient.send( + any(that: isA().having((r) => r.url, 'URL', url))), + ).thenAnswer((_) async => StreamedResponse(const Stream.empty(), 400)); - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, - silenceExceptions: true, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, + final fallbackUrl = randomUrl(fallback: true); + when( + () => mockClient.send(any( + that: isA().having((r) => r.url, 'URL', fallbackUrl))), + ).thenAnswer( + (_) async => StreamedResponse(Stream.value(testWhiteTileBytes), 200), ); - expect(startedLoadingTriggered, false); + final provider = + createDefaultImageProvider(url, fallbackUrl: fallbackUrl); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); + expect(img!.image.width, equals(256)); + expect(img.image.height, equals(256)); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Non-image response - no fallback, exceptions enabled', + 'HTTP errstatus - failed fallback, exceptions enabled', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) - .thenAnswer((_) async { - // 200 OK with html - return Uint8List.fromList(utf8.encode('Server Error')); - }); - - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: null, - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, - ); + when(() => mockClient.send(any())) + .thenAnswer((_) async => StreamedResponse(const Stream.empty(), 400)); - expect(startedLoadingTriggered, false); + final url = randomUrl(); + final fallbackUrl = randomUrl(fallback: true); + final provider = + createDefaultImageProvider(url, fallbackUrl: fallbackUrl); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNull); - final exception = tester.takeException(); - expect(exception, isInstanceOf()); - expect( - (exception as Exception).toString(), - equals('Exception: Invalid image data'), - ); - - verify(() => mockClient.readBytes(url, headers: headers)).called(1); + expect(tester.takeException(), isInstanceOf()); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Non-image response - no fallback, exceptions silenced', + 'HTTP errstatus - failed fallback, exceptions silenced', (tester) async { - final url = randomUrl(); - when(() => mockClient.readBytes(any(), headers: any(named: 'headers'))) - .thenAnswer((_) async { - // 200 OK with html - return Uint8List.fromList(utf8.encode('Server Error')); - }); + when(() => mockClient.send(any())) + .thenAnswer((_) async => StreamedResponse(const Stream.empty(), 400)); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: null, - headers: headers, - httpClient: mockClient, + final url = randomUrl(); + final fallbackUrl = randomUrl(fallback: true); + final provider = createDefaultImageProvider( + url, + fallbackUrl: fallbackUrl, silenceExceptions: true, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, ); - expect(startedLoadingTriggered, false); - final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Non-image response - successful fallback, exceptions enabled', + 'HTTP errstatus with image - optimistic decode enabled', (tester) async { + when(() => mockClient.send(any())).thenAnswer( + (_) async => StreamedResponse( + Stream.value(testWhiteTileBytes), + 400, + ), + ); + final url = randomUrl(); - when(() => mockClient.readBytes(url, headers: any(named: 'headers'))) - .thenAnswer((_) async { - // 200 OK with html - return Uint8List.fromList(utf8.encode('Server Error')); - }); + final provider = createDefaultImageProvider(url); - final fallbackUrl = randomUrl(fallback: true); - when(() => - mockClient.readBytes(fallbackUrl, headers: any(named: 'headers'))) - .thenAnswer((_) async { - return testWhiteTileBytes; - }); + final img = await tester.runAsync(() => getImageInfo(provider)); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; + expect(img, isNotNull); + expect(img!.image.width, equals(256)); + expect(img.image.height, equals(256)); + expect(tester.takeException(), isInstanceOf()); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + }, + ); + testWidgets( + 'HTTP errstatus with image - optimistic decode disabled', + (tester) async { + when(() => mockClient.send(any())).thenAnswer( + (_) async => StreamedResponse( + Stream.value(testWhiteTileBytes), + 400, + ), + ); + + final url = randomUrl(); final provider = NetworkTileImageProvider( url: url.toString(), - fallbackUrl: fallbackUrl.toString(), + fallbackUrl: null, headers: headers, httpClient: mockClient, + abortTrigger: null, silenceExceptions: false, + attemptDecodeOfHttpErrorResponses: false, cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, ); - expect(startedLoadingTriggered, false); + final img = await tester.runAsync(() => getImageInfo(provider)); + + expect(img, isNull); + expect(tester.takeException(), isInstanceOf()); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + }, + ); + + testWidgets( + 'Non-image response - no fallback, exceptions enabled', + (tester) async { + when(() => mockClient.send(any())).thenAnswer( + (_) async => StreamedResponse( + Stream.value( + Uint8List.fromList(utf8.encode('Server Error'))), + 200, + ), + ); + + final url = randomUrl(); + final provider = createDefaultImageProvider(url); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); + expect(img, isNull); + final exception = tester.takeException(); + expect(exception, isInstanceOf()); + expect( + (exception as Exception).toString(), + equals('Exception: Invalid image data'), + ); + + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + }, + timeout: defaultTimeout, + ); + + testWidgets( + 'Non-image response - no fallback, exceptions silenced', + (tester) async { + when(() => mockClient.send(any())).thenAnswer( + (_) async => StreamedResponse( + Stream.value( + Uint8List.fromList(utf8.encode('Server Error'))), + 200, + ), + ); + + final url = randomUrl(); + final provider = createDefaultImageProvider( + url, + silenceExceptions: true, + ); + + final img = await tester.runAsync(() => getImageInfo(provider)); expect(img, isNotNull); - expect(img!.image.width, equals(256)); - expect(img.image.height, equals(256)); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); testWidgets( - 'Non-image response - successful fallback, exceptions silenced', + 'Non-image response - successful fallback', (tester) async { final url = randomUrl(); - when(() => mockClient.readBytes(url, headers: any(named: 'headers'))) - .thenAnswer((_) async { - // 200 OK with html - return Uint8List.fromList(utf8.encode('Server Error')); - }); + when(() => mockClient.send( + any(that: isA().having((r) => r.url, 'URL', url)))) + .thenAnswer( + (_) async => StreamedResponse( + Stream.value( + Uint8List.fromList(utf8.encode('Server Error'))), + 200, + ), + ); final fallbackUrl = randomUrl(fallback: true); - when(() => - mockClient.readBytes(fallbackUrl, headers: any(named: 'headers'))) - .thenAnswer((_) async { - return testWhiteTileBytes; - }); - - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, + when( + () => mockClient.send(any( + that: isA().having((r) => r.url, 'URL', fallbackUrl))), + ).thenAnswer( + (_) async => StreamedResponse(Stream.value(testWhiteTileBytes), 200), ); - expect(startedLoadingTriggered, false); + final provider = createDefaultImageProvider( + url, + fallbackUrl: fallbackUrl, + ); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); + expect(img!.image.width, equals(256)); + expect(img.image.height, equals(256)); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); @@ -532,27 +709,11 @@ void main() { return Uint8List.fromList(utf8.encode('Server Error')); }); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, - silenceExceptions: false, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, - ); - - expect(startedLoadingTriggered, false); + final provider = + createDefaultImageProvider(url, fallbackUrl: fallbackUrl); final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNull); final exception = tester.takeException(); expect(exception, isInstanceOf()); @@ -561,9 +722,26 @@ void main() { equals('Exception: Invalid image data'), ); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); @@ -579,33 +757,37 @@ void main() { return Uint8List.fromList(utf8.encode('Server Error')); }); - bool startedLoadingTriggered = false; - bool finishedLoadingTriggered = false; - - final provider = NetworkTileImageProvider( - url: url.toString(), - fallbackUrl: fallbackUrl.toString(), - headers: headers, - httpClient: mockClient, + final provider = createDefaultImageProvider( + url, + fallbackUrl: fallbackUrl, silenceExceptions: true, - cachingProvider: const DisabledMapCachingProvider(), - startedLoading: () => startedLoadingTriggered = true, - finishedLoadingBytes: () => finishedLoadingTriggered = true, ); - expect(startedLoadingTriggered, false); - final img = await tester.runAsync(() => getImageInfo(provider)); - expect(startedLoadingTriggered, true); - expect(finishedLoadingTriggered, true); - expect(img, isNotNull); expect(tester.takeException(), isInstanceOf()); - verify(() => mockClient.readBytes(url, headers: headers)).called(1); - verify(() => mockClient.readBytes(fallbackUrl, headers: headers)) - .called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', url) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); + verify( + () => mockClient.send( + captureAny( + that: isA() + .having((r) => r.url, 'URL', fallbackUrl) + .having((r) => r.method, 'method', 'GET') + .having((r) => r.headers, 'headers', equals(headers)), + ), + ), + ).called(1); }, timeout: defaultTimeout, ); From 254826971f7fb2bc51cdd2d6e858433cd5b1181a Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 27 May 2025 17:25:27 +0100 Subject: [PATCH 26/49] Replace the JSON registry with a mechanism to store metadata within the same file as the tile's bytes Removed return of cache length from `isInitialised` & usage in example app --- example/lib/main.dart | 25 +-- example/lib/misc/timed_future.dart | 27 --- example/lib/pages/home.dart | 33 +-- example/lib/widgets/drawer/menu_drawer.dart | 40 ---- .../built_in/built_in_caching_provider.dart | 39 +--- .../caching/built_in/impl/native/native.dart | 205 ++++++++---------- .../workers/persistent_registry_unpacker.dart | 39 ---- .../workers/persistent_registry_writer.dart | 74 ------- .../{size_limiter.dart => size_reducer.dart} | 44 ++-- .../workers/tile_and_size_monitor_writer.dart | 150 +++++++++++++ .../workers/tile_writer_size_monitor.dart | 56 ----- .../workers/utils/size_monitor_opener.dart | 13 +- .../network/caching/built_in/impl/stub.dart | 6 +- .../caching/built_in/impl/web/web.dart | 6 +- .../network/caching/caching_provider.dart | 4 +- .../disabled/disabled_caching_provider.dart | 2 +- .../network/caching/tile_metadata.dart | 44 ++-- .../image_provider/image_provider.dart | 10 +- 18 files changed, 307 insertions(+), 510 deletions(-) delete mode 100644 example/lib/misc/timed_future.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart rename lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/{size_limiter.dart => size_reducer.dart} (61%) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index a612eaa2f..53a9aa523 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,6 +1,4 @@ import 'package:flutter/material.dart'; -import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; @@ -34,30 +32,15 @@ import 'package:flutter_map_example/pages/sliding_map.dart'; import 'package:flutter_map_example/pages/tile_builder.dart'; import 'package:flutter_map_example/pages/tile_loading_error_handle.dart'; import 'package:flutter_map_example/pages/wms_tile_layer.dart'; -import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_web_plugins/url_strategy.dart'; Future main() async { - WidgetsFlutterBinding.ensureInitialized(); - usePathUrlStrategy(); - - final cachingInstance = BuiltInMapCachingProvider.getOrCreateInstance(); - final cacheInitComplete = cachingInstance.isSupported - ? cachingInstance.isInitialised.timed() - : null; - MenuDrawer.cacheInitComplete.value = cacheInitComplete; - - runApp(MyApp(cacheInitComplete: cacheInitComplete)); + runApp(const MyApp()); } class MyApp extends StatelessWidget { - const MyApp({ - super.key, - required this.cacheInitComplete, - }); - - final TimedFuture? cacheInitComplete; + const MyApp({super.key}); @override Widget build(BuildContext context) { @@ -67,9 +50,7 @@ class MyApp extends StatelessWidget { useMaterial3: true, colorSchemeSeed: const Color(0xFF8dea88), ), - home: HomePage( - cacheInitComplete: cacheInitComplete, - ), + home: const HomePage(), routes: { CancellableTileProviderPage.route: (context) => const CancellableTileProviderPage(), diff --git a/example/lib/misc/timed_future.dart b/example/lib/misc/timed_future.dart deleted file mode 100644 index f6a984561..000000000 --- a/example/lib/misc/timed_future.dart +++ /dev/null @@ -1,27 +0,0 @@ -import 'dart:async'; - -typedef TimedFuture = ({ - Future result, - Future duration, - Future<({E result, Duration duration})> future, -}); - -extension CreateTimedFuture on Future { - /// Augments the future with the length of time it took to complete - /// - /// Note that the time is started on invocation of this method, not the actual - /// duration of the future from when it was created. - TimedFuture timed() { - final timer = Stopwatch()..start(); - final duration = Completer(); - final future = Completer<({E result, Duration duration})>(); - whenComplete(() => duration.complete((timer..stop()).elapsed)); - then( - (result) => duration.future.then( - (duration) => future.complete((result: result, duration: duration)), - ), - onError: future.completeError, - ); - return (result: this, duration: duration.future, future: future.future); - } -} diff --git a/example/lib/pages/home.dart b/example/lib/pages/home.dart index 1ecf42107..28eec1058 100644 --- a/example/lib/pages/home.dart +++ b/example/lib/pages/home.dart @@ -3,7 +3,6 @@ import 'package:flutter/material.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map_example/misc/tile_providers.dart'; -import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/widgets/drawer/floating_menu_button.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_map_example/widgets/first_start_dialog.dart'; @@ -14,12 +13,7 @@ import 'package:url_launcher/url_launcher.dart'; class HomePage extends StatefulWidget { static const String route = '/'; - const HomePage({ - super.key, - required this.cacheInitComplete, - }); - - final TimedFuture? cacheInitComplete; + const HomePage({super.key}); @override State createState() => _HomePageState(); @@ -38,35 +32,10 @@ class _HomePageState extends State { drawer: const MenuDrawer(HomePage.route), body: Stack( children: [ - if (widget.cacheInitComplete case final cacheInitComplete?) - Positioned.fill( - child: FutureBuilder( - future: cacheInitComplete.result, - builder: (context, snapshot) { - if (snapshot.connectionState == ConnectionState.done) { - return const ColoredBox(color: Color(0xFFE0E0E0)); - } - return const Center( - child: Row( - mainAxisSize: MainAxisSize.min, - spacing: 16, - children: [ - SizedBox.square( - dimension: 24, - child: CircularProgressIndicator.adaptive(), - ), - Text('Awaiting cache initialisation'), - ], - ), - ); - }, - ), - ), FlutterMap( options: const MapOptions( initialCenter: LatLng(51.5, -0.09), initialZoom: 5, - backgroundColor: Colors.transparent, ), children: [ openStreetMapTileLayer, diff --git a/example/lib/widgets/drawer/menu_drawer.dart b/example/lib/widgets/drawer/menu_drawer.dart index dfe7a82ba..cbb726b5b 100644 --- a/example/lib/widgets/drawer/menu_drawer.dart +++ b/example/lib/widgets/drawer/menu_drawer.dart @@ -1,6 +1,5 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; -import 'package:flutter_map_example/misc/timed_future.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; @@ -43,9 +42,6 @@ class MenuDrawer extends StatelessWidget { const MenuDrawer(this.currentRoute, {super.key}); - static final ValueNotifier?> cacheInitComplete = - ValueNotifier(null); - @override Widget build(BuildContext context) { return Drawer( @@ -82,42 +78,6 @@ class MenuDrawer extends StatelessWidget { textAlign: TextAlign.center, style: Theme.of(context).textTheme.bodySmall, ), - ValueListenableBuilder( - valueListenable: cacheInitComplete, - builder: (context, value, _) { - if (value == null) { - return Text( - 'No map cache in use', - style: Theme.of(context).textTheme.bodySmall, - ); - } - return FutureBuilder( - future: value.future, - builder: (context, snapshot) { - if (snapshot.hasError) { - return Text( - 'Failed to load or recover map cache', - style: Theme.of(context).textTheme.bodySmall, - ); - } - if (snapshot.hasData) { - return Text( - 'Loaded map cache of ' - '${snapshot.requireData.result!} tiles in ' - '${snapshot.requireData.duration.inMilliseconds}' - '\u00a0ms${kDebugMode ? ' (debug\u00a0mode)' : ''}', - textAlign: TextAlign.center, - style: Theme.of(context).textTheme.bodySmall, - ); - } - return Text( - 'Loading map cache...', - style: Theme.of(context).textTheme.bodySmall, - ); - }, - ); - }, - ), ], ), ), diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index b3dffa7e6..09b43b075 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -3,16 +3,12 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart' if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart'; -import 'package:uuid/data.dart'; -import 'package:uuid/rng.dart'; -import 'package:uuid/uuid.dart'; - /// Simple built-in map caching using an I/O storage mechanism, for native /// (non-web) platforms only /// -/// Uses JSON to store a centralised registry which is operated on as a map in -/// memory to maximise performance, with tile blobs stored raw as files and a -/// second file used to track the size of the cache. +/// Stores tiles as files identified with keys, containing some metadata headers +/// followed by the tile bytes, alongside a file used to track the size of the +/// cache. /// /// Usually uses HTTP headers to determine tile freshness, although /// `overrideFreshAge` can override this. @@ -44,13 +40,8 @@ abstract interface class BuiltInMapCachingProvider /// first tile load in the main memory space for the app). It is not an /// absolute limit. /// - /// This may cause some slight delay to the loading of the first tiles, - /// especially if the size is large and the cache does exceed the size. If - /// the visible delay becomes too large, disable this and manage the cache - /// size manually if necessary. - /// - /// Defaults to 800 MB. Set to `null` to disable. - int? maxCacheSize = 800_000_000, + /// Defaults to 1 GB. Set to `null` to disable. + int? maxCacheSize = 1_000_000_000, /// Override the duration of time a tile is considered fresh for /// @@ -64,7 +55,8 @@ abstract interface class BuiltInMapCachingProvider /// represent the tile image, for example, API keys contained with the query /// parameters. /// - /// The resulting key should be unique to that tile URL. + /// The resulting key should be unique to that tile URL. Keys must be usable + /// as filenames on all intended platform filesystems. /// /// Defaults to generating a UUID from the entire URL string. String Function(String url)? cacheKeyGenerator, @@ -89,28 +81,19 @@ abstract interface class BuiltInMapCachingProvider cacheDirectory: cacheDirectory, maxCacheSize: maxCacheSize, overrideFreshAge: overrideFreshAge, - cacheKeyGenerator: - cacheKeyGenerator ?? (url) => _uuid.v5(Namespace.url.value, url), + cacheKeyGenerator: cacheKeyGenerator, readOnly: readOnly, ); } static BuiltInMapCachingProviderImpl? _instance; - static final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); - /// Completes when the current instance has initialised and is ready to load /// and write tiles /// - /// See online documentation to see how to use this to preload caching to - /// remove the initial delay before loading tiles. - /// - /// Completes with: - /// * on native platforms, the number of cached tiles (at initialisation) - /// * or an error if initialisation fails and could not be recovered - /// * on web platforms, `null` (synchronously & immediately), and caching - /// will not be available + /// Completes with `null` (synchronously & immediately) on web platforms, + /// where caching is unavailable. /// /// [isSupported] will be set to determine the current platform's support. - Future get isInitialised; + Future get isInitialised; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 881d09a3e..af24f18bb 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -1,28 +1,28 @@ import 'dart:async'; -import 'dart:collection'; +import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; +import 'package:uuid/uuid.dart'; @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { - static const persistentRegistryFileName = 'registry.json'; static const sizeMonitorFileName = 'sizeMonitor.bin'; final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; - final String Function(String url) cacheKeyGenerator; + final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -36,6 +36,10 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { // This should only be called/constructed once _isInitialised.complete( () async { + if (cacheKeyGenerator == null) { + _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + } + _cacheDirectoryPath = p.join( this.cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, @@ -44,151 +48,130 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final cacheDirectory = Directory(_cacheDirectoryPath); await cacheDirectory.create(recursive: true); - final persistentRegistryFilePath = - p.join(_cacheDirectoryPath, persistentRegistryFileName); - final persistentRegistryFile = File(persistentRegistryFilePath); - final sizeMonitorFilePath = p.join(_cacheDirectoryPath, sizeMonitorFileName); - if (await persistentRegistryFile.exists()) { - final parsedCacheManager = await compute( - persistentRegistryUnpackerWorker, - persistentRegistryFilePath, - debugLabel: '[flutter_map: cache] Persistent Registry Unpacker', - ); - - if (parsedCacheManager == null) { - await cacheDirectory.delete(recursive: true); - await cacheDirectory.create(recursive: true); - _registry = HashMap(); - } else { - _registry = parsedCacheManager; - - if (maxCacheSize case final sizeLimit?) { - final currentSize = - await asyncGetOnlySizeMonitor(sizeMonitorFilePath); - - if (currentSize == null || currentSize > sizeLimit) { - (await compute( - sizeLimiterWorker, - ( - cacheDirectoryPath: _cacheDirectoryPath, - sizeMonitorFilePath: sizeMonitorFilePath, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Limiter', - )) - .forEach(_registry.remove); - } - } - } - } else { - _registry = HashMap(); - } - - final registryWorkerReceivePort = ReceivePort(); + final tileAndSizeMonitorWriterWorkerReceivePort = ReceivePort(); await Isolate.spawn( - persistentRegistryWriterWorker, + tileAndSizeMonitorWriterWorker, ( - port: registryWorkerReceivePort.sendPort, - persistentRegistryFilePath: persistentRegistryFilePath, - initialRegistry: _registry, - ), - debugName: '[flutter_map: cache] Persistent Registry Writer', - ); - final registryWorkerSendPort = - await registryWorkerReceivePort.first as SendPort; - - final tileFileWriterWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - tileWriterSizeMonitorWorker, - ( - port: tileFileWriterWorkerReceivePort.sendPort, + port: tileAndSizeMonitorWriterWorkerReceivePort.sendPort, cacheDirectoryPath: _cacheDirectoryPath, sizeMonitorFilePath: sizeMonitorFilePath, ), - debugName: '[flutter_map: cache] Tile File & Size Monitor Writer', + debugName: '[flutter_map: cache] Tile & Size Monitor Writer', ); - final tileFileWriterWorkerSendPort = - await tileFileWriterWorkerReceivePort.first as SendPort; - - _writeToPersistentRegistry = (uuid, tileInfo) => - registryWorkerSendPort.send((uuid: uuid, tileInfo: tileInfo)); - _writeTileFile = (tileFilePath, bytes) => tileFileWriterWorkerSendPort - .send((tileFilePath: tileFilePath, bytes: bytes)); - - return _registry.length; + final tileAndSizeMonitorWriterWorkerReceivePortSendPort = + await tileAndSizeMonitorWriterWorkerReceivePort.first as SendPort; + _writeTileFile = ({required path, required metadata, tileBytes}) => + tileAndSizeMonitorWriterWorkerReceivePortSendPort + .send((path: path, metadata: metadata, tileBytes: tileBytes)); + + if (maxCacheSize case final sizeLimit?) { + () async { + final currentSize = + await asyncGetOnlySizeMonitor(sizeMonitorFilePath); + + if (currentSize == null || currentSize > sizeLimit) { + final deletedSize = await compute( + sizeReducerWorker, + ( + cacheDirectoryPath: _cacheDirectoryPath, + sizeMonitorFilePath: sizeMonitorFilePath, + sizeLimit: sizeLimit, + ), + debugLabel: '[flutter_map: cache] Size Reducer', + ); + + if (deletedSize == 0) return; + tileAndSizeMonitorWriterWorkerReceivePortSendPort + .send(deletedSize); + } + }(); + } }(), ); } late final String _cacheDirectoryPath; - late final void Function(String uuid, CachedMapTileMetadata? tileInfo) - _writeToPersistentRegistry; - late final void Function(String tileFilePath, Uint8List? bytes) - _writeTileFile; - late final HashMap _registry; - - final _isInitialised = Completer(); + late final Uuid _uuid; // left un-inited if provided generator + late final void Function({ + required String path, + required CachedMapTileMetadata metadata, + Uint8List? tileBytes, + }) _writeTileFile; + + final _isInitialised = Completer(); @override - Future get isInitialised => _isInitialised.future; + Future get isInitialised => _isInitialised.future; @override bool get isSupported => true; @override - Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ) async { await isInitialised; - final uuid = cacheKeyGenerator(url); - final tileFile = File(p.join(_cacheDirectoryPath, uuid)); + final key = + cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + final tileFile = File(p.join(_cacheDirectoryPath, key)); + + if (!await tileFile.exists()) return null; + + final bytes = await tileFile.readAsBytes(); + + final firstTwoNums = bytes.buffer.asInt64List(0, 2); + final staleAt = + DateTime.fromMillisecondsSinceEpoch(firstTwoNums[0], isUtc: true); + final lastModified = firstTwoNums[1] == 0 + ? null + : DateTime.fromMillisecondsSinceEpoch(firstTwoNums[1], isUtc: true); - if (_registry[uuid] case final tileInfo? when await tileFile.exists()) { - return (bytes: await tileFile.readAsBytes(), tileInfo: tileInfo); + final etagLength = bytes.buffer.asUint16List(16, 1)[0]; + final String? etag; + if (etagLength == 0) { + etag = null; + } else { + final etagBytes = Uint8List.sublistView(bytes, 18, 18 + etagLength); + etag = const AsciiDecoder().convert(etagBytes); } - unawaited(_removeTile(uuid)); - return null; + final tileBytes = Uint8List.sublistView(bytes, 18 + etagLength); + + return ( + metadata: CachedMapTileMetadata( + staleAt: staleAt, + lastModified: lastModified, + etag: etag, + ), + bytes: tileBytes, + ); } @override Future putTile({ required String url, - required CachedMapTileMetadata tileInfo, + required CachedMapTileMetadata metadata, Uint8List? bytes, }) async { if (readOnly) return; await isInitialised; - final uuid = cacheKeyGenerator(url); - final resolvedTileInfo = overrideFreshAge != null + final key = + cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + final path = p.join(_cacheDirectoryPath, key); + + final resolvedMetadata = overrideFreshAge != null ? CachedMapTileMetadata( staleAt: DateTime.timestamp().add(overrideFreshAge!), - lastModified: tileInfo.lastModified, - etag: tileInfo.etag, + lastModified: metadata.lastModified, + etag: metadata.etag, ) - : tileInfo; - - if (bytes != null) { - final tileFilePath = p.join(_cacheDirectoryPath, uuid); - _writeTileFile(tileFilePath, bytes); - } - - _registry[uuid] = resolvedTileInfo; - _writeToPersistentRegistry(uuid, resolvedTileInfo); - } - - Future _removeTile(String uuid) async { - await isInitialised; - - final tileFilePath = p.join(_cacheDirectoryPath, uuid); - _writeTileFile(tileFilePath, null); + : metadata; - if (_registry.remove(uuid) == null) return; - _writeToPersistentRegistry(uuid, null); + _writeTileFile(path: path, metadata: resolvedMetadata, tileBytes: bytes); } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart deleted file mode 100644 index 7b602d575..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_unpacker.dart +++ /dev/null @@ -1,39 +0,0 @@ -import 'dart:collection'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:flutter_map/flutter_map.dart'; -import 'package:meta/meta.dart'; - -/// Unpack the FlatBuffer registry into a mapping of tile UUIDs to their -/// [CachedMapTileMetadata]s -/// -/// If the FlatBuffer file is invalid or the file cannot be read, this returns -/// null. -@internal -HashMap? persistentRegistryUnpackerWorker( - String persistentRegistryFilePath, -) { - final String json; - try { - json = File(persistentRegistryFilePath).readAsStringSync(); - } on FileSystemException { - return null; - } - - final Map parsed; - try { - parsed = jsonDecode(json) as Map; - } on FormatException { - return null; - } - - return HashMap.from( - parsed.map( - (key, value) => MapEntry( - key, - CachedMapTileMetadata.fromJson(value as Map), - ), - ), - ); -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart deleted file mode 100644 index 57aee9162..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/persistent_registry_writer.dart +++ /dev/null @@ -1,74 +0,0 @@ -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; -import 'dart:isolate'; - -import 'package:flutter_map/flutter_map.dart'; -import 'package:meta/meta.dart'; - -/// Isolate worker which maintains its own registry and sequences writes to -/// the persistent registry -/// -/// We cannot use [IOSink] from [File.openWrite], since we need to overwrite the -/// entire file on every write. [RandomAccessFile] allows this, and may also be -/// faster (especially for sync operations). However, it does not sequence -/// writes as [IOSink] does: attempting multiple writes at the same time throws -/// errors. If we use sync operations on every incoming update, this shouldn't -/// be an issue - instead, we use a debouncer (at 50ms, which is small enough -/// that the user should not usually terminate the isolate very close to loading -/// tiles, but also small enough to group adjacent tile loads), so manual -/// sequencing and locking is required. -@internal -Future persistentRegistryWriterWorker( - ({ - SendPort port, - String persistentRegistryFilePath, - Map initialRegistry, - }) input, -) async { - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - - final registry = input.initialRegistry; - final writer = - File(input.persistentRegistryFilePath).openSync(mode: FileMode.writeOnly); - - var writeLocker = Completer()..complete(); - var alreadyWaitingToWrite = false; - Future write() async { - if (alreadyWaitingToWrite) return; - alreadyWaitingToWrite = true; - await writeLocker.future; - writeLocker = Completer(); - alreadyWaitingToWrite = false; - - final encoded = jsonEncode(registry); - writer - ..setPositionSync(0) - ..writeStringSync(encoded) - ..truncateSync(writer.positionSync()) - ..flushSync(); - - writeLocker.complete(); - } - - write(); - - Timer createWriteDebouncer() => - Timer(const Duration(milliseconds: 50), write); - Timer? writeDebouncer; - - await for (final val in receivePort) { - final (:uuid, :tileInfo) = - val as ({String uuid, CachedMapTileMetadata? tileInfo}); - - if (tileInfo == null) { - registry.remove(uuid); - } else { - registry[uuid] = tileInfo; - } - - writeDebouncer?.cancel(); - writeDebouncer = createWriteDebouncer(); - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart similarity index 61% rename from lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart rename to lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart index f9a2591ae..b203800e7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_limiter.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart @@ -1,5 +1,4 @@ import 'dart:io'; -import 'dart:typed_data'; import 'package:collection/collection.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; @@ -7,7 +6,7 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; -typedef _SizeLimiterTile = ({String path, int size, DateTime sortKey}); +typedef _SizeReducerTile = ({String path, int size, DateTime sortKey}); /// Remove tile files from the cache directory until the total size is below the /// set limit @@ -15,11 +14,9 @@ typedef _SizeLimiterTile = ({String path, int size, DateTime sortKey}); /// Removes the least recently accessed tiles first. Tries to remove as few /// tiles as possible (largest first if last accessed at same time). /// -/// Returns removed tile UUIDs. -/// -/// This does not alter any registries in memory. +/// Returns the number of bytes deleted. @internal -Future> sizeLimiterWorker( +Future sizeReducerWorker( ({ String cacheDirectoryPath, String sizeMonitorFilePath, @@ -32,17 +29,14 @@ Future> sizeLimiterWorker( cacheDirectoryPath: input.cacheDirectoryPath, sizeMonitorFilePath: input.sizeMonitorFilePath, ); + sizeMonitor.closeSync(); - if (currentSize <= input.sizeLimit) { - sizeMonitor.closeSync(); - return []; - } + if (currentSize <= input.sizeLimit) return 0; - final tiles = await Future.wait<_SizeLimiterTile>( + final tiles = await Future.wait<_SizeReducerTile>( cacheDirectory.listSync().whereType().where((f) { final uuid = p.basename(f.absolute.path); - return uuid != BuiltInMapCachingProviderImpl.persistentRegistryFileName && - uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; + return uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; }).map((f) async { final stat = await f.stat(); // `stat.accessed` may be unstable on some OSs, but seems to work enough? @@ -50,36 +44,24 @@ Future> sizeLimiterWorker( }), ); - int compareSortKeys(_SizeLimiterTile a, _SizeLimiterTile b) => + int compareSortKeys(_SizeReducerTile a, _SizeReducerTile b) => a.sortKey.compareTo(b.sortKey); - int compareInverseSizes(_SizeLimiterTile a, _SizeLimiterTile b) => + int compareInverseSizes(_SizeReducerTile a, _SizeReducerTile b) => b.size.compareTo(a.size); tiles.sort(compareSortKeys.then(compareInverseSizes)); int i = 0; int deletedSize = 0; - final deletedFiles = >[]; - final deletedUuids = () sync* { + final deletionOperations = () sync* { while (currentSize - deletedSize > input.sizeLimit && i < tiles.length) { final tile = tiles[i++]; - final uuid = p.basename(tile.path); - deletedSize += tile.size; - deletedFiles.add(File(tile.path).delete()); - yield uuid; + yield File(tile.path).delete(); } }() .toList(growable: false); - sizeMonitor - ..setPositionSync(0) - ..writeFromSync( - Uint8List(8)..buffer.asInt64List()[0] = currentSize - deletedSize, - ) - ..flushSync() - ..closeSync(); - - await Future.wait(deletedFiles); + await Future.wait(deletionOperations); - return deletedUuids; + return deletedSize; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart new file mode 100644 index 000000000..30ff025b6 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -0,0 +1,150 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:isolate'; +import 'dart:typed_data'; + +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart'; +import 'package:meta/meta.dart'; + +/// Isolate worker which writes tile files, and updates the size monitor, +/// synchronously +@internal +Future tileAndSizeMonitorWriterWorker( + ({ + SendPort port, + String cacheDirectoryPath, + String sizeMonitorFilePath, + }) input, +) async { + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + + int currentSize; + final RandomAccessFile sizeMonitor; + (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( + cacheDirectoryPath: input.cacheDirectoryPath, + sizeMonitorFilePath: input.sizeMonitorFilePath, + ); + + void updateSizeMonitor(int deltaSize) { + currentSize += deltaSize; + sizeMonitor + ..setPositionSync(0) + ..writeFromSync(Uint8List(8)..buffer.asInt64List()[0] = currentSize) + ..flushSync(); + } + + final allocatedInt64Buffer = Uint8List(8); + final allocatedUint16Buffer = Uint8List(2); + + await for (final val in receivePort) { + if (val is int) { + updateSizeMonitor(-val); + continue; + } + + Uint8List? tileBytes; + final CachedMapTileMetadata metadata; + final String path; + (:tileBytes, :metadata, :path) = val as ({ + Uint8List? tileBytes, + CachedMapTileMetadata metadata, + String path + }); + + final tileFile = File(path); + final initialTileFileExists = tileFile.existsSync(); + final initialTileFileLength = + initialTileFileExists ? tileFile.lengthSync() : 0; + + if (!initialTileFileExists && tileBytes == null) { + // This could be caused by: + // * the tile server responding with a Not Modified status code + // incorrectly + // * the size reducer deleting the tile after we sent it's info to the + // server, and it returned Not Modified correctly + continue; + } + + final RandomAccessFile ram; + try { + ram = tileFile.openSync(mode: FileMode.append); + } on FileSystemException { + continue; + } + + ram + // We start reading from the start of the file, where we store our header + // info + ..setPositionSync(0) + // We store the stale-at header in 8 bytes... + ..writeFromSync( + allocatedInt64Buffer + ..buffer.asInt64List()[0] = metadata.staleAtMilliseconds, + ) + // ...followed by the last-modified header in 8 bytes, or '0' if null + ..writeFromSync( + allocatedInt64Buffer + ..buffer.asInt64List()[0] = metadata.lastModifiedMilliseconds ?? 0, + ); + + final initialEtagLength = + initialTileFileExists ? ram.readSync(2).buffer.asUint16List()[0] : null; + final int etagLength; + late final Uint8List etagBytes; // left unset if etagLength = 0 + if (metadata.etag == null) { + // We don't have an etag, so we write 2 bytes indicating the etag length + // is 0 + ram.writeFromSync( + allocatedUint16Buffer..buffer.asUint16List()[0] = etagLength = 0, + ); + } else { + etagBytes = const AsciiEncoder().convert(metadata.etag!); + // We store the etag length in 2 bytes... + // (unless it is too large) + ram.writeFromSync( + allocatedUint16Buffer + ..buffer.asUint16List()[0] = etagLength = + (etagBytes.lengthInBytes > 65535 ? 0 : etagBytes.lengthInBytes), + ); + } + + if (initialEtagLength != etagLength && tileBytes == null) { + // This is annoying - even if the tile bytes haven't changed, we need to + // rewrite them so they are in the right place + // To do this, we have to read the remainder of the file, skipping over + // the etag as it has not yet changed, and make it as if they were new + // bytes + ram.setPositionSync(18 + initialEtagLength!); + tileBytes ??= ram.readSync(9223372036854775807); + ram.setPositionSync(18); + } + + if (etagLength != 0) { + // ...followed by the etag itself + ram.writeFromSync(etagBytes); + } + + if (tileBytes == null) { + // If there were no updates to the tile bytes, that also implies there + // were no changes to the length of the etag, so we don't need to do + // any size updates + ram.closeSync(); + continue; + } + + // We write the new tile bytes to the file and truncate it to the end + ram.writeFromSync(tileBytes); + final finalPos = ram.positionSync(); + ram + ..truncateSync(ram.positionSync()) + ..closeSync(); + + // Then update the size monitor + if (finalPos - initialTileFileLength case final deltaSize + when deltaSize != 0) { + updateSizeMonitor(deltaSize); + } + } +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart deleted file mode 100644 index e990b8da2..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_writer_size_monitor.dart +++ /dev/null @@ -1,56 +0,0 @@ -import 'dart:io'; -import 'dart:isolate'; -import 'dart:typed_data'; - -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; -import 'package:meta/meta.dart'; - -/// Isolate worker which writes & deletes tile files, and updates the size -/// monitor, synchronously -@internal -Future tileWriterSizeMonitorWorker( - ({ - SendPort port, - String cacheDirectoryPath, - String sizeMonitorFilePath, - }) input, -) async { - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - - int currentSize; - final RandomAccessFile sizeMonitor; - (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( - cacheDirectoryPath: input.cacheDirectoryPath, - sizeMonitorFilePath: input.sizeMonitorFilePath, - ); - - final allocatedWriteBinBuffer = Uint8List(8); - - await for (final val in receivePort) { - final (:tileFilePath, :bytes) = - val as ({String tileFilePath, Uint8List? bytes}); - - final tileFile = File(tileFilePath); - final tileFileExists = tileFile.existsSync(); - - final existingTileSize = tileFileExists ? tileFile.lengthSync() : 0; - final newTileSize = bytes?.lengthInBytes ?? 0; - if (newTileSize - existingTileSize case final deltaSize - when deltaSize != 0) { - currentSize += deltaSize; - sizeMonitor - ..setPositionSync(0) - ..writeFromSync( - allocatedWriteBinBuffer..buffer.asInt64List()[0] = currentSize, - ) - ..flushSync(); - } - - if (bytes != null) { - tileFile.writeAsBytesSync(bytes); - } else if (tileFileExists) { - tileFile.deleteSync(); - } - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart index da142cc02..e4ea437e0 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart @@ -11,13 +11,8 @@ import 'package:path/path.dart' as p; @internal Future asyncGetOnlySizeMonitor(String sizeMonitorFilePath) async { final sizeMonitorFile = File(sizeMonitorFilePath); - - final sizeMonitor = await sizeMonitorFile.open(mode: FileMode.append); - await sizeMonitor.setPosition(0); - final bytes = await sizeMonitor.read(8); - - await sizeMonitor.close(); - + if (!await sizeMonitorFile.exists()) return null; + final bytes = await sizeMonitorFile.readAsBytes(); if (bytes.length == 8) return bytes.buffer.asInt64List()[0]; return null; } @@ -54,9 +49,7 @@ Future<({int currentSize, RandomAccessFile sizeMonitor})> .where( (f) { final uuid = p.basename(f.absolute.path); - return uuid != - BuiltInMapCachingProviderImpl.persistentRegistryFileName && - uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; + return uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; }, ) .map((f) => f.length()) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 87f6978ad..75a53897e 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -12,7 +12,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; - final String Function(String url) cacheKeyGenerator; + final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -31,14 +31,14 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { external bool get isSupported; @override - external Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + external Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ); @override external Future putTile({ required String url, - required CachedMapTileMetadata tileInfo, + required CachedMapTileMetadata metadata, Uint8List? bytes, }); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 6b424b02e..fd3c78334 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -9,7 +9,7 @@ class BuiltInMapCachingProviderImpl final String? cacheDirectory; final int? maxCacheSize; final Duration? overrideFreshAge; - final String Function(String url) cacheKeyGenerator; + final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -22,7 +22,5 @@ class BuiltInMapCachingProviderImpl }); @override - // False positive lint - // ignore: prefer_void_to_null - Future get isInitialised => SynchronousFuture(null); + Future get isInitialised => SynchronousFuture(null); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index 3f62a62e9..fc9c18f60 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -28,7 +28,7 @@ abstract interface class MapCachingProvider { /// /// This may throw. Tile providers should anticipate this and fallback to a /// non-caching alternative. - Future<({Uint8List bytes, CachedMapTileMetadata tileInfo})?> getTile( + Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ); @@ -38,7 +38,7 @@ abstract interface class MapCachingProvider { /// implementation specific if bytes are not supplied when required. Future putTile({ required String url, - required CachedMapTileMetadata tileInfo, + required CachedMapTileMetadata metadata, Uint8List? bytes, }); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart index 8feb5b439..cb947dbe4 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart @@ -17,7 +17,7 @@ mixin class DisabledMapCachingProvider implements MapCachingProvider { @override Never putTile({ required String url, - required CachedMapTileMetadata tileInfo, + required CachedMapTileMetadata metadata, Uint8List? bytes, }) => throw UnsupportedError('Must not be called if `isSupported` is `false`'); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index 31619a983..647c6c0dd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -15,25 +15,25 @@ import 'package:meta/meta.dart'; class CachedMapTileMetadata { /// Create new metadata CachedMapTileMetadata({ - required DateTime staleAt, - required DateTime? lastModified, + required this.staleAt, + required this.lastModified, required this.etag, - }) : _staleAt = staleAt.millisecondsSinceEpoch, - _lastModified = lastModified?.millisecondsSinceEpoch; + }) : staleAtMilliseconds = staleAt.millisecondsSinceEpoch, + lastModifiedMilliseconds = lastModified?.millisecondsSinceEpoch; - /// Decode metadata from JSON - CachedMapTileMetadata.fromJson(Map json) - : _staleAt = json['a'] as int, - _lastModified = json.containsKey('b') ? json['b'] as int : null, - etag = json.containsKey('c') ? json['c'] as String : null; + /// The calculated time at which this tile becomes stale + final DateTime staleAt; - final int _staleAt; + /// The calculated time at which this tile becomes stale, represented in + /// [DateTime.millisecondsSinceEpoch] + final int staleAtMilliseconds; /// If available, the value in [HttpHeaders.lastModifiedHeader] - DateTime? get lastModified => _lastModified == null - ? null - : DateTime.fromMillisecondsSinceEpoch(_lastModified); - final int? _lastModified; + final DateTime? lastModified; + + /// If available, the value in [HttpHeaders.lastModifiedHeader], represented + /// in [DateTime.millisecondsSinceEpoch] + final int? lastModifiedMilliseconds; /// If available, the value in [HttpHeaders.etagHeader] final String? etag; @@ -42,23 +42,17 @@ class CachedMapTileMetadata { /// /// Usually this is implemented by storing the timestamp at which the tile /// becomes stale, and comparing that to the current timestamp. - bool get isStale => DateTime.timestamp().millisecondsSinceEpoch > _staleAt; - - /// Encode the metadata to JSON - Map toJson() => { - 'a': _staleAt, - if (_lastModified != null) 'b': _lastModified, - if (etag != null) 'c': etag, - }; + bool get isStale => DateTime.timestamp().isAfter(staleAt); @override - int get hashCode => Object.hash(_staleAt, lastModified, etag); + int get hashCode => + Object.hash(staleAtMilliseconds, lastModifiedMilliseconds, etag); @override bool operator ==(Object other) => identical(this, other) || (other is CachedMapTileMetadata && - _staleAt == other._staleAt && - lastModified == other.lastModified && + staleAtMilliseconds == other.staleAtMilliseconds && + lastModifiedMilliseconds == other.lastModifiedMilliseconds && etag == other.etag); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index eacaf5f93..09d102c47 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -126,7 +126,7 @@ class NetworkTileImageProvider extends ImageProvider { } // Prepare caching provider & load cached tile if available - ({Uint8List bytes, CachedMapTileMetadata tileInfo})? cachedTile; + ({Uint8List bytes, CachedMapTileMetadata metadata})? cachedTile; final cachingProvider = this.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); if (cachingProvider.isSupported) { @@ -209,7 +209,7 @@ class NetworkTileImageProvider extends ImageProvider { cachingProvider.putTile( url: resolvedUrl, - tileInfo: CachedMapTileMetadata( + metadata: CachedMapTileMetadata( staleAt: calculateStaleAt(), lastModified: lastModified != null ? HttpDate.parse(lastModified) : null, @@ -222,7 +222,7 @@ class NetworkTileImageProvider extends ImageProvider { // Main logic // All `decodeBytes` calls should be awaited so errors may be handled try { - if (cachedTile != null && !cachedTile.tileInfo.isStale) { + if (cachedTile != null && !cachedTile.metadata.isStale) { // If we have a cached tile that's not stale, return it return await decodeBytes(cachedTile.bytes); } @@ -230,9 +230,9 @@ class NetworkTileImageProvider extends ImageProvider { // Otherwise, ask the server what's going on - supply any details we have final (:bytes, :response) = await get( additionalHeaders: { - if (cachedTile?.tileInfo.lastModified case final lastModified?) + if (cachedTile?.metadata.lastModified case final lastModified?) HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), - if (cachedTile?.tileInfo.etag case final etag?) + if (cachedTile?.metadata.etag case final etag?) HttpHeaders.ifNoneMatchHeader: etag, }, ); From 56bb2a6195396d8daa136b713e7c24fcac466a13 Mon Sep 17 00:00:00 2001 From: Luka S Date: Tue, 27 May 2025 17:32:53 +0100 Subject: [PATCH 27/49] Discard changes to example/lib/main.dart --- example/lib/main.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/lib/main.dart b/example/lib/main.dart index 53a9aa523..05d447137 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -34,7 +34,7 @@ import 'package:flutter_map_example/pages/tile_loading_error_handle.dart'; import 'package:flutter_map_example/pages/wms_tile_layer.dart'; import 'package:flutter_web_plugins/url_strategy.dart'; -Future main() async { +void main() { usePathUrlStrategy(); runApp(const MyApp()); } From 5c4899badb8fae60158c3cac02ad42a5cd6bf708 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 27 May 2025 18:05:16 +0100 Subject: [PATCH 28/49] Fixed bug --- .../impl/native/workers/tile_and_size_monitor_writer.dart | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index 30ff025b6..df6371afd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -91,6 +91,7 @@ Future tileAndSizeMonitorWriterWorker( final initialEtagLength = initialTileFileExists ? ram.readSync(2).buffer.asUint16List()[0] : null; + ram.setPositionSync(16); // we need to go back to the start of the length final int etagLength; late final Uint8List etagBytes; // left unset if etagLength = 0 if (metadata.etag == null) { @@ -117,7 +118,7 @@ Future tileAndSizeMonitorWriterWorker( // the etag as it has not yet changed, and make it as if they were new // bytes ram.setPositionSync(18 + initialEtagLength!); - tileBytes ??= ram.readSync(9223372036854775807); + tileBytes = ram.readSync(9223372036854775807); // to the end of the file ram.setPositionSync(18); } From 53e584f16b3d58fda4480d391ccc67ced1121f25 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 28 May 2025 13:26:41 +0100 Subject: [PATCH 29/49] Improved resilience to corruption Added format spec Minor improvements --- .../caching/built_in/impl/native/README.md | 39 +++++++++++++ .../workers/tile_and_size_monitor_writer.dart | 55 +++++++++++-------- .../network/caching/caching_provider.dart | 8 ++- .../image_provider/image_provider.dart | 52 +++++++++++++----- 4 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md new file mode 100644 index 000000000..29d9cc491 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -0,0 +1,39 @@ +# `BuiltInMapCachingProvider` storage spec + +The `BuiltInMapCachingProvider`, referred to as just 'built-in caching', is implemented using the filesystem for storage on native platforms. + +Cached tiles & their metadata are stored as individual keyed files. An additional file is used to improve the efficiency of tracking and reducing the cache size, called the 'size monitor'. + +## Tiles + +Tiles are stored in files, where the filename is the output of the supplied `cacheKeyGenerator` given the tile's URL. This defaults to a v5 UUID. Files have no extension. + +Also stored alongside tiles is metadata used to perform caching, namely: + +* `staleAt`: The calculated time at which the tile becomes 'stale' +* (optionally) `lastModified`: The time at which the tile was last modified on the server, based on the HTTP header +* (optionally) `etag`: A unique string identifier for the current version of that tile, using the 'etag' HTTP header + +The file format is as follows: + +1. The header containing the tile metadata +2. The tile image bytes (as responded by the server) + +The format of the header is as follows: + +1. 8-byte signed integer (Int64): the `staleAt` timestamp, represented in milliseconds since the Unix epoch in the UTC timezone +2. 8-byte signed integer (Int64) + * Where provided, the `lastModified` timestamp, represented in milliseconds since the Unix epoch in the UTC timezone, which must not be 0 + * Where not provided, the integer '0' +3. 2-byte unsigned integer (Uint16) + * Where provided, the length of the ASCII encoded `etag` in bytes + * Where not provided, the integer '0' +4. Variable number of bytes + * Where provided, the ASCII encoded `etag` (where each character is 7 bits but stored as 1 byte) with no greater than 65535 bytes + * Where not provided, no bytes + +## Size monitor + +Contains an 8-byte unsigned integer (Uint64), representing the size of all tiles (including metadata) stored in the cache in bytes. + +Named as 'sizeMonitor.bin'. diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index df6371afd..6a8e719c7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -27,6 +27,9 @@ Future tileAndSizeMonitorWriterWorker( sizeMonitorFilePath: input.sizeMonitorFilePath, ); + final allocatedInt64Buffer = Uint8List(8); + final allocatedUint16Buffer = Uint8List(2); + void updateSizeMonitor(int deltaSize) { currentSize += deltaSize; sizeMonitor @@ -35,23 +38,17 @@ Future tileAndSizeMonitorWriterWorker( ..flushSync(); } - final allocatedInt64Buffer = Uint8List(8); - final allocatedUint16Buffer = Uint8List(2); - - await for (final val in receivePort) { - if (val is int) { - updateSizeMonitor(-val); - continue; - } - + void writeTile( + ({ + Uint8List? tileBytes, + CachedMapTileMetadata metadata, + String path, + }) tileInfo, + ) { Uint8List? tileBytes; final CachedMapTileMetadata metadata; final String path; - (:tileBytes, :metadata, :path) = val as ({ - Uint8List? tileBytes, - CachedMapTileMetadata metadata, - String path - }); + (:tileBytes, :metadata, :path) = tileInfo; final tileFile = File(path); final initialTileFileExists = tileFile.existsSync(); @@ -59,19 +56,16 @@ Future tileAndSizeMonitorWriterWorker( initialTileFileExists ? tileFile.lengthSync() : 0; if (!initialTileFileExists && tileBytes == null) { - // This could be caused by: - // * the tile server responding with a Not Modified status code - // incorrectly - // * the size reducer deleting the tile after we sent it's info to the - // server, and it returned Not Modified correctly - continue; + // This should only be caused by the size reducer deleting the tile after + // we sent it's info to the server, and it returned Not Modified correctly + return; } final RandomAccessFile ram; try { ram = tileFile.openSync(mode: FileMode.append); } on FileSystemException { - continue; + return; } ram @@ -132,14 +126,14 @@ Future tileAndSizeMonitorWriterWorker( // were no changes to the length of the etag, so we don't need to do // any size updates ram.closeSync(); - continue; + return; } // We write the new tile bytes to the file and truncate it to the end ram.writeFromSync(tileBytes); final finalPos = ram.positionSync(); ram - ..truncateSync(ram.positionSync()) + ..truncateSync(finalPos) ..closeSync(); // Then update the size monitor @@ -148,4 +142,19 @@ Future tileAndSizeMonitorWriterWorker( updateSizeMonitor(deltaSize); } } + + await for (final val in receivePort) { + if (val is int) { + updateSizeMonitor(-val); + continue; + } + + writeTile( + val as ({ + Uint8List? tileBytes, + CachedMapTileMetadata metadata, + String path, + }), + ); + } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index fc9c18f60..f2843ee1c 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -26,8 +26,12 @@ abstract interface class MapCachingProvider { /// Retrieve a tile from the cache, if it exists /// - /// This may throw. Tile providers should anticipate this and fallback to a - /// non-caching alternative. + /// This may throw, for example due to an error decoding/unpacking an existing + /// cached tile. Additionally, the returned bytes are not guaranteed to form + /// a valid image - attempting to decode the bytes may throw. + /// Tile providers should anticipate this and fallback to a non-caching + /// alternative, wherever possible repairing the cache by overwriting the + /// damaged tile or removing it from the cache. Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ); diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 09d102c47..6b5ced7e7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -132,7 +132,9 @@ class NetworkTileImageProvider extends ImageProvider { if (cachingProvider.isSupported) { try { cachedTile = await cachingProvider.getTile(resolvedUrl); - } on Exception { + } catch (_) { + // This could occur due to a corrupt tile - we just try to overwrite it + // with fresh data cachedTile = null; } } @@ -222,25 +224,47 @@ class NetworkTileImageProvider extends ImageProvider { // Main logic // All `decodeBytes` calls should be awaited so errors may be handled try { + bool forceFromServer = false; if (cachedTile != null && !cachedTile.metadata.isStale) { - // If we have a cached tile that's not stale, return it - return await decodeBytes(cachedTile.bytes); + try { + // If we have a cached tile that's not stale, return it + return await decodeBytes(cachedTile.bytes); + } catch (e) { + // If the cached tile is corrupt, we proceed and get from the server + forceFromServer = true; + } } // Otherwise, ask the server what's going on - supply any details we have - final (:bytes, :response) = await get( - additionalHeaders: { - if (cachedTile?.metadata.lastModified case final lastModified?) - HttpHeaders.ifModifiedSinceHeader: HttpDate.format(lastModified), - if (cachedTile?.metadata.etag case final etag?) - HttpHeaders.ifNoneMatchHeader: etag, - }, + var (:bytes, :response) = await get( + additionalHeaders: forceFromServer + ? null + : { + if (cachedTile?.metadata.lastModified case final lastModified?) + HttpHeaders.ifModifiedSinceHeader: + HttpDate.format(lastModified), + if (cachedTile?.metadata.etag case final etag?) + HttpHeaders.ifNoneMatchHeader: etag, + }, ); // Server says nothing's changed - but might return new useful headers - if (cachedTile != null && response.statusCode == HttpStatus.notModified) { - cachePut(bytes: null, headers: response.headers); - return await decodeBytes(cachedTile.bytes); + if (!forceFromServer && + cachedTile != null && + response.statusCode == HttpStatus.notModified) { + late final Codec decodedCacheBytes; + try { + decodedCacheBytes = await decodeBytes(cachedTile.bytes); + } catch (e) { + // If the cached tile is corrupt, we get fresh from the server without + // caching, then continue + forceFromServer = true; + (:bytes, :response) = await get(); + } + if (!forceFromServer) { + cachePut(bytes: null, headers: response.headers); + return decodedCacheBytes; + } } // Server says the image has changed - store it new @@ -262,7 +286,7 @@ class NetworkTileImageProvider extends ImageProvider { evict(); try { return await decodeBytes(bytes); - } catch (err, stackTrace) { + } catch (_, stackTrace) { // If it throws, we don't want to throw the decode error, as that's not // useful for users // Instead, we throw an exception reporting the failed HTTP request, From 16c18276c4bd2fda307c501972d44cd3e8021377 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 29 May 2025 23:19:36 +0100 Subject: [PATCH 30/49] Improved resilience to corruption Improved read speeds (reduced unnecessary waiting) Heavily refactored writer, size monitor, and size limiter Added `CachedMapTileReadFailureException` Removed `BuiltInMapCachingProvider.isInitialised` --- lib/flutter_map.dart | 1 + .../built_in/built_in_caching_provider.dart | 13 +- .../caching/built_in/impl/native/README.md | 3 +- .../caching/built_in/impl/native/native.dart | 221 +++++++++------- .../impl/native/workers/size_reducer.dart | 31 +-- .../workers/tile_and_size_monitor_writer.dart | 236 +++++++++++++++--- .../workers/utils/size_monitor_opener.dart | 102 -------- .../network/caching/built_in/impl/stub.dart | 3 - .../caching/built_in/impl/web/web.dart | 4 - .../network/caching/caching_provider.dart | 16 +- .../caching/tile_read_failure_exception.dart | 44 ++++ .../image_provider/image_provider.dart | 2 +- 12 files changed, 392 insertions(+), 284 deletions(-) delete mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart create mode 100644 lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart diff --git a/lib/flutter_map.dart b/lib/flutter_map.dart index d82163df9..ad1e5eae8 100644 --- a/lib/flutter_map.dart +++ b/lib/flutter_map.dart @@ -51,6 +51,7 @@ export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/disabled/disabled_caching_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart'; +export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_provider/network/tile_provider.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_event.dart'; export 'package:flutter_map/src/layer/tile_layer/tile_update_transformer.dart'; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 09b43b075..cca82eb12 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -41,7 +41,7 @@ abstract interface class BuiltInMapCachingProvider /// absolute limit. /// /// Defaults to 1 GB. Set to `null` to disable. - int? maxCacheSize = 1_000_000_000, + int? maxCacheSize = 1_000_000, /// Override the duration of time a tile is considered fresh for /// @@ -59,6 +59,8 @@ abstract interface class BuiltInMapCachingProvider /// as filenames on all intended platform filesystems. /// /// Defaults to generating a UUID from the entire URL string. + /// + /// The callback should not throw. String Function(String url)? cacheKeyGenerator, /// Prevent any tiles from being added or updated @@ -87,13 +89,4 @@ abstract interface class BuiltInMapCachingProvider } static BuiltInMapCachingProviderImpl? _instance; - - /// Completes when the current instance has initialised and is ready to load - /// and write tiles - /// - /// Completes with `null` (synchronously & immediately) on web platforms, - /// where caching is unavailable. - /// - /// [isSupported] will be set to determine the current platform's support. - Future get isInitialised; } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md index 29d9cc491..2088d0b81 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -17,7 +17,7 @@ Also stored alongside tiles is metadata used to perform caching, namely: The file format is as follows: 1. The header containing the tile metadata -2. The tile image bytes (as responded by the server) +2. The tile image bytes (as responded by the server), no longer than 4,294,967,295 bytes The format of the header is as follows: @@ -31,6 +31,7 @@ The format of the header is as follows: 4. Variable number of bytes * Where provided, the ASCII encoded `etag` (where each character is 7 bits but stored as 1 byte) with no greater than 65535 bytes * Where not provided, no bytes +5. 4-byte unsigned integer (Uint32): the length of the tile image bytes ## Size monitor diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index af24f18bb..37c0d1374 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -2,12 +2,11 @@ import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; +import 'dart:typed_data'; import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; @@ -34,76 +33,74 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.readOnly, }) { // This should only be called/constructed once - _isInitialised.complete( - () async { - if (cacheKeyGenerator == null) { - _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + () async { + if (cacheKeyGenerator == null) { + _uuid = Uuid(goptions: GlobalOptions(MathRNG())); + } + + _cacheDirectoryPath = p.join( + this.cacheDirectory ?? + (await getApplicationCacheDirectory()).absolute.path, + 'fm_cache', + ); + final cacheDirectory = Directory(_cacheDirectoryPath!); + await cacheDirectory.create(recursive: true); + + final sizeMonitorFilePath = + p.join(_cacheDirectoryPath!, sizeMonitorFileName); + + _cacheDirectoryPathReady.complete(_cacheDirectoryPath!); + + final tileAndSizeMonitorWriterWorkerReceivePort = ReceivePort(); + SendPort? writerPort; + final writerPortReady = Completer(); + + // We can't send messages until the worker has set-up all the size + // monitoring (and potentially run the reducer) if necessary + // Reading does not depend on this. + void sendMessageToWriter(Object message) { + if (writerPort != null) { + writerPort.send(message); + } else { + writerPortReady.future.then((port) => port.send(message)); } - - _cacheDirectoryPath = p.join( - this.cacheDirectory ?? - (await getApplicationCacheDirectory()).absolute.path, - 'fm_cache', - ); - final cacheDirectory = Directory(_cacheDirectoryPath); - await cacheDirectory.create(recursive: true); - - final sizeMonitorFilePath = - p.join(_cacheDirectoryPath, sizeMonitorFileName); - - final tileAndSizeMonitorWriterWorkerReceivePort = ReceivePort(); - await Isolate.spawn( - tileAndSizeMonitorWriterWorker, - ( - port: tileAndSizeMonitorWriterWorkerReceivePort.sendPort, - cacheDirectoryPath: _cacheDirectoryPath, - sizeMonitorFilePath: sizeMonitorFilePath, - ), - debugName: '[flutter_map: cache] Tile & Size Monitor Writer', - ); - final tileAndSizeMonitorWriterWorkerReceivePortSendPort = - await tileAndSizeMonitorWriterWorkerReceivePort.first as SendPort; - _writeTileFile = ({required path, required metadata, tileBytes}) => - tileAndSizeMonitorWriterWorkerReceivePortSendPort - .send((path: path, metadata: metadata, tileBytes: tileBytes)); - - if (maxCacheSize case final sizeLimit?) { - () async { - final currentSize = - await asyncGetOnlySizeMonitor(sizeMonitorFilePath); - - if (currentSize == null || currentSize > sizeLimit) { - final deletedSize = await compute( - sizeReducerWorker, - ( - cacheDirectoryPath: _cacheDirectoryPath, - sizeMonitorFilePath: sizeMonitorFilePath, - sizeLimit: sizeLimit, - ), - debugLabel: '[flutter_map: cache] Size Reducer', - ); - - if (deletedSize == 0) return; - tileAndSizeMonitorWriterWorkerReceivePortSendPort - .send(deletedSize); - } - }(); - } - }(), - ); + } + + _writeTileFile = ({required path, required metadata, tileBytes}) => + sendMessageToWriter( + (path: path, metadata: metadata, tileBytes: tileBytes)); + _reportReadFailure = () => sendMessageToWriter(false); + + await Isolate.spawn( + tileAndSizeMonitorWriterWorker, + ( + port: tileAndSizeMonitorWriterWorkerReceivePort.sendPort, + cacheDirectoryPath: _cacheDirectoryPath!, + sizeMonitorFilePath: sizeMonitorFilePath, + sizeLimit: maxCacheSize, + ), + debugName: '[flutter_map: cache] Tile & Size Monitor Writer', + ); + + writerPort = + await tileAndSizeMonitorWriterWorkerReceivePort.first as SendPort; + writerPortReady.complete(writerPort); + }(); } - late final String _cacheDirectoryPath; + String? _cacheDirectoryPath; // ~cached version of below for instant access + final _cacheDirectoryPathReady = Completer(); + late final Uuid _uuid; // left un-inited if provided generator + late final void Function({ required String path, required CachedMapTileMetadata metadata, Uint8List? tileBytes, }) _writeTileFile; - final _isInitialised = Completer(); - @override - Future get isInitialised => _isInitialised.future; + /// See `disableSizeMonitor` in worker + late final void Function() _reportReadFailure; @override bool get isSupported => true; @@ -112,42 +109,75 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ) async { - await isInitialised; - final key = cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); - final tileFile = File(p.join(_cacheDirectoryPath, key)); + final tileFile = File( + p.join(_cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key), + ); if (!await tileFile.exists()) return null; - final bytes = await tileFile.readAsBytes(); - - final firstTwoNums = bytes.buffer.asInt64List(0, 2); - final staleAt = - DateTime.fromMillisecondsSinceEpoch(firstTwoNums[0], isUtc: true); - final lastModified = firstTwoNums[1] == 0 - ? null - : DateTime.fromMillisecondsSinceEpoch(firstTwoNums[1], isUtc: true); - - final etagLength = bytes.buffer.asUint16List(16, 1)[0]; - final String? etag; - if (etagLength == 0) { - etag = null; - } else { - final etagBytes = Uint8List.sublistView(bytes, 18, 18 + etagLength); - etag = const AsciiDecoder().convert(etagBytes); - } - - final tileBytes = Uint8List.sublistView(bytes, 18 + etagLength); + try { + final bytes = await tileFile.readAsBytes(); - return ( - metadata: CachedMapTileMetadata( - staleAt: staleAt, - lastModified: lastModified, - etag: etag, - ), - bytes: tileBytes, - ); + if (bytes.lengthInBytes < 22) { + throw CachedMapTileReadFailureException( + url: url, + description: + 'cache file (${bytes.lengthInBytes}) was shorter than the ' + 'minimum expected size', + ); + } + + final firstTwoNums = bytes.buffer.asInt64List(0, 2); + final staleAt = + DateTime.fromMillisecondsSinceEpoch(firstTwoNums[0], isUtc: true); + final lastModified = firstTwoNums[1] == 0 + ? null + : DateTime.fromMillisecondsSinceEpoch(firstTwoNums[1], isUtc: true); + + final etagLength = bytes.buffer.asUint16List(16, 1)[0]; + final String? etag; + if (etagLength == 0) { + etag = null; + } else { + final etagBytes = Uint8List.sublistView(bytes, 18, 18 + etagLength); + etag = const AsciiDecoder().convert(etagBytes); + } + + // Performing an unaligned read is a hassle + final tileBytesExpectedLength = + bytes.buffer.asByteData(18 + etagLength, 4).getUint32(0, Endian.host); + + final tileBytes = Uint8List.sublistView(bytes, 18 + etagLength + 4); + + if (tileBytes.lengthInBytes != tileBytesExpectedLength) { + throw CachedMapTileReadFailureException( + url: url, + description: + 'tile image bytes (${tileBytes.lengthInBytes}) were not of ' + 'expected length ($tileBytesExpectedLength)', + ); + } + + return ( + metadata: CachedMapTileMetadata( + staleAt: staleAt, + lastModified: lastModified, + etag: etag, + ), + bytes: tileBytes, + ); + } on CachedMapTileReadFailureException { + _reportReadFailure(); + rethrow; + } catch (error, stackTrace) { + _reportReadFailure(); + Error.throwWithStackTrace( + CachedMapTileReadFailureException(url: url, originalError: error), + stackTrace, + ); + } } @override @@ -158,11 +188,12 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { }) async { if (readOnly) return; - await isInitialised; - final key = cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); - final path = p.join(_cacheDirectoryPath, key); + final path = p.join( + _cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, + key, + ); final resolvedMetadata = overrideFreshAge != null ? CachedMapTileMetadata( diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart index b203800e7..764367d47 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart @@ -2,36 +2,25 @@ import 'dart:io'; import 'package:collection/collection.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; typedef _SizeReducerTile = ({String path, int size, DateTime sortKey}); -/// Remove tile files from the cache directory until the total size is below the -/// set limit +/// Remove tile files from the cache directory until at least [minSizeToDelete] +/// bytes have been deleted. /// /// Removes the least recently accessed tiles first. Tries to remove as few /// tiles as possible (largest first if last accessed at same time). /// -/// Returns the number of bytes deleted. +/// Returns the number of bytes actually deleted. @internal -Future sizeReducerWorker( - ({ - String cacheDirectoryPath, - String sizeMonitorFilePath, - int sizeLimit, - }) input, -) async { - final cacheDirectory = Directory(input.cacheDirectoryPath); - - final (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( - cacheDirectoryPath: input.cacheDirectoryPath, - sizeMonitorFilePath: input.sizeMonitorFilePath, - ); - sizeMonitor.closeSync(); - - if (currentSize <= input.sizeLimit) return 0; +Future sizeReducerWorker({ + required String cacheDirectoryPath, + required String sizeMonitorFilePath, + required int minSizeToDelete, +}) async { + final cacheDirectory = Directory(cacheDirectoryPath); final tiles = await Future.wait<_SizeReducerTile>( cacheDirectory.listSync().whereType().where((f) { @@ -53,7 +42,7 @@ Future sizeReducerWorker( int i = 0; int deletedSize = 0; final deletionOperations = () sync* { - while (currentSize - deletedSize > input.sizeLimit && i < tiles.length) { + while (deletedSize < minSizeToDelete && i < tiles.length) { final tile = tiles[i++]; deletedSize += tile.size; yield File(tile.path).delete(); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index 6a8e719c7..25dc6e21d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -1,11 +1,14 @@ +import 'dart:async'; import 'dart:convert'; import 'dart:io'; import 'dart:isolate'; import 'dart:typed_data'; -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; +import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart'; import 'package:meta/meta.dart'; +import 'package:path/path.dart' as p; /// Isolate worker which writes tile files, and updates the size monitor, /// synchronously @@ -15,41 +18,130 @@ Future tileAndSizeMonitorWriterWorker( SendPort port, String cacheDirectoryPath, String sizeMonitorFilePath, + int? sizeLimit, }) input, ) async { - final receivePort = ReceivePort(); - input.port.send(receivePort.sendPort); - - int currentSize; - final RandomAccessFile sizeMonitor; - (:currentSize, :sizeMonitor) = await getOrCreateSizeMonitor( - cacheDirectoryPath: input.cacheDirectoryPath, - sizeMonitorFilePath: input.sizeMonitorFilePath, - ); - - final allocatedInt64Buffer = Uint8List(8); - final allocatedUint16Buffer = Uint8List(2); + final sizeMonitorFile = File(input.sizeMonitorFilePath); + RandomAccessFile? sizeMonitor; + late int currentSize; + final allocatedInt64BufferForSizeMonitor = Uint8List(8); void updateSizeMonitor(int deltaSize) { + if (sizeMonitor == null) return; + currentSize += deltaSize; - sizeMonitor + sizeMonitor! ..setPositionSync(0) - ..writeFromSync(Uint8List(8)..buffer.asInt64List()[0] = currentSize) + ..writeFromSync( + allocatedInt64BufferForSizeMonitor + ..buffer.asInt64List()[0] = currentSize, + ) ..flushSync(); } - void writeTile( - ({ - Uint8List? tileBytes, - CachedMapTileMetadata metadata, - String path, - }) tileInfo, - ) { - Uint8List? tileBytes; - final CachedMapTileMetadata metadata; - final String path; - (:tileBytes, :metadata, :path) = tileInfo; + // This is called when a read failure occurs (potentially during writing), + // usually due to corruption of a tile. + // In this case, the size monitor cannot be made accurate without regenerating + // it. (If a tile is truncated to 10u, we can write a fresh 50u over it, + // but we cannot know that originally 40u were lost). + // We don't need the monitor until the next initialisation, where we need to + // run the size reducer, however - so we just delete it and forget about it. + void disableSizeMonitor() { + if (sizeMonitor == null) return; + + sizeMonitor!.closeSync(); + sizeMonitorFile.deleteSync(); + sizeMonitor = null; + } + + // We try to open and read the size monitor, if we have a size limit. + // If it's available, we can begin writing immediately. + // Otherwise, we need to wait for it to be regenerated, which takes some time. + // This is only neccessary for a brand new cache, an existing cache with a + // newly imposed `sizeLimit` when there was previously none, or a corrupted + // cache (where the size monitor is missing, potentially due to a read + // failure). + // We can run the size reducer in another isolate afterwards, as it returns a + // relative change to the current cache size. Writes don't need to wait for + // that expensive process. + if (input.sizeLimit case final sizeLimit?) { + Future regenerateSizeMonitor() async { + int calculatedSize = 0; + int waitingForSize = 0; + bool finishedListing = false; + final finishedCalculating = Completer(); + + await for (final file in Directory(input.cacheDirectoryPath).list()) { + if (file is! File || + p.basename(file.absolute.path) == + BuiltInMapCachingProviderImpl.sizeMonitorFileName) { + continue; + } + waitingForSize++; + file.length().then((size) { + calculatedSize += size; + waitingForSize--; + if (finishedListing && waitingForSize == 0) { + finishedCalculating.complete(); + } + }); + } + + finishedListing = true; + if (waitingForSize != 0) await finishedCalculating.future; + + sizeMonitor! + ..setPositionSync(0) + ..writeFromSync(Uint8List(8)..buffer.asInt64List()[0] = calculatedSize) + ..flushSync(); + + currentSize = calculatedSize; + } + + final sizeMonitorInitiallyExists = sizeMonitorFile.existsSync(); + sizeMonitor = sizeMonitorFile.openSync(mode: FileMode.append) + ..setPositionSync(0); + if (sizeMonitorInitiallyExists) { + try { + currentSize = sizeMonitor!.readSync(8).buffer.asInt64List()[0]; + } catch (e) { + await regenerateSizeMonitor(); + } + } else { + await regenerateSizeMonitor(); + } + + if (currentSize > sizeLimit) { + Future runSizeLimiter({ + required String cacheDirectoryPath, + required String sizeMonitorFilePath, + required int minSizeToDelete, + }) => + Isolate.run( + () => sizeReducerWorker( + cacheDirectoryPath: cacheDirectoryPath, + sizeMonitorFilePath: sizeMonitorFilePath, + minSizeToDelete: minSizeToDelete, + ), + ); + + runSizeLimiter( + cacheDirectoryPath: input.cacheDirectoryPath, + sizeMonitorFilePath: input.sizeMonitorFilePath, + minSizeToDelete: currentSize - sizeLimit, + ).then((deletedSize) => updateSizeMonitor(-deletedSize)); + } + } + + final allocatedInt64Buffer = Uint8List(8); + final allocatedUint32Buffer = Uint8List(4); + final allocatedUint16Buffer = Uint8List(2); + void writeTile({ + Uint8List? tileBytes, + required final CachedMapTileMetadata metadata, + required final String path, + }) { final tileFile = File(path); final initialTileFileExists = tileFile.existsSync(); final initialTileFileLength = @@ -61,6 +153,12 @@ Future tileAndSizeMonitorWriterWorker( return; } + if (tileBytes != null && tileBytes.lengthInBytes > 0xFFFFFFFF) { + // These bytes are too big to have a length stored in a Uint32 + // In reality, this is unlikely + return; + } + final RandomAccessFile ram; try { ram = tileFile.openSync(mode: FileMode.append); @@ -83,9 +181,28 @@ Future tileAndSizeMonitorWriterWorker( ..buffer.asInt64List()[0] = metadata.lastModifiedMilliseconds ?? 0, ); - final initialEtagLength = - initialTileFileExists ? ram.readSync(2).buffer.asUint16List()[0] : null; - ram.setPositionSync(16); // we need to go back to the start of the length + // We need to read the old etag length to compare their lengths + int? initialEtagLength; + if (initialTileFileExists) { + try { + initialEtagLength = ram.readSync(2).buffer.asUint16List()[0]; + } catch (e) { + // This implies the tile was corrupted on the previous write (the + // write was terminated unexpectedly) + // However, this shouldn't be possible in practise, since that should've + // been caught on read, which should occur before every write, causing + // a fresh overwrite with new bytes + // We try to handle it anyway by emptying the tile completely so it is + // auto-repaired on the next read + ram.truncateSync(0); + ram.closeSync(); + disableSizeMonitor(); + return; + } + + ram.setPositionSync(16); // we need to go back to the start of the length + } + final int etagLength; late final Uint8List etagBytes; // left unset if etagLength = 0 if (metadata.etag == null) { @@ -101,7 +218,7 @@ Future tileAndSizeMonitorWriterWorker( ram.writeFromSync( allocatedUint16Buffer ..buffer.asUint16List()[0] = etagLength = - (etagBytes.lengthInBytes > 65535 ? 0 : etagBytes.lengthInBytes), + (etagBytes.lengthInBytes > 0xFFFF ? 0 : etagBytes.lengthInBytes), ); } @@ -112,7 +229,29 @@ Future tileAndSizeMonitorWriterWorker( // the etag as it has not yet changed, and make it as if they were new // bytes ram.setPositionSync(18 + initialEtagLength!); - tileBytes = ram.readSync(9223372036854775807); // to the end of the file + + final int initialTileBytesLength; + try { + initialTileBytesLength = ram.readSync(4).buffer.asUint32List()[0]; + } catch (e) { + // This implies the tile was corrupted on the previous write (the + // write was terminated unexpectedly) + ram.truncateSync(0); + ram.closeSync(); + disableSizeMonitor(); + return; + } + + tileBytes = ram.readSync(initialTileBytesLength); + if (tileBytes.lengthInBytes != initialTileBytesLength) { + // This implies the tile was corrupted on the previous write (the + // write was terminated unexpectedly whilst writing tile bytes) + ram.truncateSync(0); + ram.closeSync(); + disableSizeMonitor(); + return; + } + ram.setPositionSync(18); } @@ -129,10 +268,17 @@ Future tileAndSizeMonitorWriterWorker( return; } - // We write the new tile bytes to the file and truncate it to the end + // We store the length of the tile bytes in 4 bytes... + ram.writeFromSync( + allocatedUint32Buffer..buffer.asUint32List()[0] = tileBytes.lengthInBytes, + ); + + // ...followed by the tile bytes ram.writeFromSync(tileBytes); final finalPos = ram.positionSync(); ram + // We truncate the tile in case the bytes have been moved forward or are + // shorter than previously ..truncateSync(finalPos) ..closeSync(); @@ -143,18 +289,26 @@ Future tileAndSizeMonitorWriterWorker( } } + // Now we're ready to recieve write messages + final receivePort = ReceivePort(); + input.port.send(receivePort.sendPort); + await for (final val in receivePort) { - if (val is int) { - updateSizeMonitor(-val); + if (val + case ( + :final Uint8List? tileBytes, + :final CachedMapTileMetadata metadata, + :final String path, + )) { + writeTile(path: path, metadata: metadata, tileBytes: tileBytes); continue; } - - writeTile( - val as ({ - Uint8List? tileBytes, - CachedMapTileMetadata metadata, - String path, - }), + if (val is bool && !val) { + disableSizeMonitor(); + continue; + } + throw UnsupportedError( + 'Message was not in the correct format for a tile write', ); } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart deleted file mode 100644 index e4ea437e0..000000000 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/utils/size_monitor_opener.dart +++ /dev/null @@ -1,102 +0,0 @@ -import 'dart:async'; -import 'dart:io'; -import 'dart:typed_data'; - -import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart'; -import 'package:meta/meta.dart'; -import 'package:path/path.dart' as p; - -/// Asynchronously read the existing size monitor if available, -/// returning `null` if unavailable -@internal -Future asyncGetOnlySizeMonitor(String sizeMonitorFilePath) async { - final sizeMonitorFile = File(sizeMonitorFilePath); - if (!await sizeMonitorFile.exists()) return null; - final bytes = await sizeMonitorFile.readAsBytes(); - if (bytes.length == 8) return bytes.buffer.asInt64List()[0]; - return null; -} - -/// Opens and reads the existing size monitor if available -/// -/// If one does not exist, it calculates the current cache size and writes it -/// to a new size monitor. -/// -/// The returned [RandomAccessFile] is open - closure is the responsibility of -/// the caller. -@internal -Future<({int currentSize, RandomAccessFile sizeMonitor})> - getOrCreateSizeMonitor({ - required String cacheDirectoryPath, - required String sizeMonitorFilePath, -}) async { - final sizeMonitorFile = File(sizeMonitorFilePath); - - final sizeMonitor = sizeMonitorFile.openSync(mode: FileMode.append) - ..setPositionSync(0); - final bytes = sizeMonitor.readSync(8); - - if (bytes.length == 8) { - return ( - currentSize: bytes.buffer.asInt64List()[0], - sizeMonitor: sizeMonitor, - ); - } - - final calculatedCurrentSize = await Directory(cacheDirectoryPath) - .listSync() - .whereType() - .where( - (f) { - final uuid = p.basename(f.absolute.path); - return uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; - }, - ) - .map((f) => f.length()) - .asyncFold(0, (v, l) => v + l); - - sizeMonitor - ..setPositionSync(0) - ..writeFromSync( - Uint8List(8)..buffer.asInt64List()[0] = calculatedCurrentSize, - ) - ..flushSync(); - - return (currentSize: calculatedCurrentSize, sizeMonitor: sizeMonitor); -} - -extension _AsyncFold on Iterable> { - /// Reduces a collection of [Future]s to a single value by iteratively - /// combining each element of the collection when it completes with an - /// existing value - /// - /// The result must not depend on the order of completetion and [combine] - /// calls. - Future asyncFold( - T initialValue, - T Function(T previousValue, E element) combine, - ) async { - var value = initialValue; - - bool hasFinishedIterating = false; - int waiting = 0; - final completer = Completer(); - - for (final element in this) { - waiting++; - unawaited( - element.then((result) { - value = combine(value, result); - waiting--; - if (hasFinishedIterating && waiting == 0) completer.complete(); - }), - ); - } - - if (waiting == 0) return value; - - hasFinishedIterating = true; - await completer.future; - return value; - } -} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 75a53897e..6037997bd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -24,9 +24,6 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.readOnly, }); - @override - external Future get isInitialised; - @override external bool get isSupported; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index fd3c78334..2d1df706d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -1,4 +1,3 @@ -import 'package:flutter/foundation.dart'; import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; @@ -20,7 +19,4 @@ class BuiltInMapCachingProviderImpl required this.cacheKeyGenerator, required this.readOnly, }); - - @override - Future get isInitialised => SynchronousFuture(null); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index f2843ee1c..f2b8a112f 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -26,12 +26,16 @@ abstract interface class MapCachingProvider { /// Retrieve a tile from the cache, if it exists /// - /// This may throw, for example due to an error decoding/unpacking an existing - /// cached tile. Additionally, the returned bytes are not guaranteed to form - /// a valid image - attempting to decode the bytes may throw. - /// Tile providers should anticipate this and fallback to a non-caching - /// alternative, wherever possible repairing the cache by overwriting the - /// damaged tile or removing it from the cache. + /// Returns `null` if the tile was not present in the cache. + /// + /// If the tile was present, but could not be correctly read (for example, due + /// to an unexpected corruption), this may throw + /// [CachedMapTileReadFailureException]. Additionally, any returned tile image + /// `bytes` are not guaranteed to form a valid image - attempting to decode + /// the bytes may also throw. + /// Tile providers should anticipate these exceptions and fallback to a + /// non-caching alternative, wherever possible repairing or replacing the tile + /// with a fresh & valid one. Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart new file mode 100644 index 000000000..17fe8e075 --- /dev/null +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart @@ -0,0 +1,44 @@ +/// Indicates that the tile with the given URL was present in the cache, but +/// could not be correctly read +/// +/// This may be due to an unexpected corruption. It should not be thrown when +/// the tile was written correctly. +/// +/// Tile providers should catch this exception. Wherever possible, they should +/// repair or replace the tile with a fresh & valid one. +/// +/// The absence of this exception does not necessarily mean that the returned +/// tile image bytes are valid, only that all the correctly written information +/// was successfully read. +/// +/// This exception is not usually for external consumption, except for tile +/// provider implementations. +class CachedMapTileReadFailureException implements Exception { + /// Create an exception which indicates the tile with the given URL was + /// present in the cache, but could not be correctly read + /// + /// Usually, one of [description] or [originalError] should be provided. + const CachedMapTileReadFailureException({ + required this.url, + this.description, + this.originalError, + }); + + /// URL of the failed tile + final String url; + + /// An optional description of the read failure which caused this to be thrown + /// + /// Usually, one of [description] or [originalError] should be provided. + final String? description; + + /// If available, the original error/exception which caused this to be thrown + /// (if not thrown manually) + /// + /// Usually, one of [description] or [originalError] should be provided. + final Object? originalError; + + @override + String toString() => + 'Failed to read cached tile for $url: ${description ?? originalError}'; +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 6b5ced7e7..2bd1d75d6 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -132,7 +132,7 @@ class NetworkTileImageProvider extends ImageProvider { if (cachingProvider.isSupported) { try { cachedTile = await cachingProvider.getTile(resolvedUrl); - } catch (_) { + } on CachedMapTileReadFailureException { // This could occur due to a corrupt tile - we just try to overwrite it // with fresh data cachedTile = null; From 5f85c711f123a0c34e3edef11a7467e87620202c Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Thu, 29 May 2025 23:20:01 +0100 Subject: [PATCH 31/49] Fix unintended change --- .../network/caching/built_in/built_in_caching_provider.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index cca82eb12..cfce96aae 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -41,7 +41,7 @@ abstract interface class BuiltInMapCachingProvider /// absolute limit. /// /// Defaults to 1 GB. Set to `null` to disable. - int? maxCacheSize = 1_000_000, + int? maxCacheSize = 1_000_000_000, /// Override the duration of time a tile is considered fresh for /// From 3b350dc77aaf2c63c5be22edcee2f9b3202a1fd9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sat, 31 May 2025 10:18:28 +0100 Subject: [PATCH 32/49] Fixed minor bug --- .../caching/built_in/impl/native/native.dart | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 37c0d1374..4ec4726fc 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -38,20 +38,20 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _uuid = Uuid(goptions: GlobalOptions(MathRNG())); } - _cacheDirectoryPath = p.join( + final cacheDirectoryPath = p.join( this.cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, 'fm_cache', ); - final cacheDirectory = Directory(_cacheDirectoryPath!); + final cacheDirectory = Directory(cacheDirectoryPath); await cacheDirectory.create(recursive: true); final sizeMonitorFilePath = - p.join(_cacheDirectoryPath!, sizeMonitorFileName); + p.join(cacheDirectoryPath, sizeMonitorFileName); - _cacheDirectoryPathReady.complete(_cacheDirectoryPath!); + _cacheDirectoryPath = cacheDirectoryPath; + _cacheDirectoryPathReady.complete(cacheDirectoryPath); - final tileAndSizeMonitorWriterWorkerReceivePort = ReceivePort(); SendPort? writerPort; final writerPortReady = Completer(); @@ -71,19 +71,19 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { (path: path, metadata: metadata, tileBytes: tileBytes)); _reportReadFailure = () => sendMessageToWriter(false); + final writerReceivePort = ReceivePort(); await Isolate.spawn( tileAndSizeMonitorWriterWorker, ( - port: tileAndSizeMonitorWriterWorkerReceivePort.sendPort, - cacheDirectoryPath: _cacheDirectoryPath!, + port: writerReceivePort.sendPort, + cacheDirectoryPath: cacheDirectoryPath, sizeMonitorFilePath: sizeMonitorFilePath, sizeLimit: maxCacheSize, ), debugName: '[flutter_map: cache] Tile & Size Monitor Writer', ); - writerPort = - await tileAndSizeMonitorWriterWorkerReceivePort.first as SendPort; + writerPort = await writerReceivePort.first as SendPort; writerPortReady.complete(writerPort); }(); } From e6ba6d8c2121c94d0a4a4b21b8f0dd66dbc1d3a9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 1 Jun 2025 23:15:04 +0100 Subject: [PATCH 33/49] Minor improvements & bug fixes --- .../built_in/built_in_caching_provider.dart | 61 +++++++------ .../caching/built_in/impl/native/native.dart | 74 ++++++++-------- .../workers/tile_and_size_monitor_writer.dart | 88 ++++++++++--------- .../network/caching/built_in/impl/stub.dart | 4 +- .../caching/built_in/impl/web/web.dart | 4 +- .../network/caching/caching_provider.dart | 7 +- .../caching/tile_read_failure_exception.dart | 4 +- .../image_provider/image_provider.dart | 12 +-- 8 files changed, 132 insertions(+), 122 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index cfce96aae..5c39945da 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -19,53 +19,64 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b /// For more information, see the online documentation. abstract interface class BuiltInMapCachingProvider implements MapCachingProvider { - /// if a singleton instance exists, return it, otherwise create a new - /// singleton instance (and start asynchronously initialising it) + /// If an instance exists, return it, otherwise create a new instance /// - /// If an instance already exists, the provided configuration will be ignored. + /// The provided configuration will only be respected if an instance does not + /// already exist. /// /// See individual properties for more information about configuration. factory BuiltInMapCachingProvider.getOrCreateInstance({ - /// Path to the caching directory to use + /// Path to the directory to use to store cached tiles & other related files /// - /// This must be accessible to the program. + /// The provider actually uses the 'fm_cache' directory created as a child + /// of the path specified here. + /// + /// The program must have rights/permissions to access the path. + /// + /// The path does not have to exist, it will be recursively created if + /// missing. + /// + /// All files and directories within the path will be liable to deletion by + /// the size reducer. /// /// Defaults to a platform provided cache directory, which may be cleared by /// the OS at any time. String? cacheDirectory, - /// Preferred maximum size (in bytes) of the cache + /// Maximum total size of cached tiles, in bytes /// - /// This is applied when the internal caching mechanism is created (on the - /// first tile load in the main memory space for the app). It is not an - /// absolute limit. + /// This is applied only when the instance is created, by running the size + /// reducer. This runs in the background (and so does not delay reads or + /// writes). The cache size may exceed this limit while the program is + /// running. + /// + /// Disabling the size limit may improve write performance. /// /// Defaults to 1 GB. Set to `null` to disable. int? maxCacheSize = 1_000_000_000, - /// Override the duration of time a tile is considered fresh for + /// Function to convert a tile's URL to a key used to uniquely identify the + /// tile /// - /// Defaults to `null`: use duration calculated from each tile's HTTP - /// headers. - Duration? overrideFreshAge, - - /// Function to convert a tile URL to a key used in the cache + /// Where parts of the URL are volatile or do not represent the tile's + /// contents/image - for example, API keys contained with the query + /// parameters - this should be modified to remove the volatile portions. /// - /// This may be useful where parts of the URL are volatile or do not - /// represent the tile image, for example, API keys contained with the query - /// parameters. - /// - /// The resulting key should be unique to that tile URL. Keys must be usable - /// as filenames on all intended platform filesystems. + /// Keys must be usable as filenames on all intended platform filesystems. + /// The callback should not throw. /// /// Defaults to generating a UUID from the entire URL string. + String Function(String url)? tileKeyGenerator, + + /// Override the duration of time a tile is considered fresh for /// - /// The callback should not throw. - String Function(String url)? cacheKeyGenerator, + /// Defaults to `null`: use duration calculated from each tile's HTTP + /// headers. + Duration? overrideFreshAge, /// Prevent any tiles from being added or updated /// - /// Does not disable the size limiter if the cache size is larger than + /// Does not disable the size reducer if the cache size is larger than /// `maxCacheSize`. /// /// Defaults to `false`. @@ -83,7 +94,7 @@ abstract interface class BuiltInMapCachingProvider cacheDirectory: cacheDirectory, maxCacheSize: maxCacheSize, overrideFreshAge: overrideFreshAge, - cacheKeyGenerator: cacheKeyGenerator, + tileKeyGenerator: tileKeyGenerator, readOnly: readOnly, ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 4ec4726fc..cd0f7a9e3 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -20,8 +20,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; + final String Function(String url)? tileKeyGenerator; final Duration? overrideFreshAge; - final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -29,22 +29,20 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, - required this.cacheKeyGenerator, + required this.tileKeyGenerator, required this.readOnly, }) { // This should only be called/constructed once () async { - if (cacheKeyGenerator == null) { + if (tileKeyGenerator == null) { _uuid = Uuid(goptions: GlobalOptions(MathRNG())); } final cacheDirectoryPath = p.join( - this.cacheDirectory ?? - (await getApplicationCacheDirectory()).absolute.path, + cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, 'fm_cache', ); - final cacheDirectory = Directory(cacheDirectoryPath); - await cacheDirectory.create(recursive: true); + await Directory(cacheDirectoryPath).create(recursive: true); final sizeMonitorFilePath = p.join(cacheDirectoryPath, sizeMonitorFileName); @@ -59,16 +57,13 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { // monitoring (and potentially run the reducer) if necessary // Reading does not depend on this. void sendMessageToWriter(Object message) { - if (writerPort != null) { - writerPort.send(message); - } else { - writerPortReady.future.then((port) => port.send(message)); - } + if (writerPort != null) return writerPort.send(message); + writerPortReady.future.then((port) => port.send(message)); } - _writeTileFile = ({required path, required metadata, tileBytes}) => - sendMessageToWriter( - (path: path, metadata: metadata, tileBytes: tileBytes)); + _writeTileFile = (path, metadata, tileBytes) => sendMessageToWriter( + (path: path, metadata: metadata, tileBytes: tileBytes), + ); _reportReadFailure = () => sendMessageToWriter(false); final writerReceivePort = ReceivePort(); @@ -78,7 +73,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { port: writerReceivePort.sendPort, cacheDirectoryPath: cacheDirectoryPath, sizeMonitorFilePath: sizeMonitorFilePath, - sizeLimit: maxCacheSize, + maxCacheSize: maxCacheSize, ), debugName: '[flutter_map: cache] Tile & Size Monitor Writer', ); @@ -93,14 +88,14 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { late final Uuid _uuid; // left un-inited if provided generator - late final void Function({ - required String path, - required CachedMapTileMetadata metadata, + late final void Function( + String path, + CachedMapTileMetadata metadata, Uint8List? tileBytes, - }) _writeTileFile; + ) _writeTileFile; - /// See `disableSizeMonitor` in worker - late final void Function() _reportReadFailure; + late final void Function() + _reportReadFailure; // See `disableSizeMonitor` in worker @override bool get isSupported => true; @@ -110,7 +105,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { String url, ) async { final key = - cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + tileKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); final tileFile = File( p.join(_cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key), ); @@ -121,7 +116,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final bytes = await tileFile.readAsBytes(); if (bytes.lengthInBytes < 22) { - throw CachedMapTileReadFailureException( + throw CachedMapTileReadFailure( url: url, description: 'cache file (${bytes.lengthInBytes}) was shorter than the ' @@ -145,14 +140,13 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { etag = const AsciiDecoder().convert(etagBytes); } - // Performing an unaligned read is a hassle - final tileBytesExpectedLength = + final tileBytesExpectedLength = // Perform an unaligned read bytes.buffer.asByteData(18 + etagLength, 4).getUint32(0, Endian.host); final tileBytes = Uint8List.sublistView(bytes, 18 + etagLength + 4); if (tileBytes.lengthInBytes != tileBytesExpectedLength) { - throw CachedMapTileReadFailureException( + throw CachedMapTileReadFailure( url: url, description: 'tile image bytes (${tileBytes.lengthInBytes}) were not of ' @@ -168,13 +162,13 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { ), bytes: tileBytes, ); - } on CachedMapTileReadFailureException { + } on CachedMapTileReadFailure { _reportReadFailure(); rethrow; } catch (error, stackTrace) { _reportReadFailure(); Error.throwWithStackTrace( - CachedMapTileReadFailureException(url: url, originalError: error), + CachedMapTileReadFailure(url: url, originalError: error), stackTrace, ); } @@ -189,20 +183,22 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { if (readOnly) return; final key = - cacheKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + tileKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); final path = p.join( _cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key, ); - final resolvedMetadata = overrideFreshAge != null - ? CachedMapTileMetadata( - staleAt: DateTime.timestamp().add(overrideFreshAge!), - lastModified: metadata.lastModified, - etag: metadata.etag, - ) - : metadata; - - _writeTileFile(path: path, metadata: resolvedMetadata, tileBytes: bytes); + _writeTileFile( + path, + overrideFreshAge != null + ? CachedMapTileMetadata( + staleAt: DateTime.timestamp().add(overrideFreshAge!), + lastModified: metadata.lastModified, + etag: metadata.etag, + ) + : metadata, + bytes, + ); } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index 25dc6e21d..a415d7650 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -10,31 +10,33 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/t import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; -/// Isolate worker which writes tile files, and updates the size monitor, -/// synchronously +/// Isolate worker which writes tile files, maintains the size monitor, and +/// (if necessary) starts the size reducer +/// +/// Follows the storage spec described in README.md. @internal Future tileAndSizeMonitorWriterWorker( ({ SendPort port, String cacheDirectoryPath, String sizeMonitorFilePath, - int? sizeLimit, + int? maxCacheSize, }) input, ) async { + //! SIZE MONITOR HANDLING + final sizeMonitorFile = File(input.sizeMonitorFilePath); RandomAccessFile? sizeMonitor; late int currentSize; - final allocatedInt64BufferForSizeMonitor = Uint8List(8); + final allocUint64BufferSizeMonitor = Uint8List(8); void updateSizeMonitor(int deltaSize) { if (sizeMonitor == null) return; - currentSize += deltaSize; sizeMonitor! ..setPositionSync(0) ..writeFromSync( - allocatedInt64BufferForSizeMonitor - ..buffer.asInt64List()[0] = currentSize, + allocUint64BufferSizeMonitor..buffer.asUint64List()[0] = currentSize, ) ..flushSync(); } @@ -48,7 +50,6 @@ Future tileAndSizeMonitorWriterWorker( // run the size reducer, however - so we just delete it and forget about it. void disableSizeMonitor() { if (sizeMonitor == null) return; - sizeMonitor!.closeSync(); sizeMonitorFile.deleteSync(); sizeMonitor = null; @@ -58,13 +59,13 @@ Future tileAndSizeMonitorWriterWorker( // If it's available, we can begin writing immediately. // Otherwise, we need to wait for it to be regenerated, which takes some time. // This is only neccessary for a brand new cache, an existing cache with a - // newly imposed `sizeLimit` when there was previously none, or a corrupted + // newly imposed `maxCacheSize` when there was previously none, or a corrupted // cache (where the size monitor is missing, potentially due to a read // failure). // We can run the size reducer in another isolate afterwards, as it returns a // relative change to the current cache size. Writes don't need to wait for // that expensive process. - if (input.sizeLimit case final sizeLimit?) { + if (input.maxCacheSize case final maxCacheSize?) { Future regenerateSizeMonitor() async { int calculatedSize = 0; int waitingForSize = 0; @@ -92,7 +93,7 @@ Future tileAndSizeMonitorWriterWorker( sizeMonitor! ..setPositionSync(0) - ..writeFromSync(Uint8List(8)..buffer.asInt64List()[0] = calculatedSize) + ..writeFromSync(Uint8List(8)..buffer.asUint64List()[0] = calculatedSize) ..flushSync(); currentSize = calculatedSize; @@ -103,16 +104,16 @@ Future tileAndSizeMonitorWriterWorker( ..setPositionSync(0); if (sizeMonitorInitiallyExists) { try { - currentSize = sizeMonitor!.readSync(8).buffer.asInt64List()[0]; - } catch (e) { + currentSize = sizeMonitor!.readSync(8).buffer.asUint64List()[0]; + } catch (_) { await regenerateSizeMonitor(); } } else { await regenerateSizeMonitor(); } - if (currentSize > sizeLimit) { - Future runSizeLimiter({ + if (currentSize > maxCacheSize) { + Future runSizeReducer({ required String cacheDirectoryPath, required String sizeMonitorFilePath, required int minSizeToDelete, @@ -123,24 +124,26 @@ Future tileAndSizeMonitorWriterWorker( sizeMonitorFilePath: sizeMonitorFilePath, minSizeToDelete: minSizeToDelete, ), + debugName: '[flutter_map: cache] Size Reducer', ); - runSizeLimiter( + runSizeReducer( cacheDirectoryPath: input.cacheDirectoryPath, sizeMonitorFilePath: input.sizeMonitorFilePath, - minSizeToDelete: currentSize - sizeLimit, + minSizeToDelete: currentSize - maxCacheSize, ).then((deletedSize) => updateSizeMonitor(-deletedSize)); } } - final allocatedInt64Buffer = Uint8List(8); - final allocatedUint32Buffer = Uint8List(4); - final allocatedUint16Buffer = Uint8List(2); + //! TILE WRITING + final allocInt64BufferTileWrite = Uint8List(8); + final allocUint32BufferTileWrite = Uint8List(4); + final allocUint16BufferTileWrite = Uint8List(2); void writeTile({ - Uint8List? tileBytes, - required final CachedMapTileMetadata metadata, required final String path, + required final CachedMapTileMetadata metadata, + Uint8List? tileBytes, }) { final tileFile = File(path); final initialTileFileExists = tileFile.existsSync(); @@ -170,14 +173,15 @@ Future tileAndSizeMonitorWriterWorker( // We start reading from the start of the file, where we store our header // info ..setPositionSync(0) - // We store the stale-at header in 8 bytes... + // We store the stale-at header in 8 signed bytes... ..writeFromSync( - allocatedInt64Buffer + allocInt64BufferTileWrite ..buffer.asInt64List()[0] = metadata.staleAtMilliseconds, ) - // ...followed by the last-modified header in 8 bytes, or '0' if null + // ...followed by the last-modified header in 8 signed bytes, or '0' if + // null ..writeFromSync( - allocatedInt64Buffer + allocInt64BufferTileWrite ..buffer.asInt64List()[0] = metadata.lastModifiedMilliseconds ?? 0, ); @@ -186,7 +190,7 @@ Future tileAndSizeMonitorWriterWorker( if (initialTileFileExists) { try { initialEtagLength = ram.readSync(2).buffer.asUint16List()[0]; - } catch (e) { + } catch (_) { // This implies the tile was corrupted on the previous write (the // write was terminated unexpectedly) // However, this shouldn't be possible in practise, since that should've @@ -206,17 +210,16 @@ Future tileAndSizeMonitorWriterWorker( final int etagLength; late final Uint8List etagBytes; // left unset if etagLength = 0 if (metadata.etag == null) { - // We don't have an etag, so we write 2 bytes indicating the etag length - // is 0 + // We don't have an etag, so we write 2 unsigned bytes indicating the etag + // length is 0 ram.writeFromSync( - allocatedUint16Buffer..buffer.asUint16List()[0] = etagLength = 0, + allocUint16BufferTileWrite..buffer.asUint16List()[0] = etagLength = 0, ); } else { etagBytes = const AsciiEncoder().convert(metadata.etag!); - // We store the etag length in 2 bytes... - // (unless it is too large) + // We store the etag length in 2 signed bytes (unless it is too large)... ram.writeFromSync( - allocatedUint16Buffer + allocUint16BufferTileWrite ..buffer.asUint16List()[0] = etagLength = (etagBytes.lengthInBytes > 0xFFFF ? 0 : etagBytes.lengthInBytes), ); @@ -233,7 +236,7 @@ Future tileAndSizeMonitorWriterWorker( final int initialTileBytesLength; try { initialTileBytesLength = ram.readSync(4).buffer.asUint32List()[0]; - } catch (e) { + } catch (_) { // This implies the tile was corrupted on the previous write (the // write was terminated unexpectedly) ram.truncateSync(0); @@ -268,9 +271,10 @@ Future tileAndSizeMonitorWriterWorker( return; } - // We store the length of the tile bytes in 4 bytes... + // We store the length of the tile bytes in 4 unsigned bytes... ram.writeFromSync( - allocatedUint32Buffer..buffer.asUint32List()[0] = tileBytes.lengthInBytes, + allocUint32BufferTileWrite + ..buffer.asUint32List()[0] = tileBytes.lengthInBytes, ); // ...followed by the tile bytes @@ -289,16 +293,18 @@ Future tileAndSizeMonitorWriterWorker( } } - // Now we're ready to recieve write messages + //! COMMS HANDLING + + // Now we're ready to recieve commands final receivePort = ReceivePort(); input.port.send(receivePort.sendPort); await for (final val in receivePort) { if (val case ( - :final Uint8List? tileBytes, - :final CachedMapTileMetadata metadata, :final String path, + :final CachedMapTileMetadata metadata, + :final Uint8List? tileBytes, )) { writeTile(path: path, metadata: metadata, tileBytes: tileBytes); continue; @@ -307,8 +313,6 @@ Future tileAndSizeMonitorWriterWorker( disableSizeMonitor(); continue; } - throw UnsupportedError( - 'Message was not in the correct format for a tile write', - ); + throw UnsupportedError('Command was in unknown format'); } } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 6037997bd..6b538bde9 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -11,8 +11,8 @@ import 'package:meta/meta.dart'; class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; + final String Function(String url)? tileKeyGenerator; final Duration? overrideFreshAge; - final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -20,7 +20,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, - required this.cacheKeyGenerator, + required this.tileKeyGenerator, required this.readOnly, }); diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index 2d1df706d..c5255f168 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -7,8 +7,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; + final String Function(String url)? tileKeyGenerator; final Duration? overrideFreshAge; - final String Function(String url)? cacheKeyGenerator; final bool readOnly; @internal @@ -16,7 +16,7 @@ class BuiltInMapCachingProviderImpl required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, - required this.cacheKeyGenerator, + required this.tileKeyGenerator, required this.readOnly, }); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index f2b8a112f..d1b840ffc 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -29,10 +29,9 @@ abstract interface class MapCachingProvider { /// Returns `null` if the tile was not present in the cache. /// /// If the tile was present, but could not be correctly read (for example, due - /// to an unexpected corruption), this may throw - /// [CachedMapTileReadFailureException]. Additionally, any returned tile image - /// `bytes` are not guaranteed to form a valid image - attempting to decode - /// the bytes may also throw. + /// to an unexpected corruption), this may throw [CachedMapTileReadFailure]. + /// Additionally, any returned tile image `bytes` are not guaranteed to form a + /// valid image - attempting to decode the bytes may also throw. /// Tile providers should anticipate these exceptions and fallback to a /// non-caching alternative, wherever possible repairing or replacing the tile /// with a fresh & valid one. diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart index 17fe8e075..40c071cc2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_read_failure_exception.dart @@ -13,12 +13,12 @@ /// /// This exception is not usually for external consumption, except for tile /// provider implementations. -class CachedMapTileReadFailureException implements Exception { +class CachedMapTileReadFailure implements Exception { /// Create an exception which indicates the tile with the given URL was /// present in the cache, but could not be correctly read /// /// Usually, one of [description] or [originalError] should be provided. - const CachedMapTileReadFailureException({ + const CachedMapTileReadFailure({ required this.url, this.description, this.originalError, diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 2bd1d75d6..34add1018 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -132,7 +132,7 @@ class NetworkTileImageProvider extends ImageProvider { if (cachingProvider.isSupported) { try { cachedTile = await cachingProvider.getTile(resolvedUrl); - } on CachedMapTileReadFailureException { + } on CachedMapTileReadFailure { // This could occur due to a corrupt tile - we just try to overwrite it // with fresh data cachedTile = null; @@ -229,7 +229,7 @@ class NetworkTileImageProvider extends ImageProvider { try { // If we have a cached tile that's not stale, return it return await decodeBytes(cachedTile.bytes); - } catch (e) { + } catch (_) { // If the cached tile is corrupt, we proceed and get from the server forceFromServer = true; } @@ -255,7 +255,7 @@ class NetworkTileImageProvider extends ImageProvider { late final Codec decodedCacheBytes; try { decodedCacheBytes = await decodeBytes(cachedTile.bytes); - } catch (e) { + } catch (_) { // If the cached tile is corrupt, we get fresh from the server without // caching, then continue forceFromServer = true; @@ -313,7 +313,7 @@ class NetworkTileImageProvider extends ImageProvider { chunkEvents.close(); return await decodeBytes(TileProvider.transparentImage); } */ - on ClientException catch (e) { + on ClientException catch (err) { // This could be a wide range of issues, potentially ours, potentially // network, etc. @@ -324,7 +324,7 @@ class NetworkTileImageProvider extends ImageProvider { // This can occur when the map/tile layer is disposed early - in older // versions, we used manual tracking to avoid disposing too early, but now // we just attempt to catch (it's cleaner & easier) - if (e.message.contains('closed') || e.message.contains('cancel')) { + if (err.message.contains('closed') || err.message.contains('cancel')) { return await decodeBytes(TileProvider.transparentImage); } @@ -334,7 +334,7 @@ class NetworkTileImageProvider extends ImageProvider { return await decodeBytes(TileProvider.transparentImage); } return _loadImage(key, chunkEvents, decode, useFallback: true); - } catch (e) { + } catch (_) { // Non-specific catch to catch decoding errors, the manually thrown HTTP // exception, etc. From 35d06077b868a1ab7998e328441df7577b78400b Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Mon, 2 Jun 2025 12:59:15 +0100 Subject: [PATCH 34/49] Move default `tileKeyGenerator` implementation into a static util method --- .../built_in/built_in_caching_provider.dart | 18 ++++++++++++++++-- .../caching/built_in/impl/native/native.dart | 18 +++--------------- .../network/caching/built_in/impl/stub.dart | 2 +- .../network/caching/built_in/impl/web/web.dart | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 5c39945da..5af52ce7d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -2,6 +2,9 @@ import 'package:flutter_map/flutter_map.dart'; import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart' if (dart.library.io) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart' if (dart.library.js_interop) 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart'; +import 'package:uuid/data.dart'; +import 'package:uuid/rng.dart'; +import 'package:uuid/uuid.dart'; /// Simple built-in map caching using an I/O storage mechanism, for native /// (non-web) platforms only @@ -65,7 +68,8 @@ abstract interface class BuiltInMapCachingProvider /// Keys must be usable as filenames on all intended platform filesystems. /// The callback should not throw. /// - /// Defaults to generating a UUID from the entire URL string. + /// Defaults to using [uuidTileKeyGenerator], which custom implementations + /// may utilise. String Function(String url)? tileKeyGenerator, /// Override the duration of time a tile is considered fresh for @@ -94,10 +98,20 @@ abstract interface class BuiltInMapCachingProvider cacheDirectory: cacheDirectory, maxCacheSize: maxCacheSize, overrideFreshAge: overrideFreshAge, - tileKeyGenerator: tileKeyGenerator, + tileKeyGenerator: tileKeyGenerator ?? uuidTileKeyGenerator, readOnly: readOnly, ); } static BuiltInMapCachingProviderImpl? _instance; + + /// Default `tileKeyGenerator` which generates v5 UUIDs from input strings + /// + /// May be utilised in custom `tileKeyGenerator` implementations. + /// + /// See [BuiltInMapCachingProvider.getOrCreateInstance]'s parameter for more + /// info. + static String uuidTileKeyGenerator(String url) => + _uuid.v5(Namespace.url.value, url); + static final _uuid = Uuid(goptions: GlobalOptions(MathRNG())); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index cd0f7a9e3..683000b29 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -10,9 +10,6 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; import 'package:path_provider/path_provider.dart'; -import 'package:uuid/data.dart'; -import 'package:uuid/rng.dart'; -import 'package:uuid/uuid.dart'; @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { @@ -20,7 +17,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; - final String Function(String url)? tileKeyGenerator; + final String Function(String url) tileKeyGenerator; final Duration? overrideFreshAge; final bool readOnly; @@ -34,10 +31,6 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { }) { // This should only be called/constructed once () async { - if (tileKeyGenerator == null) { - _uuid = Uuid(goptions: GlobalOptions(MathRNG())); - } - final cacheDirectoryPath = p.join( cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, 'fm_cache', @@ -86,14 +79,11 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { String? _cacheDirectoryPath; // ~cached version of below for instant access final _cacheDirectoryPathReady = Completer(); - late final Uuid _uuid; // left un-inited if provided generator - late final void Function( String path, CachedMapTileMetadata metadata, Uint8List? tileBytes, ) _writeTileFile; - late final void Function() _reportReadFailure; // See `disableSizeMonitor` in worker @@ -104,8 +94,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, ) async { - final key = - tileKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + final key = tileKeyGenerator(url); final tileFile = File( p.join(_cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key), ); @@ -182,8 +171,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { }) async { if (readOnly) return; - final key = - tileKeyGenerator?.call(url) ?? _uuid.v5(Namespace.url.value, url); + final key = tileKeyGenerator(url); final path = p.join( _cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key, diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 6b538bde9..66c55d6d2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -11,7 +11,7 @@ import 'package:meta/meta.dart'; class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; - final String Function(String url)? tileKeyGenerator; + final String Function(String url) tileKeyGenerator; final Duration? overrideFreshAge; final bool readOnly; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index c5255f168..e3a391a34 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -7,7 +7,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final String? cacheDirectory; final int? maxCacheSize; - final String Function(String url)? tileKeyGenerator; + final String Function(String url) tileKeyGenerator; final Duration? overrideFreshAge; final bool readOnly; From 849720452de8911023ac974087727be6215859e5 Mon Sep 17 00:00:00 2001 From: Luka S Date: Sun, 8 Jun 2025 16:19:29 +0100 Subject: [PATCH 35/49] Discard changes to lib/src/layer/tile_layer/tile_layer.dart --- lib/src/layer/tile_layer/tile_layer.dart | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_layer.dart b/lib/src/layer/tile_layer/tile_layer.dart index 2caecec42..ca73239eb 100644 --- a/lib/src/layer/tile_layer/tile_layer.dart +++ b/lib/src/layer/tile_layer/tile_layer.dart @@ -362,17 +362,20 @@ class _TileLayerState extends State with TickerProviderStateMixin { false && (kReleaseMode || kProfileMode) && !_unblockOpenStreetMapUrl; void _warnOpenStreetMapUrl() { if (!_isOpenStreetMapUrl || !kDebugMode || _unblockOpenStreetMapUrl) return; - Logger(printer: PrettyPrinter(methodCount: 0)).w( + Logger(printer: PrettyPrinter(methodCount: 0)).e( '''\x1B[1m\x1B[3mflutter_map\x1B[0m flutter_map wants to help keep map data available for everyone. We use the public OpenStreetMap tile servers in our code examples & demo app, but they are NOT free to use by everyone. +In an upcoming non-major release, requests to 'tile.openstreetmap.org' or +'tile.osm.org' will be blocked by default in release mode. Please review https://operations.osmfoundation.org/policies/tiles/ to see if your project is compliant with their Tile Usage Policy. For more information, see https://docs.fleaflet.dev/tile-servers/using-openstreetmap-direct. It describes in additional detail why we feel it is important to do this, how -you can disable this warning, the timeframes for this new policy, and how we're -working to reduce requests without any extra work from you.''', +you can unblock the tile servers if your use-case is acceptable, the timeframes +for this new policy, and how we're working to reduce requests without any extra +work from you.''', ); } From 5c783296faf542f9f98bec377ab25a2e49596283 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 8 Jun 2025 17:07:34 +0100 Subject: [PATCH 36/49] Prevent fallback tiles from being cached Ensure internal image chunk event stream is always closed on response completion Minor clean-up --- example/pubspec.lock | 20 +++---- .../image_provider/image_provider.dart | 58 ++++++++++--------- .../tile_provider/network/tile_provider.dart | 20 +------ 3 files changed, 43 insertions(+), 55 deletions(-) diff --git a/example/pubspec.lock b/example/pubspec.lock index 79656615d..3665b6ea2 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -187,26 +187,26 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "6bb818ecbdffe216e81182c2f0714a2e62b593f4a4f13098713ff1685dfb6ab0" + sha256: "8dcda04c3fc16c14f48a7bb586d4be1f0d1572731b6d81d51772ef47c02081e0" url: "https://pub.dev" source: hosted - version: "10.0.9" + version: "11.0.1" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: f8b613e7e6a13ec79cfdc0e97638fddb3ab848452eff057653abd3edba760573 + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" url: "https://pub.dev" source: hosted - version: "3.0.9" + version: "3.0.10" leak_tracker_testing: dependency: transitive description: name: leak_tracker_testing - sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3" + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" url: "https://pub.dev" source: hosted - version: "3.0.1" + version: "3.0.2" lints: dependency: transitive description: @@ -464,10 +464,10 @@ packages: dependency: transitive description: name: test_api - sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" url: "https://pub.dev" source: hosted - version: "0.7.4" + version: "0.7.6" typed_data: dependency: transitive description: @@ -560,10 +560,10 @@ packages: dependency: transitive description: name: vector_math - sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.2.0" vm_service: dependency: transitive description: diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 34add1018..80a16f1f2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -14,7 +14,7 @@ import 'package:meta/meta.dart'; /// /// Supports falling back to a secondary URL, if the primary URL fetch fails. /// Note that specifying a [fallbackUrl] will prevent this image provider from -/// being cached. +/// being cached in memory. @immutable @internal class NetworkTileImageProvider extends ImageProvider { @@ -27,6 +27,8 @@ class NetworkTileImageProvider extends ImageProvider { /// If this is non-null, [operator==] will always return `false` (except if /// the two objects are [identical]). Therefore, if this is non-null, this /// image provider will not be cached in memory. + /// + /// If the fallback is used, it will not be cached with the [cachingProvider]. final String? fallbackUrl; /// The headers to include with the tile fetch request @@ -90,7 +92,11 @@ class NetworkTileImageProvider extends ImageProvider { final chunkEvents = StreamController(); return MultiFrameImageStreamCompleter( - codec: _loadImage(key, chunkEvents, decode), + codec: _loadImage(key, chunkEvents.sink, decode) + ..then( + (_) => unawaited(chunkEvents.close()), + onError: (_) => unawaited(chunkEvents.close()), + ), chunkEvents: chunkEvents.stream, scale: 1, debugLabel: key.url, @@ -102,9 +108,10 @@ class NetworkTileImageProvider extends ImageProvider { ); } + // TODO: Support cancellation Future _loadImage( NetworkTileImageProvider key, - StreamController chunkEvents, + StreamSink chunkEvents, ImageDecoderCallback decode, { bool useFallback = false, }) async { @@ -125,25 +132,10 @@ class NetworkTileImageProvider extends ImageProvider { rethrow; } - // Prepare caching provider & load cached tile if available - ({Uint8List bytes, CachedMapTileMetadata metadata})? cachedTile; - final cachingProvider = - this.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); - if (cachingProvider.isSupported) { - try { - cachedTile = await cachingProvider.getTile(resolvedUrl); - } on CachedMapTileReadFailure { - // This could occur due to a corrupt tile - we just try to overwrite it - // with fresh data - cachedTile = null; - } - } - // Create method to get bytes from server Future<({Uint8List bytes, StreamedResponse response})> get({ Map? additionalHeaders, }) async { - // TODO: Support cancellation // final request = AbortableRequest('GET', uri, abortTrigger: abortTrigger); final request = Request('GET', uri); @@ -165,12 +157,26 @@ class NetworkTileImageProvider extends ImageProvider { return (bytes: bytes, response: response); } + // Prepare caching provider & load cached tile if available + ({Uint8List bytes, CachedMapTileMetadata metadata})? cachedTile; + final cachingProvider = + this.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); + if (cachingProvider.isSupported) { + try { + cachedTile = await cachingProvider.getTile(resolvedUrl); + } on CachedMapTileReadFailure { + // This could occur due to a corrupt tile - we just try to overwrite it + // with fresh data + cachedTile = null; + } + } + // Create method to interact with cache void cachePut({ required Uint8List? bytes, required Map headers, }) { - if (!cachingProvider.isSupported) return; + if (useFallback || !cachingProvider.isSupported) return; final lastModified = headers[HttpHeaders.lastModifiedHeader]; final etag = headers[HttpHeaders.etagHeader]; @@ -274,10 +280,13 @@ class NetworkTileImageProvider extends ImageProvider { } // It's likely an error at this point - // If the user has disabled attempted-decode, we just throw and catch - // below - // Otherwise we try to decode it anyway, without memory caching - if (!attemptDecodeOfHttpErrorResponses) { + // However, some servers may produce error responses with useful bodies, + // perhaps intentionally (such as an "API Key Required" message) + // Therefore, if there is a body, and the user allows it, we attempt to + // decode the body bytes as an image (although we don't cache if + // successful) + // Otherwise, we just throw early + if (!attemptDecodeOfHttpErrorResponses || bytes.isEmpty) { throw NetworkImageLoadException( statusCode: response.statusCode, uri: uri, @@ -305,7 +314,6 @@ class NetworkTileImageProvider extends ImageProvider { ); } } - // TODO: Support cancellation /* on AbortedRequest { // This is a planned exception, we just quit silently @@ -329,7 +337,6 @@ class NetworkTileImageProvider extends ImageProvider { } if (useFallback || fallbackUrl == null) { - chunkEvents.close(); if (!silenceExceptions) rethrow; return await decodeBytes(TileProvider.transparentImage); } @@ -341,7 +348,6 @@ class NetworkTileImageProvider extends ImageProvider { evict(); if (useFallback || fallbackUrl == null) { - chunkEvents.close(); if (!silenceExceptions) rethrow; return await decodeBytes(TileProvider.transparentImage); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index e7896b890..8526529bb 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -49,7 +49,7 @@ class NetworkTileProvider extends TileProvider { /// Whether to optimistically attempt to decode HTTP responses that have a /// non-successful status code as an image /// - /// If the decode is unnsuccessful, the behaviour depends on + /// If the decode is unsuccessful, the behaviour depends on /// [silenceExceptions]. /// /// Defaults to `true`. @@ -75,7 +75,6 @@ class NetworkTileProvider extends TileProvider { final bool _isInternallyCreatedClient; @override - // TODO: True when abortable bool get supportsCancelLoading => false; @override @@ -94,23 +93,6 @@ class NetworkTileProvider extends TileProvider { cachingProvider: cachingProvider, ); - /*@override - ImageProvider getImageWithCancelLoadingSupport( - TileCoordinates coordinates, - TileLayer options, - Future cancelLoading, - ) => - NetworkTileImageProvider( - url: getTileUrl(coordinates, options), - fallbackUrl: getTileFallbackUrl(coordinates, options), - headers: headers, - httpClient: _httpClient, - abortTrigger: cancelLoading, - silenceExceptions: silenceExceptions, - attemptDecodeOfHttpErrorResponses: attemptDecodeOfHttpErrorResponses, - cachingProvider: cachingProvider, - );*/ - @override Future dispose() async { if (_isInternallyCreatedClient) _httpClient.close(); From 507ff51623ec11d93c404bedc5eed4cf6cebe7f3 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 8 Jun 2025 18:23:28 +0100 Subject: [PATCH 37/49] Expose `calculateStaleAt` via `CachedMapTileMetadata.fromHttpHeaders` factory --- .../workers/tile_and_size_monitor_writer.dart | 5 +- .../network/caching/tile_metadata.dart | 87 +++++++++++++++---- .../image_provider/image_provider.dart | 48 +--------- 3 files changed, 74 insertions(+), 66 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index a415d7650..467a6c8fe 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -176,13 +176,14 @@ Future tileAndSizeMonitorWriterWorker( // We store the stale-at header in 8 signed bytes... ..writeFromSync( allocInt64BufferTileWrite - ..buffer.asInt64List()[0] = metadata.staleAtMilliseconds, + ..buffer.asInt64List()[0] = metadata.staleAt.millisecondsSinceEpoch, ) // ...followed by the last-modified header in 8 signed bytes, or '0' if // null ..writeFromSync( allocInt64BufferTileWrite - ..buffer.asInt64List()[0] = metadata.lastModifiedMilliseconds ?? 0, + ..buffer.asInt64List()[0] = + metadata.lastModified?.millisecondsSinceEpoch ?? 0, ); // We need to read the old etag length to compare their lengths diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart index 647c6c0dd..858c65fd2 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/tile_metadata.dart @@ -1,4 +1,5 @@ -import 'dart:io' show HttpHeaders; // web safe! +import 'dart:io' show HttpHeaders, HttpDate; // web safe! +import 'dart:math'; import 'package:flutter_map/flutter_map.dart'; import 'package:meta/meta.dart'; @@ -14,26 +15,77 @@ import 'package:meta/meta.dart'; @immutable class CachedMapTileMetadata { /// Create new metadata - CachedMapTileMetadata({ + const CachedMapTileMetadata({ required this.staleAt, required this.lastModified, required this.etag, - }) : staleAtMilliseconds = staleAt.millisecondsSinceEpoch, - lastModifiedMilliseconds = lastModified?.millisecondsSinceEpoch; + }); - /// The calculated time at which this tile becomes stale - final DateTime staleAt; + /// Create new metadata based off an HTTP response's headers + /// + /// Where a response does not include enough information to calculate the + /// freshness age, [fallbackFreshnessAge] is used. + factory CachedMapTileMetadata.fromHttpHeaders( + Map headers, { + Duration fallbackFreshnessAge = const Duration(days: 7), + }) { + // There is no guarantee that this meets the HTTP specification - however, + // it was designed with it in mind + DateTime calculateStaleAt() { + final addToNow = DateTime.timestamp().add; - /// The calculated time at which this tile becomes stale, represented in - /// [DateTime.millisecondsSinceEpoch] - final int staleAtMilliseconds; + if (headers[HttpHeaders.cacheControlHeader]?.toLowerCase() + case final cacheControl?) { + final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; - /// If available, the value in [HttpHeaders.lastModifiedHeader] - final DateTime? lastModified; + if (maxAge == null) { + if (headers[HttpHeaders.expiresHeader]?.toLowerCase() + case final expires?) { + return HttpDate.parse(expires); + } + + return addToNow(fallbackFreshnessAge); + } + + if (headers[HttpHeaders.ageHeader] case final currentAge?) { + return addToNow( + Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), + ); + } + + final estimatedAge = max( + 0, + DateTime.timestamp() + .difference(HttpDate.parse(headers[HttpHeaders.dateHeader]!)) + .inSeconds, + ); + return addToNow(Duration(seconds: int.parse(maxAge) - estimatedAge)); + } + + return addToNow(fallbackFreshnessAge); + } - /// If available, the value in [HttpHeaders.lastModifiedHeader], represented - /// in [DateTime.millisecondsSinceEpoch] - final int? lastModifiedMilliseconds; + final lastModified = headers[HttpHeaders.lastModifiedHeader]; + final etag = headers[HttpHeaders.etagHeader]; + + return CachedMapTileMetadata( + staleAt: calculateStaleAt(), + lastModified: lastModified != null ? HttpDate.parse(lastModified) : null, + etag: etag, + ); + } + + /// The calculated time at which this tile becomes stale (UTC) + /// + /// Tile providers should use [isStale] to check whether a tile is stale, + /// instead of manually comparing this to the current timestamp. + /// + /// This may have been calculated based off an HTTP response's headers using + /// [CachedMapTileMetadata.fromHttpHeaders], or it may be custom. + final DateTime staleAt; + + /// If available, the value in [HttpHeaders.lastModifiedHeader] (UTC) + final DateTime? lastModified; /// If available, the value in [HttpHeaders.etagHeader] final String? etag; @@ -45,14 +97,13 @@ class CachedMapTileMetadata { bool get isStale => DateTime.timestamp().isAfter(staleAt); @override - int get hashCode => - Object.hash(staleAtMilliseconds, lastModifiedMilliseconds, etag); + int get hashCode => Object.hash(staleAt, lastModified, etag); @override bool operator ==(Object other) => identical(this, other) || (other is CachedMapTileMetadata && - staleAtMilliseconds == other.staleAtMilliseconds && - lastModifiedMilliseconds == other.lastModifiedMilliseconds && + staleAt == other.staleAt && + lastModified == other.lastModified && etag == other.etag); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 80a16f1f2..06aa195c7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -1,6 +1,5 @@ import 'dart:async'; import 'dart:io' show HttpHeaders, HttpDate, HttpStatus; // this is web safe! -import 'dart:math'; import 'dart:ui'; import 'package:flutter/foundation.dart'; @@ -171,58 +170,15 @@ class NetworkTileImageProvider extends ImageProvider { } } - // Create method to interact with cache + // Create method to write response to cache when applicable void cachePut({ required Uint8List? bytes, required Map headers, }) { if (useFallback || !cachingProvider.isSupported) return; - - final lastModified = headers[HttpHeaders.lastModifiedHeader]; - final etag = headers[HttpHeaders.etagHeader]; - - DateTime calculateStaleAt() { - final addToNow = DateTime.timestamp().add; - - if (headers[HttpHeaders.cacheControlHeader]?.toLowerCase() - case final cacheControl?) { - final maxAge = RegExp(r'max-age=(\d+)').firstMatch(cacheControl)?[1]; - - if (maxAge == null) { - if (headers[HttpHeaders.expiresHeader]?.toLowerCase() - case final expires?) { - return HttpDate.parse(expires); - } - - return addToNow(const Duration(days: 7)); - } - - if (headers[HttpHeaders.ageHeader] case final currentAge?) { - return addToNow( - Duration(seconds: int.parse(maxAge) - int.parse(currentAge)), - ); - } - - final estimatedAge = max( - 0, - DateTime.timestamp() - .difference(HttpDate.parse(headers[HttpHeaders.dateHeader]!)) - .inSeconds, - ); - return addToNow(Duration(seconds: int.parse(maxAge) - estimatedAge)); - } - - return addToNow(const Duration(days: 7)); - } - cachingProvider.putTile( url: resolvedUrl, - metadata: CachedMapTileMetadata( - staleAt: calculateStaleAt(), - lastModified: - lastModified != null ? HttpDate.parse(lastModified) : null, - etag: etag, - ), + metadata: CachedMapTileMetadata.fromHttpHeaders(headers), bytes: bytes, ); } From 7d0a1a74e367202436a0fa42e0e72b882a0a8c0c Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Sun, 15 Jun 2025 14:32:06 +0100 Subject: [PATCH 38/49] Added uninitialisation & cache deletion support Minor internal renaming --- example/pubspec.lock | 10 +-- .../built_in/built_in_caching_provider.dart | 36 +++++++++- .../caching/built_in/impl/native/native.dart | 65 +++++++++++++++---- .../workers/tile_and_size_monitor_writer.dart | 13 ++-- .../network/caching/built_in/impl/stub.dart | 8 ++- .../caching/built_in/impl/web/web.dart | 8 ++- 6 files changed, 114 insertions(+), 26 deletions(-) diff --git a/example/pubspec.lock b/example/pubspec.lock index b189034b9..cd5e3cfa9 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -464,10 +464,10 @@ packages: dependency: transitive description: name: test_api - sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" + sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd url: "https://pub.dev" source: hosted - version: "0.7.6" + version: "0.7.4" typed_data: dependency: transitive description: @@ -560,10 +560,10 @@ packages: dependency: transitive description: name: vector_math - sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b + sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" url: "https://pub.dev" source: hosted - version: "2.2.0" + version: "2.1.4" vm_service: dependency: transitive description: @@ -597,5 +597,5 @@ packages: source: hosted version: "1.1.0" sdks: - dart: ">=3.7.0 <4.0.0" + dart: ">=3.8.0 <4.0.0" flutter: ">=3.27.0" diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart index 5af52ce7d..e29b6fbf8 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/built_in_caching_provider.dart @@ -19,6 +19,8 @@ import 'package:uuid/uuid.dart'; /// This is enabled by default in flutter_map, when using the /// [NetworkTileProvider] (or cancellable version). /// +/// It is safe to use all public methods when running on web - they will noop. +/// /// For more information, see the online documentation. abstract interface class BuiltInMapCachingProvider implements MapCachingProvider { @@ -94,17 +96,49 @@ abstract interface class BuiltInMapCachingProvider overrideFreshAge == null || overrideFreshAge > Duration.zero, '`overrideFreshAge` must be greater than 0 or disabled', ); - return _instance ??= BuiltInMapCachingProviderImpl.createAndInitialise( + return _instance ??= BuiltInMapCachingProviderImpl.create( cacheDirectory: cacheDirectory, maxCacheSize: maxCacheSize, overrideFreshAge: overrideFreshAge, tileKeyGenerator: tileKeyGenerator ?? uuidTileKeyGenerator, readOnly: readOnly, + resetSingleton: () => _instance = null, ); } static BuiltInMapCachingProviderImpl? _instance; + /// Destroy this caching provider instance + /// + /// This means that all workers will be terminated and caching will be + /// unavailable until the next time + /// [BuiltInMapCachingProvider.getOrCreateInstance] is called (which may be + /// on the next tile load by default). + /// + /// If [deleteCache] is `true` (defaults to `false`), then the entire + /// `cacheDirectory` and its contents will be deleted. + /// + /// Completes when fully uninitialised. It is not necessary to wait for this + /// to complete before calling [BuiltInMapCachingProvider.getOrCreateInstance] + /// again (to create a new instance). + /// + /// --- + /// + /// It is usually safe to let the caching provider 'naturally' terminate with + /// the program when the system terminates the program after the app is + /// closed. Therefore, this method is not required to be called at the end of + /// the app's lifecycle. + /// + /// This method is provided to: + /// * allow cache provider's configuration to be changed + /// * allow the cache to be deleted from within the app + /// * facilitate testing + /// + /// Note that the cache may also be deleted when the program is not running + /// by deleteting the `cacheDirectory` - for example, by the user choosing to + /// clear the app's cache through the operating system (by default). + Future destroy({bool deleteCache = false}); + /// Default `tileKeyGenerator` which generates v5 UUIDs from input strings /// /// May be utilised in custom `tileKeyGenerator` implementations. diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index 683000b29..cdda1f4fb 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -21,15 +21,25 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final Duration? overrideFreshAge; final bool readOnly; + final void Function() resetSingleton; + @internal - BuiltInMapCachingProviderImpl.createAndInitialise({ + BuiltInMapCachingProviderImpl.create({ required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, required this.tileKeyGenerator, required this.readOnly, + required this.resetSingleton, }) { - // This should only be called/constructed once + Future Function()? killWorker; + bool earlyUninitialiseRequested = false; + _killWorker = () { + if (killWorker != null) return killWorker!(); + earlyUninitialiseRequested = true; + return _cacheDirectoryPathReady.future; + }; + () async { final cacheDirectoryPath = p.join( cacheDirectory ?? (await getApplicationCacheDirectory()).absolute.path, @@ -43,27 +53,31 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _cacheDirectoryPath = cacheDirectoryPath; _cacheDirectoryPathReady.complete(cacheDirectoryPath); - SendPort? writerPort; - final writerPortReady = Completer(); + if (earlyUninitialiseRequested) return; + + SendPort? workerPort; + final workerPortReady = Completer(); // We can't send messages until the worker has set-up all the size // monitoring (and potentially run the reducer) if necessary // Reading does not depend on this. - void sendMessageToWriter(Object message) { - if (writerPort != null) return writerPort.send(message); - writerPortReady.future.then((port) => port.send(message)); + void sendMessageToWorker(Object? message) { + if (workerPort != null) return workerPort!.send(message); + workerPortReady.future.then((port) => port.send(message)); } - _writeTileFile = (path, metadata, tileBytes) => sendMessageToWriter( + _writeTileFile = (path, metadata, tileBytes) => sendMessageToWorker( (path: path, metadata: metadata, tileBytes: tileBytes), ); - _reportReadFailure = () => sendMessageToWriter(false); + _reportReadFailure = () => sendMessageToWorker(false); + + final workerReceivePort = ReceivePort(); + final workerExited = Completer(); - final writerReceivePort = ReceivePort(); await Isolate.spawn( tileAndSizeMonitorWriterWorker, ( - port: writerReceivePort.sendPort, + port: workerReceivePort.sendPort, cacheDirectoryPath: cacheDirectoryPath, sizeMonitorFilePath: sizeMonitorFilePath, maxCacheSize: maxCacheSize, @@ -71,8 +85,23 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { debugName: '[flutter_map: cache] Tile & Size Monitor Writer', ); - writerPort = await writerReceivePort.first as SendPort; - writerPortReady.complete(writerPort); + workerReceivePort.listen( + (response) { + if (response is SendPort && workerPort == null) { + return workerPortReady.complete(workerPort = response); + } + if (response == null) { + return workerReceivePort.close(); + } + + throw UnsupportedError('Response was in unknown format'); + }, + onDone: workerExited.complete, + ); + killWorker = () { + sendMessageToWorker(null); + return workerExited.future; + }; }(); } @@ -86,10 +115,20 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { ) _writeTileFile; late final void Function() _reportReadFailure; // See `disableSizeMonitor` in worker + late final Future Function() _killWorker; @override bool get isSupported => true; + @override + Future destroy({bool deleteCache = false}) async { + resetSingleton(); + await _killWorker(); + if (deleteCache) { + await Directory(_cacheDirectoryPath!).delete(recursive: true); + } + } + @override Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( String url, diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index 467a6c8fe..f60c109ce 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -308,12 +308,15 @@ Future tileAndSizeMonitorWriterWorker( :final Uint8List? tileBytes, )) { writeTile(path: path, metadata: metadata, tileBytes: tileBytes); - continue; - } - if (val is bool && !val) { + } else if (val == false) { disableSizeMonitor(); - continue; + } else if (val == null) { + receivePort.close(); + } else { + throw UnsupportedError('Command was in unknown format'); } - throw UnsupportedError('Command was in unknown format'); } + + sizeMonitor?.closeSync(); + Isolate.exit(input.port); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 66c55d6d2..8a24496a1 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -15,15 +15,21 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { final Duration? overrideFreshAge; final bool readOnly; + final void Function() resetSingleton; + @internal - const BuiltInMapCachingProviderImpl.createAndInitialise({ + const BuiltInMapCachingProviderImpl.create({ required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, required this.tileKeyGenerator, required this.readOnly, + required this.resetSingleton, }); + @override + external Future destroy({bool deleteCache = false}); + @override external bool get isSupported; diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart index e3a391a34..5d2fc3252 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/web/web.dart @@ -11,12 +11,18 @@ class BuiltInMapCachingProviderImpl final Duration? overrideFreshAge; final bool readOnly; + final void Function() resetSingleton; + @internal - const BuiltInMapCachingProviderImpl.createAndInitialise({ + const BuiltInMapCachingProviderImpl.create({ required this.cacheDirectory, required this.maxCacheSize, required this.overrideFreshAge, required this.tileKeyGenerator, required this.readOnly, + required this.resetSingleton, }); + + @override + Future destroy({bool deleteCache = false}) => Future.value(); } From 175386619f35a5e660ab13f9a2ae78053c1c52ce Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 8 Jul 2025 20:09:26 +0100 Subject: [PATCH 39/49] Integrated abortable HTTP requests ('package:http ^1.5.0-beta') into image provider Updated examples to fit planned removal of CNTP --- example/lib/main.dart | 6 +- example/lib/misc/tile_providers.dart | 4 - .../lib/pages/abort_unnecessary_requests.dart | 71 ++++++++++++++++++ .../lib/pages/animated_map_controller.dart | 2 - .../lib/pages/cancellable_tile_provider.dart | 74 ------------------- example/lib/pages/fallback_url_page.dart | 2 - example/lib/pages/reset_tile_layer.dart | 2 - example/lib/pages/tile_builder.dart | 2 - .../lib/pages/tile_loading_error_handle.dart | 3 +- example/lib/widgets/drawer/menu_drawer.dart | 6 +- example/pubspec.lock | 64 +++++----------- example/pubspec.yaml | 1 - .../caching/built_in/impl/native/README.md | 6 +- .../image_provider/image_provider.dart | 11 +-- .../tile_provider/network/tile_provider.dart | 38 +++++++++- pubspec.yaml | 2 +- .../network_image_provider_test.dart | 4 +- 17 files changed, 145 insertions(+), 153 deletions(-) create mode 100644 example/lib/pages/abort_unnecessary_requests.dart delete mode 100644 example/lib/pages/cancellable_tile_provider.dart diff --git a/example/lib/main.dart b/example/lib/main.dart index 05d447137..44d2d0d84 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,7 +1,7 @@ import 'package:flutter/material.dart'; +import 'package:flutter_map_example/pages/abort_unnecessary_requests.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; -import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; import 'package:flutter_map_example/pages/circle.dart'; import 'package:flutter_map_example/pages/debouncing_tile_update_transformer.dart'; import 'package:flutter_map_example/pages/epsg3996_crs.dart'; @@ -52,8 +52,8 @@ class MyApp extends StatelessWidget { ), home: const HomePage(), routes: { - CancellableTileProviderPage.route: (context) => - const CancellableTileProviderPage(), + AbortUnnecessaryRequestsPage.route: (context) => + const AbortUnnecessaryRequestsPage(), PolylinePage.route: (context) => const PolylinePage(), SingleWorldPolysPage.route: (context) => const SingleWorldPolysPage(), PolylinePerfStressPage.route: (context) => diff --git a/example/lib/misc/tile_providers.dart b/example/lib/misc/tile_providers.dart index e0ea30153..7c7f2aee1 100644 --- a/example/lib/misc/tile_providers.dart +++ b/example/lib/misc/tile_providers.dart @@ -1,15 +1,11 @@ import 'package:flutter_map/flutter_map.dart'; import 'package:http/http.dart'; import 'package:http/retry.dart'; -//import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; final httpClient = RetryClient(Client()); TileLayer get openStreetMapTileLayer => TileLayer( urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', - // Use the recommended flutter_map_cancellable_tile_provider package to - // support the cancellation of loading tiles. - // TODO: change tileProvider: NetworkTileProvider(httpClient: httpClient), ); diff --git a/example/lib/pages/abort_unnecessary_requests.dart b/example/lib/pages/abort_unnecessary_requests.dart new file mode 100644 index 000000000..899fd96fa --- /dev/null +++ b/example/lib/pages/abort_unnecessary_requests.dart @@ -0,0 +1,71 @@ +import 'package:flutter/material.dart'; +import 'package:flutter_map/flutter_map.dart'; +import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; +import 'package:flutter_map_example/widgets/notice_banner.dart'; +import 'package:latlong2/latlong.dart'; + +class AbortUnnecessaryRequestsPage extends StatefulWidget { + static const String route = '/abort_unnecessary_requests_page'; + + const AbortUnnecessaryRequestsPage({super.key}); + + @override + State createState() => + _AbortUnnecessaryRequestsPage(); +} + +class _AbortUnnecessaryRequestsPage + extends State { + bool _abortingEnabled = true; + + @override + Widget build(BuildContext context) { + return Scaffold( + appBar: AppBar(title: const Text('Abort Unnecessary Requests')), + drawer: const MenuDrawer(AbortUnnecessaryRequestsPage.route), + body: Column( + children: [ + Padding( + padding: const EdgeInsets.symmetric(horizontal: 8), + child: Center( + child: Switch.adaptive( + value: _abortingEnabled, + onChanged: (value) => setState(() => _abortingEnabled = value), + ), + ), + ), + const NoticeBanner.recommendation( + text: 'Since v8.2.0, in-flight HTTP requests for tiles which are ' + 'no longer displayed are cancelled by default.', + url: 'https://docs.fleaflet.dev/layers/tile-layer/tile-providers', + sizeTransition: 870, + ), + Expanded( + child: FlutterMap( + options: MapOptions( + initialCenter: const LatLng(51.5, -0.09), + initialZoom: 5, + cameraConstraint: CameraConstraint.contain( + bounds: LatLngBounds( + const LatLng(-90, -180), + const LatLng(90, 180), + ), + ), + ), + children: [ + TileLayer( + key: ValueKey(_abortingEnabled), + urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', + userAgentPackageName: 'dev.fleaflet.flutter_map.example', + tileProvider: NetworkTileProvider( + abortUnneededRequests: _abortingEnabled, + ), + ), + ], + ), + ), + ], + ), + ); + } +} diff --git a/example/lib/pages/animated_map_controller.dart b/example/lib/pages/animated_map_controller.dart index b4cc4fdb6..97a6f7e61 100644 --- a/example/lib/pages/animated_map_controller.dart +++ b/example/lib/pages/animated_map_controller.dart @@ -1,6 +1,5 @@ import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:latlong2/latlong.dart'; @@ -182,7 +181,6 @@ class AnimatedMapControllerPageState extends State urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', - tileProvider: CancellableNetworkTileProvider(), tileUpdateTransformer: _animatedMoveTileUpdateTransformer, ), const MarkerLayer(markers: _markers), diff --git a/example/lib/pages/cancellable_tile_provider.dart b/example/lib/pages/cancellable_tile_provider.dart deleted file mode 100644 index aac67c461..000000000 --- a/example/lib/pages/cancellable_tile_provider.dart +++ /dev/null @@ -1,74 +0,0 @@ -import 'package:flutter/material.dart'; -import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; -import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; -import 'package:flutter_map_example/widgets/notice_banner.dart'; -import 'package:latlong2/latlong.dart'; - -class CancellableTileProviderPage extends StatefulWidget { - static const String route = '/cancellable_tile_provider_page'; - - const CancellableTileProviderPage({super.key}); - - @override - State createState() => - _CancellableTileProviderPageState(); -} - -class _CancellableTileProviderPageState - extends State { - bool _providerEnabled = false; - - @override - Widget build(BuildContext context) { - return Scaffold( - appBar: AppBar(title: const Text('Cancellable Tile Provider')), - drawer: const MenuDrawer(CancellableTileProviderPage.route), - body: Column( - children: [ - Padding( - padding: const EdgeInsets.symmetric(horizontal: 8), - child: ConstrainedBox( - constraints: const BoxConstraints(maxWidth: 420), - child: SwitchListTile.adaptive( - title: const Text('Use CancellableNetworkTileProvider'), - value: _providerEnabled, - onChanged: (value) => setState(() => _providerEnabled = value), - ), - ), - ), - const NoticeBanner.recommendation( - text: - 'This tile provider cancels unnecessary HTTP requests, which can help performance (especially on the web)', - url: - 'https://docs.fleaflet.dev/layers/tile-layer/tile-providers#cancellablenetworktileprovider', - sizeTransition: 905, - ), - Expanded( - child: FlutterMap( - options: MapOptions( - initialCenter: const LatLng(51.5, -0.09), - initialZoom: 5, - cameraConstraint: CameraConstraint.contain( - bounds: LatLngBounds( - const LatLng(-90, -180), - const LatLng(90, 180), - ), - ), - ), - children: [ - TileLayer( - urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', - userAgentPackageName: 'dev.fleaflet.flutter_map.example', - tileProvider: _providerEnabled - ? CancellableNetworkTileProvider() - : null, - ), - ], - ), - ), - ], - ), - ); - } -} diff --git a/example/lib/pages/fallback_url_page.dart b/example/lib/pages/fallback_url_page.dart index 49289772c..908a2e6fe 100644 --- a/example/lib/pages/fallback_url_page.dart +++ b/example/lib/pages/fallback_url_page.dart @@ -1,6 +1,5 @@ import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_map_example/widgets/notice_banner.dart'; import 'package:latlong2/latlong.dart'; @@ -44,7 +43,6 @@ class FallbackUrlPage extends StatelessWidget { 'https://not-a-real-provider-url.local/{z}/{x}/{y}.png', fallbackUrl: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', - tileProvider: CancellableNetworkTileProvider(), ), ], ), diff --git a/example/lib/pages/reset_tile_layer.dart b/example/lib/pages/reset_tile_layer.dart index e30abb7eb..8ebda37bf 100644 --- a/example/lib/pages/reset_tile_layer.dart +++ b/example/lib/pages/reset_tile_layer.dart @@ -2,7 +2,6 @@ import 'dart:async'; import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:latlong2/latlong.dart'; @@ -63,7 +62,6 @@ class ResetTileLayerPageState extends State { urlTemplate: layerToggle ? layer1 : layer2, subdomains: layerToggle ? const [] : const ['a', 'b', 'c'], userAgentPackageName: 'dev.fleaflet.flutter_map.example', - tileProvider: CancellableNetworkTileProvider(), ), const MarkerLayer( markers: [ diff --git a/example/lib/pages/tile_builder.dart b/example/lib/pages/tile_builder.dart index 25ddf4dc4..1054dd732 100644 --- a/example/lib/pages/tile_builder.dart +++ b/example/lib/pages/tile_builder.dart @@ -1,6 +1,5 @@ import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:latlong2/latlong.dart'; @@ -120,7 +119,6 @@ class TileBuilderPageState extends State { urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', - tileProvider: CancellableNetworkTileProvider(), tileBuilder: tileBuilder, ), ), diff --git a/example/lib/pages/tile_loading_error_handle.dart b/example/lib/pages/tile_loading_error_handle.dart index 4b328d0ea..5a34f8af2 100644 --- a/example/lib/pages/tile_loading_error_handle.dart +++ b/example/lib/pages/tile_loading_error_handle.dart @@ -1,6 +1,5 @@ import 'package:flutter/material.dart'; import 'package:flutter_map/flutter_map.dart'; -import 'package:flutter_map_cancellable_tile_provider/flutter_map_cancellable_tile_provider.dart'; import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:latlong2/latlong.dart'; @@ -67,7 +66,7 @@ class TileLoadingErrorHandleState extends State { // or use the recommended tile provider tileProvider: _simulateTileLoadErrors ? _SimulateErrorsTileProvider() - : CancellableNetworkTileProvider(), + : NetworkTileProvider(), ), ], ); diff --git a/example/lib/widgets/drawer/menu_drawer.dart b/example/lib/widgets/drawer/menu_drawer.dart index cbb726b5b..4548cfcb8 100644 --- a/example/lib/widgets/drawer/menu_drawer.dart +++ b/example/lib/widgets/drawer/menu_drawer.dart @@ -1,8 +1,8 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; +import 'package:flutter_map_example/pages/abort_unnecessary_requests.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; -import 'package:flutter_map_example/pages/cancellable_tile_provider.dart'; import 'package:flutter_map_example/pages/circle.dart'; import 'package:flutter_map_example/pages/debouncing_tile_update_transformer.dart'; import 'package:flutter_map_example/pages/epsg3996_crs.dart'; @@ -162,8 +162,8 @@ class MenuDrawer extends StatelessWidget { currentRoute: currentRoute, ), MenuItemWidget( - caption: 'Cancellable Tile Provider', - routeName: CancellableTileProviderPage.route, + caption: 'Abort Unnecessary Requests', + routeName: AbortUnnecessaryRequestsPage.route, currentRoute: currentRoute, ), MenuItemWidget( diff --git a/example/pubspec.lock b/example/pubspec.lock index cd5e3cfa9..9ddf3867b 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -57,22 +57,6 @@ packages: url: "https://pub.dev" source: hosted version: "1.2.0" - dio: - dependency: transitive - description: - name: dio - sha256: "253a18bbd4851fecba42f7343a1df3a9a4c1d31a2c1b37e221086b4fa8c8dbc9" - url: "https://pub.dev" - source: hosted - version: "5.8.0+1" - dio_web_adapter: - dependency: transitive - description: - name: dio_web_adapter - sha256: "7586e476d70caecaf1686d21eee7247ea43ef5c345eab9e0cc3583ff13378d78" - url: "https://pub.dev" - source: hosted - version: "2.1.1" fake_async: dependency: transitive description: @@ -125,14 +109,6 @@ packages: relative: true source: path version: "8.1.1" - flutter_map_cancellable_tile_provider: - dependency: "direct main" - description: - name: flutter_map_cancellable_tile_provider - sha256: "801760c104a3cfd9268cda7c9b1241223247e8182613a7e060ef4ffc0d825ac8" - url: "https://pub.dev" - source: hosted - version: "3.1.0" flutter_test: dependency: "direct dev" description: flutter @@ -155,10 +131,10 @@ packages: dependency: "direct main" description: name: http - sha256: fe7ab022b76f3034adc518fb6ea04a82387620e19977665ea18d30a1cf43442f + sha256: "559ab0d950643c9a04512f4c4bd1c435dcb5af1aa9c848f3bf5867347044328a" url: "https://pub.dev" source: hosted - version: "1.3.0" + version: "1.5.0-beta" http_parser: dependency: transitive description: @@ -227,10 +203,10 @@ packages: dependency: transitive description: name: logger - sha256: be4b23575aac7ebf01f225a241eb7f6b5641eeaf43c6a8613510fc2f8cf187d1 + sha256: "2621da01aabaf223f8f961e751f2c943dbb374dc3559b982f200ccedadaa6999" url: "https://pub.dev" source: hosted - version: "2.5.0" + version: "2.6.0" matcher: dependency: transitive description: @@ -355,18 +331,18 @@ packages: dependency: "direct main" description: name: shared_preferences - sha256: "846849e3e9b68f3ef4b60c60cf4b3e02e9321bc7f4d8c4692cf87ffa82fc8a3a" + sha256: "6e8bf70b7fef813df4e9a36f658ac46d107db4b4cfe1048b477d4e453a8159f5" url: "https://pub.dev" source: hosted - version: "2.5.2" + version: "2.5.3" shared_preferences_android: dependency: transitive description: name: shared_preferences_android - sha256: "9f9f3d372d4304723e6136663bb291c0b93f5e4c8a4a6314347f481a33bda2b1" + sha256: "20cbd561f743a342c76c151d6ddb93a9ce6005751e7aa458baad3858bfbfb6ac" url: "https://pub.dev" source: hosted - version: "2.4.7" + version: "2.4.10" shared_preferences_foundation: dependency: transitive description: @@ -464,10 +440,10 @@ packages: dependency: transitive description: name: test_api - sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + sha256: "522f00f556e73044315fa4585ec3270f1808a4b186c936e612cab0b565ff1e00" url: "https://pub.dev" source: hosted - version: "0.7.4" + version: "0.7.6" typed_data: dependency: transitive description: @@ -496,18 +472,18 @@ packages: dependency: transitive description: name: url_launcher_android - sha256: "6fc2f56536ee873eeb867ad176ae15f304ccccc357848b351f6f0d8d4a40d193" + sha256: "8582d7f6fe14d2652b4c45c9b6c14c0b678c2af2d083a11b604caeba51930d79" url: "https://pub.dev" source: hosted - version: "6.3.14" + version: "6.3.16" url_launcher_ios: dependency: transitive description: name: url_launcher_ios - sha256: "16a513b6c12bb419304e72ea0ae2ab4fed569920d1c7cb850263fe3acc824626" + sha256: "7f2022359d4c099eea7df3fdf739f7d3d3b9faf3166fb1dd390775176e0b76cb" url: "https://pub.dev" source: hosted - version: "6.3.2" + version: "6.3.3" url_launcher_linux: dependency: transitive description: @@ -536,10 +512,10 @@ packages: dependency: transitive description: name: url_launcher_web - sha256: "3ba963161bd0fe395917ba881d320b9c4f6dd3c4a233da62ab18a5025c85f1e9" + sha256: "4bd2b7b4dc4d4d0b94e5babfffbca8eac1a126c7f3d6ecbc1a11013faa3abba2" url: "https://pub.dev" source: hosted - version: "2.4.0" + version: "2.4.1" url_launcher_windows: dependency: transitive description: @@ -560,18 +536,18 @@ packages: dependency: transitive description: name: vector_math - sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.2.0" vm_service: dependency: transitive description: name: vm_service - sha256: ddfa8d30d89985b96407efce8acbdd124701f96741f2d981ca860662f1c0dc02 + sha256: "45caa6c5917fa127b5dbcfbd1fa60b14e583afdc08bfc96dda38886ca252eb60" url: "https://pub.dev" source: hosted - version: "15.0.0" + version: "15.0.2" web: dependency: transitive description: diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 20050a09d..a8011e6b1 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -11,7 +11,6 @@ dependencies: flutter: sdk: flutter flutter_map: - flutter_map_cancellable_tile_provider: ^3.1.0 flutter_web_plugins: sdk: flutter http: ^1.2.2 diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md index 2088d0b81..1a67deba7 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -37,4 +37,8 @@ The format of the header is as follows: Contains an 8-byte unsigned integer (Uint64), representing the size of all tiles (including metadata) stored in the cache in bytes. -Named as 'sizeMonitor.bin'. +This size monitor should stay in sync with the actual size of the cache - as calculating the cache size using I/O operations is expensive and slow. Therefore, if it might go out of sync with reality for any reason (such as a detected read failure, indicating a corrupted tile likely of a different length to what is accounted for in the size monitor), then it must be disabled. Since it is only used on startup, it is recalculated using the expensive method on the next startup. + +Whilst it is being calculated (which should happen on the first initialisation of the cache, or when required as above), writes must be delayed. Reads can still occur. + +Named 'sizeMonitor.bin'. diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 06aa195c7..2708d6324 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -107,7 +107,6 @@ class NetworkTileImageProvider extends ImageProvider { ); } - // TODO: Support cancellation Future _loadImage( NetworkTileImageProvider key, StreamSink chunkEvents, @@ -135,8 +134,7 @@ class NetworkTileImageProvider extends ImageProvider { Future<({Uint8List bytes, StreamedResponse response})> get({ Map? additionalHeaders, }) async { - // final request = AbortableRequest('GET', uri, abortTrigger: abortTrigger); - final request = Request('GET', uri); + final request = AbortableRequest('GET', uri, abortTrigger: abortTrigger); request.headers.addAll(headers); if (additionalHeaders != null) request.headers.addAll(additionalHeaders); @@ -269,15 +267,12 @@ class NetworkTileImageProvider extends ImageProvider { stackTrace, ); } - } - /* on AbortedRequest { + } on RequestAbortedException { // This is a planned exception, we just quit silently evict(); - chunkEvents.close(); return await decodeBytes(TileProvider.transparentImage); - } */ - on ClientException catch (err) { + } on ClientException catch (err) { // This could be a wide range of issues, potentially ours, potentially // network, etc. diff --git a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index 8526529bb..6e48b11cd 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -36,6 +36,7 @@ class NetworkTileProvider extends TileProvider { Client? httpClient, this.silenceExceptions = false, this.attemptDecodeOfHttpErrorResponses = true, + this.abortUnneededRequests = true, this.cachingProvider, }) : _isInternallyCreatedClient = httpClient == null, _httpClient = httpClient ?? RetryClient(Client()); @@ -55,6 +56,36 @@ class NetworkTileProvider extends TileProvider { /// Defaults to `true`. final bool attemptDecodeOfHttpErrorResponses; + /// Whether to abort HTTP requests for tiles that are no longer needed + /// + /// For example, tiles may be pruned from an intermediate zoom level during a + /// user's fast zoom. When disabled, the request for each tile that has been + /// pruned still needs to complete and be processed. When enabled, those + /// tiles' requests can be aborted before they are fully loaded. + /// + /// > [!TIP] + /// > This functionality replaces the 'flutter_map_cancellable_tile_provider' + /// > package. + /// + /// This may have multiple advantages: + /// * It may improve tile loading speeds + /// * It may reduce the user's consumption of a metered network connection + /// * It may reduce the user's consumption of storage capacity in the + /// [cachingProvider] + /// * It may reduce unnecessary tile requests, reducing tile server costs + /// * It may negligibly improve app performance in general + /// + /// This is likely to be more effective on web platforms (where + /// `BrowserClient` is used) and with clients or servers with limited numbers + /// of simultaneous connections or slow traffic speeds, but is also likely to + /// have a positive effect everywhere. If an HTTP client is used which does + /// not support the standard method of request aborting, this has no effect. + /// + /// Defaults to `true`. It is recommended to enable this functionality, unless + /// you suspect it is causing problems; in this case, please report the issue + /// to flutter_map. + final bool abortUnneededRequests; + /// Caching provider used to get cached tiles /// /// See online documentation for more information about built-in caching. @@ -75,19 +106,20 @@ class NetworkTileProvider extends TileProvider { final bool _isInternallyCreatedClient; @override - bool get supportsCancelLoading => false; + bool get supportsCancelLoading => true; @override - ImageProvider getImage( + ImageProvider getImageWithCancelLoadingSupport( TileCoordinates coordinates, TileLayer options, + Future cancelLoading, ) => NetworkTileImageProvider( url: getTileUrl(coordinates, options), fallbackUrl: getTileFallbackUrl(coordinates, options), headers: headers, httpClient: _httpClient, - abortTrigger: null, + abortTrigger: abortUnneededRequests ? cancelLoading : null, silenceExceptions: silenceExceptions, attemptDecodeOfHttpErrorResponses: attemptDecodeOfHttpErrorResponses, cachingProvider: cachingProvider, diff --git a/pubspec.yaml b/pubspec.yaml index 87a20ceeb..ff86379d9 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -31,7 +31,7 @@ dependencies: dart_earcut: ^1.1.0 flutter: sdk: flutter - http: ^1.2.1 + http: ^1.5.0-beta latlong2: ^0.9.1 logger: ^2.0.0 meta: ^1.11.0 diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index c8c6a32db..badb935d8 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -41,6 +41,8 @@ Uri randomUrl({bool fallback = false}) { } } +// TODO: Write tests to test aborting? + void main() { const headers = { 'user-agent': 'flutter_map', @@ -70,7 +72,7 @@ void main() { setUpAll(() { // Ensure the Mock library has example values for Uri. registerFallbackValue(Uri()); - registerFallbackValue(Request('GET', Uri())); // TODO: Abortable? + registerFallbackValue(Request('GET', Uri())); }); // We expect a request to be made to the correct URL with the appropriate headers. From 4489316ca1b06c1dfef45d3fb01073d31736171f Mon Sep 17 00:00:00 2001 From: Luka S Date: Tue, 8 Jul 2025 22:07:12 +0100 Subject: [PATCH 40/49] Apply 'terminating period' suggestions from code review Co-authored-by: Joscha Eckert <34318751+josxha@users.noreply.github.com> --- .../native/workers/tile_and_size_monitor_writer.dart | 2 +- .../tile_provider/network/tile_provider.dart | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index f60c109ce..7eaac477d 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -11,7 +11,7 @@ import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; /// Isolate worker which writes tile files, maintains the size monitor, and -/// (if necessary) starts the size reducer +/// (if necessary) starts the size reducer. /// /// Follows the storage spec described in README.md. @internal diff --git a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index 6e48b11cd..69e9ea2ca 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -6,7 +6,7 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/image_pro import 'package:http/http.dart'; import 'package:http/retry.dart'; -/// [TileProvider] to fetch tiles from the network +/// [TileProvider] to fetch tiles from the network. /// /// By default, a [RetryClient] is used to retry failed requests. 'dart:http' /// or 'dart:io' might be needed to override this. @@ -19,7 +19,7 @@ import 'package:http/retry.dart'; /// HTTP requests on the web is /// [not yet supported in Dart](https://github.com/dart-lang/http/issues/424). class NetworkTileProvider extends TileProvider { - /// [TileProvider] to fetch tiles from the network + /// [TileProvider] to fetch tiles from the network. /// /// By default, a [RetryClient] is used to retry failed requests. 'dart:http' /// or 'dart:io' might be needed to override this. @@ -42,7 +42,7 @@ class NetworkTileProvider extends TileProvider { _httpClient = httpClient ?? RetryClient(Client()); /// Whether to ignore exceptions and errors that occur whilst fetching tiles - /// over the network, and just return a transparent tile + /// over the network, and just return a transparent tile. /// /// Defaults to `false`. final bool silenceExceptions; @@ -56,7 +56,7 @@ class NetworkTileProvider extends TileProvider { /// Defaults to `true`. final bool attemptDecodeOfHttpErrorResponses; - /// Whether to abort HTTP requests for tiles that are no longer needed + /// Whether to abort HTTP requests for tiles that are no longer needed. /// /// For example, tiles may be pruned from an intermediate zoom level during a /// user's fast zoom. When disabled, the request for each tile that has been @@ -86,7 +86,7 @@ class NetworkTileProvider extends TileProvider { /// to flutter_map. final bool abortUnneededRequests; - /// Caching provider used to get cached tiles + /// Caching provider used to get cached tiles. /// /// See online documentation for more information about built-in caching. /// From f519ee234a4450f82fd52fc8b611b8c4ed63efdc Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 8 Jul 2025 22:12:08 +0100 Subject: [PATCH 41/49] Rename "unneeded" & "unnecessary" to "obsolete" --- example/lib/main.dart | 6 +++--- ...ests.dart => abort_obsolete_requests.dart} | 19 +++++++++---------- example/lib/widgets/drawer/menu_drawer.dart | 6 +++--- .../tile_provider/network/tile_provider.dart | 10 +++++----- 4 files changed, 20 insertions(+), 21 deletions(-) rename example/lib/pages/{abort_unnecessary_requests.dart => abort_obsolete_requests.dart} (76%) diff --git a/example/lib/main.dart b/example/lib/main.dart index 44d2d0d84..969ef5af1 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -1,5 +1,5 @@ import 'package:flutter/material.dart'; -import 'package:flutter_map_example/pages/abort_unnecessary_requests.dart'; +import 'package:flutter_map_example/pages/abort_obsolete_requests.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/circle.dart'; @@ -52,8 +52,8 @@ class MyApp extends StatelessWidget { ), home: const HomePage(), routes: { - AbortUnnecessaryRequestsPage.route: (context) => - const AbortUnnecessaryRequestsPage(), + AbortObsoleteRequestsPage.route: (context) => + const AbortObsoleteRequestsPage(), PolylinePage.route: (context) => const PolylinePage(), SingleWorldPolysPage.route: (context) => const SingleWorldPolysPage(), PolylinePerfStressPage.route: (context) => diff --git a/example/lib/pages/abort_unnecessary_requests.dart b/example/lib/pages/abort_obsolete_requests.dart similarity index 76% rename from example/lib/pages/abort_unnecessary_requests.dart rename to example/lib/pages/abort_obsolete_requests.dart index 899fd96fa..f96d8f97a 100644 --- a/example/lib/pages/abort_unnecessary_requests.dart +++ b/example/lib/pages/abort_obsolete_requests.dart @@ -4,25 +4,24 @@ import 'package:flutter_map_example/widgets/drawer/menu_drawer.dart'; import 'package:flutter_map_example/widgets/notice_banner.dart'; import 'package:latlong2/latlong.dart'; -class AbortUnnecessaryRequestsPage extends StatefulWidget { - static const String route = '/abort_unnecessary_requests_page'; +class AbortObsoleteRequestsPage extends StatefulWidget { + static const String route = '/abort_obsolete_requests_page'; - const AbortUnnecessaryRequestsPage({super.key}); + const AbortObsoleteRequestsPage({super.key}); @override - State createState() => + State createState() => _AbortUnnecessaryRequestsPage(); } -class _AbortUnnecessaryRequestsPage - extends State { +class _AbortUnnecessaryRequestsPage extends State { bool _abortingEnabled = true; @override Widget build(BuildContext context) { return Scaffold( - appBar: AppBar(title: const Text('Abort Unnecessary Requests')), - drawer: const MenuDrawer(AbortUnnecessaryRequestsPage.route), + appBar: AppBar(title: const Text('Abort Obsolete Requests')), + drawer: const MenuDrawer(AbortObsoleteRequestsPage.route), body: Column( children: [ Padding( @@ -36,7 +35,7 @@ class _AbortUnnecessaryRequestsPage ), const NoticeBanner.recommendation( text: 'Since v8.2.0, in-flight HTTP requests for tiles which are ' - 'no longer displayed are cancelled by default.', + 'no longer displayed are aborted by default.', url: 'https://docs.fleaflet.dev/layers/tile-layer/tile-providers', sizeTransition: 870, ), @@ -58,7 +57,7 @@ class _AbortUnnecessaryRequestsPage urlTemplate: 'https://tile.openstreetmap.org/{z}/{x}/{y}.png', userAgentPackageName: 'dev.fleaflet.flutter_map.example', tileProvider: NetworkTileProvider( - abortUnneededRequests: _abortingEnabled, + abortObsoleteRequests: _abortingEnabled, ), ), ], diff --git a/example/lib/widgets/drawer/menu_drawer.dart b/example/lib/widgets/drawer/menu_drawer.dart index 4548cfcb8..b150476e9 100644 --- a/example/lib/widgets/drawer/menu_drawer.dart +++ b/example/lib/widgets/drawer/menu_drawer.dart @@ -1,6 +1,6 @@ import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; -import 'package:flutter_map_example/pages/abort_unnecessary_requests.dart'; +import 'package:flutter_map_example/pages/abort_obsolete_requests.dart'; import 'package:flutter_map_example/pages/animated_map_controller.dart'; import 'package:flutter_map_example/pages/bundled_offline_map.dart'; import 'package:flutter_map_example/pages/circle.dart'; @@ -162,8 +162,8 @@ class MenuDrawer extends StatelessWidget { currentRoute: currentRoute, ), MenuItemWidget( - caption: 'Abort Unnecessary Requests', - routeName: AbortUnnecessaryRequestsPage.route, + caption: 'Abort Obsolete Requests', + routeName: AbortObsoleteRequestsPage.route, currentRoute: currentRoute, ), MenuItemWidget( diff --git a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart index 69e9ea2ca..3a678edc3 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/tile_provider.dart @@ -36,7 +36,7 @@ class NetworkTileProvider extends TileProvider { Client? httpClient, this.silenceExceptions = false, this.attemptDecodeOfHttpErrorResponses = true, - this.abortUnneededRequests = true, + this.abortObsoleteRequests = true, this.cachingProvider, }) : _isInternallyCreatedClient = httpClient == null, _httpClient = httpClient ?? RetryClient(Client()); @@ -56,7 +56,7 @@ class NetworkTileProvider extends TileProvider { /// Defaults to `true`. final bool attemptDecodeOfHttpErrorResponses; - /// Whether to abort HTTP requests for tiles that are no longer needed. + /// Whether to abort HTTP requests for tiles that will no longer be displayed. /// /// For example, tiles may be pruned from an intermediate zoom level during a /// user's fast zoom. When disabled, the request for each tile that has been @@ -65,7 +65,7 @@ class NetworkTileProvider extends TileProvider { /// /// > [!TIP] /// > This functionality replaces the 'flutter_map_cancellable_tile_provider' - /// > package. + /// > plugin package. /// /// This may have multiple advantages: /// * It may improve tile loading speeds @@ -84,7 +84,7 @@ class NetworkTileProvider extends TileProvider { /// Defaults to `true`. It is recommended to enable this functionality, unless /// you suspect it is causing problems; in this case, please report the issue /// to flutter_map. - final bool abortUnneededRequests; + final bool abortObsoleteRequests; /// Caching provider used to get cached tiles. /// @@ -119,7 +119,7 @@ class NetworkTileProvider extends TileProvider { fallbackUrl: getTileFallbackUrl(coordinates, options), headers: headers, httpClient: _httpClient, - abortTrigger: abortUnneededRequests ? cancelLoading : null, + abortTrigger: abortObsoleteRequests ? cancelLoading : null, silenceExceptions: silenceExceptions, attemptDecodeOfHttpErrorResponses: attemptDecodeOfHttpErrorResponses, cachingProvider: cachingProvider, From 92dac52b0ceed3ffad85fdd2a67f3d4118e462f1 Mon Sep 17 00:00:00 2001 From: Luka S Date: Tue, 8 Jul 2025 22:19:58 +0100 Subject: [PATCH 42/49] Simplify image provider's `hashCode` implementation Co-authored-by: Joscha Eckert <34318751+josxha@users.noreply.github.com> --- .../tile_provider/network/image_provider/image_provider.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 2708d6324..fc12889f6 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -322,5 +322,5 @@ class NetworkTileImageProvider extends ImageProvider { @override int get hashCode => - Object.hashAll([url, if (fallbackUrl != null) fallbackUrl]); + Object.hash(url, fallbackUrl); } From c9ff03bb8f9ddf3d7c9f8a8a9f44e690e49089a9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 8 Jul 2025 22:23:52 +0100 Subject: [PATCH 43/49] Fix formatting --- .../tile_provider/network/image_provider/image_provider.dart | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index fc12889f6..159ccd435 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -321,6 +321,5 @@ class NetworkTileImageProvider extends ImageProvider { url == other.url); @override - int get hashCode => - Object.hash(url, fallbackUrl); + int get hashCode => Object.hash(url, fallbackUrl); } From 64753d53fbc6f4552a7f5d9b47764ce3e4e9696c Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 8 Jul 2025 22:27:11 +0100 Subject: [PATCH 44/49] Added reasoning for avoiding databases --- .../network/caching/built_in/impl/native/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md index 1a67deba7..e3f1f545a 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -2,6 +2,13 @@ The `BuiltInMapCachingProvider`, referred to as just 'built-in caching', is implemented using the filesystem for storage on native platforms. +The filesystem is used over alternatives such as databases because: + +* Cached tiles can be read immediately without any preprocessing step; writing to the cache does need to potentially wait for an 'initialisation' +* It is lightweight and adds only additional packages to a shipped app, not binaries +* It is likely to be more resilient across platforms than trying to ship a binary +* The capabilities of a database in terms of relationships are not required + Cached tiles & their metadata are stored as individual keyed files. An additional file is used to improve the efficiency of tracking and reducing the cache size, called the 'size monitor'. ## Tiles From 7c2e3692b74f93afcabf2de20699c5bff451d797 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Tue, 8 Jul 2025 23:38:01 +0100 Subject: [PATCH 45/49] Added file format signature and version to cached tile specification --- .../caching/built_in/impl/native/README.md | 16 ++--- .../caching/built_in/impl/native/native.dart | 63 ++++++++++++++----- .../workers/tile_and_size_monitor_writer.dart | 15 +++-- 3 files changed, 65 insertions(+), 29 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md index e3f1f545a..42e6ce11b 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -11,7 +11,7 @@ The filesystem is used over alternatives such as databases because: Cached tiles & their metadata are stored as individual keyed files. An additional file is used to improve the efficiency of tracking and reducing the cache size, called the 'size monitor'. -## Tiles +## Tiles (format v1) Tiles are stored in files, where the filename is the output of the supplied `cacheKeyGenerator` given the tile's URL. This defaults to a v5 UUID. Files have no extension. @@ -28,17 +28,19 @@ The file format is as follows: The format of the header is as follows: -1. 8-byte signed integer (Int64): the `staleAt` timestamp, represented in milliseconds since the Unix epoch in the UTC timezone -2. 8-byte signed integer (Int64) +1. (position 0) 6-byte ASCII encoded string: file format identifier "FMBICT" ("FlutterMapBuiltInCacheTile") +2. (position 6) 2-byte unsigned integer (Uint16): the format version (1) +3. (position 8) 8-byte signed integer (Int64): the `staleAt` timestamp, represented in milliseconds since the Unix epoch in the UTC timezone +4. (position 16) 8-byte signed integer (Int64) * Where provided, the `lastModified` timestamp, represented in milliseconds since the Unix epoch in the UTC timezone, which must not be 0 * Where not provided, the integer '0' -3. 2-byte unsigned integer (Uint16) +5. (position 24) 2-byte unsigned integer (Uint16) * Where provided, the length of the ASCII encoded `etag` in bytes * Where not provided, the integer '0' -4. Variable number of bytes +6. (position 26) Variable number of bytes * Where provided, the ASCII encoded `etag` (where each character is 7 bits but stored as 1 byte) with no greater than 65535 bytes * Where not provided, no bytes -5. 4-byte unsigned integer (Uint32): the length of the tile image bytes +7. (position 26 + 2-byte value read from position 24) 4-byte unsigned integer (Uint32): the length of the tile image bytes ## Size monitor @@ -48,4 +50,4 @@ This size monitor should stay in sync with the actual size of the cache - as cal Whilst it is being calculated (which should happen on the first initialisation of the cache, or when required as above), writes must be delayed. Reads can still occur. -Named 'sizeMonitor.bin'. +Named 'sizeMonitor.bin'. Does not contain an indentifier/signature. diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index cdda1f4fb..b96217749 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -14,6 +14,7 @@ import 'package:path_provider/path_provider.dart'; @internal class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { static const sizeMonitorFileName = 'sizeMonitor.bin'; + static const tileFileFormatSignature = [70, 77, 66, 73, 67, 84]; final String? cacheDirectory; final int? maxCacheSize; @@ -117,6 +118,8 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { _reportReadFailure; // See `disableSizeMonitor` in worker late final Future Function() _killWorker; + final _asciiDecoder = const AsciiDecoder(); + @override bool get isSupported => true; @@ -143,42 +146,68 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { try { final bytes = await tileFile.readAsBytes(); - if (bytes.lengthInBytes < 22) { + if (bytes.lengthInBytes < 30) { + throw CachedMapTileReadFailure( + url: url, + description: 'file was shorter than the min. expected size (found ' + '${bytes.lengthInBytes} bytes, expected >= 30 bytes)', + ); + } + + final formatSignature = bytes.buffer.asUint8List(0, 6); + + if (formatSignature[0] != tileFileFormatSignature[0] || + formatSignature[1] != tileFileFormatSignature[1] || + formatSignature[2] != tileFileFormatSignature[2] || + formatSignature[3] != tileFileFormatSignature[3] || + formatSignature[4] != tileFileFormatSignature[4] || + formatSignature[5] != tileFileFormatSignature[5]) { throw CachedMapTileReadFailure( url: url, description: - 'cache file (${bytes.lengthInBytes}) was shorter than the ' - 'minimum expected size', + 'file did not contain expected format signature at start (found ' + '$formatSignature, expected $tileFileFormatSignature)', ); } - final firstTwoNums = bytes.buffer.asInt64List(0, 2); + final version = bytes.buffer.asUint16List(6, 1)[0]; + + if (version != 1) { + throw CachedMapTileReadFailure( + url: url, + description: + 'cache file was of a different version (found v$version, ' + 'expected v1)', + ); + } + + final timestamps = bytes.buffer.asInt64List(8, 2); final staleAt = - DateTime.fromMillisecondsSinceEpoch(firstTwoNums[0], isUtc: true); - final lastModified = firstTwoNums[1] == 0 + DateTime.fromMillisecondsSinceEpoch(timestamps[0], isUtc: true); + final lastModified = timestamps[1] == 0 ? null - : DateTime.fromMillisecondsSinceEpoch(firstTwoNums[1], isUtc: true); + : DateTime.fromMillisecondsSinceEpoch(timestamps[1], isUtc: true); - final etagLength = bytes.buffer.asUint16List(16, 1)[0]; + final etagLength = bytes.buffer.asUint16List(24, 1)[0]; final String? etag; if (etagLength == 0) { etag = null; } else { - final etagBytes = Uint8List.sublistView(bytes, 18, 18 + etagLength); - etag = const AsciiDecoder().convert(etagBytes); + final etagBytes = Uint8List.sublistView(bytes, 26, 26 + etagLength); + etag = _asciiDecoder.convert(etagBytes); } final tileBytesExpectedLength = // Perform an unaligned read - bytes.buffer.asByteData(18 + etagLength, 4).getUint32(0, Endian.host); - - final tileBytes = Uint8List.sublistView(bytes, 18 + etagLength + 4); - + bytes.buffer.asByteData(26 + etagLength, 4).getUint32(0, Endian.host); + // We read the remainder of the file, rather than just reading the + // specified number of bytes + final tileBytes = Uint8List.sublistView(bytes, 26 + etagLength + 4); if (tileBytes.lengthInBytes != tileBytesExpectedLength) { throw CachedMapTileReadFailure( url: url, - description: - 'tile image bytes (${tileBytes.lengthInBytes}) were not of ' - 'expected length ($tileBytesExpectedLength)', + description: 'tile image bytes were not of expected length (found ' + '${tileBytes.lengthInBytes} bytes, expected ' + '$tileBytesExpectedLength bytes)', ); } diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index 7eaac477d..d0eb1f943 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -140,6 +140,7 @@ Future tileAndSizeMonitorWriterWorker( final allocInt64BufferTileWrite = Uint8List(8); final allocUint32BufferTileWrite = Uint8List(4); final allocUint16BufferTileWrite = Uint8List(2); + final asciiEncoder = const AsciiEncoder(); void writeTile({ required final String path, required final CachedMapTileMetadata metadata, @@ -170,9 +171,13 @@ Future tileAndSizeMonitorWriterWorker( } ram - // We start reading from the start of the file, where we store our header + // We start writing to the start of the file, where we store our header // info ..setPositionSync(0) + // Identify the file format in 6 bytes + ..writeFromSync(BuiltInMapCachingProviderImpl.tileFileFormatSignature) + // Identify the format version (v1) in 2 bytes + ..writeFromSync(allocUint16BufferTileWrite..buffer.asUint16List()[0] = 1) // We store the stale-at header in 8 signed bytes... ..writeFromSync( allocInt64BufferTileWrite @@ -205,7 +210,7 @@ Future tileAndSizeMonitorWriterWorker( return; } - ram.setPositionSync(16); // we need to go back to the start of the length + ram.setPositionSync(24); // we need to go back to the start of the length } final int etagLength; @@ -217,7 +222,7 @@ Future tileAndSizeMonitorWriterWorker( allocUint16BufferTileWrite..buffer.asUint16List()[0] = etagLength = 0, ); } else { - etagBytes = const AsciiEncoder().convert(metadata.etag!); + etagBytes = asciiEncoder.convert(metadata.etag!); // We store the etag length in 2 signed bytes (unless it is too large)... ram.writeFromSync( allocUint16BufferTileWrite @@ -232,7 +237,7 @@ Future tileAndSizeMonitorWriterWorker( // To do this, we have to read the remainder of the file, skipping over // the etag as it has not yet changed, and make it as if they were new // bytes - ram.setPositionSync(18 + initialEtagLength!); + ram.setPositionSync(26 + initialEtagLength!); final int initialTileBytesLength; try { @@ -256,7 +261,7 @@ Future tileAndSizeMonitorWriterWorker( return; } - ram.setPositionSync(18); + ram.setPositionSync(26); } if (etagLength != 0) { From e2239dc6fb1cb1e27c1fa17869afeffa37abad6a Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 9 Jul 2025 00:00:33 +0100 Subject: [PATCH 46/49] Convert `_SizeReducerTile` into an object Fixed bug with size reducer attempting to delete locked files Minor style tweaks --- .../impl/native/workers/size_reducer.dart | 47 ++++++++++++++++--- .../workers/tile_and_size_monitor_writer.dart | 21 +++++---- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart index 764367d47..b4f8e9391 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/size_reducer.dart @@ -5,8 +5,6 @@ import 'package:flutter_map/src/layer/tile_layer/tile_provider/network/caching/b import 'package:meta/meta.dart'; import 'package:path/path.dart' as p; -typedef _SizeReducerTile = ({String path, int size, DateTime sortKey}); - /// Remove tile files from the cache directory until at least [minSizeToDelete] /// bytes have been deleted. /// @@ -22,14 +20,19 @@ Future sizeReducerWorker({ }) async { final cacheDirectory = Directory(cacheDirectoryPath); - final tiles = await Future.wait<_SizeReducerTile>( + final tiles = await Future.wait( cacheDirectory.listSync().whereType().where((f) { final uuid = p.basename(f.absolute.path); return uuid != BuiltInMapCachingProviderImpl.sizeMonitorFileName; }).map((f) async { - final stat = await f.stat(); // `stat.accessed` may be unstable on some OSs, but seems to work enough? - return (path: f.absolute.path, size: stat.size, sortKey: stat.accessed); + final stat = await f.stat(); + + return _SizeReducerTile( + path: f.absolute.path, + size: stat.size, + sortKey: stat.accessed, + ); }), ); @@ -45,7 +48,18 @@ Future sizeReducerWorker({ while (deletedSize < minSizeToDelete && i < tiles.length) { final tile = tiles[i++]; deletedSize += tile.size; - yield File(tile.path).delete(); + yield File(tile.path).delete().then((_) {}, onError: (_) { + // We might not be able to delete the tile if its being read/just been + // read, because "another process" has obtained a lock on the tile. + // (jaffaketchup) (it's difficult to prove whether this is the case, but + // it makes sense) + // This could be seen as a useful feature: the tiles which the user sees + // when they start the app remain cached. + // In reality, this is unlikely to occur unless the size limit is really + // small (since other older tiles will be deleted first, which shouldn't + // be locked). + // This silences the error. + }); } }() .toList(growable: false); @@ -54,3 +68,24 @@ Future sizeReducerWorker({ return deletedSize; } + +@immutable +class _SizeReducerTile { + final String path; // We assume the path is unique for equality purposes + final int size; + final DateTime sortKey; + + const _SizeReducerTile({ + required this.path, + required this.size, + required this.sortKey, + }); + + @override + int get hashCode => path.hashCode; + + @override + bool operator ==(Object other) => + identical(this, other) || + (other is _SizeReducerTile && other.path == path); +} diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart index d0eb1f943..7e2a67117 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/workers/tile_and_size_monitor_writer.dart @@ -204,8 +204,9 @@ Future tileAndSizeMonitorWriterWorker( // a fresh overwrite with new bytes // We try to handle it anyway by emptying the tile completely so it is // auto-repaired on the next read - ram.truncateSync(0); - ram.closeSync(); + ram + ..truncateSync(0) + ..closeSync(); disableSizeMonitor(); return; } @@ -245,8 +246,9 @@ Future tileAndSizeMonitorWriterWorker( } catch (_) { // This implies the tile was corrupted on the previous write (the // write was terminated unexpectedly) - ram.truncateSync(0); - ram.closeSync(); + ram + ..truncateSync(0) + ..closeSync(); disableSizeMonitor(); return; } @@ -255,8 +257,9 @@ Future tileAndSizeMonitorWriterWorker( if (tileBytes.lengthInBytes != initialTileBytesLength) { // This implies the tile was corrupted on the previous write (the // write was terminated unexpectedly whilst writing tile bytes) - ram.truncateSync(0); - ram.closeSync(); + ram + ..truncateSync(0) + ..closeSync(); disableSizeMonitor(); return; } @@ -285,15 +288,15 @@ Future tileAndSizeMonitorWriterWorker( // ...followed by the tile bytes ram.writeFromSync(tileBytes); - final finalPos = ram.positionSync(); + final finalPosition = ram.positionSync(); ram // We truncate the tile in case the bytes have been moved forward or are // shorter than previously - ..truncateSync(finalPos) + ..truncateSync(finalPosition) ..closeSync(); // Then update the size monitor - if (finalPos - initialTileFileLength case final deltaSize + if (finalPosition - initialTileFileLength case final deltaSize when deltaSize != 0) { updateSizeMonitor(deltaSize); } From e371fc8c15b2ab0890be5f4c9b857d104c4091d7 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 9 Jul 2025 00:02:42 +0100 Subject: [PATCH 47/49] Added TODO --- .../tile_layer/tile_provider/network_image_provider_test.dart | 1 + 1 file changed, 1 insertion(+) diff --git a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart index badb935d8..2001fb3b1 100644 --- a/test/layer/tile_layer/tile_provider/network_image_provider_test.dart +++ b/test/layer/tile_layer/tile_provider/network_image_provider_test.dart @@ -42,6 +42,7 @@ Uri randomUrl({bool fallback = false}) { } // TODO: Write tests to test aborting? +// TODO: Write tests to test built-in caching void main() { const headers = { From 4372897c4d1e1f4a6dbb95a439f9d271959772b9 Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 9 Jul 2025 22:17:30 +0100 Subject: [PATCH 48/49] Minor cache README refinement --- .../network/caching/built_in/impl/native/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md index 42e6ce11b..c9e539884 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/README.md @@ -6,8 +6,6 @@ The filesystem is used over alternatives such as databases because: * Cached tiles can be read immediately without any preprocessing step; writing to the cache does need to potentially wait for an 'initialisation' * It is lightweight and adds only additional packages to a shipped app, not binaries -* It is likely to be more resilient across platforms than trying to ship a binary -* The capabilities of a database in terms of relationships are not required Cached tiles & their metadata are stored as individual keyed files. An additional file is used to improve the efficiency of tracking and reducing the cache size, called the 'size monitor'. From ddc736e6777fe32862bb1f898b296dc7586e5cfa Mon Sep 17 00:00:00 2001 From: JaffaKetchup Date: Wed, 9 Jul 2025 22:22:24 +0100 Subject: [PATCH 49/49] Add `CachedMapTile` typedef --- .../network/caching/built_in/impl/native/native.dart | 4 +--- .../tile_provider/network/caching/built_in/impl/stub.dart | 4 +--- .../tile_provider/network/caching/caching_provider.dart | 7 ++++--- .../network/image_provider/image_provider.dart | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart index b96217749..f073aa3e0 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/native/native.dart @@ -133,9 +133,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { } @override - Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( - String url, - ) async { + Future getTile(String url) async { final key = tileKeyGenerator(url); final tileFile = File( p.join(_cacheDirectoryPath ?? await _cacheDirectoryPathReady.future, key), diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart index 8a24496a1..ff1e250e9 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/built_in/impl/stub.dart @@ -34,9 +34,7 @@ class BuiltInMapCachingProviderImpl implements BuiltInMapCachingProvider { external bool get isSupported; @override - external Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( - String url, - ); + external Future getTile(String url); @override external Future putTile({ diff --git a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart index d1b840ffc..1335b2a88 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/caching/caching_provider.dart @@ -35,9 +35,7 @@ abstract interface class MapCachingProvider { /// Tile providers should anticipate these exceptions and fallback to a /// non-caching alternative, wherever possible repairing or replacing the tile /// with a fresh & valid one. - Future<({Uint8List bytes, CachedMapTileMetadata metadata})?> getTile( - String url, - ); + Future getTile(String url); /// Add or update a tile in the cache /// @@ -49,3 +47,6 @@ abstract interface class MapCachingProvider { Uint8List? bytes, }); } + +/// A tile's bytes and metadata returned from [MapCachingProvider.getTile] +typedef CachedMapTile = ({Uint8List bytes, CachedMapTileMetadata metadata}); diff --git a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart index 159ccd435..515059d14 100644 --- a/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart +++ b/lib/src/layer/tile_layer/tile_provider/network/image_provider/image_provider.dart @@ -155,7 +155,7 @@ class NetworkTileImageProvider extends ImageProvider { } // Prepare caching provider & load cached tile if available - ({Uint8List bytes, CachedMapTileMetadata metadata})? cachedTile; + CachedMapTile? cachedTile; final cachingProvider = this.cachingProvider ?? BuiltInMapCachingProvider.getOrCreateInstance(); if (cachingProvider.isSupported) {