From 08ec70b70dc398ef5cf1c8a1ac02d205708bf3cf Mon Sep 17 00:00:00 2001 From: lifegpc Date: Mon, 27 May 2024 10:39:18 +0000 Subject: [PATCH] Add support to get size for image_cache --- lib/platform/image_cache_io.dart | 14 +- lib/platform/image_cache_web.dart | 99 ++++++++++++++- lib/platform/web/indexed_db.dart | 205 ++++++++++++++++++++++++++++++ 3 files changed, 307 insertions(+), 11 deletions(-) create mode 100644 lib/platform/web/indexed_db.dart diff --git a/lib/platform/image_cache_io.dart b/lib/platform/image_cache_io.dart index 02d0d0e..5317c70 100644 --- a/lib/platform/image_cache_io.dart +++ b/lib/platform/image_cache_io.dart @@ -21,7 +21,7 @@ PRIMARY KEY(url) );"""; const _allTables = ['images']; -final _log = Logger("ImageCachesDb"); +final _log = Logger("ImageCachesIO"); class ImageCaches { Database? _db; @@ -155,12 +155,17 @@ class ImageCaches { if (needDeleted.isNotEmpty) await _optimize(); } + Future _updateSize() async { + final re = await _db!.rawQuery("SELECT SUM(size) AS sizes FROM images;"); + _size = re.isEmpty ? 0 : ((re[0]["sizes"] as int?) ?? 0); + } + Future init() async { sqfliteFfiInit(); _db = await databaseFactoryFfi.openDatabase(await _filePath); await _createDir(); if (!(await _checkDatabase())) await _createTable(); - await updateSize(); + await _updateSize(); _inited = true; } @@ -232,12 +237,13 @@ class ImageCaches { } Future updateSize({bool clear = false}) async { + if (!_inited) return; if (clear) await _removeUnexist(); - final re = await _db!.rawQuery("SELECT SUM(size) AS sizes FROM images;"); - _size = re.isEmpty ? 0 : ((re[0]["sizes"] as int?) ?? 0); + await _updateSize(); } Future clear() async { + if (!_inited) return; int offset = 0; late List> records; do { diff --git a/lib/platform/image_cache_web.dart b/lib/platform/image_cache_web.dart index b1ab898..15bb945 100644 --- a/lib/platform/image_cache_web.dart +++ b/lib/platform/image_cache_web.dart @@ -1,24 +1,66 @@ import 'dart:js_interop'; import 'dart:js_interop_unsafe'; import 'dart:typed_data'; +import 'package:logging/logging.dart'; import 'package:web/web.dart'; +import 'web/indexed_db.dart'; + +final _log = Logger("ImageCachesWeb"); class ImageCaches { - Cache? cache; + late Cache cache; ImageCaches(); int _size = 0; int get size => _size; + late IndexedDb _db; + bool _inited = false; + Future _updateSize() async { + int total = 0; + await _db.openCursor("images", (cur) { + final v = cur.value as JSObject; + final size = v.getProperty("size".toJS) as JSNumber; + total += size.toDartInt; + cur.continue_(); + }); + _size = total; + } + + Future _removeUnexist() async { + final urls = (await _db.getAllKeys("images")) + .toDart + .map((e) => (e as JSString).toDart) + .toList(); + for (final uri in urls) { + final init = RequestInit(credentials: 'include'); + final req = Request(URL(uri), init); + final opts = CacheQueryOptions(ignoreVary: true); + final re = await cache.match(req, opts).toDart; + if (re == null) await _db.delete("images", uri.toJS); + } + } + Future init() async { cache = await window.caches.open("image_caches").toDart; + _db = IndexedDb("image_caches", (event, db) { + _log.info( + "upgrade image_caches from ${event.oldVersion} to ${event.newVersion}"); + if (event.oldVersion.isNaN || event.oldVersion < 1) { + final opts = IDBObjectStoreParameters(keyPath: 'url'.toJS); + db.createObjectStore('images', opts); + } + }, 1); + await _db.init(); + await _updateSize(); + _inited = true; } Future<(Uint8List, Map>, String?)?> getCache( String uri) async { - if (cache == null) return null; + if (!_inited) return null; final init = RequestInit(credentials: 'include'); final req = Request(URL(uri), init); final opts = CacheQueryOptions(ignoreVary: true); - final re = await cache!.match(req, opts).toDart; + final re = await cache.match(req, opts).toDart; if (re == null || re!.body == null) return null; final ab = await re!.arrayBuffer().toDart; final buffer = ab.toDart; @@ -30,21 +72,64 @@ class ImageCaches { } forEach?.callAsFunction(he, forE.toJS); + final lastUsed = DateTime.now().millisecondsSinceEpoch; + try { + final data = await _db.get("images", uri.toJS) as JSObject?; + if (data != null) { + data!.setProperty("last_used".toJS, lastUsed.toJS); + await _db.put("images", data!); + } else { + _log.info("Can not find record for $uri in database."); + } + } catch (e) { + _log.warning("Failed to set last_used to $lastUsed for $uri: $e"); + } return (buffer.asUint8List(), h, null); } Future putCache(String uri, Uint8List data, Map> headers, String? realUri) async { - if (cache == null) return; + if (!_inited) return; final he = JSObject(); for (final e in headers.entries) { he.setProperty(e.key.toJS, e.value.map((e) => e.toJS).toList().toJS); } final opts = ResponseInit(status: 200, statusText: 'OK', headers: he); + final lastUsed = DateTime.now().millisecondsSinceEpoch; final res = Response(data.toJS, opts); - await cache!.put(uri.toJS, res).toDart; + await cache.put(uri.toJS, res).toDart; + final oObj = (await _db.get("images", uri.toJS)) as JSObject?; + final obj = JSObject(); + obj.setProperty("url".toJS, uri.toJS); + obj.setProperty("size".toJS, data.length.toJS); + obj.setProperty("last_used".toJS, lastUsed.toJS); + await _db.put("images", obj); + if (oObj == null) { + _size += data.length; + } else { + final originalSize = + (oObj!.getProperty("size".toJS) as JSNumber).toDartInt; + _size += (data.length - originalSize); + } } - Future updateSize({bool clear = false}) async {} - Future clear() async {} + Future updateSize({bool clear = false}) async { + if (!_inited) return; + if (clear) await _removeUnexist(); + await _updateSize(); + } + + Future clear() async { + if (!_inited) return; + await _db.openKeyCursor("images", (cur) { + final uri = (cur.key as JSString).toDart; + final init = RequestInit(credentials: 'include'); + final req = Request(URL(uri), init); + final opts = CacheQueryOptions(ignoreVary: true); + cache.delete(req, opts); + cur.continue_(); + }); + await _db.clear("images"); + _size = 0; + } } diff --git a/lib/platform/web/indexed_db.dart b/lib/platform/web/indexed_db.dart new file mode 100644 index 0000000..6cff1b2 --- /dev/null +++ b/lib/platform/web/indexed_db.dart @@ -0,0 +1,205 @@ +import 'dart:async'; +import 'dart:js_interop'; +import 'package:web/web.dart'; + +Future makeStoragePersist() async { + final storage = window.navigator.storage; + bool peristed = (await storage.persisted().toDart).toDart; + if (!peristed) { + peristed = (await storage.persist().toDart).toDart; + } + return peristed; +} + +class IndexedDb { + late IDBDatabase _db; + bool _inited = false; + bool get inited => _inited; + final String dbName; + final int? version; + final void Function(IDBVersionChangeEvent, IDBDatabase) onUpgradeNeeded; + IndexedDb(this.dbName, this.onUpgradeNeeded, this.version); + Future _waitRequest(IDBRequest request) async { + bool ok = false; + bool handled = false; + void onsuccess(Event _) { + ok = true; + handled = true; + } + + void onerror(Event _) { + handled = true; + } + + request.onsuccess = onsuccess.toJS; + request.onerror = onerror.toJS; + FutureOr waitResult() { + if (handled) return ok; + return Future.delayed(const Duration(milliseconds: 1), waitResult); + } + + await Future.microtask(waitResult); + if (ok) { + return request.result; + } else { + _inited = false; + throw request.error ?? ""; + } + } + + Future init() async { + makeStoragePersist(); + final req = version != null + ? window.indexedDB.open(dbName, version!) + : window.indexedDB.open(dbName); + void onupgradeneeded(IDBVersionChangeEvent event) { + onUpgradeNeeded(event, req.result as IDBDatabase); + } + + req.onupgradeneeded = onupgradeneeded.toJS; + bool ok = false; + bool handled = false; + void onsuccess(Event _) { + ok = true; + handled = true; + } + + void onerror(Event _) { + handled = true; + } + + req.onsuccess = onsuccess.toJS; + req.onerror = onerror.toJS; + FutureOr waitResult() { + if (handled) return ok; + return Future.delayed(const Duration(milliseconds: 1), waitResult); + } + + await Future.microtask(waitResult); + if (ok) { + _db = req.result as IDBDatabase; + } else { + throw req.error ?? ""; + } + _inited = true; + } + + Future clear(String table) async { + final tx = _db.transaction([table.toJS].toJS, 'readwrite'); + final store = tx.objectStore(table); + final req = store.clear(); + await _waitRequest(req); + } + + Future delete(String table, JSAny key) async { + final tx = _db.transaction([table.toJS].toJS, 'readwrite'); + final store = tx.objectStore(table); + final req = store.delete(key); + await _waitRequest(req); + } + + Future get(String table, JSAny key) async { + await init(); + final tx = _db.transaction([table.toJS].toJS, 'readonly'); + final store = tx.objectStore(table); + final req = store.get(key); + return await _waitRequest(req); + } + + Future> getAllKeys(String table, + {JSAny? query, int? count}) async { + await init(); + final tx = _db.transaction([table.toJS].toJS, 'readonly'); + final store = tx.objectStore(table); + final req = query != null && count != null + ? store.getAllKeys(query, count!) + : query != null + ? store.getAllKeys(query) + : store.getAllKeys(); + final r = await _waitRequest(req); + return r as JSArray; + } + + Future openCursor( + String table, void Function(IDBCursorWithValue) callback, + {JSAny? query, String? direction, bool readwrite = false}) async { + await init(); + final tx = _db.transaction( + [table.toJS].toJS, readwrite ? 'readwrite' : 'readonly'); + final store = tx.objectStore(table); + final req = query != null && direction != null + ? store.openCursor(query, direction!) + : query != null + ? store.openCursor(query) + : store.openCursor(); + bool ok = false; + bool handled = false; + void onsuccess(Event _) { + if (req.result != null) { + final cursor = req.result as IDBCursorWithValue; + callback(cursor); + } else { + ok = true; + handled = true; + } + } + + void onerror(Event _) { + handled = true; + } + + req.onsuccess = onsuccess.toJS; + req.onerror = onerror.toJS; + FutureOr waitResult() { + if (handled) return ok; + return Future.delayed(const Duration(milliseconds: 1), waitResult); + } + + await Future.microtask(waitResult); + } + + Future openKeyCursor(String table, void Function(IDBCursor) callback, + {JSAny? query, String? direction, bool readwrite = false}) async { + await init(); + final tx = _db.transaction( + [table.toJS].toJS, readwrite ? 'readwrite' : 'readonly'); + final store = tx.objectStore(table); + final req = query != null && direction != null + ? store.openKeyCursor(query, direction!) + : query != null + ? store.openKeyCursor(query) + : store.openKeyCursor(); + bool ok = false; + bool handled = false; + void onsuccess(Event _) { + if (req.result != null) { + final cursor = req.result as IDBCursor; + callback(cursor); + } else { + ok = true; + handled = true; + } + } + + void onerror(Event _) { + handled = true; + } + + req.onsuccess = onsuccess.toJS; + req.onerror = onerror.toJS; + FutureOr waitResult() { + if (handled) return ok; + return Future.delayed(const Duration(milliseconds: 1), waitResult); + } + + await Future.microtask(waitResult); + } + + Future put(String table, JSAny value, {JSAny? key}) async { + await init(); + final tx = _db.transaction([table.toJS].toJS, 'readwrite'); + final store = tx.objectStore(table); + final req = key == null ? store.put(value) : store.put(value, key); + return await _waitRequest(req); + } +}