From 3a609256e0f2a37af21d18e908e95f3f7de7acd3 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 26 May 2024 18:23:14 +0800 Subject: [PATCH] Add support to cache thumbnail images to disk --- .github/workflows/linux.yml | 2 +- lib/api/client.dart | 10 +- lib/components/thumbnail.dart | 31 +++++++ lib/globals.dart | 17 ++++ lib/l10n/app_en.arb | 3 +- lib/l10n/app_zh.arb | 3 +- lib/main.dart | 1 + lib/platform/image_cache.dart | 1 + lib/platform/image_cache_ffi.dart | 149 ++++++++++++++++++++++++++++++ lib/platform/image_cache_web.dart | 45 +++++++++ lib/settings.dart | 34 +++++++ pubspec.lock | 34 ++++++- pubspec.yaml | 3 + 13 files changed, 324 insertions(+), 9 deletions(-) create mode 100644 lib/platform/image_cache.dart create mode 100644 lib/platform/image_cache_ffi.dart create mode 100644 lib/platform/image_cache_web.dart diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ab2cd16..d374199 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -28,7 +28,7 @@ jobs: with: channel: stable - name: Install dependencies - run: sudo apt-get update && sudo apt-get install -y libgtk-3-dev liblzma-dev libstdc++-12-dev ninja-build + run: sudo apt-get update && sudo apt-get install -y libgtk-3-dev liblzma-dev libstdc++-12-dev ninja-build libsqlite3-dev - name: Build run: flutter build linux --release - name: Package files diff --git a/lib/api/client.dart b/lib/api/client.dart index b2c952f..46cd3a5 100644 --- a/lib/api/client.dart +++ b/lib/api/client.dart @@ -315,11 +315,11 @@ class EHApi extends __EHApi { ThumbnailAlign? align}) { final uri = Uri.parse(_combineBaseUrls(_dio.options.baseUrl, baseUrl)); final queryParameters = { - r'max': max, - r'width': width, - r'height': height, - r'quality': quality, - r'force': force, + r'max': max?.toString(), + r'width': width?.toString(), + r'height': height?.toString(), + r'quality': quality?.toString(), + r'force': force?.toString(), r'method': method?.name, r'align': align?.name, }; diff --git a/lib/components/thumbnail.dart b/lib/components/thumbnail.dart index 773a3b5..00e4a4a 100644 --- a/lib/components/thumbnail.dart +++ b/lib/components/thumbnail.dart @@ -84,6 +84,7 @@ class _Thumbnail extends State { Color? _iconColor; double? _iconSize; bool _disposed = false; + String? _originalUrl; void _onNsfwChanged(dynamic args) { final arguments = args as (String, bool)?; if (arguments == null) return; @@ -134,6 +135,28 @@ class _Thumbnail extends State { .files[token]![0]! .id; } + _originalUrl ??= api.getThumbnailUrl(_fileId!, + max: widget._max, + width: widget._width, + height: widget._height, + method: ThumbnailMethod.contain, + align: ThumbnailAlign.center); + if (isImageCacheEnabled) { + try { + final cache = await imageCaches.getCache(_originalUrl!); + if (cache != null) { + setState(() { + _data = cache!.$1; + _uri = cache!.$3 ?? _originalUrl; + _isLoading = false; + _cancel = null; + }); + return; + } + } catch (e) { + _log.warning("Failed to get cache for $_originalUrl: $e"); + } + } final re = await api.getThumbnail(_fileId!, max: widget._max, width: widget._width, @@ -148,6 +171,14 @@ class _Thumbnail extends State { _uri = re.response.realUri.toString(); final data = Uint8List.fromList(re.data); if (!_cancel!.isCancelled) { + if (isImageCacheEnabled) { + try { + await imageCaches.putCache( + _originalUrl!, data, re.response.headers.map, _uri); + } catch (e) { + _log.warning("Failed to put cache for $_originalUrl: $e"); + } + } setState(() { _isLoading = false; _data = data; diff --git a/lib/globals.dart b/lib/globals.dart index 0567720..2912f35 100644 --- a/lib/globals.dart +++ b/lib/globals.dart @@ -22,6 +22,7 @@ import 'main.dart'; import 'platform/clipboard.dart'; import 'platform/display.dart'; import 'platform/get_jar.dart'; +import 'platform/image_cache.dart'; import 'platform/path.dart'; import 'platform/set_title.dart'; import 'tags.dart'; @@ -38,6 +39,7 @@ final dio = Dio() Config? _prefs; EHApi? _api; PersistCookieJar? _jar; +ImageCaches? _imageCaches; Future prepareJar() async { final jar = PersistCookieJar(storage: FileStorage(await getJarPath())); @@ -66,6 +68,21 @@ Config get prefs { return _prefs!; } +final _globalLog = Logger("global"); + +Future prepareImageCaches() async { + _imageCaches = ImageCaches(); + try { + await _imageCaches!.init(); + } catch (e) { + _globalLog.warning("Failed to initiailzed image caches: $e"); + } +} + +ImageCaches get imageCaches => _imageCaches!; + +bool get isImageCacheEnabled => prefs.getBool("enableImageCache") ?? true; + void initApi(String baseUrl) { _api = EHApi(dio, baseUrl: baseUrl); } diff --git a/lib/l10n/app_en.arb b/lib/l10n/app_en.arb index 93f66a3..40b9c32 100644 --- a/lib/l10n/app_en.arb +++ b/lib/l10n/app_en.arb @@ -195,5 +195,6 @@ "dlUseAvgSpeed": "Show average speed in task details.", "refresh": "Refresh", "originalImg": "Original image", - "overwriteDefaultConfig": "Overwrite default config" + "overwriteDefaultConfig": "Overwrite default config", + "enableImageCache": "Cache images to disk." } diff --git a/lib/l10n/app_zh.arb b/lib/l10n/app_zh.arb index 63711b3..bb4c138 100644 --- a/lib/l10n/app_zh.arb +++ b/lib/l10n/app_zh.arb @@ -195,5 +195,6 @@ "dlUseAvgSpeed": "在任务详情中显示平均速度。", "refresh": "刷新", "originalImg": "原图", - "overwriteDefaultConfig": "覆盖默认设置" + "overwriteDefaultConfig": "覆盖默认设置", + "enableImageCache": "将图片缓存到本地硬盘。" } diff --git a/lib/main.dart b/lib/main.dart index f383b23..ee4e9b1 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -251,6 +251,7 @@ void main() async { if (prefs.getBool("preventScreenCapture") ?? false) { await platformDisplay.enableProtect(); } + await prepareImageCaches(); GoRouter.optionURLReflectsImperativeAPIs = true; runApp(const MainApp()); } diff --git a/lib/platform/image_cache.dart b/lib/platform/image_cache.dart new file mode 100644 index 0000000..efe8913 --- /dev/null +++ b/lib/platform/image_cache.dart @@ -0,0 +1 @@ +export './image_cache_ffi.dart' if (dart.library.html) './image_cache_web.dart'; diff --git a/lib/platform/image_cache_ffi.dart b/lib/platform/image_cache_ffi.dart new file mode 100644 index 0000000..c95eb3d --- /dev/null +++ b/lib/platform/image_cache_ffi.dart @@ -0,0 +1,149 @@ +import 'dart:convert'; +import 'dart:io' as io; +import 'dart:typed_data'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; +import 'package:logging/logging.dart'; +import 'package:path/path.dart' as path; +import 'package:path_provider/path_provider.dart'; +import 'package:sqflite_common_ffi/sqflite_ffi.dart'; +import '../globals.dart'; +import '../utils.dart'; + +const _imagesTable = """CREATE TABLE images ( +url TEXT, +path TEXT, +last_used INT, +headers TEXT, +realUrl TEXT, +PRIMARY KEY(url) +);"""; +const _allTables = ['images']; + +final _log = Logger("ImageCachesDb"); + +class ImageCaches { + Database? _db; + final _fs = const LocalFileSystem(); + Directory? _cacheDir; + String? _exeDir; + final Set _existingTable = {}; + bool _inited = false; + ImageCaches(); + Future _desktopFilePath() async { + final String? exe = await platformPath.getCurrentExe(); + if (exe == null) return null; + _exeDir = path.dirname(exe); + return path.join(_exeDir!, "image_caches.db"); + } + + Future get _filePath async { + if (isWindows || isLinux) { + try { + final tmp = await _desktopFilePath(); + if (tmp != null) return tmp; + } catch (e) { + _log.warning("Failed to get database file location."); + } + } + final io.Directory appSupportDir = await getApplicationSupportDirectory(); + return path.join(appSupportDir.path, "image_caches.db"); + } + + Future get cacheDir async { + if (_cacheDir != null) return _cacheDir!; + if ((isWindows || isLinux) && _exeDir != null) { + return _cacheDir = _fs.directory(path.join(_exeDir!, "image_caches")); + } + final io.Directory cacheDir = await getApplicationCacheDirectory(); + return _cacheDir = _fs.directory(path.join(cacheDir.path, "image_caches")); + } + + Future _createDir() async { + final dir = await cacheDir; + if (!(await dir.exists())) { + await dir.create(recursive: true); + } + } + + Future _checkDatabase() async { + await _updateExistsTable(); + final v = await _db!.getVersion(); + _log.fine("Database version: $v"); + if (_allTables.length != _existingTable.length || + !_allTables.every((e) => _existingTable.contains(e))) { + return false; + } + return true; + } + + Future _createTable() async { + if (!_existingTable.contains("images")) { + await _db!.execute(_imagesTable); + } + await _updateExistsTable(); + } + + Future _updateExistsTable() async { + _existingTable.clear(); + final cur = await _db! + .query("sqlite_master", where: 'type = ?', whereArgs: ['table']); + for (final c in cur) { + _existingTable.add(c["name"]! as String); + } + } + + Future init() async { + sqfliteFfiInit(); + _db = await databaseFactoryFfi.openDatabase(await _filePath); + await _createDir(); + if (!(await _checkDatabase())) await _createTable(); + _inited = true; + } + + Future<(Uint8List, Map>, String?)?> getCache( + String uri) async { + if (!_inited) return null; + final d = await _db!.query("images", where: 'url = ?', whereArgs: [uri]); + if (d.isEmpty) return null; + final data = d.first; + final path = data["path"] as String; + final header = data["headers"] as String; + final realUrl = data["readUrl"] as String?; + final lastUsed = DateTime.now().millisecondsSinceEpoch; + try { + await _db!.rawUpdate( + "UPDATE images SET last_used = ? WHERE url = ?;", [lastUsed, uri]); + } catch (e) { + _log.warning("Failed to set last_used to $lastUsed for $uri."); + } + final f = _fs.file(path); + final da = await f.readAsBytes(); + final h = jsonDecode(header) as Map; + return ( + da, + h.map((k, v) => MapEntry(k, (v as List).cast())), + realUrl + ); + } + + Future putCache(String uri, Uint8List data, + Map> headers, String? realUri) async { + if (!_inited) return; + final u = Uri.parse(uri); + final dir = await cacheDir; + final p = path.join(dir.path, u.host.isEmpty ? "nohost" : u.host, + u.path.substring(1) + u.query); + final d = _fs.directory(path.dirname(p)); + if (!(await d.exists())) { + await d.create(recursive: true); + } + final f = _fs.file(p); + await f.writeAsBytes(data.toList()); + final lastUsed = DateTime.now().millisecondsSinceEpoch; + final header = jsonEncode(headers); + await _db!.rawInsert( + "INSERT OR REPLACE INTO images VALUES (?, ?, ?, ?, ?);", + [uri, p, lastUsed, header, realUri]); + } +} diff --git a/lib/platform/image_cache_web.dart b/lib/platform/image_cache_web.dart new file mode 100644 index 0000000..0b75c0c --- /dev/null +++ b/lib/platform/image_cache_web.dart @@ -0,0 +1,45 @@ +import 'dart:js_interop'; +import 'dart:js_interop_unsafe'; +import 'dart:typed_data'; +import 'package:web/web.dart'; + +class ImageCaches { + Cache? cache; + ImageCaches(); + Future init() async { + cache = await window.caches.open("image_caches").toDart; + } + + Future<(Uint8List, Map>, String?)?> getCache( + String uri) async { + if (cache == null) return null; + final init = RequestInit(credentials: 'include'); + final req = Request(URL(uri), init); + final opts = CacheQueryOptions(ignoreVary: true); + final re = await cache!.match(req, opts).toDart; + if (re == null || re!.body == null) return null; + final ab = await re!.arrayBuffer().toDart; + final buffer = ab.toDart; + final he = re!.headers; + final forEach = he.getProperty("forEach".toJS) as JSFunction?; + Map> h = {}; + void forE(JSString value, JSString key, Headers headers) { + h[key.toDart] = value.toDart.split(","); + } + + forEach?.callAsFunction(he, forE.toJS); + return (buffer.asUint8List(), h, null); + } + + Future putCache(String uri, Uint8List data, + Map> headers, String? realUri) async { + if (cache == null) return; + final he = JSObject(); + for (final e in headers.entries) { + he.setProperty(e.key.toJS, e.value.map((e) => e.toJS).toList().toJS); + } + final opts = ResponseInit(status: 200, statusText: 'OK', headers: he); + final res = Response(data.toJS, opts); + await cache!.put(uri.toJS, res).toDart; + } +} diff --git a/lib/settings.dart b/lib/settings.dart index d38f23b..d2a8af6 100644 --- a/lib/settings.dart +++ b/lib/settings.dart @@ -25,6 +25,7 @@ class _SettingsPage extends State with ThemeModeWidget { bool _oriShowTranslatedTag = false; bool _oriUseTitleJpn = false; bool _oriDlUseAvgSpeed = false; + bool _oriEnableImageCache = false; bool _displayAd = false; Lang _lang = Lang.system; bool _preventScreenCapture = false; @@ -32,6 +33,7 @@ class _SettingsPage extends State with ThemeModeWidget { bool _showTranslatedTag = false; bool _useTitleJpn = false; bool _dlUseAvgSpeed = false; + bool _enableImageCache = false; @override void initState() { super.initState(); @@ -92,6 +94,14 @@ class _SettingsPage extends State with ThemeModeWidget { _oriDlUseAvgSpeed = false; _dlUseAvgSpeed = false; } + try { + _oriEnableImageCache = prefs.getBool("enableImageCache") ?? true; + _enableImageCache = _oriEnableImageCache; + } catch (e) { + _log.warning("Failed to get enableImageCache:", e); + _oriEnableImageCache = true; + _enableImageCache = true; + } } void fallback(BuildContext context) { @@ -110,6 +120,7 @@ class _SettingsPage extends State with ThemeModeWidget { _showTranslatedTag = _oriLang.toLocale().languageCode == "zh"; _preventScreenCapture = false; _dlUseAvgSpeed = false; + _enableImageCache = true; }); } @@ -178,6 +189,14 @@ class _SettingsPage extends State with ThemeModeWidget { _oriDlUseAvgSpeed = _dlUseAvgSpeed; } } + if (_enableImageCache != _oriEnableImageCache) { + if (!await prefs.setBool("enableImageCache", _enableImageCache)) { + re = false; + _log.warning("Failed to save enableImageCache."); + } else { + _oriEnableImageCache = _enableImageCache; + } + } return re; } @@ -331,6 +350,21 @@ class _SettingsPage extends State with ThemeModeWidget { .dlUseAvgSpeed), ), ), + Container( + padding: const EdgeInsets.symmetric(vertical: 8), + child: CheckboxMenuButton( + value: _enableImageCache, + onChanged: (bool? value) { + if (value != null) { + setState(() { + _enableImageCache = value; + }); + } + }, + child: Text(AppLocalizations.of(context)! + .enableImageCache), + ), + ), Row( mainAxisAlignment: MainAxisAlignment.center, children: [ diff --git a/pubspec.lock b/pubspec.lock index 2962fcf..a0b589c 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -875,6 +875,30 @@ packages: url: "https://pub.dev" source: hosted version: "7.0.0" + sqflite_common: + dependency: transitive + description: + name: sqflite_common + sha256: "3da423ce7baf868be70e2c0976c28a1bb2f73644268b7ffa7d2e08eab71f16a4" + url: "https://pub.dev" + source: hosted + version: "2.5.4" + sqflite_common_ffi: + dependency: "direct main" + description: + name: sqflite_common_ffi + sha256: "4d6137c29e930d6e4a8ff373989dd9de7bac12e3bc87bce950f6e844e8ad3bb5" + url: "https://pub.dev" + source: hosted + version: "2.3.3" + sqlite3: + dependency: transitive + description: + name: sqlite3 + sha256: b384f598b813b347c5a7e5ffad82cbaff1bec3d1561af267041e66f6f0899295 + url: "https://pub.dev" + source: hosted + version: "2.4.3" stack_trace: dependency: transitive description: @@ -931,6 +955,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.8.15" + synchronized: + dependency: transitive + description: + name: synchronized + sha256: "539ef412b170d65ecdafd780f924e5be3f60032a1128df156adad6c5b373d558" + url: "https://pub.dev" + source: hosted + version: "3.1.0+1" term_glyph: dependency: transitive description: @@ -1028,7 +1060,7 @@ packages: source: hosted version: "1.1.0" web: - dependency: transitive + dependency: "direct main" description: name: web sha256: "97da13628db363c635202ad97068d47c5b8aa555808e7a9411963c533b449b27" diff --git a/pubspec.yaml b/pubspec.yaml index 9d51878..84e2449 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -38,10 +38,12 @@ dependencies: photo_view: ^0.15.0 retrofit: ^4.0.1 shared_preferences: ^2.2.0 + sqflite_common_ffi: ^2.3.3 super_clipboard: ^0.8.15 super_context_menu: ^0.8.15 ua_parser_js: ^1.0.1 user_agent_analyzer: ^5.0.0 + web: ^0.5.0 web_socket_channel: ^3.0.0 window_manager: ^0.3.6 @@ -59,6 +61,7 @@ dev_dependencies: build_runner: ^2.4.6 retrofit_generator: ^8.1.0 json_serializable: ^6.7.1 + sqflite_common_ffi: ^2.3.3 flutter: generate: true