From 5ab38375d402c1941022a1a473e503ee62ef9107 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Thu, 6 Jun 2024 15:26:28 +0800 Subject: [PATCH] Add new settings check file hash --- config.ts | 5 +++++ import_map.json | 3 ++- page/SinglePage_test.ts | 10 ++++++++-- tasks/download.ts | 24 ++++++++++++++++++++++-- utils.ts | 30 +++++++++++++++++++++++++++++- utils_test.ts | 9 +++++++++ 6 files changed, 75 insertions(+), 6 deletions(-) diff --git a/config.ts b/config.ts index 92a9e43..9053226 100644 --- a/config.ts +++ b/config.ts @@ -36,6 +36,7 @@ export type ConfigType = { eh_metadata_cache_time: number; random_file_secret?: string; use_path_based_img_url: boolean; + check_file_hash: boolean; }; export enum ThumbnailMethod { @@ -208,6 +209,9 @@ export class Config { get use_path_based_img_url() { return this._return_bool("use_path_based_img_url") ?? true; } + get check_file_hash() { + return this._return_bool("check_file_hash") ?? true; + } to_json(): ConfigType { return { cookies: typeof this.cookies === "string", @@ -243,6 +247,7 @@ export class Config { eh_metadata_cache_time: this.eh_metadata_cache_time, random_file_secret: this.random_file_secret, use_path_based_img_url: this.use_path_based_img_url, + check_file_hash: this.check_file_hash, }; } } diff --git a/import_map.json b/import_map.json index 1aca991..3822d39 100644 --- a/import_map.json +++ b/import_map.json @@ -34,6 +34,7 @@ "async/": "https://deno.land/x/async@v2.1.0/", "@twind/core": "https://esm.sh/@twind/core@1.1.3", "@twind/preset-tailwind": "https://esm.sh/@twind/preset-tailwind@1.1.4/", - "@twind/preset-autoprefix": "https://esm.sh/@twind/preset-autoprefix@1.0.7/" + "@twind/preset-autoprefix": "https://esm.sh/@twind/preset-autoprefix@1.0.7/", + "@lifegpc/sha1": "jsr:/@lifegpc/sha1@1.0.0" } } diff --git a/page/SinglePage_test.ts b/page/SinglePage_test.ts index ab9b552..35777d2 100644 --- a/page/SinglePage_test.ts +++ b/page/SinglePage_test.ts @@ -1,11 +1,11 @@ import { assert, assertEquals } from "@std/assert"; import { Client } from "../client.ts"; import { load_settings } from "../config.ts"; -import { API_PERMISSION } from "../test_base.ts"; +import { SHA1 } from "@lifegpc/sha1"; +import { getHashFromUrl } from "../utils.ts"; Deno.test({ name: "SinglePage_test", - permissions: API_PERMISSION, }, async () => { const cfg = await load_settings("./config.json"); const client = new Client(cfg); @@ -38,4 +38,10 @@ Deno.test({ assertEquals(re2.origin_xres, 4893); assertEquals(re2.origin_yres, 3446); console.log(np.nl, re.nl, re2.nl); + console.log(re2.img_url); + const res = await client.request(re2.img_url, "GET"); + const data = await res.arrayBuffer(); + const h = (new SHA1()).update(new Uint8Array(data)).digest_hex(); + const oh = getHashFromUrl(re2.img_url); + assertEquals(h, oh); }); diff --git a/tasks/download.ts b/tasks/download.ts index b5b5a42..73071c4 100644 --- a/tasks/download.ts +++ b/tasks/download.ts @@ -9,7 +9,7 @@ import { TaskType, } from "../task.ts"; import { TaskManager } from "../task_manager.ts"; -import { RecoverableError } from "../utils.ts"; +import { calFileSha1, getHashFromUrl, RecoverableError } from "../utils.ts"; import { add_suffix_to_path, asyncEvery, @@ -181,6 +181,12 @@ class DownloadManager { } } +class HashError extends RecoverableError { + constructor() { + super("Hash error"); + } +} + interface Image { data: unknown; index: number; @@ -402,6 +408,17 @@ export async function download_task( null; } } + if (manager.cfg.check_file_hash) { + const url = re.url; + const hash = getHashFromUrl(url); + const fhash = await calFileSha1(path); + if (hash != fhash) { + console.warn( + `Hash not matched: file hash ${fhash}, original hash ${hash}, url ${url}`, + ); + throw new HashError(); + } + } } const errors: unknown[] = []; function try_download(a: number) { @@ -426,7 +443,10 @@ export async function download_task( return; } } - if (e instanceof TimeoutError) { + if ( + e instanceof TimeoutError || + e instanceof HashError + ) { m.remove_details(i.index); reject(e); return; diff --git a/utils.ts b/utils.ts index cd3cdd2..2dce152 100644 --- a/utils.ts +++ b/utils.ts @@ -3,6 +3,7 @@ import { extname } from "@std/path"; import { initParser } from "deno_dom/wasm-noinit"; import { configure } from "zipjs/index.js"; import { MD5 } from "lifegpc-md5"; +import { SHA1 } from "@lifegpc/sha1"; export function sleep(time: number): Promise { return new Promise((r) => { @@ -191,7 +192,25 @@ export async function calFileMd5(p: string | URL) { do { readed = await f.read(buf); if (readed) { - h.update(buf.slice(0, readed)); + h.update(buf, readed); + } + } while (readed !== null); + return h.digest_hex(); + } finally { + f.close(); + } +} + +export async function calFileSha1(p: string | URL) { + const h = new SHA1(); + const f = await Deno.open(p); + try { + const buf = new Uint8Array(4096); + let readed: number | null = null; + do { + readed = await f.read(buf); + if (readed) { + h.update(buf, readed); } } while (readed !== null); return h.digest_hex(); @@ -313,3 +332,12 @@ export function isNumNaN(num: number | bigint) { export function compareNum(num1: number | bigint, num2: number | bigint) { return num1 == num2 ? 0 : num1 < num2 ? -1 : 1; } + +const HASH_PATTERN = /^\/h\/([0-9a-f]+)/; + +export function getHashFromUrl(url: string | URL) { + const u = typeof url === "string" ? new URL(url) : url; + const m = u.pathname.match(HASH_PATTERN); + if (m) return m[1]; + throw Error(`URL ${url} not contains hash info.`); +} diff --git a/utils_test.ts b/utils_test.ts index 9f3dd28..1387901 100644 --- a/utils_test.ts +++ b/utils_test.ts @@ -6,6 +6,7 @@ import { asyncFilter, asyncForEach, calFileMd5, + calFileSha1, compareNum, filterFilename, map, @@ -17,6 +18,7 @@ import { toJSON, } from "./utils.ts"; import { md5 } from "lifegpc-md5"; +import { sha1 } from "@lifegpc/sha1"; Deno.test("promiseState_test", async () => { const p1 = new Promise((res) => setTimeout(() => res(100), 100)); @@ -104,6 +106,13 @@ Deno.test("calFileMd5_test", async () => { assertEquals(await calFileMd5("./test/test.txt"), md5(text)); }); +Deno.test("calFileSha1_test", async () => { + await sure_dir(); + const text = `Hello World.te${Math.random()}dsadasd`; + await Deno.writeTextFile("./test/testsha1.txt", text); + assertEquals(await calFileSha1("./test/testsha1.txt"), sha1(text)); +}); + Deno.test("asyncEvery_test", async () => { // @ts-ignore: FUCKED UP const list = [];