Add new settings check file hash

config.ts

@@ -36,6 +36,7 @@ export type ConfigType = {
     eh_metadata_cache_time: number;
     random_file_secret?: string;
     use_path_based_img_url: boolean;
+    check_file_hash: boolean;
 };
 
 export enum ThumbnailMethod {
@@ -208,6 +209,9 @@ export class Config {
     get use_path_based_img_url() {
         return this._return_bool("use_path_based_img_url") ?? true;
     }
+    get check_file_hash() {
+        return this._return_bool("check_file_hash") ?? true;
+    }
     to_json(): ConfigType {
         return {
             cookies: typeof this.cookies === "string",
@@ -243,6 +247,7 @@ export class Config {
             eh_metadata_cache_time: this.eh_metadata_cache_time,
             random_file_secret: this.random_file_secret,
             use_path_based_img_url: this.use_path_based_img_url,
+            check_file_hash: this.check_file_hash,
         };
     }
 }

import_map.json

@@ -34,6 +34,7 @@
         "async/": "https://deno.land/x/[email protected]/",
         "@twind/core": "https://esm.sh/@twind/[email protected]",
         "@twind/preset-tailwind": "https://esm.sh/@twind/[email protected]/",
-        "@twind/preset-autoprefix": "https://esm.sh/@twind/[email protected]/"
+        "@twind/preset-autoprefix": "https://esm.sh/@twind/[email protected]/",
+        "@lifegpc/sha1": "jsr:/@lifegpc/[email protected]"
     }
 }

page/SinglePage_test.ts

@@ -1,11 +1,11 @@
 import { assert, assertEquals } from "@std/assert";
 import { Client } from "../client.ts";
 import { load_settings } from "../config.ts";
-import { API_PERMISSION } from "../test_base.ts";
+import { SHA1 } from "@lifegpc/sha1";
+import { getHashFromUrl } from "../utils.ts";
 
 Deno.test({
     name: "SinglePage_test",
-    permissions: API_PERMISSION,
 }, async () => {
     const cfg = await load_settings("./config.json");
     const client = new Client(cfg);
@@ -38,4 +38,10 @@ Deno.test({
     assertEquals(re2.origin_xres, 4893);
     assertEquals(re2.origin_yres, 3446);
     console.log(np.nl, re.nl, re2.nl);
+    console.log(re2.img_url);
+    const res = await client.request(re2.img_url, "GET");
+    const data = await res.arrayBuffer();
+    const h = (new SHA1()).update(new Uint8Array(data)).digest_hex();
+    const oh = getHashFromUrl(re2.img_url);
+    assertEquals(h, oh);
 });

tasks/download.ts

@@ -9,7 +9,7 @@ import {
     TaskType,
 } from "../task.ts";
 import { TaskManager } from "../task_manager.ts";
-import { RecoverableError } from "../utils.ts";
+import { calFileSha1, getHashFromUrl, RecoverableError } from "../utils.ts";
 import {
     add_suffix_to_path,
     asyncEvery,
@@ -181,6 +181,12 @@ class DownloadManager {
     }
 }
 
+class HashError extends RecoverableError {
+    constructor() {
+        super("Hash error");
+    }
+}
+
 interface Image {
     data: unknown;
     index: number;
@@ -402,6 +408,17 @@ export async function download_task(
                                 null;
                             }
                         }
+                        if (manager.cfg.check_file_hash) {
+                            const url = re.url;
+                            const hash = getHashFromUrl(url);
+                            const fhash = await calFileSha1(path);
+                            if (hash != fhash) {
+                                console.warn(
+                                    `Hash not matched: file hash ${fhash}, original hash ${hash}, url ${url}`,
+                                );
+                                throw new HashError();
+                            }
+                        }
                     }
                     const errors: unknown[] = [];
                     function try_download(a: number) {
@@ -426,7 +443,10 @@ export async function download_task(
                                     return;
                                 }
                             }
-                            if (e instanceof TimeoutError) {
+                            if (
+                                e instanceof TimeoutError ||
+                                e instanceof HashError
+                            ) {
                                 m.remove_details(i.index);
                                 reject(e);
                                 return;

utils.ts

@@ -3,6 +3,7 @@ import { extname } from "@std/path";
 import { initParser } from "deno_dom/wasm-noinit";
 import { configure } from "zipjs/index.js";
 import { MD5 } from "lifegpc-md5";
+import { SHA1 } from "@lifegpc/sha1";
 
 export function sleep(time: number): Promise<undefined> {
     return new Promise((r) => {
@@ -191,7 +192,25 @@ export async function calFileMd5(p: string | URL) {
         do {
             readed = await f.read(buf);
             if (readed) {
-                h.update(buf.slice(0, readed));
+                h.update(buf, readed);
+            }
+        } while (readed !== null);
+        return h.digest_hex();
+    } finally {
+        f.close();
+    }
+}
+
+export async function calFileSha1(p: string | URL) {
+    const h = new SHA1();
+    const f = await Deno.open(p);
+    try {
+        const buf = new Uint8Array(4096);
+        let readed: number | null = null;
+        do {
+            readed = await f.read(buf);
+            if (readed) {
+                h.update(buf, readed);
             }
         } while (readed !== null);
         return h.digest_hex();
@@ -313,3 +332,12 @@ export function isNumNaN(num: number | bigint) {
 export function compareNum(num1: number | bigint, num2: number | bigint) {
     return num1 == num2 ? 0 : num1 < num2 ? -1 : 1;
 }
+
+const HASH_PATTERN = /^\/h\/([0-9a-f]+)/;
+
+export function getHashFromUrl(url: string | URL) {
+    const u = typeof url === "string" ? new URL(url) : url;
+    const m = u.pathname.match(HASH_PATTERN);
+    if (m) return m[1];
+    throw Error(`URL ${url} not contains hash info.`);
+}

utils_test.ts

@@ -6,6 +6,7 @@ import {
     asyncFilter,
     asyncForEach,
     calFileMd5,
+    calFileSha1,
     compareNum,
     filterFilename,
     map,
@@ -17,6 +18,7 @@ import {
     toJSON,
 } from "./utils.ts";
 import { md5 } from "lifegpc-md5";
+import { sha1 } from "@lifegpc/sha1";
 
 Deno.test("promiseState_test", async () => {
     const p1 = new Promise((res) => setTimeout(() => res(100), 100));
@@ -104,6 +106,13 @@ Deno.test("calFileMd5_test", async () => {
     assertEquals(await calFileMd5("./test/test.txt"), md5(text));
 });
 
+Deno.test("calFileSha1_test", async () => {
+    await sure_dir();
+    const text = `Hello World.te${Math.random()}dsadasd`;
+    await Deno.writeTextFile("./test/testsha1.txt", text);
+    assertEquals(await calFileSha1("./test/testsha1.txt"), sha1(text));
+});
+
 Deno.test("asyncEvery_test", async () => {
     // @ts-ignore: FUCKED UP
     const list = [];