Save chapter to database now use hash to reduce same chapter

package.json

@@ -2,6 +2,7 @@
   "name": "bookdownload",
   "dependencies": {
     "@ant-design/icons": "^6.1.0",
+    "@stablelib/sha256": "^2.0.1",
     "@types/chrome": "^0.1.36",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",

src/db/indexedDb.ts

@@ -1,6 +1,8 @@
 import { IndexedDbConfig } from "../config";
 import type { QdChapterInfo } from "../types";
 import { compress } from "../utils";
+import { hash_qdchapter_info } from "../utils/qd";
+import type { Db } from "./interfaces";
 
 async function make_storage_persist() {
     const persisted = await navigator.storage.persisted();
@@ -23,14 +25,64 @@ async function save_data<T>(db: IDBDatabase, storeName: string, data: T, key?: I
     });
 }
 
+type QdChapterKey = [number, number, number];
+type QdChapterHashKey = [number, number, string];
+
+async function get_data<T>(db: IDBDatabase, storeName: string, key: IDBValidKey | IDBKeyRange, index?: string): Promise<T | undefined> {
+    return new Promise<T | undefined>((resolve, reject) => {
+        const tx = db.transaction(storeName, 'readonly');
+        const store = tx.objectStore(storeName);
+        const req = index ? store.index(index).get(key) : store.get(key);
+        req.onsuccess = () => {
+            resolve(req.result);
+        }
+        req.onerror = () => {
+            reject(req.error);
+        }
+    });
+}
+
+type GetAllOptions = {
+    index?: string;
+    count?: number;
+}
+
+async function get_datas<T>(db: IDBDatabase, storeName: string, key?: IDBValidKey | IDBKeyRange, options?: GetAllOptions): Promise<T[]> {
+    return new Promise<T[]>((resolve, reject) => {
+        const tx = db.transaction(storeName, 'readonly');
+        const store = tx.objectStore(storeName);
+        const req = options?.index ? store.index(options.index).getAll(key, options.count) : store.getAll(key, options?.count);
+        req.onsuccess = () => {
+            resolve(req.result);
+        }
+        req.onerror = () => {
+            reject(req.error);
+        }
+    });
+}
+
+async function get_keys<K extends IDBValidKey = IDBValidKey>(db: IDBDatabase, storeName: string, query?: IDBValidKey | IDBKeyRange, options?: GetAllOptions): Promise<K[]> {
+    return new Promise<K[]>((resolve, reject) => {
+        const tx = db.transaction(storeName, 'readonly');
+        const store = tx.objectStore(storeName);
+        const req = options?.index ? store.index(options.index).getAllKeys(query, options.count) : store.getAllKeys(query, options?.count);
+        req.onsuccess = () => {
+            resolve(req.result as K[]);
+        }
+        req.onerror = () => {
+            reject(req.error);
+        }
+    });
+}
 type CompressedQdChapterInfo = {
     compressed: Uint8Array;
     bookId: number;
     id: number;
     time: number;
+    hash: string;
 }
 
-export class IndexedDb {
+export class IndexedDb implements Db {
     compress: boolean;
     _qddb?: IDBDatabase;
     constructor(cfg: IndexedDbConfig) {
@@ -54,6 +106,7 @@ export class IndexedDb {
                     const chapters = db.createObjectStore('chapters', { keyPath: ['id', 'bookId', 'time'] });
                     chapters.createIndex('bookId', 'bookId');
                     chapters.createIndex('id', 'id');
+                    chapters.createIndex('hash', ['id', 'bookId', 'hash']);
                 }
             }
             dbreq.onerror = () => {
@@ -70,7 +123,15 @@ export class IndexedDb {
         await this.init_qddb();
     }
     async saveQdChapter(info: QdChapterInfo) {
+        const hash = hash_qdchapter_info(info);
+        const key: QdChapterHashKey = [info.id, info.bookId, hash];
+        const existed = await get_data<CompressedQdChapterInfo | QdChapterInfo>(this.qddb, 'chapters', key, 'hash');
+        if (existed) {
+            console.log(`Chapter ${info.id} of book ${info.bookId} already exists in database, skipping`);
+            return;
+        }
         if (this.compress) {
+            info.hash = undefined;
             const data = JSON.stringify(info);
             const encoded = new TextEncoder().encode(data);
             const compressed = await compress(encoded);
@@ -79,9 +140,11 @@ export class IndexedDb {
                 bookId: info.bookId,
                 id: info.id,
                 time: info.time,
+                hash,
             }
             await save_data(this.qddb, 'chapters', compressedInfo);
         } else {
+            info.hash = hash;
             await save_data(this.qddb, 'chapters', info);
         }
     }

src/db/interfaces.ts

@@ -4,6 +4,10 @@ import type { QdChapterInfo } from "../types";
 
 export interface Db {
     init(): Promise<void>;
+    /**
+     * Save chapter info to database.
+     * @param info Chapter info to save. if id, bookId and hash are matched in the database, skip saving.
+     */
     saveQdChapter(info: QdChapterInfo): Promise<void>;
     close(): void;
 }

src/types.ts

@@ -19,6 +19,7 @@ export type QdChapterInfo = {
     contents?: string[];
     /**Timestamp of the chapter */
     time: number;
+    hash?: string;
 }
 
 export type SendMessageMap = {

src/utils.ts

@@ -131,3 +131,7 @@ export async function decompress(data: BufferSource, method: CompressionFormat =
     }
     return result;
 }
+
+export function ToHex(bytes: Uint8Array): string {
+    return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join('');
+}

src/utils/qd.ts

@@ -0,0 +1,29 @@
+import { SHA256 } from "@stablelib/sha256";
+import type { QdChapterInfo } from "../types";
+import { get_chapter_content, ToHex } from "../utils";
+
+export function hash_qdchapter_info(info: QdChapterInfo): string {
+    const encoder = new TextEncoder();
+    const h = new SHA256();
+    h.update(encoder.encode(info.bookId.toString() + '\n'));
+    h.update(encoder.encode(info.bookInfo.bookName + '\n'));
+    h.update(encoder.encode(info.bookInfo.authorId.toString() + '\n'));
+    h.update(encoder.encode(info.bookInfo.authorName + '\n'));
+    h.update(encoder.encode(info.id.toString() + '\n'));
+    h.update(encoder.encode(info.chapterInfo.vipStatus.toString() + '\n'));
+    h.update(encoder.encode(info.chapterInfo.isBuy.toString() + '\n'));
+    h.update(encoder.encode(info.chapterInfo.updateTime + '\n'));
+    h.update(encoder.encode(info.chapterInfo.chapterName + '\n'));
+    if (info.contents) {
+        for (const line of info.contents) {
+            h.update(encoder.encode(line + '\n'));
+        }
+    } else {
+        const content = get_chapter_content(info.chapterInfo.content);
+        for (const line of content) {
+            h.update(encoder.encode(line + '\n'));
+        }
+    }
+    const hash = h.digest();
+    return ToHex(hash);
+}

yarn.lock

@@ -605,6 +605,37 @@
     "@rc-component/util" "^1.4.0"
     clsx "^2.1.1"
 
+"@stablelib/binary@^2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@stablelib/binary/-/binary-2.0.1.tgz#65c36a24e2c65f375e4c5c4cb340b9112d9badb6"
+  integrity sha512-U9iAO8lXgEDONsA0zPPSgcf3HUBNAqHiJmSHgZz62OvC3Hi2Bhc5kTnQ3S1/L+sthDTHtCMhcEiklmIly6uQ3w==
+  dependencies:
+    "@stablelib/int" "^2.0.1"
+
+"@stablelib/hash@^2.0.0":
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/@stablelib/hash/-/hash-2.0.0.tgz#7b74c372dc07187e273844e970a475f1338e92cf"
+  integrity sha512-u3WPSqGido8lwJuMcrBgM5K54LrPGhkWAdtsyccf7dGsLixAZUds77zOAbu7bvKPwQlmoByH0txBi5rTmEKuHg==
+
+"@stablelib/int@^2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@stablelib/int/-/int-2.0.1.tgz#daf262843b158e6bb99ec029a14378ecdda2230f"
+  integrity sha512-Ht63fQp3wz/F8U4AlXEPb7hfJOIILs8Lq55jgtD7KueWtyjhVuzcsGLSTAWtZs3XJDZYdF1WcSKn+kBtbzupww==
+
+"@stablelib/sha256@^2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@stablelib/sha256/-/sha256-2.0.1.tgz#9455c64e3d14d3ebd59523ab6424f728a0fd8347"
+  integrity sha512-LA1PaLDc6Lv72ppA4PEZ7abDE741KfG7k7QhBiUyIfViMqrwWv8HqQQFPeuPfS4k2OxFv++IAgc8HlvdBatD+w==
+  dependencies:
+    "@stablelib/binary" "^2.0.1"
+    "@stablelib/hash" "^2.0.0"
+    "@stablelib/wipe" "^2.0.1"
+
+"@stablelib/wipe@^2.0.1":
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/@stablelib/wipe/-/wipe-2.0.1.tgz#9bc1d20519aa4fc513fe1992cf8061bab33c3049"
+  integrity sha512-1eU2K9EgOcV4qc9jcP6G72xxZxEm5PfeI5H55l08W95b4oRJaqhmlWRc4xZAm6IVSKhVNxMi66V67hCzzuMTAg==
+
 "@types/chrome@^0.1.36":
   version "0.1.36"
   resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.1.36.tgz#c2b91964c9d93c6d690335441289abdda5ab3130"