From e3554324366db339483017903fa343deff53e463 Mon Sep 17 00:00:00 2001 From: lifegpc Date: Sun, 15 Feb 2026 10:43:47 +0800 Subject: [PATCH] Save chapter to database now use hash to reduce same chapter --- package.json | 1 + src/db/indexedDb.ts | 65 +++++++++++++++++++++++++++++++++++++++++++- src/db/interfaces.ts | 4 +++ src/types.ts | 1 + src/utils.ts | 4 +++ src/utils/qd.ts | 29 ++++++++++++++++++++ yarn.lock | 31 +++++++++++++++++++++ 7 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 src/utils/qd.ts diff --git a/package.json b/package.json index c57a984..9fe4d94 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "bookdownload", "dependencies": { "@ant-design/icons": "^6.1.0", + "@stablelib/sha256": "^2.0.1", "@types/chrome": "^0.1.36", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", diff --git a/src/db/indexedDb.ts b/src/db/indexedDb.ts index ebe77c5..c483a96 100644 --- a/src/db/indexedDb.ts +++ b/src/db/indexedDb.ts @@ -1,6 +1,8 @@ import { IndexedDbConfig } from "../config"; import type { QdChapterInfo } from "../types"; import { compress } from "../utils"; +import { hash_qdchapter_info } from "../utils/qd"; +import type { Db } from "./interfaces"; async function make_storage_persist() { const persisted = await navigator.storage.persisted(); @@ -23,14 +25,64 @@ async function save_data(db: IDBDatabase, storeName: string, data: T, key?: I }); } +type QdChapterKey = [number, number, number]; +type QdChapterHashKey = [number, number, string]; + +async function get_data(db: IDBDatabase, storeName: string, key: IDBValidKey | IDBKeyRange, index?: string): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(storeName, 'readonly'); + const store = tx.objectStore(storeName); + const req = index ? store.index(index).get(key) : store.get(key); + req.onsuccess = () => { + resolve(req.result); + } + req.onerror = () => { + reject(req.error); + } + }); +} + +type GetAllOptions = { + index?: string; + count?: number; +} + +async function get_datas(db: IDBDatabase, storeName: string, key?: IDBValidKey | IDBKeyRange, options?: GetAllOptions): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(storeName, 'readonly'); + const store = tx.objectStore(storeName); + const req = options?.index ? store.index(options.index).getAll(key, options.count) : store.getAll(key, options?.count); + req.onsuccess = () => { + resolve(req.result); + } + req.onerror = () => { + reject(req.error); + } + }); +} + +async function get_keys(db: IDBDatabase, storeName: string, query?: IDBValidKey | IDBKeyRange, options?: GetAllOptions): Promise { + return new Promise((resolve, reject) => { + const tx = db.transaction(storeName, 'readonly'); + const store = tx.objectStore(storeName); + const req = options?.index ? store.index(options.index).getAllKeys(query, options.count) : store.getAllKeys(query, options?.count); + req.onsuccess = () => { + resolve(req.result as K[]); + } + req.onerror = () => { + reject(req.error); + } + }); +} type CompressedQdChapterInfo = { compressed: Uint8Array; bookId: number; id: number; time: number; + hash: string; } -export class IndexedDb { +export class IndexedDb implements Db { compress: boolean; _qddb?: IDBDatabase; constructor(cfg: IndexedDbConfig) { @@ -54,6 +106,7 @@ export class IndexedDb { const chapters = db.createObjectStore('chapters', { keyPath: ['id', 'bookId', 'time'] }); chapters.createIndex('bookId', 'bookId'); chapters.createIndex('id', 'id'); + chapters.createIndex('hash', ['id', 'bookId', 'hash']); } } dbreq.onerror = () => { @@ -70,7 +123,15 @@ export class IndexedDb { await this.init_qddb(); } async saveQdChapter(info: QdChapterInfo) { + const hash = hash_qdchapter_info(info); + const key: QdChapterHashKey = [info.id, info.bookId, hash]; + const existed = await get_data(this.qddb, 'chapters', key, 'hash'); + if (existed) { + console.log(`Chapter ${info.id} of book ${info.bookId} already exists in database, skipping`); + return; + } if (this.compress) { + info.hash = undefined; const data = JSON.stringify(info); const encoded = new TextEncoder().encode(data); const compressed = await compress(encoded); @@ -79,9 +140,11 @@ export class IndexedDb { bookId: info.bookId, id: info.id, time: info.time, + hash, } await save_data(this.qddb, 'chapters', compressedInfo); } else { + info.hash = hash; await save_data(this.qddb, 'chapters', info); } } diff --git a/src/db/interfaces.ts b/src/db/interfaces.ts index d38a4c2..91728b6 100644 --- a/src/db/interfaces.ts +++ b/src/db/interfaces.ts @@ -4,6 +4,10 @@ import type { QdChapterInfo } from "../types"; export interface Db { init(): Promise; + /** + * Save chapter info to database. + * @param info Chapter info to save. if id, bookId and hash are matched in the database, skip saving. + */ saveQdChapter(info: QdChapterInfo): Promise; close(): void; } diff --git a/src/types.ts b/src/types.ts index ff08181..6bab927 100644 --- a/src/types.ts +++ b/src/types.ts @@ -19,6 +19,7 @@ export type QdChapterInfo = { contents?: string[]; /**Timestamp of the chapter */ time: number; + hash?: string; } export type SendMessageMap = { diff --git a/src/utils.ts b/src/utils.ts index 399ca98..93645cd 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -131,3 +131,7 @@ export async function decompress(data: BufferSource, method: CompressionFormat = } return result; } + +export function ToHex(bytes: Uint8Array): string { + return Array.from(bytes).map(b => b.toString(16).padStart(2, '0')).join(''); +} diff --git a/src/utils/qd.ts b/src/utils/qd.ts new file mode 100644 index 0000000..1f0f5cc --- /dev/null +++ b/src/utils/qd.ts @@ -0,0 +1,29 @@ +import { SHA256 } from "@stablelib/sha256"; +import type { QdChapterInfo } from "../types"; +import { get_chapter_content, ToHex } from "../utils"; + +export function hash_qdchapter_info(info: QdChapterInfo): string { + const encoder = new TextEncoder(); + const h = new SHA256(); + h.update(encoder.encode(info.bookId.toString() + '\n')); + h.update(encoder.encode(info.bookInfo.bookName + '\n')); + h.update(encoder.encode(info.bookInfo.authorId.toString() + '\n')); + h.update(encoder.encode(info.bookInfo.authorName + '\n')); + h.update(encoder.encode(info.id.toString() + '\n')); + h.update(encoder.encode(info.chapterInfo.vipStatus.toString() + '\n')); + h.update(encoder.encode(info.chapterInfo.isBuy.toString() + '\n')); + h.update(encoder.encode(info.chapterInfo.updateTime + '\n')); + h.update(encoder.encode(info.chapterInfo.chapterName + '\n')); + if (info.contents) { + for (const line of info.contents) { + h.update(encoder.encode(line + '\n')); + } + } else { + const content = get_chapter_content(info.chapterInfo.content); + for (const line of content) { + h.update(encoder.encode(line + '\n')); + } + } + const hash = h.digest(); + return ToHex(hash); +} diff --git a/yarn.lock b/yarn.lock index e27830f..de5c9a7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -605,6 +605,37 @@ "@rc-component/util" "^1.4.0" clsx "^2.1.1" +"@stablelib/binary@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@stablelib/binary/-/binary-2.0.1.tgz#65c36a24e2c65f375e4c5c4cb340b9112d9badb6" + integrity sha512-U9iAO8lXgEDONsA0zPPSgcf3HUBNAqHiJmSHgZz62OvC3Hi2Bhc5kTnQ3S1/L+sthDTHtCMhcEiklmIly6uQ3w== + dependencies: + "@stablelib/int" "^2.0.1" + +"@stablelib/hash@^2.0.0": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@stablelib/hash/-/hash-2.0.0.tgz#7b74c372dc07187e273844e970a475f1338e92cf" + integrity sha512-u3WPSqGido8lwJuMcrBgM5K54LrPGhkWAdtsyccf7dGsLixAZUds77zOAbu7bvKPwQlmoByH0txBi5rTmEKuHg== + +"@stablelib/int@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@stablelib/int/-/int-2.0.1.tgz#daf262843b158e6bb99ec029a14378ecdda2230f" + integrity sha512-Ht63fQp3wz/F8U4AlXEPb7hfJOIILs8Lq55jgtD7KueWtyjhVuzcsGLSTAWtZs3XJDZYdF1WcSKn+kBtbzupww== + +"@stablelib/sha256@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@stablelib/sha256/-/sha256-2.0.1.tgz#9455c64e3d14d3ebd59523ab6424f728a0fd8347" + integrity sha512-LA1PaLDc6Lv72ppA4PEZ7abDE741KfG7k7QhBiUyIfViMqrwWv8HqQQFPeuPfS4k2OxFv++IAgc8HlvdBatD+w== + dependencies: + "@stablelib/binary" "^2.0.1" + "@stablelib/hash" "^2.0.0" + "@stablelib/wipe" "^2.0.1" + +"@stablelib/wipe@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@stablelib/wipe/-/wipe-2.0.1.tgz#9bc1d20519aa4fc513fe1992cf8061bab33c3049" + integrity sha512-1eU2K9EgOcV4qc9jcP6G72xxZxEm5PfeI5H55l08W95b4oRJaqhmlWRc4xZAm6IVSKhVNxMi66V67hCzzuMTAg== + "@types/chrome@^0.1.36": version "0.1.36" resolved "https://registry.yarnpkg.com/@types/chrome/-/chrome-0.1.36.tgz#c2b91964c9d93c6d690335441289abdda5ab3130"