From ded0770aead1f021fd8abb3303f059f6e610a115 Mon Sep 17 00:00:00 2001 From: johnathan <952508490@qq.com> Date: Thu, 26 Jun 2025 15:36:05 +0800 Subject: [PATCH] Add Page Worker Composables --- src/composables/useCloudExporter.ts | 2 +- src/composables/usePageWorker.ts | 170 ++++++++++++++++++ src/global.d.ts | 2 + src/logic/error-handler.ts | 6 +- src/logic/execute-script.ts | 2 +- src/logic/page-worker/amazon.ts | 46 ++--- src/logic/page-worker/base.ts | 9 + src/logic/page-worker/homedepot.ts | 21 +-- src/logic/page-worker/types.ts | 21 +-- src/logic/storages/amazon.ts | 2 + src/logic/upload.ts | 7 +- src/logic/web-injectors/amazon.ts | 11 +- src/options/views/AmazonResultTable.vue | 11 +- .../views/AmazonEntries/DetailPageEntry.vue | 71 +++----- .../views/AmazonEntries/ReviewPageEntry.vue | 50 ++---- .../views/AmazonEntries/SearchPageEntry.vue | 40 ++--- src/sidepanel/views/AmazonSidepanel.vue | 9 +- 17 files changed, 307 insertions(+), 173 deletions(-) create mode 100644 src/composables/usePageWorker.ts create mode 100644 src/logic/page-worker/base.ts diff --git a/src/composables/useCloudExporter.ts b/src/composables/useCloudExporter.ts index 7991f8c..dee6484 100644 --- a/src/composables/useCloudExporter.ts +++ b/src/composables/useCloudExporter.ts @@ -87,7 +87,7 @@ class ExportExcelPipeline { export type DataFragment = { data: Array>; - imageColumn?: string; + imageColumn?: string | string[]; name?: string; }; diff --git a/src/composables/usePageWorker.ts b/src/composables/usePageWorker.ts new file mode 100644 index 0000000..6f0f3c1 --- /dev/null +++ b/src/composables/usePageWorker.ts @@ -0,0 +1,170 @@ +import { amazon } from '~/logic/page-worker'; +import { uploadImage } from '~/logic/upload'; +import { useLongTask } from './useLongTask'; + +export interface AmazonPageWorkerSettings { + searchItems?: Ref; + detailItems?: Ref>; + reviewItems?: Ref>; + commitChangeIngerval?: number; +} + +class PageWorkerFactory { + public amazonWorker: ReturnType | null = null; + + public amazonWorkerSettings: AmazonPageWorkerSettings = {}; + + public buildAmazonPageWorker() { + const { isRunning, startTask } = useLongTask(); + + const worker = amazon.getAmazonPageWorker(); + + const searchCache = [] as AmazonSearchItem[]; + const detailCache = new Map(); + const reviewCache = new Map(); + + const unsubscribeFuncs = [] as (() => void)[]; + + onMounted(() => { + unsubscribeFuncs.push( + ...[ + worker.on('error', () => { + worker.stop(); + }), + worker.on('item-links-collected', ({ objs }) => { + updateSearchCache(objs); + }), + worker.on('item-base-info-collected', (ev) => { + updateDetailCache(ev); + }), + worker.on('item-category-rank-collected', (ev) => { + updateDetailCache(ev); + }), + worker.on('item-images-collected', (ev) => { + updateDetailCache(ev); + }), + worker.on('item-top-reviews-collected', (ev) => { + updateDetailCache(ev); + }), + worker.on('item-aplus-screenshot-collect', (ev) => { + uploadImage(ev.base64data, `${ev.asin}.png`).then((url) => { + url && updateDetailCache({ asin: ev.asin, aplus: url }); + }); + }), + worker.on('item-review-collected', (ev) => { + updateReviews(ev); + }), + ], + ); + }); + + onUnmounted(() => { + unsubscribeFuncs.forEach((unsubscribe) => unsubscribe()); + unsubscribeFuncs.splice(0, unsubscribeFuncs.length); + }); + + const updateSearchCache = (data: AmazonSearchItem[]) => { + searchCache.push(...data); + }; + + const updateDetailCache = (data: { asin: string } & Partial) => { + const asin = data.asin; + if (detailCache.has(data.asin)) { + const origin = detailCache.get(data.asin); + detailCache.set(asin, { ...origin, ...data } as AmazonDetailItem); + } else { + detailCache.set(asin, data as AmazonDetailItem); + } + }; + + const updateReviews = (data: { asin: string; reviews: AmazonReview[] }) => { + const { asin, reviews } = data; + const values = reviewCache.get(asin) || []; + const ids = new Set(values.map((item) => item.id)); + for (const review of reviews) { + ids.has(review.id) || values.push(review); + } + reviewCache.set(asin, values); + }; + + const commitChange = () => { + const { searchItems, detailItems, reviewItems } = this.amazonWorkerSettings; + if (typeof searchItems !== 'undefined') { + searchItems.value = searchItems.value.concat(searchCache); + searchCache.splice(0, searchCache.length); + } + if (typeof detailItems !== 'undefined') { + for (const [k, v] of detailCache.entries()) { + detailItems.value.delete(k); // Trigger update + detailItems.value.set(k, v); + } + detailCache.clear(); + } + if (typeof reviewItems !== 'undefined') { + for (const [asin, reviews] of reviewCache.entries()) { + if (reviewItems.value.has(asin)) { + const adds = new Set(reviews.map((x) => x.id)); + const newReviews = reviewItems.value + .get(asin)! + .filter((x) => !adds.has(x.id)) + .concat(reviews); + newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo))); + reviewItems.value.delete(asin); // Trigger update + reviewItems.value.set(asin, newReviews); + } else { + reviewItems.value.set(asin, reviews); + } + } + reviewCache.clear(); + } + }; + + const taskWrapper = any>(func: T) => { + const { commitChangeIngerval = 1500 } = this.amazonWorkerSettings; + searchCache.splice(0, searchCache.length); + detailCache.clear(); + reviewCache.clear(); + return (...params: Parameters) => + startTask(async () => { + const interval = setInterval(() => commitChange(), commitChangeIngerval); + await func(...params); + clearInterval(interval); + commitChange(); + }); + }; + + const runDetailPageTask = taskWrapper(worker.runDetailPageTask.bind(worker)); + const runSearchPageTask = taskWrapper(worker.runSearchPageTask.bind(worker)); + const runReviewPageTask = taskWrapper(worker.runReviewPageTask.bind(worker)); + + return { + isRunning, + runSearchPageTask, + runDetailPageTask, + runReviewPageTask, + on: worker.on.bind(worker), + off: worker.off.bind(worker), + once: worker.once.bind(worker), + stop: worker.stop.bind(worker), + }; + } + + loadAmazonPageWorker(settings?: AmazonPageWorkerSettings) { + if (settings) { + this.amazonWorkerSettings = { ...this.amazonWorkerSettings, ...settings }; + } + if (!this.amazonWorker) { + this.amazonWorker = this.buildAmazonPageWorker(); + } + return this.amazonWorker; + } +} + +const facotry = new PageWorkerFactory(); + +export function usePageWorker(type: 'amazon', settings?: AmazonPageWorkerSettings) { + if (type === 'amazon') { + return facotry.loadAmazonPageWorker(settings); + } + throw new Error(`Unsupported page worker type: ${type}`); +} diff --git a/src/global.d.ts b/src/global.d.ts index e4145b6..8ebfc04 100644 --- a/src/global.d.ts +++ b/src/global.d.ts @@ -47,6 +47,8 @@ declare type AmazonSearchItem = { declare type AmazonDetailItem = { asin: string; title: string; + timestamp: string; + broughtInfo?: string; price?: string; rating?: number; ratingCount?: number; diff --git a/src/logic/error-handler.ts b/src/logic/error-handler.ts index 87bdb41..2b07691 100644 --- a/src/logic/error-handler.ts +++ b/src/logic/error-handler.ts @@ -1,7 +1,5 @@ -import Emittery from 'emittery'; - export interface ErrorChannelContainer { - channel: Emittery<{ error: { message: string } }>; + emit: (event: 'error', error: { message: string }) => void; } /** @@ -18,7 +16,7 @@ export function withErrorHandling( try { return await originalMethod.call(this, ...args); // 调用原有方法 } catch (error) { - this.channel.emit('error', { message: `发生未知错误：${error}` }); + this.emit('error', { message: `发生未知错误：${error}` }); throw error; } }; diff --git a/src/logic/execute-script.ts b/src/logic/execute-script.ts index c03411c..e195552 100644 --- a/src/logic/execute-script.ts +++ b/src/logic/execute-script.ts @@ -49,11 +49,11 @@ export async function exec>( return new Promise(async (resolve, reject) => { if (isFirefox) { while (true) { + await new Promise((r) => setTimeout(r, 200)); const tab = await browser.tabs.get(tabId); if (tab.status === 'complete') { break; } - await new Promise((r) => setTimeout(r, 100)); } } setTimeout(() => reject('脚本运行超时'), timeout); diff --git a/src/logic/page-worker/amazon.ts b/src/logic/page-worker/amazon.ts index e6c5eef..84ad3f0 100644 --- a/src/logic/page-worker/amazon.ts +++ b/src/logic/page-worker/amazon.ts @@ -1,4 +1,3 @@ -import Emittery from 'emittery'; import type { AmazonPageWorker, AmazonPageWorkerEvents, LanchTaskBaseOptions } from './types'; import type { Tabs } from 'webextension-polyfill'; import { withErrorHandling } from '../error-handler'; @@ -8,12 +7,16 @@ import { AmazonSearchPageInjector, } from '~/logic/web-injectors/amazon'; import { isForbiddenUrl } from '~/env'; +import { BaseWorker } from './base'; /** * AmazonPageWorkerImpl can run on background & sidepanel & popup, * **can't** run on content script! */ -class AmazonPageWorkerImpl implements AmazonPageWorker { +class AmazonPageWorkerImpl + extends BaseWorker + implements AmazonPageWorker +{ //#region Singleton private static _instance: AmazonPageWorker | null = null; public static getInstance() { @@ -22,13 +25,11 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { } return this._instance; } - private constructor() {} + protected constructor() { + super(); + } //#endregion - private readonly _controlChannel = new Emittery<{ interrupt: undefined }>(); - - public readonly channel = new Emittery(); - private async getCurrentTab(): Promise { const tab = await browser.tabs .query({ active: true, currentWindow: true }) @@ -55,7 +56,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { const hasNextPage = await injector.determineHasNextPage(); await new Promise((resolve) => setTimeout(resolve, 1000)); if (data === null || typeof hasNextPage !== 'boolean') { - this.channel.emit('error', { message: '爬取单页信息失败', url: tab.url }); + this.emit('error', { message: '爬取单页信息失败', url: tab.url }); throw new Error('爬取单页信息失败'); } return { data, hasNextPage, page }; @@ -90,10 +91,10 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { keywords, page, rank: offset + 1 + i, - createTime: new Date().toLocaleString(), + createTime: dayjs().format('YYYY/M/D HH:mm:ss'), asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0], })); - this.channel.emit('item-links-collected', { objs }); + this.emit('item-links-collected', { objs }); offset += data.length; if (!hasNextPage) { break; @@ -131,10 +132,11 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { //#region Fetch Base Info const baseInfo = await injector.getBaseInfo(); const ratingInfo = await injector.getRatingInfo(); - this.channel.emit('item-base-info-collected', { + await this.emit('item-base-info-collected', { asin: params.asin, ...baseInfo, ...ratingInfo, + timestamp: dayjs().format('YYYY/M/D HH:mm:ss'), }); //#endregion //#region Fetch Category Rank Info @@ -158,7 +160,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { info['category2'] = { name, rank }; } } - this.channel.emit('item-category-rank-collected', { + await this.emit('item-category-rank-collected', { asin: params.asin, ...info, }); @@ -167,16 +169,16 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { //#region Fetch Goods' Images const imageUrls = await injector.getImageUrls(); imageUrls.length > 0 && - this.channel.emit('item-images-collected', { + (await this.emit('item-images-collected', { asin: params.asin, imageUrls: Array.from(new Set(imageUrls)), - }); + })); await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds. //#endregion //#region Fetch Top Reviews // const reviews = await injector.getTopReviews(); // reviews.length > 0 && - // this.channel.emit('item-top-reviews-collected', { + // await this.emit('item-top-reviews-collected', { // asin: params.asin, // topReviews: reviews, // }); @@ -184,7 +186,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { // #region Get APlus Sreen shot if (aplus && (await injector.scanAPlus())) { const { b64: base64data } = await injector.captureAPlus(); - this.channel.emit('item-aplus-screenshot-collect', { asin: params.asin, base64data }); + await this.emit('item-aplus-screenshot-collect', { asin: params.asin, base64data }); } // #endregion } @@ -203,7 +205,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { while (true) { await injector.waitForPageLoad(); const reviews = await injector.getSinglePageReviews(); - reviews.length > 0 && this.channel.emit('item-review-collected', { asin, reviews }); + reviews.length > 0 && this.emit('item-review-collected', { asin, reviews }); const hasNextPage = await injector.jumpToNextPageIfExist(); if (!hasNextPage) { break; @@ -220,7 +222,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { const { progress } = options; let remains = [...keywordsList]; let interrupt = false; - const unsubscribe = this._controlChannel.on('interrupt', () => { + const unsubscribe = this.on('interrupt', () => { interrupt = true; }); while (remains.length > 0 && !interrupt) { @@ -232,14 +234,14 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { unsubscribe(); } - public async runDetaiPageTask( + public async runDetailPageTask( asins: string[], options: LanchTaskBaseOptions & { aplus?: boolean } = {}, ): Promise { const { progress, aplus = false } = options; const remains = [...asins]; let interrupt = false; - const unsubscribe = this._controlChannel.on('interrupt', () => { + const unsubscribe = this.on('interrupt', () => { interrupt = true; }); while (remains.length > 0 && !interrupt) { @@ -257,7 +259,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { const { progress } = options; const remains = [...asins]; let interrupt = false; - const unsubscribe = this._controlChannel.on('interrupt', () => { + const unsubscribe = this.on('interrupt', () => { interrupt = true; }); while (remains.length > 0 && !interrupt) { @@ -269,7 +271,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker { } public stop(): Promise { - return this._controlChannel.emit('interrupt'); + return this.emit('interrupt'); } } diff --git a/src/logic/page-worker/base.ts b/src/logic/page-worker/base.ts new file mode 100644 index 0000000..65db471 --- /dev/null +++ b/src/logic/page-worker/base.ts @@ -0,0 +1,9 @@ +import Emittery from 'emittery'; + +@Emittery.mixin('_emittery') +export class BaseWorker { + declare on: Emittery['on']; + declare off: Emittery['off']; + declare emit: Emittery['emit']; + declare once: Emittery['once']; +} diff --git a/src/logic/page-worker/homedepot.ts b/src/logic/page-worker/homedepot.ts index 0a545ec..c267acc 100644 --- a/src/logic/page-worker/homedepot.ts +++ b/src/logic/page-worker/homedepot.ts @@ -1,10 +1,13 @@ -import Emittery from 'emittery'; import type { HomedepotEvents, HomedepotWorker, LanchTaskBaseOptions } from './types'; import { Tabs } from 'webextension-polyfill'; import { withErrorHandling } from '../error-handler'; import { HomedepotDetailPageInjector } from '~/logic/web-injectors/homedepot'; +import { BaseWorker } from './base'; -class HomedepotWorkerImpl implements HomedepotWorker { +class HomedepotWorkerImpl + extends BaseWorker + implements HomedepotWorker +{ private static _instance: HomedepotWorker | null = null; public static getInstance() { if (!HomedepotWorkerImpl._instance) { @@ -12,11 +15,9 @@ class HomedepotWorkerImpl implements HomedepotWorker { } return HomedepotWorkerImpl._instance as HomedepotWorker; } - protected constructor() {} - - readonly channel: Emittery = new Emittery(); - - private readonly _controlChannel = new Emittery<{ interrupt: undefined }>(); + protected constructor() { + super(); + } private async createNewTab(url?: string): Promise { const tab = await browser.tabs.create({ url, active: true }); @@ -30,7 +31,7 @@ class HomedepotWorkerImpl implements HomedepotWorker { const injector = new HomedepotDetailPageInjector(tab); await injector.waitForPageLoad(); const info = await injector.getInfo(); - this.channel.emit('detail-item-collected', { item: { OSMID, ...info } }); + await this.emit('detail-item-collected', { item: { OSMID, ...info } }); setTimeout(() => { browser.tabs.remove(tab.id!); }, 1000); @@ -40,7 +41,7 @@ class HomedepotWorkerImpl implements HomedepotWorker { const { progress } = options; const remains = [...OSMIDs]; let interrupt = false; - const unsubscribe = this._controlChannel.on('interrupt', () => { + const unsubscribe = this.on('interrupt', () => { interrupt = true; }); while (remains.length > 0 && !interrupt) { @@ -52,7 +53,7 @@ class HomedepotWorkerImpl implements HomedepotWorker { } stop(): Promise { - return this._controlChannel.emit('interrupt'); + return this.emit('interrupt'); } } diff --git a/src/logic/page-worker/types.ts b/src/logic/page-worker/types.ts index bd5a959..220f70f 100644 --- a/src/logic/page-worker/types.ts +++ b/src/logic/page-worker/types.ts @@ -1,5 +1,7 @@ import type Emittery from 'emittery'; +type Listener = Pick, 'on' | 'off' | 'once'>; + export type LanchTaskBaseOptions = { progress?: (remains: string[]) => Promise | void }; export interface AmazonPageWorkerEvents { @@ -12,7 +14,7 @@ export interface AmazonPageWorkerEvents { */ ['item-base-info-collected']: Pick< AmazonDetailItem, - 'asin' | 'title' | 'price' | 'rating' | 'ratingCount' + 'asin' | 'title' | 'broughtInfo' | 'price' | 'rating' | 'ratingCount' | 'timestamp' >; /** * The event is fired when worker @@ -40,13 +42,7 @@ export interface AmazonPageWorkerEvents { ['error']: { message: string; url?: string }; } -export interface AmazonPageWorker { - /** - * The channel for communication with the Amazon page worker. - * This is an instance of Emittery, which allows for event-based communication. - */ - readonly channel: Emittery; - +export interface AmazonPageWorker extends Listener { /** * Browsing goods search page and collect links to those goods. * @param keywordsList - The keywords list to search for on Amazon. @@ -59,7 +55,7 @@ export interface AmazonPageWorker { * @param asins Amazon Standard Identification Numbers. * @param options The Options Specify Behaviors. */ - runDetaiPageTask( + runDetailPageTask( asins: string[], options?: LanchTaskBaseOptions & { aplus?: boolean }, ): Promise; @@ -88,12 +84,7 @@ export interface HomedepotEvents { ['error']: { message: string; url?: string }; } -export interface HomedepotWorker { - /** - * The channel for communication with the Homedepot page worker. - */ - readonly channel: Emittery; - +export interface HomedepotWorker extends Listener { /** * Browsing goods detail page and collect target information */ diff --git a/src/logic/storages/amazon.ts b/src/logic/storages/amazon.ts index 6fa4619..5f91928 100644 --- a/src/logic/storages/amazon.ts +++ b/src/logic/storages/amazon.ts @@ -71,6 +71,7 @@ export const allItems = computed({ const detailItemsProps: (keyof AmazonDetailItem)[] = [ 'asin', 'title', + 'timestamp', 'price', 'category1', 'category2', @@ -78,6 +79,7 @@ export const allItems = computed({ 'rating', 'ratingCount', 'topReviews', + 'aplus', ]; detailItems.value = newValue .filter((row) => row.hasDetail) diff --git a/src/logic/upload.ts b/src/logic/upload.ts index 7dbcf42..07fd9f2 100644 --- a/src/logic/upload.ts +++ b/src/logic/upload.ts @@ -3,9 +3,12 @@ import { remoteHost } from '~/env'; export async function uploadImage( base64String: string, filename: string, + contentType: string = 'image/png', ): Promise { // Remove the data URL prefix if present - const base64Data = base64String.replace(/^data:image\/png;base64,/, ''); + const base64Data = base64String.startsWith(`data:${contentType};base64,`) + ? base64String.split(',', 2)[1] + : base64String; // Convert base64 to binary const byteCharacters = atob(base64Data); const byteNumbers = new Array(byteCharacters.length); @@ -14,7 +17,7 @@ export async function uploadImage( } const byteArray = new Uint8Array(byteNumbers); // Create a Blob from the byte array - const blob = new Blob([byteArray], { type: 'image/png' }); + const blob = new Blob([byteArray], { type: contentType }); // Create FormData and append the file const formData = new FormData(); formData.append('file', blob, filename); diff --git a/src/logic/web-injectors/amazon.ts b/src/logic/web-injectors/amazon.ts index 9741a1c..e87c23b 100644 --- a/src/logic/web-injectors/amazon.ts +++ b/src/logic/web-injectors/amazon.ts @@ -168,7 +168,10 @@ export class AmazonDetailPageInjector extends BaseInjector { const price = document.querySelector( '.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen', )?.innerText; - return { title, price }; + const broughtInfo = document.querySelector( + `#social-proofing-faceout-title-tk_bought`, + )?.innerText; + return { title, price, broughtInfo }; }); } @@ -303,7 +306,11 @@ export class AmazonDetailPageInjector extends BaseInjector { public async scanAPlus() { return this.run(async () => { const aplusEl = document.querySelector('#aplus_feature_div'); - if (!aplusEl) { + if ( + !aplusEl || + aplusEl.getClientRects().length === 0 || + aplusEl.getClientRects()[0].height === 0 + ) { return false; } while (aplusEl.getClientRects().length === 0) { diff --git a/src/options/views/AmazonResultTable.vue b/src/options/views/AmazonResultTable.vue index 4e1ea3d..579c411 100644 --- a/src/options/views/AmazonResultTable.vue +++ b/src/options/views/AmazonResultTable.vue @@ -149,12 +149,13 @@ const extraHeaders: Header[] = [ { prop: 'imageUrls', label: '商品图片链接', - formatOutputValue: (val: string[] | undefined, _i, rowData) => { - if (!val) return undefined; - return rowData.aplus ? val.concat([rowData.aplus]).join(';') : val.join(';'); - }, + formatOutputValue: (val?: string[]) => val?.join(';'), parseImportValue: (val?: string) => val?.split(';'), }, + { + prop: 'aplus', + label: 'A+截图', + }, ]; const reviewHeaders: Header[] = [ @@ -244,7 +245,7 @@ const handleCloudExport = async () => { const mappedData1 = await castRecordsByHeaders(items, itemHeaders); const mappedData2 = await castRecordsByHeaders(reviews, reviewHeaders); const fragments = [ - { data: mappedData1, imageColumn: '商品图片链接', name: 'items' }, + { data: mappedData1, imageColumn: ['商品图片链接', 'A+截图'], name: 'items' }, { data: mappedData2, imageColumn: '图片链接', name: 'reviews' }, ]; const filename = await cloudExporter.doExport(fragments); diff --git a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue index d58bbd5..dc86cdc 100644 --- a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue +++ b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue @@ -1,30 +1,17 @@