Add Page Worker Composables

This commit is contained in:
johnathan 2025-06-26 15:36:05 +08:00
parent da5e76aae7
commit ded0770aea
17 changed files with 307 additions and 173 deletions

View File

@ -87,7 +87,7 @@ class ExportExcelPipeline {
export type DataFragment = { export type DataFragment = {
data: Array<Record<string, any>>; data: Array<Record<string, any>>;
imageColumn?: string; imageColumn?: string | string[];
name?: string; name?: string;
}; };

View File

@ -0,0 +1,170 @@
import { amazon } from '~/logic/page-worker';
import { uploadImage } from '~/logic/upload';
import { useLongTask } from './useLongTask';
export interface AmazonPageWorkerSettings {
searchItems?: Ref<AmazonSearchItem[]>;
detailItems?: Ref<Map<string, AmazonDetailItem>>;
reviewItems?: Ref<Map<string, AmazonReview[]>>;
commitChangeIngerval?: number;
}
class PageWorkerFactory {
public amazonWorker: ReturnType<typeof this.buildAmazonPageWorker> | null = null;
public amazonWorkerSettings: AmazonPageWorkerSettings = {};
public buildAmazonPageWorker() {
const { isRunning, startTask } = useLongTask();
const worker = amazon.getAmazonPageWorker();
const searchCache = [] as AmazonSearchItem[];
const detailCache = new Map<string, AmazonDetailItem>();
const reviewCache = new Map<string, AmazonReview[]>();
const unsubscribeFuncs = [] as (() => void)[];
onMounted(() => {
unsubscribeFuncs.push(
...[
worker.on('error', () => {
worker.stop();
}),
worker.on('item-links-collected', ({ objs }) => {
updateSearchCache(objs);
}),
worker.on('item-base-info-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-category-rank-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-images-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-top-reviews-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-aplus-screenshot-collect', (ev) => {
uploadImage(ev.base64data, `${ev.asin}.png`).then((url) => {
url && updateDetailCache({ asin: ev.asin, aplus: url });
});
}),
worker.on('item-review-collected', (ev) => {
updateReviews(ev);
}),
],
);
});
onUnmounted(() => {
unsubscribeFuncs.forEach((unsubscribe) => unsubscribe());
unsubscribeFuncs.splice(0, unsubscribeFuncs.length);
});
const updateSearchCache = (data: AmazonSearchItem[]) => {
searchCache.push(...data);
};
const updateDetailCache = (data: { asin: string } & Partial<AmazonDetailItem>) => {
const asin = data.asin;
if (detailCache.has(data.asin)) {
const origin = detailCache.get(data.asin);
detailCache.set(asin, { ...origin, ...data } as AmazonDetailItem);
} else {
detailCache.set(asin, data as AmazonDetailItem);
}
};
const updateReviews = (data: { asin: string; reviews: AmazonReview[] }) => {
const { asin, reviews } = data;
const values = reviewCache.get(asin) || [];
const ids = new Set(values.map((item) => item.id));
for (const review of reviews) {
ids.has(review.id) || values.push(review);
}
reviewCache.set(asin, values);
};
const commitChange = () => {
const { searchItems, detailItems, reviewItems } = this.amazonWorkerSettings;
if (typeof searchItems !== 'undefined') {
searchItems.value = searchItems.value.concat(searchCache);
searchCache.splice(0, searchCache.length);
}
if (typeof detailItems !== 'undefined') {
for (const [k, v] of detailCache.entries()) {
detailItems.value.delete(k); // Trigger update
detailItems.value.set(k, v);
}
detailCache.clear();
}
if (typeof reviewItems !== 'undefined') {
for (const [asin, reviews] of reviewCache.entries()) {
if (reviewItems.value.has(asin)) {
const adds = new Set(reviews.map((x) => x.id));
const newReviews = reviewItems.value
.get(asin)!
.filter((x) => !adds.has(x.id))
.concat(reviews);
newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo)));
reviewItems.value.delete(asin); // Trigger update
reviewItems.value.set(asin, newReviews);
} else {
reviewItems.value.set(asin, reviews);
}
}
reviewCache.clear();
}
};
const taskWrapper = <T extends (...params: any) => any>(func: T) => {
const { commitChangeIngerval = 1500 } = this.amazonWorkerSettings;
searchCache.splice(0, searchCache.length);
detailCache.clear();
reviewCache.clear();
return (...params: Parameters<T>) =>
startTask(async () => {
const interval = setInterval(() => commitChange(), commitChangeIngerval);
await func(...params);
clearInterval(interval);
commitChange();
});
};
const runDetailPageTask = taskWrapper(worker.runDetailPageTask.bind(worker));
const runSearchPageTask = taskWrapper(worker.runSearchPageTask.bind(worker));
const runReviewPageTask = taskWrapper(worker.runReviewPageTask.bind(worker));
return {
isRunning,
runSearchPageTask,
runDetailPageTask,
runReviewPageTask,
on: worker.on.bind(worker),
off: worker.off.bind(worker),
once: worker.once.bind(worker),
stop: worker.stop.bind(worker),
};
}
loadAmazonPageWorker(settings?: AmazonPageWorkerSettings) {
if (settings) {
this.amazonWorkerSettings = { ...this.amazonWorkerSettings, ...settings };
}
if (!this.amazonWorker) {
this.amazonWorker = this.buildAmazonPageWorker();
}
return this.amazonWorker;
}
}
const facotry = new PageWorkerFactory();
export function usePageWorker(type: 'amazon', settings?: AmazonPageWorkerSettings) {
if (type === 'amazon') {
return facotry.loadAmazonPageWorker(settings);
}
throw new Error(`Unsupported page worker type: ${type}`);
}

2
src/global.d.ts vendored
View File

@ -47,6 +47,8 @@ declare type AmazonSearchItem = {
declare type AmazonDetailItem = { declare type AmazonDetailItem = {
asin: string; asin: string;
title: string; title: string;
timestamp: string;
broughtInfo?: string;
price?: string; price?: string;
rating?: number; rating?: number;
ratingCount?: number; ratingCount?: number;

View File

@ -1,7 +1,5 @@
import Emittery from 'emittery';
export interface ErrorChannelContainer { export interface ErrorChannelContainer {
channel: Emittery<{ error: { message: string } }>; emit: (event: 'error', error: { message: string }) => void;
} }
/** /**
@ -18,7 +16,7 @@ export function withErrorHandling(
try { try {
return await originalMethod.call(this, ...args); // 调用原有方法 return await originalMethod.call(this, ...args); // 调用原有方法
} catch (error) { } catch (error) {
this.channel.emit('error', { message: `发生未知错误:${error}` }); this.emit('error', { message: `发生未知错误:${error}` });
throw error; throw error;
} }
}; };

View File

@ -49,11 +49,11 @@ export async function exec<T, P extends Record<string, unknown>>(
return new Promise<T>(async (resolve, reject) => { return new Promise<T>(async (resolve, reject) => {
if (isFirefox) { if (isFirefox) {
while (true) { while (true) {
await new Promise<void>((r) => setTimeout(r, 200));
const tab = await browser.tabs.get(tabId); const tab = await browser.tabs.get(tabId);
if (tab.status === 'complete') { if (tab.status === 'complete') {
break; break;
} }
await new Promise<void>((r) => setTimeout(r, 100));
} }
} }
setTimeout(() => reject('脚本运行超时'), timeout); setTimeout(() => reject('脚本运行超时'), timeout);

View File

@ -1,4 +1,3 @@
import Emittery from 'emittery';
import type { AmazonPageWorker, AmazonPageWorkerEvents, LanchTaskBaseOptions } from './types'; import type { AmazonPageWorker, AmazonPageWorkerEvents, LanchTaskBaseOptions } from './types';
import type { Tabs } from 'webextension-polyfill'; import type { Tabs } from 'webextension-polyfill';
import { withErrorHandling } from '../error-handler'; import { withErrorHandling } from '../error-handler';
@ -8,12 +7,16 @@ import {
AmazonSearchPageInjector, AmazonSearchPageInjector,
} from '~/logic/web-injectors/amazon'; } from '~/logic/web-injectors/amazon';
import { isForbiddenUrl } from '~/env'; import { isForbiddenUrl } from '~/env';
import { BaseWorker } from './base';
/** /**
* AmazonPageWorkerImpl can run on background & sidepanel & popup, * AmazonPageWorkerImpl can run on background & sidepanel & popup,
* **can't** run on content script! * **can't** run on content script!
*/ */
class AmazonPageWorkerImpl implements AmazonPageWorker { class AmazonPageWorkerImpl
extends BaseWorker<AmazonPageWorkerEvents & { interrupt: undefined }>
implements AmazonPageWorker
{
//#region Singleton //#region Singleton
private static _instance: AmazonPageWorker | null = null; private static _instance: AmazonPageWorker | null = null;
public static getInstance() { public static getInstance() {
@ -22,13 +25,11 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
} }
return this._instance; return this._instance;
} }
private constructor() {} protected constructor() {
super();
}
//#endregion //#endregion
private readonly _controlChannel = new Emittery<{ interrupt: undefined }>();
public readonly channel = new Emittery<AmazonPageWorkerEvents>();
private async getCurrentTab(): Promise<Tabs.Tab> { private async getCurrentTab(): Promise<Tabs.Tab> {
const tab = await browser.tabs const tab = await browser.tabs
.query({ active: true, currentWindow: true }) .query({ active: true, currentWindow: true })
@ -55,7 +56,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
const hasNextPage = await injector.determineHasNextPage(); const hasNextPage = await injector.determineHasNextPage();
await new Promise((resolve) => setTimeout(resolve, 1000)); await new Promise((resolve) => setTimeout(resolve, 1000));
if (data === null || typeof hasNextPage !== 'boolean') { if (data === null || typeof hasNextPage !== 'boolean') {
this.channel.emit('error', { message: '爬取单页信息失败', url: tab.url }); this.emit('error', { message: '爬取单页信息失败', url: tab.url });
throw new Error('爬取单页信息失败'); throw new Error('爬取单页信息失败');
} }
return { data, hasNextPage, page }; return { data, hasNextPage, page };
@ -90,10 +91,10 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
keywords, keywords,
page, page,
rank: offset + 1 + i, rank: offset + 1 + i,
createTime: new Date().toLocaleString(), createTime: dayjs().format('YYYY/M/D HH:mm:ss'),
asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0], asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0],
})); }));
this.channel.emit('item-links-collected', { objs }); this.emit('item-links-collected', { objs });
offset += data.length; offset += data.length;
if (!hasNextPage) { if (!hasNextPage) {
break; break;
@ -131,10 +132,11 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
//#region Fetch Base Info //#region Fetch Base Info
const baseInfo = await injector.getBaseInfo(); const baseInfo = await injector.getBaseInfo();
const ratingInfo = await injector.getRatingInfo(); const ratingInfo = await injector.getRatingInfo();
this.channel.emit('item-base-info-collected', { await this.emit('item-base-info-collected', {
asin: params.asin, asin: params.asin,
...baseInfo, ...baseInfo,
...ratingInfo, ...ratingInfo,
timestamp: dayjs().format('YYYY/M/D HH:mm:ss'),
}); });
//#endregion //#endregion
//#region Fetch Category Rank Info //#region Fetch Category Rank Info
@ -158,7 +160,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
info['category2'] = { name, rank }; info['category2'] = { name, rank };
} }
} }
this.channel.emit('item-category-rank-collected', { await this.emit('item-category-rank-collected', {
asin: params.asin, asin: params.asin,
...info, ...info,
}); });
@ -167,16 +169,16 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
//#region Fetch Goods' Images //#region Fetch Goods' Images
const imageUrls = await injector.getImageUrls(); const imageUrls = await injector.getImageUrls();
imageUrls.length > 0 && imageUrls.length > 0 &&
this.channel.emit('item-images-collected', { (await this.emit('item-images-collected', {
asin: params.asin, asin: params.asin,
imageUrls: Array.from(new Set(imageUrls)), imageUrls: Array.from(new Set(imageUrls)),
}); }));
await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds. await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds.
//#endregion //#endregion
//#region Fetch Top Reviews //#region Fetch Top Reviews
// const reviews = await injector.getTopReviews(); // const reviews = await injector.getTopReviews();
// reviews.length > 0 && // reviews.length > 0 &&
// this.channel.emit('item-top-reviews-collected', { // await this.emit('item-top-reviews-collected', {
// asin: params.asin, // asin: params.asin,
// topReviews: reviews, // topReviews: reviews,
// }); // });
@ -184,7 +186,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
// #region Get APlus Sreen shot // #region Get APlus Sreen shot
if (aplus && (await injector.scanAPlus())) { if (aplus && (await injector.scanAPlus())) {
const { b64: base64data } = await injector.captureAPlus(); const { b64: base64data } = await injector.captureAPlus();
this.channel.emit('item-aplus-screenshot-collect', { asin: params.asin, base64data }); await this.emit('item-aplus-screenshot-collect', { asin: params.asin, base64data });
} }
// #endregion // #endregion
} }
@ -203,7 +205,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
while (true) { while (true) {
await injector.waitForPageLoad(); await injector.waitForPageLoad();
const reviews = await injector.getSinglePageReviews(); const reviews = await injector.getSinglePageReviews();
reviews.length > 0 && this.channel.emit('item-review-collected', { asin, reviews }); reviews.length > 0 && this.emit('item-review-collected', { asin, reviews });
const hasNextPage = await injector.jumpToNextPageIfExist(); const hasNextPage = await injector.jumpToNextPageIfExist();
if (!hasNextPage) { if (!hasNextPage) {
break; break;
@ -220,7 +222,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
const { progress } = options; const { progress } = options;
let remains = [...keywordsList]; let remains = [...keywordsList];
let interrupt = false; let interrupt = false;
const unsubscribe = this._controlChannel.on('interrupt', () => { const unsubscribe = this.on('interrupt', () => {
interrupt = true; interrupt = true;
}); });
while (remains.length > 0 && !interrupt) { while (remains.length > 0 && !interrupt) {
@ -232,14 +234,14 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
unsubscribe(); unsubscribe();
} }
public async runDetaiPageTask( public async runDetailPageTask(
asins: string[], asins: string[],
options: LanchTaskBaseOptions & { aplus?: boolean } = {}, options: LanchTaskBaseOptions & { aplus?: boolean } = {},
): Promise<void> { ): Promise<void> {
const { progress, aplus = false } = options; const { progress, aplus = false } = options;
const remains = [...asins]; const remains = [...asins];
let interrupt = false; let interrupt = false;
const unsubscribe = this._controlChannel.on('interrupt', () => { const unsubscribe = this.on('interrupt', () => {
interrupt = true; interrupt = true;
}); });
while (remains.length > 0 && !interrupt) { while (remains.length > 0 && !interrupt) {
@ -257,7 +259,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
const { progress } = options; const { progress } = options;
const remains = [...asins]; const remains = [...asins];
let interrupt = false; let interrupt = false;
const unsubscribe = this._controlChannel.on('interrupt', () => { const unsubscribe = this.on('interrupt', () => {
interrupt = true; interrupt = true;
}); });
while (remains.length > 0 && !interrupt) { while (remains.length > 0 && !interrupt) {
@ -269,7 +271,7 @@ class AmazonPageWorkerImpl implements AmazonPageWorker {
} }
public stop(): Promise<void> { public stop(): Promise<void> {
return this._controlChannel.emit('interrupt'); return this.emit('interrupt');
} }
} }

View File

@ -0,0 +1,9 @@
import Emittery from 'emittery';
@Emittery.mixin('_emittery')
export class BaseWorker<EV> {
declare on: Emittery<EV>['on'];
declare off: Emittery<EV>['off'];
declare emit: Emittery<EV>['emit'];
declare once: Emittery<EV>['once'];
}

View File

@ -1,10 +1,13 @@
import Emittery from 'emittery';
import type { HomedepotEvents, HomedepotWorker, LanchTaskBaseOptions } from './types'; import type { HomedepotEvents, HomedepotWorker, LanchTaskBaseOptions } from './types';
import { Tabs } from 'webextension-polyfill'; import { Tabs } from 'webextension-polyfill';
import { withErrorHandling } from '../error-handler'; import { withErrorHandling } from '../error-handler';
import { HomedepotDetailPageInjector } from '~/logic/web-injectors/homedepot'; import { HomedepotDetailPageInjector } from '~/logic/web-injectors/homedepot';
import { BaseWorker } from './base';
class HomedepotWorkerImpl implements HomedepotWorker { class HomedepotWorkerImpl
extends BaseWorker<HomedepotEvents & { interrupt: undefined }>
implements HomedepotWorker
{
private static _instance: HomedepotWorker | null = null; private static _instance: HomedepotWorker | null = null;
public static getInstance() { public static getInstance() {
if (!HomedepotWorkerImpl._instance) { if (!HomedepotWorkerImpl._instance) {
@ -12,11 +15,9 @@ class HomedepotWorkerImpl implements HomedepotWorker {
} }
return HomedepotWorkerImpl._instance as HomedepotWorker; return HomedepotWorkerImpl._instance as HomedepotWorker;
} }
protected constructor() {} protected constructor() {
super();
readonly channel: Emittery<HomedepotEvents> = new Emittery(); }
private readonly _controlChannel = new Emittery<{ interrupt: undefined }>();
private async createNewTab(url?: string): Promise<Tabs.Tab> { private async createNewTab(url?: string): Promise<Tabs.Tab> {
const tab = await browser.tabs.create({ url, active: true }); const tab = await browser.tabs.create({ url, active: true });
@ -30,7 +31,7 @@ class HomedepotWorkerImpl implements HomedepotWorker {
const injector = new HomedepotDetailPageInjector(tab); const injector = new HomedepotDetailPageInjector(tab);
await injector.waitForPageLoad(); await injector.waitForPageLoad();
const info = await injector.getInfo(); const info = await injector.getInfo();
this.channel.emit('detail-item-collected', { item: { OSMID, ...info } }); await this.emit('detail-item-collected', { item: { OSMID, ...info } });
setTimeout(() => { setTimeout(() => {
browser.tabs.remove(tab.id!); browser.tabs.remove(tab.id!);
}, 1000); }, 1000);
@ -40,7 +41,7 @@ class HomedepotWorkerImpl implements HomedepotWorker {
const { progress } = options; const { progress } = options;
const remains = [...OSMIDs]; const remains = [...OSMIDs];
let interrupt = false; let interrupt = false;
const unsubscribe = this._controlChannel.on('interrupt', () => { const unsubscribe = this.on('interrupt', () => {
interrupt = true; interrupt = true;
}); });
while (remains.length > 0 && !interrupt) { while (remains.length > 0 && !interrupt) {
@ -52,7 +53,7 @@ class HomedepotWorkerImpl implements HomedepotWorker {
} }
stop(): Promise<void> { stop(): Promise<void> {
return this._controlChannel.emit('interrupt'); return this.emit('interrupt');
} }
} }

View File

@ -1,5 +1,7 @@
import type Emittery from 'emittery'; import type Emittery from 'emittery';
type Listener<T> = Pick<Emittery<T>, 'on' | 'off' | 'once'>;
export type LanchTaskBaseOptions = { progress?: (remains: string[]) => Promise<void> | void }; export type LanchTaskBaseOptions = { progress?: (remains: string[]) => Promise<void> | void };
export interface AmazonPageWorkerEvents { export interface AmazonPageWorkerEvents {
@ -12,7 +14,7 @@ export interface AmazonPageWorkerEvents {
*/ */
['item-base-info-collected']: Pick< ['item-base-info-collected']: Pick<
AmazonDetailItem, AmazonDetailItem,
'asin' | 'title' | 'price' | 'rating' | 'ratingCount' 'asin' | 'title' | 'broughtInfo' | 'price' | 'rating' | 'ratingCount' | 'timestamp'
>; >;
/** /**
* The event is fired when worker * The event is fired when worker
@ -40,13 +42,7 @@ export interface AmazonPageWorkerEvents {
['error']: { message: string; url?: string }; ['error']: { message: string; url?: string };
} }
export interface AmazonPageWorker { export interface AmazonPageWorker extends Listener<AmazonPageWorkerEvents> {
/**
* The channel for communication with the Amazon page worker.
* This is an instance of Emittery, which allows for event-based communication.
*/
readonly channel: Emittery<AmazonPageWorkerEvents>;
/** /**
* Browsing goods search page and collect links to those goods. * Browsing goods search page and collect links to those goods.
* @param keywordsList - The keywords list to search for on Amazon. * @param keywordsList - The keywords list to search for on Amazon.
@ -59,7 +55,7 @@ export interface AmazonPageWorker {
* @param asins Amazon Standard Identification Numbers. * @param asins Amazon Standard Identification Numbers.
* @param options The Options Specify Behaviors. * @param options The Options Specify Behaviors.
*/ */
runDetaiPageTask( runDetailPageTask(
asins: string[], asins: string[],
options?: LanchTaskBaseOptions & { aplus?: boolean }, options?: LanchTaskBaseOptions & { aplus?: boolean },
): Promise<void>; ): Promise<void>;
@ -88,12 +84,7 @@ export interface HomedepotEvents {
['error']: { message: string; url?: string }; ['error']: { message: string; url?: string };
} }
export interface HomedepotWorker { export interface HomedepotWorker extends Listener<HomedepotEvents> {
/**
* The channel for communication with the Homedepot page worker.
*/
readonly channel: Emittery<HomedepotEvents>;
/** /**
* Browsing goods detail page and collect target information * Browsing goods detail page and collect target information
*/ */

View File

@ -71,6 +71,7 @@ export const allItems = computed({
const detailItemsProps: (keyof AmazonDetailItem)[] = [ const detailItemsProps: (keyof AmazonDetailItem)[] = [
'asin', 'asin',
'title', 'title',
'timestamp',
'price', 'price',
'category1', 'category1',
'category2', 'category2',
@ -78,6 +79,7 @@ export const allItems = computed({
'rating', 'rating',
'ratingCount', 'ratingCount',
'topReviews', 'topReviews',
'aplus',
]; ];
detailItems.value = newValue detailItems.value = newValue
.filter((row) => row.hasDetail) .filter((row) => row.hasDetail)

View File

@ -3,9 +3,12 @@ import { remoteHost } from '~/env';
export async function uploadImage( export async function uploadImage(
base64String: string, base64String: string,
filename: string, filename: string,
contentType: string = 'image/png',
): Promise<string | undefined> { ): Promise<string | undefined> {
// Remove the data URL prefix if present // Remove the data URL prefix if present
const base64Data = base64String.replace(/^data:image\/png;base64,/, ''); const base64Data = base64String.startsWith(`data:${contentType};base64,`)
? base64String.split(',', 2)[1]
: base64String;
// Convert base64 to binary // Convert base64 to binary
const byteCharacters = atob(base64Data); const byteCharacters = atob(base64Data);
const byteNumbers = new Array(byteCharacters.length); const byteNumbers = new Array(byteCharacters.length);
@ -14,7 +17,7 @@ export async function uploadImage(
} }
const byteArray = new Uint8Array(byteNumbers); const byteArray = new Uint8Array(byteNumbers);
// Create a Blob from the byte array // Create a Blob from the byte array
const blob = new Blob([byteArray], { type: 'image/png' }); const blob = new Blob([byteArray], { type: contentType });
// Create FormData and append the file // Create FormData and append the file
const formData = new FormData(); const formData = new FormData();
formData.append('file', blob, filename); formData.append('file', blob, filename);

View File

@ -168,7 +168,10 @@ export class AmazonDetailPageInjector extends BaseInjector {
const price = document.querySelector<HTMLElement>( const price = document.querySelector<HTMLElement>(
'.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen', '.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen',
)?.innerText; )?.innerText;
return { title, price }; const broughtInfo = document.querySelector<HTMLElement>(
`#social-proofing-faceout-title-tk_bought`,
)?.innerText;
return { title, price, broughtInfo };
}); });
} }
@ -303,7 +306,11 @@ export class AmazonDetailPageInjector extends BaseInjector {
public async scanAPlus() { public async scanAPlus() {
return this.run(async () => { return this.run(async () => {
const aplusEl = document.querySelector<HTMLElement>('#aplus_feature_div'); const aplusEl = document.querySelector<HTMLElement>('#aplus_feature_div');
if (!aplusEl) { if (
!aplusEl ||
aplusEl.getClientRects().length === 0 ||
aplusEl.getClientRects()[0].height === 0
) {
return false; return false;
} }
while (aplusEl.getClientRects().length === 0) { while (aplusEl.getClientRects().length === 0) {

View File

@ -149,12 +149,13 @@ const extraHeaders: Header<AmazonItem>[] = [
{ {
prop: 'imageUrls', prop: 'imageUrls',
label: '商品图片链接', label: '商品图片链接',
formatOutputValue: (val: string[] | undefined, _i, rowData) => { formatOutputValue: (val?: string[]) => val?.join(';'),
if (!val) return undefined;
return rowData.aplus ? val.concat([rowData.aplus]).join(';') : val.join(';');
},
parseImportValue: (val?: string) => val?.split(';'), parseImportValue: (val?: string) => val?.split(';'),
}, },
{
prop: 'aplus',
label: 'A+截图',
},
]; ];
const reviewHeaders: Header<AmazonReview>[] = [ const reviewHeaders: Header<AmazonReview>[] = [
@ -244,7 +245,7 @@ const handleCloudExport = async () => {
const mappedData1 = await castRecordsByHeaders(items, itemHeaders); const mappedData1 = await castRecordsByHeaders(items, itemHeaders);
const mappedData2 = await castRecordsByHeaders(reviews, reviewHeaders); const mappedData2 = await castRecordsByHeaders(reviews, reviewHeaders);
const fragments = [ const fragments = [
{ data: mappedData1, imageColumn: '商品图片链接', name: 'items' }, { data: mappedData1, imageColumn: ['商品图片链接', 'A+截图'], name: 'items' },
{ data: mappedData2, imageColumn: '图片链接', name: 'reviews' }, { data: mappedData2, imageColumn: '图片链接', name: 'reviews' },
]; ];
const filename = await cloudExporter.doExport(fragments); const filename = await cloudExporter.doExport(fragments);

View File

@ -1,30 +1,17 @@
<script setup lang="ts"> <script setup lang="ts">
import type { Timeline } from '~/components/ProgressReport.vue'; import type { Timeline } from '~/components/ProgressReport.vue';
import { useLongTask } from '~/composables/useLongTask'; import { usePageWorker } from '~/composables/usePageWorker';
import { amazon as pageWorker } from '~/logic/page-worker';
import { detailAsinInput, detailItems, detailWorkerSettings } from '~/logic/storages/amazon'; import { detailAsinInput, detailItems, detailWorkerSettings } from '~/logic/storages/amazon';
import { uploadImage } from '~/logic/upload';
const message = useMessage(); const message = useMessage();
const timelines = ref<Timeline[]>([]); const timelines = ref<Timeline[]>([]);
const { isRunning, startTask } = useLongTask();
const emit = defineEmits<{
start: [];
stop: [];
}>();
watch(isRunning, (newVal) => {
newVal ? emit('start') : emit('stop');
});
const asinInputRef = useTemplateRef('asin-input'); const asinInputRef = useTemplateRef('asin-input');
//#region Page Worker Code //#region Page Worker Code
const worker = pageWorker.getAmazonPageWorker(); // Page Worker const worker = usePageWorker('amazon', { detailItems });
worker.channel.on('error', ({ message: msg }) => { worker.on('error', ({ message: msg }) => {
timelines.value.push({ timelines.value.push({
type: 'error', type: 'error',
title: '错误发生', title: '错误发生',
@ -32,18 +19,16 @@ worker.channel.on('error', ({ message: msg }) => {
content: msg, content: msg,
}); });
message.error(msg); message.error(msg);
worker.stop();
}); });
worker.channel.on('item-base-info-collected', (ev) => { worker.on('item-base-info-collected', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}基本信息`, title: `商品${ev.asin}基本信息`,
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `标题: ${ev.title};价格:${ev.price}; 评分: ${ev.rating}; 评论数: ${ev.ratingCount}`, content: `标题: ${ev.title};价格:${ev.price}; 评分: ${ev.rating}; 评论数: ${ev.ratingCount}`,
}); });
updateDetailItems(ev);
}); });
worker.channel.on('item-category-rank-collected', (ev) => { worker.on('item-category-rank-collected', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}分类排名`, title: `商品${ev.asin}分类排名`,
@ -53,50 +38,34 @@ worker.channel.on('item-category-rank-collected', (ev) => {
ev.category2 ? `#${ev.category2.rank} in ${ev.category2.name}` : '', ev.category2 ? `#${ev.category2.rank} in ${ev.category2.name}` : '',
].join('\n'), ].join('\n'),
}); });
updateDetailItems(ev);
}); });
worker.channel.on('item-images-collected', (ev) => { worker.on('item-images-collected', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}图像`, title: `商品${ev.asin}图像`,
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `图片数: ${ev.imageUrls!.length}`, content: `图片数: ${ev.imageUrls!.length}`,
}); });
updateDetailItems(ev);
}); });
worker.channel.on('item-top-reviews-collected', (ev) => { worker.on('item-top-reviews-collected', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}精选评论`, title: `商品${ev.asin}精选评论`,
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `精选评论数: ${ev.topReviews!.length}`, content: `精选评论数: ${ev.topReviews!.length}`,
}); });
updateDetailItems(ev);
}); });
worker.channel.on('item-aplus-screenshot-collect', (ev) => { worker.on('item-aplus-screenshot-collect', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}A+`, title: `商品${ev.asin} A+信息`,
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `获取到A+`, content: `获取到A+信息`,
});
uploadImage(ev.base64data, `${ev.asin}.png`).then((url) => {
url && updateDetailItems({ asin: ev.asin, aplus: url });
}); });
}); });
const updateDetailItems = (row: { asin: string } & Partial<AmazonDetailItem>) => {
const asin = row.asin;
if (detailItems.value.has(row.asin)) {
const origin = detailItems.value.get(row.asin);
detailItems.value.set(asin, { ...origin, ...row } as AmazonDetailItem);
} else {
detailItems.value.set(asin, row as AmazonDetailItem);
}
};
//#endregion //#endregion
const task = async () => { const launch = async () => {
const asinList = detailAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0); const asinList = detailAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0);
timelines.value = [ timelines.value = [
{ {
@ -106,7 +75,7 @@ const task = async () => {
content: '开始数据采集', content: '开始数据采集',
}, },
]; ];
await worker.runDetaiPageTask(asinList, { await worker.runDetailPageTask(asinList, {
progress: (remains) => { progress: (remains) => {
detailAsinInput.value = remains.join('\n'); detailAsinInput.value = remains.join('\n');
}, },
@ -121,11 +90,11 @@ const task = async () => {
}; };
const handleStart = () => { const handleStart = () => {
asinInputRef.value?.validate().then(async (success) => success && startTask(task)); asinInputRef.value?.validate().then(async (success) => success && launch());
}; };
const handleInterrupt = () => { const handleInterrupt = () => {
if (!isRunning.value) return; if (!worker.isRunning.value) return;
worker.stop(); worker.stop();
message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 }); message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 });
}; };
@ -135,7 +104,7 @@ const handleInterrupt = () => {
<div class="detail-page-entry"> <div class="detail-page-entry">
<header-title>Amazon Detail</header-title> <header-title>Amazon Detail</header-title>
<div class="interative-section"> <div class="interative-section">
<ids-input v-model="detailAsinInput" :disabled="isRunning" ref="asin-input"> <ids-input v-model="detailAsinInput" :disabled="worker.isRunning.value" ref="asin-input">
<template #extra-settings> <template #extra-settings>
<div class="setting-panel"> <div class="setting-panel">
<n-form label-placement="left"> <n-form label-placement="left">
@ -146,7 +115,13 @@ const handleInterrupt = () => {
</div> </div>
</template> </template>
</ids-input> </ids-input>
<n-button v-if="!isRunning" round size="large" type="primary" @click="handleStart"> <n-button
v-if="!worker.isRunning.value"
round
size="large"
type="primary"
@click="handleStart"
>
<template #icon> <template #icon>
<ant-design-thunderbolt-outlined /> <ant-design-thunderbolt-outlined />
</template> </template>
@ -159,7 +134,7 @@ const handleInterrupt = () => {
停止 停止
</n-button> </n-button>
</div> </div>
<div v-if="isRunning" class="running-tip-section"> <div v-if="worker.isRunning.value" class="running-tip-section">
<n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert> <n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert>
</div> </div>
<progress-report class="progress-report" :timelines="timelines" /> <progress-report class="progress-report" :timelines="timelines" />

View File

@ -1,38 +1,24 @@
<script lang="ts" setup> <script lang="ts" setup>
import type { Timeline } from '~/components/ProgressReport.vue'; import type { Timeline } from '~/components/ProgressReport.vue';
import { useLongTask } from '~/composables/useLongTask'; import { usePageWorker } from '~/composables/usePageWorker';
import { amazon as pageWorker } from '~/logic/page-worker';
import { reviewAsinInput, reviewItems } from '~/logic/storages/amazon'; import { reviewAsinInput, reviewItems } from '~/logic/storages/amazon';
const { isRunning, startTask } = useLongTask(); const worker = usePageWorker('amazon', { reviewItems });
worker.on('error', ({ message: msg }) => {
const emit = defineEmits<{
start: [];
stop: [];
}>();
watch(isRunning, (newVal) => {
newVal ? emit('start') : emit('stop');
});
const worker = pageWorker.getAmazonPageWorker();
worker.channel.on('error', ({ message: msg }) => {
timelines.value.push({ timelines.value.push({
type: 'error', type: 'error',
title: '错误发生', title: '错误发生',
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: msg, content: msg,
}); });
worker.stop();
}); });
worker.channel.on('item-review-collected', (ev) => { worker.on('item-review-collected', (ev) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: `商品${ev.asin}评价`, title: `商品${ev.asin}评价`,
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `获取到 ${ev.reviews.length} 条评价`, content: `获取到 ${ev.reviews.length} 条评价`,
}); });
updateReviews(ev);
}); });
const asinInputRef = useTemplateRef('asin-input'); const asinInputRef = useTemplateRef('asin-input');
@ -41,7 +27,7 @@ const message = useMessage();
const timelines = ref<Timeline[]>([]); const timelines = ref<Timeline[]>([]);
const task = async () => { const launch = async () => {
const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0); const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0);
timelines.value = [ timelines.value = [
{ {
@ -65,33 +51,27 @@ const task = async () => {
}; };
const handleStart = () => { const handleStart = () => {
asinInputRef.value?.validate().then(async (success) => success && startTask(task)); asinInputRef.value?.validate().then(async (success) => success && launch());
}; };
const handleInterrupt = () => { const handleInterrupt = () => {
worker.stop(); worker.stop();
message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 }); message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 });
}; };
const updateReviews = (params: { asin: string; reviews: AmazonReview[] }) => {
const { asin, reviews } = params;
const values = toRaw(reviewItems.value).get(asin) || [];
const ids = new Set(values.map((item) => item.id));
for (const review of reviews) {
ids.has(review.id) || values.push(review);
}
values.sort((a, b) => dayjs(b.dateInfo).valueOf() - dayjs(a.dateInfo).valueOf());
reviewItems.value.delete(asin);
reviewItems.value.set(asin, values);
};
</script> </script>
<template> <template>
<div class="review-page-entry"> <div class="review-page-entry">
<header-title>Amazon Review</header-title> <header-title>Amazon Review</header-title>
<div class="interative-section"> <div class="interative-section">
<ids-input v-model="reviewAsinInput" :disabled="isRunning" ref="asin-input" /> <ids-input v-model="reviewAsinInput" :disabled="worker.isRunning.value" ref="asin-input" />
<n-button v-if="!isRunning" round size="large" type="primary" @click="handleStart"> <n-button
v-if="!worker.isRunning.value"
round
size="large"
type="primary"
@click="handleStart"
>
<template #icon> <template #icon>
<ant-design-thunderbolt-outlined /> <ant-design-thunderbolt-outlined />
</template> </template>
@ -104,7 +84,7 @@ const updateReviews = (params: { asin: string; reviews: AmazonReview[] }) => {
停止 停止
</n-button> </n-button>
</div> </div>
<div v-if="isRunning" class="running-tip-section"> <div v-if="worker.isRunning.value" class="running-tip-section">
<n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert> <n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert>
</div> </div>
<progress-report class="progress-report" :timelines="timelines" /> <progress-report class="progress-report" :timelines="timelines" />

View File

@ -1,26 +1,14 @@
<script setup lang="ts"> <script setup lang="ts">
import { keywordsList } from '~/logic/storages/amazon'; import { keywordsList } from '~/logic/storages/amazon';
import { amazon as pageWorker } from '~/logic/page-worker';
import { NButton } from 'naive-ui';
import { searchItems } from '~/logic/storages/amazon'; import { searchItems } from '~/logic/storages/amazon';
import { useLongTask } from '~/composables/useLongTask';
import type { Timeline } from '~/components/ProgressReport.vue'; import type { Timeline } from '~/components/ProgressReport.vue';
import { usePageWorker } from '~/composables/usePageWorker';
const message = useMessage(); const message = useMessage();
const { isRunning, startTask } = useLongTask();
const emit = defineEmits<{
start: [];
stop: [];
}>();
watch(isRunning, (newVal) => {
newVal ? emit('start') : emit('stop');
});
//#region Initial Page Worker //#region Initial Page Worker
const worker = pageWorker.getAmazonPageWorker(); const worker = usePageWorker('amazon', { searchItems });
worker.channel.on('error', ({ message: msg }) => { worker.on('error', ({ message: msg }) => {
timelines.value.push({ timelines.value.push({
type: 'error', type: 'error',
title: '错误发生', title: '错误发生',
@ -28,22 +16,20 @@ worker.channel.on('error', ({ message: msg }) => {
content: msg, content: msg,
}); });
message.error(msg); message.error(msg);
worker.stop();
}); });
worker.channel.on('item-links-collected', ({ objs }) => { worker.on('item-links-collected', ({ objs }) => {
timelines.value.push({ timelines.value.push({
type: 'success', type: 'success',
title: '检测到数据', title: '检测到数据',
time: new Date().toLocaleString(), time: new Date().toLocaleString(),
content: `成功采集到 ${objs.length} 条数据`, content: `成功采集到 ${objs.length} 条数据`,
}); });
searchItems.value = searchItems.value.concat(objs); // Add records
}); });
//#endregion //#endregion
const timelines = ref<Timeline[]>([]); const timelines = ref<Timeline[]>([]);
const task = async () => { const launch = async () => {
const kws = unref(keywordsList); const kws = unref(keywordsList);
timelines.value = [ timelines.value = [
{ {
@ -79,11 +65,11 @@ const handleStart = async () => {
if (keywordsList.value.length === 0) { if (keywordsList.value.length === 0) {
return; return;
} }
startTask(task); launch();
}; };
const handleInterrupt = () => { const handleInterrupt = () => {
if (!isRunning.value) return; if (!worker.isRunning.value) return;
worker.stop(); worker.stop();
message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 }); message.info('已触发中断,正在等待当前任务完成。', { duration: 2000 });
}; };
@ -94,7 +80,7 @@ const handleInterrupt = () => {
<header-title>Amazon Search</header-title> <header-title>Amazon Search</header-title>
<div class="interactive-section"> <div class="interactive-section">
<n-dynamic-input <n-dynamic-input
:disabled="isRunning" :disabled="worker.isRunning.value"
v-model:value="keywordsList" v-model:value="keywordsList"
:min="1" :min="1"
:max="10" :max="10"
@ -104,7 +90,13 @@ const handleInterrupt = () => {
round round
placeholder="请输入关键词采集信息" placeholder="请输入关键词采集信息"
/> />
<n-button v-if="!isRunning" type="primary" round size="large" @click="handleStart"> <n-button
v-if="!worker.isRunning.value"
type="primary"
round
size="large"
@click="handleStart"
>
<template #icon> <template #icon>
<n-icon> <n-icon>
<ant-design-thunderbolt-outlined /> <ant-design-thunderbolt-outlined />
@ -121,7 +113,7 @@ const handleInterrupt = () => {
中断 中断
</n-button> </n-button>
</div> </div>
<div v-if="isRunning" class="running-tip-section"> <div v-if="worker.isRunning.value" class="running-tip-section">
<n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert> <n-alert title="Warning" type="warning"> 警告在插件运行期间请勿与浏览器交互 </n-alert>
</div> </div>
<progress-report class="progress-report" :timelines="timelines" /> <progress-report class="progress-report" :timelines="timelines" />

View File

@ -2,6 +2,7 @@
import DetailPageEntry from './AmazonEntries/DetailPageEntry.vue'; import DetailPageEntry from './AmazonEntries/DetailPageEntry.vue';
import SearchPageEntry from './AmazonEntries/SearchPageEntry.vue'; import SearchPageEntry from './AmazonEntries/SearchPageEntry.vue';
import ReviewPageEntry from './AmazonEntries/ReviewPageEntry.vue'; import ReviewPageEntry from './AmazonEntries/ReviewPageEntry.vue';
import { usePageWorker } from '~/composables/usePageWorker';
const tabs = [ const tabs = [
{ {
@ -24,21 +25,21 @@ const currentComponent = computed(() => {
return tab ? tab.component : null; return tab ? tab.component : null;
}); });
const running = ref(false); const worker = usePageWorker('amazon');
</script> </script>
<template> <template>
<div class="side-panel"> <div class="side-panel">
<header class="header-menu"> <header class="header-menu">
<n-tabs <n-tabs
:tab-style="{ cursor: running ? 'not-allowed' : undefined }" :tab-style="{ cursor: worker.isRunning.value ? 'not-allowed' : undefined }"
placement="top" placement="top"
:default-value="tabs[0].name" :default-value="tabs[0].name"
type="segment" type="segment"
:value="selectedTab" :value="selectedTab"
@update:value=" @update:value="
(val) => { (val) => {
if (!running && tabs.findIndex((t) => t.name === val) !== -1) { if (!worker.isRunning.value && tabs.findIndex((t) => t.name === val) !== -1) {
selectedTab = val; selectedTab = val;
} }
} }
@ -49,7 +50,7 @@ const running = ref(false);
</header> </header>
<main class="main-content"> <main class="main-content">
<keep-alive> <keep-alive>
<Component :is="currentComponent" @start="running = true" @stop="running = false" /> <Component :is="currentComponent" />
</keep-alive> </keep-alive>
</main> </main>
</div> </div>