This commit is contained in:
johnathan 2025-07-17 18:16:11 +08:00
parent 219b34661a
commit cbdc3efd22
19 changed files with 789 additions and 1577 deletions

View File

@ -27,16 +27,14 @@
"prepare": "husky"
},
"devDependencies": {
"@iconify/json": "^2.2.356",
"@iconify/json": "^2.2.359",
"@types/fs-extra": "^11.0.4",
"@types/gulp-terser": "^1.2.6",
"@types/node": "^22.16.0",
"@types/node": "^22.16.4",
"@types/webextension-polyfill": "^0.12.3",
"@vitejs/plugin-vue": "^6.0.0",
"@vitejs/plugin-vue-jsx": "^5.0.1",
"@vue/test-utils": "^2.4.6",
"@vueuse/core": "^12.8.2",
"@zumer/snapdom": "^1.8.0",
"@vueuse/core": "^13.5.0",
"@zumer/snapdom": "^1.9.5",
"alova": "^3.3.4",
"chokidar": "^4.0.3",
"cross-env": "^7.0.3",
@ -50,17 +48,19 @@
"jsdom": "^26.1.0",
"kolorist": "^1.8.0",
"lint-staged": "^16.1.2",
"markdown-it-anchor": "^9.2.0",
"markdown-it-toc-done-right": "^4.2.0",
"naive-ui": "^2.42.0",
"npm-run-all": "^4.1.5",
"prettier": "3.5.3",
"prettier": "3.6.2",
"rimraf": "^6.0.1",
"sass-embedded": "^1.89.2",
"typescript": "^5.8.3",
"unplugin-auto-import": "^19.3.0",
"unplugin-icons": "^22.1.0",
"unplugin-vue-components": "^28.8.0",
"vite": "^7.0.2",
"vite-plugin-md": "^0.21.5",
"unplugin-vue-markdown": "^29.1.0",
"vite": "^7.0.4",
"vitest": "^3.2.4",
"vue": "^3.5.17",
"vue-demi": "^0.14.10",

1936
pnpm-lock.yaml generated

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,4 @@
<script setup lang="ts">
import { useElementBounding, useParentElement } from '@vueuse/core';
import type { EllipsisProps } from 'naive-ui';
export type TableColumn =
@ -18,17 +17,6 @@ export type TableColumn =
renderExpand: (row: any) => VNode;
};
const parentEl = useParentElement();
const currentRootEl = useTemplateRef('result-table');
const { height: parentHeight } = useElementBounding(parentEl);
const tableHeight = computed(() => {
let contentHeight = 0;
currentRootEl.value?.childNodes.forEach((element) => {
contentHeight += (element as Element).getBoundingClientRect().height;
});
return ~~Math.max(parentHeight.value, contentHeight);
});
const props = withDefaults(
defineProps<{
records: Record<string, unknown>[];
@ -58,8 +46,13 @@ function generateUUID() {
</script>
<template>
<div class="result-table" ref="result-table" :style="{ height: `${tableHeight}px` }">
<n-card class="result-content-container">
<div class="result-table">
<n-card
class="result-content-container"
ref="card"
header-class="result-table-header"
content-class="result-table-main-content"
>
<template #header><slot name="header" /></template>
<template #header-extra><slot name="header-extra" /></template>
<n-empty v-if="itemView.records.length === 0" size="huge">
@ -95,6 +88,11 @@ function generateUUID() {
<style scoped lang="scss">
.result-content-container {
height: 100%;
:deep(*) {
transition-duration: 0ms;
}
:deep(.n-card__content:has(.n-empty)) {
display: flex;
flex-direction: column;

View File

@ -1,3 +1,5 @@
/** 数据转换工具类 */
export function flattenObject(obj: Record<string, unknown>) {
const mappedEnties: [string[], unknown][] = [];
const stack: string[][] = Object.keys(obj).map((k) => [k]);

81
src/logic/crypto.ts Normal file
View File

@ -0,0 +1,81 @@
/** 数据加密工具类 */
/** 辅助函数Base64转ArrayBuffer */
function base64ToArrayBuffer(base64: string): ArrayBuffer {
const binaryString = atob(base64);
const bytes = new Uint8Array(binaryString.length);
for (let i = 0; i < binaryString.length; i++) {
bytes[i] = binaryString.charCodeAt(i);
}
return bytes.buffer;
}
/** 辅助函数ArrayBuffer转Base64 */
function arrayBufferToBase64(buffer: ArrayBuffer): string {
let binary = '';
const bytes = new Uint8Array(buffer);
for (let i = 0; i < bytes.byteLength; i++) {
binary += String.fromCharCode(bytes[i]);
}
return btoa(binary);
}
/** 生成RSA密钥对 */
export async function generateKeyPair(): Promise<CryptoKeyPair> {
return crypto.subtle.generateKey(
{
name: 'RSASSA-PKCS1-v1_5',
modulusLength: 2048,
publicExponent: new Uint8Array([0x01, 0x00, 0x01]),
hash: { name: 'SHA-256' },
},
true,
['sign', 'verify'],
);
}
/** 使用私钥对数据进行签名 */
export async function signData(privateKey: CryptoKey, data: string) {
// 将数据转换为ArrayBuffer
const encoder = new TextEncoder();
const encodedData = encoder.encode(data);
// 使用私钥签名
const signature = await crypto.subtle.sign(
{
name: 'RSASSA-PKCS1-v1_5',
},
privateKey,
encodedData,
);
// 将签名转换为Base64以便传输
return arrayBufferToBase64(signature);
}
/** 验证签名 */
export async function verifySignature(
publicKey: CryptoKey,
signature: string,
data: string,
): Promise<boolean> {
const encoder = new TextEncoder();
const encodedData: BufferSource = encoder.encode(data);
const signatureBuffer: ArrayBuffer = base64ToArrayBuffer(signature);
return await crypto.subtle.verify(
{
name: 'RSASSA-PKCS1-v1_5',
},
publicKey,
signatureBuffer,
encodedData,
);
}
/**导出秘钥为Base64 */
export async function exportKey(key: CryptoKey): Promise<string> {
const exportedPublicKey = await crypto.subtle.exportKey('spki', key);
const publicKeyBase64 = arrayBufferToBase64(exportedPublicKey);
return publicKeyBase64;
}

View File

@ -1,3 +1,4 @@
/** Excel工具类 */
import excel from 'exceljs';
class Worksheet {

View File

@ -1,3 +1,5 @@
/** 内容脚本注入工具类 */
type ExecOptions = {
timeout?: number;
};

View File

@ -1,3 +1,4 @@
/** 上传图片工具类 */
import { remoteHost } from '~/env';
export async function uploadImage(

View File

@ -1,3 +1,5 @@
[[toc]]
# 软件目的
本软件的开发旨在自动采集Amazon电商平台的数据并提供导出和预览数据的功能。

View File

@ -1,31 +1,41 @@
import { useLongTask } from '~/composables/useLongTask';
import amazon from '../impls/amazon';
import { uploadImage } from '~/logic/upload';
import { uploadImage } from '~/logic/upload-image';
import { detailItems, reviewItems, searchItems } from '~/storages/amazon';
import { createGlobalState } from '@vueuse/core';
import { useAmazonService } from '~/services/amazon';
import { LanchTaskBaseOptions } from '../interfaces';
import { LanchTaskBaseOptions, WorkerComposable } from '../interfaces/common';
import { AmazonPageWorker } from '../interfaces/amazon';
/** Settings interface for the Amazon page worker */
export interface AmazonPageWorkerSettings {
objects?: ('search' | 'detail' | 'review')[];
commitChangeIngerval?: number;
commitChangeInterval?: number;
}
/** Main function to build the Amazon page worker composable */
function buildAmazonPageWorker() {
// Reactive settings object
const settings = shallowRef<AmazonPageWorkerSettings>({});
// Long task management
const { isRunning, startTask } = useLongTask();
// Amazon service instance
const service = useAmazonService();
// Get the worker instance from implementation
const worker = amazon.getAmazonPageWorker();
// Caches for different item types
const searchCache = [] as AmazonSearchItem[];
const detailCache = new Map<string, AmazonDetailItem>();
const reviewCache = new Map<string, AmazonReview[]>();
// Add search items to cache
const updateSearchCache = (data: AmazonSearchItem[]) => {
searchCache.push(...data);
};
// Update or add detail item in cache
const updateDetailCache = (data: { asin: string } & Partial<AmazonDetailItem>) => {
const asin = data.asin;
if (detailCache.has(data.asin)) {
@ -36,7 +46,8 @@ function buildAmazonPageWorker() {
}
};
const updateReviews = (data: { asin: string; reviews: AmazonReview[] }) => {
// Update or add reviews in cache
const updateReviewCache = (data: { asin: string; reviews: AmazonReview[] }) => {
const { asin, reviews } = data;
const values = reviewCache.get(asin) || [];
const ids = new Set(values.map((item) => item.id));
@ -46,13 +57,23 @@ function buildAmazonPageWorker() {
reviewCache.set(asin, values);
};
// Clear all caches
const clearAllCaches = () => {
searchCache.splice(0, searchCache.length);
detailCache.clear();
reviewCache.clear();
};
// Commit cached changes to storage and service
const commitChange = async () => {
const { objects } = settings.value;
// Commit search items
if (objects?.includes('search')) {
searchItems.value = searchItems.value.concat(searchCache);
await service.commitSearchItems(searchCache);
searchCache.splice(0, searchCache.length);
}
// Commit detail items
if (objects?.includes('detail')) {
for (const [k, v] of detailCache.entries()) {
if (detailItems.value.has(k)) {
@ -65,6 +86,7 @@ function buildAmazonPageWorker() {
await service.commitDetailItems(detailCache);
detailCache.clear();
}
// Commit reviews
if (objects?.includes('review')) {
for (const [asin, reviews] of reviewCache.entries()) {
if (reviewItems.value.has(asin)) {
@ -82,54 +104,65 @@ function buildAmazonPageWorker() {
}
};
const unsubscribeFuncs = [] as (() => void)[];
// Store unsubscribe functions for worker events
const unsubscribes: (() => void)[] = [];
// Register all relevant worker event handlers
function registerWorkerEvents() {
return [
// Stop worker on error
worker.on('error', () => {
worker.stop();
}),
// Collect search item links
worker.on('item-links-collected', ({ objs }) => {
updateSearchCache(objs);
}),
// Collect base info for detail items
worker.on('item-base-info-collected', (ev) => {
updateDetailCache(ev);
}),
// Collect category rank for detail items
worker.on('item-category-rank-collected', (ev) => {
updateDetailCache(ev);
}),
// Collect images for detail items
worker.on('item-images-collected', (ev) => {
updateDetailCache(ev);
}),
// Collect A+ content screenshots and upload
worker.on('item-aplus-screenshot-collect', async (ev) => {
const url = await uploadImage(ev.base64data, `${ev.asin}.png`);
url && updateDetailCache({ asin: ev.asin, aplus: url });
}),
// Collect extra info for detail items
worker.on('item-extra-info-collect', (ev) => {
updateDetailCache({ asin: ev.asin, ...ev.info });
}),
// Collect reviews
worker.on('item-review-collected', (ev) => {
updateReviewCache(ev);
}),
];
}
// Register event handlers on mount
onMounted(() => {
unsubscribeFuncs.push(
...[
worker.on('error', () => {
worker.stop();
}),
worker.on('item-links-collected', ({ objs }) => {
updateSearchCache(objs);
}),
worker.on('item-base-info-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-category-rank-collected', (ev) => {
updateDetailCache(ev);
}),
worker.on('item-images-collected', (ev) => {
updateDetailCache(ev);
}),
// worker.on('item-top-reviews-collected', (ev) => {
// updateDetailCache(ev);
// }),
worker.on('item-aplus-screenshot-collect', async (ev) => {
const url = await uploadImage(ev.base64data, `${ev.asin}.png`);
url && updateDetailCache({ asin: ev.asin, aplus: url });
}),
worker.on('item-extra-info-collect', (ev) => {
updateDetailCache({ asin: ev.asin, ...ev.info });
}),
worker.on('item-review-collected', (ev) => {
updateReviews(ev);
}),
],
);
unsubscribes.push(...registerWorkerEvents());
});
// Unregister event handlers on unmount
onUnmounted(() => {
unsubscribeFuncs.forEach((unsubscribe) => unsubscribe());
unsubscribeFuncs.splice(0, unsubscribeFuncs.length);
unsubscribes.forEach((unsubscribe) => unsubscribe());
unsubscribes.splice(0, unsubscribes.length);
});
/**Commit change by interval time */
/**
* Task wrapper: commit changes by interval during task execution
*/
const taskWrapper1 = <T extends (...params: any[]) => Promise<void>>(func: T) => {
const { commitChangeIngerval = 10000 } = settings.value;
searchCache.splice(0, searchCache.length);
detailCache.clear();
reviewCache.clear();
const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value;
clearAllCaches();
return (...params: Parameters<T>) =>
startTask(async () => {
const interval = setInterval(() => commitChange(), commitChangeIngerval);
@ -139,13 +172,13 @@ function buildAmazonPageWorker() {
});
};
/**Commit changes in the end of task unit */
/**
* Task wrapper: commit changes at the end of each progress step
*/
const taskWrapper2 = <T extends (input: any, options?: LanchTaskBaseOptions) => Promise<void>>(
func: T,
) => {
searchCache.splice(0, searchCache.length);
detailCache.clear();
reviewCache.clear();
clearAllCaches();
return (...params: Parameters<T>) =>
startTask(async () => {
if (!params?.[1]) {
@ -167,10 +200,12 @@ function buildAmazonPageWorker() {
});
};
// Wrapped task runners for each page type
const runSearchPageTask = taskWrapper1(worker.runSearchPageTask.bind(worker));
const runDetailPageTask = taskWrapper2(worker.runDetailPageTask.bind(worker));
const runReviewPageTask = taskWrapper1(worker.runReviewPageTask.bind(worker));
// Expose API for the composable
return {
settings,
isRunning,
@ -181,7 +216,8 @@ function buildAmazonPageWorker() {
off: worker.off.bind(worker),
once: worker.once.bind(worker),
stop: worker.stop.bind(worker),
};
} as WorkerComposable<AmazonPageWorker, AmazonPageWorkerSettings>;
}
/** Create a global state composable for the Amazon worker */
export const useAmazonWorker = createGlobalState(buildAmazonPageWorker);

View File

@ -2,6 +2,8 @@ import { useLongTask } from '~/composables/useLongTask';
import { detailItems as homedepotDetailItems } from '~/storages/homedepot';
import homedepot from '../impls/homedepot';
import { createGlobalState } from '@vueuse/core';
import { WorkerComposable } from '../interfaces/common';
import { HomedepotWorker } from '../interfaces/homedepot';
export interface HomedepotWorkerSettings {
objects?: 'detail'[];
@ -77,7 +79,7 @@ function buildHomedepotWorker() {
off: worker.off.bind(worker),
once: worker.once.bind(worker),
stop: worker.stop.bind(worker),
};
} as WorkerComposable<HomedepotWorker, HomedepotWorkerSettings>;
}
export const useHomedepotWorker = createGlobalState(buildHomedepotWorker);

View File

@ -1,4 +1,3 @@
import type { AmazonPageWorker, AmazonPageWorkerEvents, LanchTaskBaseOptions } from '../interfaces';
import type { Tabs } from 'webextension-polyfill';
import { withErrorHandling } from '../error-handler';
import {
@ -8,6 +7,8 @@ import {
} from '../web-injectors/amazon';
import { isForbiddenUrl } from '~/env';
import { BaseWorker } from './base';
import { AmazonPageWorker, AmazonPageWorkerEvents } from '../interfaces/amazon';
import { LanchTaskBaseOptions } from '../interfaces/common';
/**
* AmazonPageWorkerImpl can run on background & sidepanel & popup,
@ -250,7 +251,7 @@ class AmazonPageWorkerImpl
asins: string[],
options: LanchTaskBaseOptions & { aplus?: boolean; extra?: boolean } = {},
): Promise<void> {
const { progress, aplus = false } = options;
const { progress } = options;
const remains = [...asins];
let interrupt = false;
const unsubscribe = this.on('interrupt', () => {

View File

@ -1,8 +1,9 @@
import type { HomedepotEvents, HomedepotWorker, LanchTaskBaseOptions } from '../interfaces';
import { Tabs } from 'webextension-polyfill';
import { withErrorHandling } from '../error-handler';
import { HomedepotDetailPageInjector } from '../web-injectors/homedepot';
import { BaseWorker } from './base';
import { LanchTaskBaseOptions } from '../interfaces/common';
import { HomedepotEvents, HomedepotWorker } from '../interfaces/homedepot';
class HomedepotWorkerImpl
extends BaseWorker<HomedepotEvents & { interrupt: undefined }>

View File

@ -1,8 +1,4 @@
import type Emittery from 'emittery';
type Listener<T> = Pick<Emittery<T>, 'on' | 'off' | 'once'>;
export type LanchTaskBaseOptions = { progress?: (remains: string[]) => Promise<void> | void };
import { LanchTaskBaseOptions, Listener } from './common';
export interface AmazonPageWorkerEvents {
/** The event is fired when worker collected links to items on the Amazon search page. */
@ -53,14 +49,14 @@ export interface AmazonPageWorkerEvents {
export interface AmazonPageWorker extends Listener<AmazonPageWorkerEvents> {
/**
* Browsing goods search page and collect links to those goods.
* Browsing item search pages and collect links to those items.
* @param keywordsList - The keywords list to search for on Amazon.
* @param options The Options Specify Behaviors.
*/
runSearchPageTask(keywordsList: string[], options?: LanchTaskBaseOptions): Promise<void>;
/**
* Browsing goods detail page and collect target information.
* Browsing item detail pages and collect target information.
* @param asins Amazon Standard Identification Numbers.
* @param options The Options Specify Behaviors.
*/
@ -70,7 +66,7 @@ export interface AmazonPageWorker extends Listener<AmazonPageWorkerEvents> {
): Promise<void>;
/**
* Browsing goods review page and collect target information.
* Browsing item review pages and collect target information.
* @param asins Amazon Standard Identification Numbers.
* @param options The Options Specify Behaviors.
*/
@ -84,54 +80,3 @@ export interface AmazonPageWorker extends Listener<AmazonPageWorkerEvents> {
*/
stop(): Promise<void>;
}
export interface HomedepotEvents {
/** The event is fired when detail items collect */
['detail-item-collected']: { item: HomedepotDetailItem };
/** The event is fired when reviews collect */
['review-collected']: { reviews: HomedepotReview[] };
/** The event is fired when error occurs. */
['error']: { message: string; url?: string };
}
export interface HomedepotWorker extends Listener<HomedepotEvents> {
/**
* Browsing goods detail page and collect target information
*/
runDetailPageTask(
OSMIDs: string[],
options?: LanchTaskBaseOptions & { review?: boolean },
): Promise<void>;
/**
* Stop the worker.
*/
stop(): Promise<void>;
}
export interface LowesEvents {
/** The event is fired when detail items collect */
['detail-item-collected']: { item: LowesDetailItem };
/** The event is fired when error occurs. */
['error']: { message: string; url?: string };
}
export interface LowesWorker {
/**
* The channel for communication with the Lowes page worker.
*/
readonly channel: Emittery<LowesEvents>;
/**
* Browsing goods detail page and collect target information
*/
runDetailPageTask(urls: string[], options?: LanchTaskBaseOptions): Promise<void>;
/**
* Stop the worker.
*/
stop(): Promise<void>;
}

View File

@ -0,0 +1,35 @@
import type Emittery from 'emittery';
export type Listener<T> = Pick<Emittery<T>, 'on' | 'off' | 'once'>;
export type LanchTaskBaseOptions = { progress?: (remains: string[]) => Promise<void> | void };
export interface LowesEvents {
/** The event is fired when detail items collect */
['detail-item-collected']: { item: LowesDetailItem };
/** The event is fired when error occurs. */
['error']: { message: string; url?: string };
}
export interface LowesWorker {
/**
* The channel for communication with the Lowes page worker.
*/
readonly channel: Emittery<LowesEvents>;
/**
* Browsing goods detail page and collect target information
*/
runDetailPageTask(urls: string[], options?: LanchTaskBaseOptions): Promise<void>;
/**
* Stop the worker.
*/
stop(): Promise<void>;
}
export type WorkerComposable<Base, S = {}> = Base & {
settings: Ref<S>;
isRunning: Ref<boolean>;
};

View File

@ -0,0 +1,27 @@
import { LanchTaskBaseOptions, Listener } from './common';
export interface HomedepotEvents {
/** The event is fired when detail items collect */
['detail-item-collected']: { item: HomedepotDetailItem };
/** The event is fired when reviews collect */
['review-collected']: { reviews: HomedepotReview[] };
/** The event is fired when error occurs. */
['error']: { message: string; url?: string };
}
export interface HomedepotWorker extends Listener<HomedepotEvents> {
/**
* Browsing goods detail page and collect target information
*/
runDetailPageTask(
OSMIDs: string[],
options?: LanchTaskBaseOptions & { review?: boolean },
): Promise<void>;
/**
* Stop the worker.
*/
stop(): Promise<void>;
}

View File

@ -2,12 +2,25 @@ import { createAlova } from 'alova';
import adapterFetch from 'alova/fetch';
import { remoteHost } from '~/env';
import VueHook from 'alova/vue';
import { exportKey, generateKeyPair, signData } from '~/logic/crypto';
const keyPairPromise = generateKeyPair();
const publicKeyBase64Promise = keyPairPromise.then(async ({ publicKey }) => exportKey(publicKey));
const httpClient = createAlova({
baseURL: `http://${remoteHost}`,
requestAdapter: adapterFetch(),
timeout: 10000,
statesHook: VueHook,
beforeRequest: async (method) => {
// 生成签名
const { privateKey } = await keyPairPromise;
const publicKeyBase64Data = await publicKeyBase64Promise;
const dataToSign = method.data ? JSON.stringify(method.data) : '';
const signature = await signData(privateKey, dataToSign);
method.config.headers['x-signature'] = signature;
method.config.headers['x-public-key'] = publicKeyBase64Data;
},
responded: {
onSuccess: (response) => response.json(),
onError: (response: Response) => {

View File

@ -1,7 +1,7 @@
{
"compilerOptions": {
"incremental": false,
"useDefineForClassFields": false, // Could **not** be true because of vue-router
"useDefineForClassFields": false, // Could **not** be true because of the conflict between vue-router and sidepanel
"target": "es2016",
"jsx": "preserve",
"jsxImportSource": "vue",

View File

@ -10,9 +10,11 @@ import IconsResolver from 'unplugin-icons/resolver';
import Components from 'unplugin-vue-components/vite';
import AutoImport from 'unplugin-auto-import/vite';
import { NaiveUiResolver } from 'unplugin-vue-components/resolvers';
import Markdown from 'unplugin-vue-markdown/vite';
import MarkdownItAnchor from 'markdown-it-anchor';
import markdownItTocDoneRight from 'markdown-it-toc-done-right';
import { isDev, outputDir, port, r } from './scripts/utils.js';
import packageJson from './package.json';
import Markdown from 'vite-plugin-md';
export const sharedConfig: UserConfig = {
root: r('src'),
@ -30,7 +32,12 @@ export const sharedConfig: UserConfig = {
Vue({
include: [/\.vue$/, /\.md$/],
}),
Markdown(),
Markdown({
markdownItSetup(md) {
md.use(MarkdownItAnchor, {});
md.use(markdownItTocDoneRight);
},
}),
VueJsx(),
AutoImport({
imports: [