From bcbdc321c8a6fdb9d07cd1b48c0115e9cf085d71 Mon Sep 17 00:00:00 2001 From: syrup <1032771964@qq.com> Date: Tue, 1 Sep 2026 17:21:04 +0800 Subject: [PATCH] feat: add Amazon collection metadata and reliable sync --- .gitignore | 1 + README.md | 16 +- package.json | 11 +- scripts/manifest.ts | 10 +- scripts/pack.ts | 25 +- scripts/test-amazon-detail-images.ts | 51 ++ scripts/test-amazon-detail-page-ready.ts | 49 ++ scripts/test-amazon-page-navigation.ts | 81 +++ scripts/test-amazon-review-page.ts | 82 +++ scripts/test-amazon-search-page-ready.ts | 65 ++ scripts/test-amazon-search-pagination.ts | 33 + scripts/test-amazon-search-parser.ts | 104 ++++ scripts/test-execute-script.ts | 57 ++ scripts/verify-extension-runtime.mjs | 71 +++ scripts/verify-extension-sync-e2e.mjs | 115 ++++ src/background/amazon-sync.ts | 214 +++++++ src/background/main.ts | 1 + src/composables/useLongTask.ts | 6 +- src/env.ts | 9 +- src/logic/amazon-sync-client.ts | 9 + src/logic/amazon-sync-shared.ts | 12 + src/logic/execute-script.ts | 68 +-- src/options/views/AmazonResultTable.vue | 92 ++- src/options/views/help/guide.md | 12 +- src/page-worker/composables/amazon.ts | 160 +++-- src/page-worker/impls/amazon.ts | 249 ++++++-- src/page-worker/interfaces/amazon.ts | 9 +- src/page-worker/interfaces/common.ts | 1 + src/page-worker/page-navigation.ts | 58 ++ .../web-injectors/amazon-detail-images.ts | 103 ++++ .../web-injectors/amazon-detail-page-ready.ts | 32 + .../web-injectors/amazon-review-page.ts | 73 +++ .../web-injectors/amazon-search-page-ready.ts | 46 ++ .../web-injectors/amazon-search-pagination.ts | 12 + .../web-injectors/amazon-search-parser.ts | 340 +++++++++++ src/page-worker/web-injectors/amazon.ts | 567 ++---------------- .../views/AmazonEntries/DetailPageEntry.vue | 21 +- .../views/AmazonEntries/ReviewPageEntry.vue | 32 +- src/storages/amazon-sync.ts | 8 + src/storages/amazon.ts | 25 +- src/types/amazon-sync.d.ts | 25 + src/types/amazon.d.ts | 43 ++ 42 files changed, 2310 insertions(+), 688 deletions(-) create mode 100644 scripts/test-amazon-detail-images.ts create mode 100644 scripts/test-amazon-detail-page-ready.ts create mode 100644 scripts/test-amazon-page-navigation.ts create mode 100644 scripts/test-amazon-review-page.ts create mode 100644 scripts/test-amazon-search-page-ready.ts create mode 100644 scripts/test-amazon-search-pagination.ts create mode 100644 scripts/test-amazon-search-parser.ts create mode 100644 scripts/test-execute-script.ts create mode 100644 scripts/verify-extension-runtime.mjs create mode 100644 scripts/verify-extension-sync-e2e.mjs create mode 100644 src/background/amazon-sync.ts create mode 100644 src/logic/amazon-sync-client.ts create mode 100644 src/logic/amazon-sync-shared.ts create mode 100644 src/page-worker/page-navigation.ts create mode 100644 src/page-worker/web-injectors/amazon-detail-images.ts create mode 100644 src/page-worker/web-injectors/amazon-detail-page-ready.ts create mode 100644 src/page-worker/web-injectors/amazon-review-page.ts create mode 100644 src/page-worker/web-injectors/amazon-search-page-ready.ts create mode 100644 src/page-worker/web-injectors/amazon-search-pagination.ts create mode 100644 src/page-worker/web-injectors/amazon-search-parser.ts create mode 100644 src/storages/amazon-sync.ts create mode 100644 src/types/amazon-sync.d.ts diff --git a/.gitignore b/.gitignore index 5fc4637..063bb37 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ dist-ssr extension/manifest.json extension-firefox/manifest.json node_modules +.runtime src/auto-imports.d.ts src/components.d.ts .eslintcache diff --git a/README.md b/README.md index 8aef4a2..ae74eea 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Azon Seeker is a browser extension designed to collect and manage product inform - 🛒 **Collect Product Data:** Easily gather product details and reviews from Amazon and other websites. - 📦 **Import/Export:** Supports importing and exporting data in Excel format for further analysis. - ☁️ **Cloud Export:** Export your data to the cloud for backup or sharing. +- 🔄 **Reliable Server Sync:** Collected Amazon data is durably queued, retried in the background, and removed from the queue only after the server confirms persistence. - 📊 **Data Table:** View, filter, and manage collected data in a user-friendly table. - 🖼️ **Image Preview:** Hover to preview product images. - 🌐 **Multi-browser Support:** Works with Chromium-based browsers and Firefox. @@ -80,4 +81,17 @@ pnpm run build-firefox # For Firefox pnpm run pack:7z ``` -After the build is completed, the extension package will be generated in `build/extension.7z` +After the build is completed, the Chromium package will be generated in +`build/extension-v.7z`. + +## Amazon Server Synchronization + +- Development builds use `127.0.0.1:8001` by default. +- Production builds use `amzplugin.rekeymed.com` over HTTPS/WSS by default. +- Override either target at build time with `VITE_REMOTE_HOST=host:port`. +- The extension persists unsent batches in `browser.storage.local` and retries once per minute. +- Search and detail records include a client-generated `syncId`; the server uses it as a unique idempotency key. +- The side panel displays pending, syncing, synchronized, and failed states and provides a manual retry action. + +Before publishing a production package, deploy the backend migrations and verify that the +production `/openapi.json` contains `syncId`, `adType`, and `strikethroughPrice`. diff --git a/package.json b/package.json index fc861e2..b05926a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "azon-seeker", - "displayName": "Azon Seeker v0.7.1.10-beta", + "displayName": "Azon Seeker v0.7.1.12-beta", "version": "0.7.2", "private": true, "description": "Starter modify by honestfox101 and PetrichorFun", @@ -23,6 +23,15 @@ "clear": "rimraf --glob extension/dist extension/manifest.json extension.* ", "clear-firefox": "rimraf --glob extension-firefox/dist extension-firefox/manifest.json extension.*", "typecheck": "tsc --noEmit", + "test:amazon-parser": "esno scripts/test-amazon-search-parser.ts", + "test:amazon-page-ready": "esno scripts/test-amazon-search-page-ready.ts", + "test:amazon-pagination": "esno scripts/test-amazon-search-pagination.ts", + "test:amazon-detail-images": "esno scripts/test-amazon-detail-images.ts", + "test:amazon-detail-page-ready": "esno scripts/test-amazon-detail-page-ready.ts", + "test:amazon-review-page": "esno scripts/test-amazon-review-page.ts", + "test:amazon-page-navigation": "esno scripts/test-amazon-page-navigation.ts", + "test:execute-script": "esno scripts/test-execute-script.ts", + "test:extension-runtime": "node scripts/verify-extension-runtime.mjs", "prepare": "husky" }, "devDependencies": { diff --git a/scripts/manifest.ts b/scripts/manifest.ts index 06fba1d..9b2ceca 100644 --- a/scripts/manifest.ts +++ b/scripts/manifest.ts @@ -30,7 +30,15 @@ async function getManifest() { 48: './assets/icon-512.png', 128: './assets/icon-512.png', }, - permissions: ['tabs', 'storage', 'activeTab', 'scripting', 'unlimitedStorage', 'contextMenus'], + permissions: [ + 'tabs', + 'storage', + 'activeTab', + 'scripting', + 'unlimitedStorage', + 'contextMenus', + 'alarms', + ], content_security_policy: { extension_pages: isDev ? `script-src \'self\' http://localhost:${port}; object-src \'self\';` diff --git a/scripts/pack.ts b/scripts/pack.ts index ef65341..329fc56 100644 --- a/scripts/pack.ts +++ b/scripts/pack.ts @@ -1,17 +1,24 @@ import _7z from '7zip-min'; import { basename } from 'path'; +import fs from 'fs-extra'; import { r, log } from './utils'; import packageJson from '../package.json'; const scriptPath = ['extension', 'extension-firefox']; -scriptPath.forEach(async (src) => { - const srcPath = r(src); - const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`); - try { - const output = await _7z.pack(srcPath, distPath); - log('Info', `stdout of the 7za command execution ${output}`); - } catch (err) { - log('Error', `${err}`); +async function main() { + for (const src of scriptPath) { + const srcPath = r(src); + if (!(await fs.pathExists(r(src, 'manifest.json')))) continue; + const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`); + try { + const output = await _7z.pack(srcPath, distPath); + log('Info', `stdout of the 7za command execution ${output}`); + } catch (err) { + log('Error', `${err}`); + process.exitCode = 1; + } } -}); +} + +void main(); diff --git a/scripts/test-amazon-detail-images.ts b/scripts/test-amazon-detail-images.ts new file mode 100644 index 0000000..1cf42bc --- /dev/null +++ b/scripts/test-amazon-detail-images.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { JSDOM } from 'jsdom'; +import { collectAmazonDetailImageUrls } from '../src/page-worker/web-injectors/amazon-detail-images'; + +async function runWithHtml(html: string) { + const dom = new JSDOM(html, { url: 'https://www.amazon.com/dp/B0DMNDL5NH' }); + const previousDocument = globalThis.document; + const previousMouseEvent = globalThis.MouseEvent; + Object.assign(globalThis, { + document: dom.window.document, + MouseEvent: dom.window.MouseEvent, + }); + try { + return await collectAmazonDetailImageUrls(); + } finally { + Object.assign(globalThis, { + document: previousDocument, + MouseEvent: previousMouseEvent, + }); + dom.window.close(); + } +} + +async function main() { + const scriptUrls = await runWithHtml(` + +`); + assert.deepEqual(scriptUrls, [ + 'https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg', + 'https://m.media-amazon.com/images/I/second._AC_SL1500_.jpg', + ]); + + const landingUrl = 'https://m.media-amazon.com/images/I/landing._AC_SL1500_.jpg'; + const landingUrls = await runWithHtml( + ``, + ); + assert.deepEqual(landingUrls, [landingUrl]); + + console.log('Amazon detail image parser tests passed.'); +} + +void main(); diff --git a/scripts/test-amazon-detail-page-ready.ts b/scripts/test-amazon-detail-page-ready.ts new file mode 100644 index 0000000..663d118 --- /dev/null +++ b/scripts/test-amazon-detail-page-ready.ts @@ -0,0 +1,49 @@ +import { JSDOM } from 'jsdom'; +import { waitForAmazonDetailPageReady } from '../src/page-worker/web-injectors/amazon-detail-page-ready'; + +const installDom = (html: string, url = 'https://www.amazon.com/dp/B0DMNDL5NH') => { + const dom = new JSDOM(html, { url }); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + return dom; +}; + +const main = async () => { + const modernDetail = installDom('Ready product'); + await waitForAmazonDetailPageReady(); + modernDetail.window.close(); + + const legacyDetail = installDom('Legacy product'); + await waitForAmazonDetailPageReady(); + legacyDetail.window.close(); + + const signIn = installDom('
Sign in
', 'https://www.amazon.com/ap/signin'); + await waitForAmazonDetailPageReady().then( + () => { + throw new Error('Sign-in page should fail immediately'); + }, + (error) => { + if (!String(error).includes('需要重新登录')) throw error; + }, + ); + signIn.window.close(); + + const captcha = installDom(''); + await waitForAmazonDetailPageReady().then( + () => { + throw new Error('Captcha page should fail immediately'); + }, + (error) => { + if (!String(error).includes('验证码')) throw error; + }, + ); + captcha.window.close(); + + console.log('Amazon detail page readiness tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-page-navigation.ts b/scripts/test-amazon-page-navigation.ts new file mode 100644 index 0000000..a06ba24 --- /dev/null +++ b/scripts/test-amazon-page-navigation.ts @@ -0,0 +1,81 @@ +import { + extractAmazonDetailAsin, + hasReachedAmazonDetailPage, + hasReachedAmazonNavigationTarget, +} from '../src/page-worker/page-navigation'; + +const assert = (condition: boolean, message: string) => { + if (!condition) throw new Error(message); +}; + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews&filterByStar=one_star', + ), + 'An unfiltered completed page must not satisfy a star-filter navigation.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_viewopt_sr?filterByStar=one_star&pageNumber=2', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_paging_btm_next_2?filterByStar=one_star&pageNumber=2', + ), + 'Tracking ref rewrites must not prevent matching the requested review page.', +); + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=1', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=2', + ), + 'A previous review page must not satisfy the next-page navigation.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/s?k=security+camera&page=17&ref=sr_pg_17', + 'https://www.amazon.com/s?ref=sr_pg_17&page=17&k=security+camera', + ), + 'Search navigation should tolerate query ordering differences.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH/ref=sr_1_1', + 'https://www.amazon.com/dp/B0DMNDL5NH', + ), + 'A canonical product slug must still match the requested detail ASIN.', +); + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH', + 'https://www.amazon.com/dp/B0BKPNRLC8', + ), + 'The previous detail ASIN must not satisfy the next-ASIN navigation.', +); + +assert( + hasReachedAmazonDetailPage( + 'https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1', + 'https://www.amazon.com/dp/B0D2MC2865', + ), + 'A canonical Amazon detail redirect must be accepted as a loaded detail page.', +); + +assert( + !hasReachedAmazonDetailPage( + 'https://www.amazon.com/', + 'https://www.amazon.com/dp/B0D2MC2865', + ), + 'The Amazon home page must not be accepted as a detail redirect.', +); + +assert( + extractAmazonDetailAsin('https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1') === + 'B0CCXB29LF', + 'The resolved ASIN must be extracted from canonical detail URLs.', +); + +console.log('Amazon page navigation tests passed.'); diff --git a/scripts/test-amazon-review-page.ts b/scripts/test-amazon-review-page.ts new file mode 100644 index 0000000..4c44405 --- /dev/null +++ b/scripts/test-amazon-review-page.ts @@ -0,0 +1,82 @@ +import { JSDOM } from 'jsdom'; +import { + collectAmazonNextReviewPageUrl, + isAmazonReviewRatingFilterIgnored, + waitForAmazonReviewPageReady, +} from '../src/page-worker/web-injectors/amazon-review-page'; + +const installDom = (html: string, url = 'https://www.amazon.com/product-reviews/B08XM6S6ZL') => { + const dom = new JSDOM(html, { url }); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + return dom; +}; + +const main = async () => { + if ( + !isAmazonReviewRatingFilterIgnored( + [{ rating: '5.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }], + 3, + ) + ) { + throw new Error('Mixed ratings must be detected when Amazon ignores filterByStar'); + } + if ( + isAmazonReviewRatingFilterIgnored( + [{ rating: '3.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }], + 3, + ) + ) { + throw new Error('A correctly filtered review page must not be treated as limited'); + } + + const readyDom = installDom('
'); + await waitForAmazonReviewPageReady({}); + readyDom.window.close(); + + const paginationDom = installDom( + '
', + ); + const nextUrl = await collectAmazonNextReviewPageUrl(); + if (nextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2') { + throw new Error(`Unexpected next review URL: ${nextUrl}`); + } + paginationDom.window.close(); + + const standalonePaginationDom = installDom( + '
', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2', + ); + const standaloneNextUrl = await collectAmazonNextReviewPageUrl(); + if (standaloneNextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=3') { + throw new Error(`Unexpected standalone review pagination URL: ${standaloneNextUrl}`); + } + standalonePaginationDom.window.close(); + + const lastPageDom = installDom( + '
', + ); + if ((await collectAmazonNextReviewPageUrl()) !== null) { + throw new Error('Disabled review pagination must not return a URL'); + } + lastPageDom.window.close(); + + const signInDom = installDom('
Sign in
', 'https://www.amazon.com/ap/signin'); + await waitForAmazonReviewPageReady({}).then( + () => { + throw new Error('Sign-in page should fail immediately'); + }, + (error) => { + if (!String(error).includes('需要重新登录')) throw error; + }, + ); + signInDom.window.close(); + + console.log('Amazon review page tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-page-ready.ts b/scripts/test-amazon-search-page-ready.ts new file mode 100644 index 0000000..ec40af2 --- /dev/null +++ b/scripts/test-amazon-search-page-ready.ts @@ -0,0 +1,65 @@ +import { JSDOM } from 'jsdom'; +import { waitForAmazonSearchPageReady } from '../src/page-worker/web-injectors/amazon-search-page-ready'; + +const main = async () => { + const dom = new JSDOM( + '
Next
', + { url: 'https://www.amazon.com/s?k=security+camera' }, + ); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + + const started = Date.now(); + await waitForAmazonSearchPageReady(); + const elapsed = Date.now() - started; + dom.window.close(); + + if (elapsed >= 2000) { + throw new Error(`Ready search results were delayed by a persistent spinner (${elapsed}ms)`); + } + + const errorDom = new JSDOM( + '

SORRY

Something went wrong on our end.

Please go back and try again.

', + { url: 'https://www.amazon.com/s?k=protein+powder' }, + ); + Object.defineProperty(errorDom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: errorDom.window.document, window: errorDom.window }); + await waitForAmazonSearchPageReady().then( + () => { + throw new Error('Amazon transient error page must not be accepted as an empty result page'); + }, + (error) => { + if (!String(error).includes('临时错误页')) throw error; + }, + ); + errorDom.window.close(); + + const captchaDom = new JSDOM('', { + url: 'https://www.amazon.com/s?k=protein+powder', + }); + Object.defineProperty(captchaDom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: captchaDom.window.document, window: captchaDom.window }); + await waitForAmazonSearchPageReady().then( + () => { + throw new Error('Amazon captcha page must fail immediately'); + }, + (error) => { + if (!String(error).includes('验证码')) throw error; + }, + ); + captchaDom.window.close(); + + console.log('Amazon search page-ready tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-pagination.ts b/scripts/test-amazon-search-pagination.ts new file mode 100644 index 0000000..71475c0 --- /dev/null +++ b/scripts/test-amazon-search-pagination.ts @@ -0,0 +1,33 @@ +import { JSDOM } from 'jsdom'; +import { collectAmazonNextPageUrl } from '../src/page-worker/web-injectors/amazon-search-pagination'; + +const test = async (name: string, html: string, expected: string | null) => { + const dom = new JSDOM(html, { url: 'https://www.amazon.com/s?k=security+camera&page=1' }); + const previousDocument = globalThis.document; + const previousWindow = globalThis.window; + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + try { + const actual = await collectAmazonNextPageUrl(); + if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`); + } finally { + Object.assign(globalThis, { document: previousDocument, window: previousWindow }); + dom.window.close(); + } +}; + +const main = async () => { + await test( + 'active next page', + 'Next', + 'https://www.amazon.com/s?k=security+camera&page=2', + ); + await test( + 'disabled next page', + 'Next', + null, + ); + await test('missing next page', '
No pagination
', null); + console.log('Amazon search pagination tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-parser.ts b/scripts/test-amazon-search-parser.ts new file mode 100644 index 0000000..993732f --- /dev/null +++ b/scripts/test-amazon-search-parser.ts @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import { JSDOM } from 'jsdom'; +import { collectAmazonSearchPageData } from '../src/page-worker/web-injectors/amazon-search-parser'; + +const html = ` +
+
+ Sponsored +
+

Brand product one

+ +
+
+

Brand product two

+ +
+
+ +
+
+
+ Amazon's Choice +

Organic product

+ + $29.99 +
+ +
+ +

Sponsored product

+ + $33.93 + $39.99 + $0.28 +
+ +
+

Second organic product

+ +
+
+ +
+ Sponsored +
+

Unknown sponsored product

+ +
+
+
+ + +
+`; + +const dom = new JSDOM(html, { url: 'https://www.amazon.com/s?k=mirror' }); +Object.assign(globalThis, { + document: dom.window.document, + Node: dom.window.Node, + Element: dom.window.Element, + HTMLElement: dom.window.HTMLElement, + HTMLAnchorElement: dom.window.HTMLAnchorElement, + HTMLImageElement: dom.window.HTMLImageElement, +}); + +async function main() { + const items = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + assert.equal(items.length, 7); + + const byAsin = new Map(items.map((item) => [item.asin, item])); + assert.equal(byAsin.get('B000000003')?.adType, 'organic'); + assert.equal(byAsin.get('B000000004')?.adType, 'sponsoredProducts'); + assert.equal(byAsin.get('B000000004')?.price, '$33.93'); + assert.equal(byAsin.get('B000000004')?.strikethroughPrice, '$39.99'); + assert.equal(byAsin.get('B000000004')?.link, 'https://www.amazon.com/dp/B000000004'); + assert.equal(byAsin.get('B000000004')?.placement, 'middleOfSearch'); + assert.equal(byAsin.get('B000000001')?.adType, 'sponsoredBrands'); + assert.equal(byAsin.get('B000000002')?.adType, 'sponsoredBrands'); + assert.equal(byAsin.get('B000000001')?.placement, 'topOfSearch'); + assert.equal(byAsin.get('B000000001')?.placementId, byAsin.get('B000000002')?.placementId); + assert.equal(byAsin.get('B000000001')?.adProductIndex, 1); + assert.equal(byAsin.get('B000000002')?.adProductIndex, 2); + assert.equal(byAsin.get('B000000005')?.adType, 'unknownSponsored'); + assert.equal(byAsin.get('B000000005')?.placement, 'bottomOfSearch'); + assert.equal(byAsin.get('B000000006')?.adType, 'sponsoredDisplay'); + assert.equal(byAsin.get('B000000006')?.placement, 'sidebar'); + assert.deepEqual( + items.map((item) => item.pageRank), + [1, 2, 3, 4, 5, 6, 7], + ); + + console.log('Amazon search parser fixtures passed.'); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/test-execute-script.ts b/scripts/test-execute-script.ts new file mode 100644 index 0000000..80fde65 --- /dev/null +++ b/scripts/test-execute-script.ts @@ -0,0 +1,57 @@ +import { exec } from '../src/logic/execute-script'; + +const setBrowser = ( + executeScript: () => Promise, + getTab: () => Promise<{ status: string }> = async () => ({ status: 'complete' }), +) => { + Object.assign(globalThis, { + browser: { + tabs: { get: getTab }, + scripting: { executeScript }, + }, + }); +}; + +const main = async () => { + setBrowser(async () => { + throw new Error('Frame was removed'); + }); + const rejectionStarted = Date.now(); + await exec(1, async () => true, undefined, { timeout: 1000 }).then( + () => { + throw new Error('Expected executeScript to reject'); + }, + (error) => { + if (!String(error).includes('Frame was removed')) throw error; + if (Date.now() - rejectionStarted >= 1000) { + throw new Error('The original executeScript error was delayed until the timeout'); + } + }, + ); + + setBrowser(async () => [{ result: 42 }]); + const result = await exec(1, async () => 42, undefined, { timeout: 1000 }); + if (result !== 42) throw new Error(`Expected 42, received ${result}`); + + let injectionAttempted = false; + setBrowser( + async () => { + injectionAttempted = true; + return [{ result: true }]; + }, + async () => ({ status: 'loading' }), + ); + await exec(1, async () => true, undefined, { timeout: 100 }).then( + () => { + throw new Error('A perpetually loading page should time out before injection'); + }, + (error) => { + if (!String(error).includes('页面加载超时')) throw error; + }, + ); + if (injectionAttempted) throw new Error('Script was injected into a still-loading page'); + + console.log('Execute-script error propagation tests passed.'); +}; + +void main(); diff --git a/scripts/verify-extension-runtime.mjs b/scripts/verify-extension-runtime.mjs new file mode 100644 index 0000000..6951ffd --- /dev/null +++ b/scripts/verify-extension-runtime.mjs @@ -0,0 +1,71 @@ +const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9225'; +const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) => + response.json(), +); +const worker = targets.find( + (target) => target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'), +); +if (!worker) throw new Error('Azon Seeker background service worker was not found.'); + +const socket = new WebSocket(worker.webSocketDebuggerUrl); +const pending = new Map(); +let nextId = 1; + +socket.addEventListener('message', (event) => { + const message = JSON.parse(event.data); + if (!message.id || !pending.has(message.id)) return; + const { resolve, reject } = pending.get(message.id); + pending.delete(message.id); + message.error ? reject(new Error(message.error.message)) : resolve(message.result); +}); + +await new Promise((resolve, reject) => { + socket.addEventListener('open', resolve, { once: true }); + socket.addEventListener('error', reject, { once: true }); +}); + +function request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); +} + +const evaluation = await request('Runtime.evaluate', { + expression: `(async () => ({ + manifest: chrome.runtime.getManifest(), + alarm: await chrome.alarms.get('amazon-sync-retry'), + storage: await chrome.storage.local.get([ + 'amazon-sync-queue-v1', + 'amazon-sync-state-v1' + ]) + }))()`, + awaitPromise: true, + returnByValue: true, +}); +socket.close(); + +if (evaluation.exceptionDetails) { + throw new Error(evaluation.exceptionDetails.text || 'Extension evaluation failed.'); +} +const runtime = evaluation.result.value; +if (runtime.manifest.version !== '0.7.2') throw new Error('Unexpected extension version.'); +if (!runtime.manifest.permissions.includes('alarms')) throw new Error('Missing alarms permission.'); +if (runtime.alarm?.name !== 'amazon-sync-retry') throw new Error('Retry alarm was not created.'); +const state = JSON.parse(runtime.storage['amazon-sync-state-v1']); +if (state.pendingCount !== 0 || state.status !== 'synced') { + throw new Error(`Unexpected initial sync state: ${JSON.stringify(state)}`); +} +console.log( + JSON.stringify( + { + extensionId: new URL(worker.url).host, + version: runtime.manifest.version, + retryAlarm: runtime.alarm.name, + syncState: state, + }, + null, + 2, + ), +); diff --git a/scripts/verify-extension-sync-e2e.mjs b/scripts/verify-extension-sync-e2e.mjs new file mode 100644 index 0000000..a02dba3 --- /dev/null +++ b/scripts/verify-extension-sync-e2e.mjs @@ -0,0 +1,115 @@ +const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9226'; +const syncId = process.env.AZON_E2E_SYNC_ID; +const expectFailure = process.env.AZON_EXPECT_SYNC_FAILURE === '1'; +if (!syncId) throw new Error('AZON_E2E_SYNC_ID is required.'); + +let worker; +for (let attempt = 0; attempt < 60 && !worker; attempt++) { + const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) => + response.json(), + ); + worker = targets.find( + (target) => + target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'), + ); + if (!worker) await new Promise((resolve) => setTimeout(resolve, 250)); +} +if (!worker) throw new Error('Azon Seeker background service worker was not found.'); + +const socket = new WebSocket(worker.webSocketDebuggerUrl); +const pending = new Map(); +let nextId = 1; +socket.addEventListener('message', (event) => { + const message = JSON.parse(event.data); + if (!message.id || !pending.has(message.id)) return; + const { resolve, reject } = pending.get(message.id); + pending.delete(message.id); + message.error ? reject(new Error(message.error.message)) : resolve(message.result); +}); +await new Promise((resolve, reject) => { + socket.addEventListener('open', resolve, { once: true }); + socket.addEventListener('error', reject, { once: true }); +}); +function request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); +} +async function evaluate(expression) { + const result = await request('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (result.exceptionDetails) throw new Error(result.exceptionDetails.text); + return result.result.value; +} + +const searchItem = { + syncId, + asin: 'B000000098', + keywords: 'extension queue e2e', + rank: 1, + page: 1, + pageRank: 1, + link: 'https://www.amazon.com/dp/B000000098', + title: 'Extension queue end-to-end fixture', + price: '$33.93', + strikethroughPrice: '$39.99', + adType: 'sponsoredDisplay', + adRank: 1, + placement: 'sidebar', + placementRank: 1, + placementId: 'extension-e2e-placement', + adProductIndex: 1, + adFormat: 'standard', + imageSrc: 'https://example.com/extension-e2e.jpg', + createTime: '2026/08/31 11:45:00', +}; +await evaluate(`(async () => { + await chrome.storage.local.set({ + 'amazon-sync-queue-v1': ${JSON.stringify({ + searchItems: [searchItem], + detailItems: [], + reviews: [], + })}, + 'amazon-sync-state-v1': JSON.stringify({status: 'pending', pendingCount: 1}) + }); + await chrome.alarms.create('amazon-sync-retry', { when: Date.now() + 200 }); + return true; +})()`); + +let state; +for (let attempt = 0; attempt < 30; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 500)); + state = await evaluate(`(async () => { + const stored = await chrome.storage.local.get(['amazon-sync-queue-v1', 'amazon-sync-state-v1']); + return { + queue: stored['amazon-sync-queue-v1'], + state: JSON.parse(stored['amazon-sync-state-v1']) + }; + })()`); + if (state.state.status === 'failed') { + if (expectFailure) break; + throw new Error(state.state.lastError); + } + if (state.state.status === 'synced' && state.state.pendingCount === 0) break; +} +socket.close(); +if (expectFailure) { + if ( + state.state.status !== 'failed' || + state.state.pendingCount !== 1 || + state.queue.searchItems.length !== 1 + ) { + throw new Error(`Failed synchronization was not retained: ${JSON.stringify(state)}`); + } + console.log(JSON.stringify({ syncId, retained: true, ...state.state }, null, 2)); + process.exit(0); +} +if (state.state.status !== 'synced' || state.queue.searchItems.length !== 0) { + throw new Error(`Queue did not drain: ${JSON.stringify(state)}`); +} +console.log(JSON.stringify({ syncId, ...state.state }, null, 2)); diff --git a/src/background/amazon-sync.ts b/src/background/amazon-sync.ts new file mode 100644 index 0000000..1bf7b7e --- /dev/null +++ b/src/background/amazon-sync.ts @@ -0,0 +1,214 @@ +import { onMessage } from 'webext-bridge/background'; +import { flattenObject } from '~/logic/convert'; +import { exportKey, generateKeyPair, signData } from '~/logic/crypto'; +import { remoteHost } from '~/env'; +import { + AMAZON_SYNC_ALARM, + AMAZON_SYNC_QUEUE_KEY, + AMAZON_SYNC_STATE_KEY, + EMPTY_AMAZON_SYNC_STATE, +} from '~/logic/amazon-sync-shared'; + +type AmazonSyncQueue = Required; + +const emptyQueue = (): AmazonSyncQueue => ({ searchItems: [], detailItems: [], reviews: [] }); +const keyPairPromise = generateKeyPair(); +const publicKeyPromise = keyPairPromise.then(({ publicKey }) => exportKey(publicKey)); +let queueMutation: Promise = Promise.resolve(); +let flushOperation: Promise | undefined; +let flushAgain = false; + +function mutateQueue(operation: () => Promise): Promise { + const result = queueMutation.then(operation, operation); + queueMutation = result.then( + () => undefined, + () => undefined, + ); + return result; +} + +async function readQueue(): Promise { + const stored = await browser.storage.local.get(AMAZON_SYNC_QUEUE_KEY); + return { ...emptyQueue(), ...(stored[AMAZON_SYNC_QUEUE_KEY] as AmazonSyncQueue | undefined) }; +} + +function pendingCount(queue: AmazonSyncQueue) { + return queue.searchItems.length + queue.detailItems.length + queue.reviews.length; +} + +async function writeQueue(queue: AmazonSyncQueue) { + await browser.storage.local.set({ [AMAZON_SYNC_QUEUE_KEY]: queue }); +} + +async function writeState(state: AmazonSyncState) { + await browser.storage.local.set({ [AMAZON_SYNC_STATE_KEY]: JSON.stringify(state) }); + return state; +} + +function mergeUnique(current: T[], incoming: T[], key: (item: T) => string | undefined) { + const values = new Map(current.map((item) => [key(item), item])); + for (const item of incoming) values.set(key(item), item); + values.delete(undefined); + return Array.from(values.values()); +} + +async function signedPost(path: string, data: unknown[]) { + if (data.length === 0) return; + const body = JSON.stringify(data); + const { privateKey } = await keyPairPromise; + const response = await fetch(`https://${remoteHost}${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-signature': await signData(privateKey, body), + 'x-public-key': await publicKeyPromise, + }, + body, + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) throw new Error(`${path} HTTP ${response.status}`); + const result = (await response.json()) as { success?: boolean; count?: number }; + if (result.success !== true || result.count !== data.length) { + throw new Error(`${path} 返回的确认数量不匹配`); + } +} + +function serializeDetailItems(items: AmazonDetailItem[]) { + return items.map((item) => + flattenObject( + item.imageUrls + ? { + ...item, + imageUrls: item.imageUrls.join(';'), + } + : item, + ), + ); +} + +function serializeReviews(items: ({ asin: string } & AmazonReview)[]) { + return items.map((item) => ({ ...item, imageSrc: item.imageSrc?.join(';') })); +} + +async function removeSynced( + field: keyof AmazonSyncQueue, + sentKeys: Set, + key: (item: any) => string | undefined, +) { + await mutateQueue(async () => { + const current = await readQueue(); + (current[field] as any[]) = (current[field] as any[]).filter((item) => { + const value = key(item); + return !value || !sentKeys.has(value); + }); + await writeQueue(current); + }); +} + +async function flushQueueInternal(): Promise { + let queue = await mutateQueue(readQueue); + if (pendingCount(queue) === 0) { + return mutateQueue(async () => { + const current = await readQueue(); + return writeState({ + ...EMPTY_AMAZON_SYNC_STATE, + status: pendingCount(current) === 0 ? 'synced' : 'pending', + pendingCount: pendingCount(current), + }); + }); + } + await writeState({ status: 'syncing', pendingCount: pendingCount(queue) }); + try { + const searchSnapshot = queue.searchItems; + await signedPost('/amazon/search_items', searchSnapshot); + await removeSynced( + 'searchItems', + new Set(searchSnapshot.map((item) => item.syncId).filter(Boolean) as string[]), + (item: AmazonSearchItem) => item.syncId, + ); + + queue = await mutateQueue(readQueue); + const detailSnapshot = queue.detailItems; + await signedPost('/amazon/detail_items', serializeDetailItems(detailSnapshot)); + await removeSynced( + 'detailItems', + new Set(detailSnapshot.map((item) => item.syncId).filter(Boolean) as string[]), + (item: AmazonDetailItem) => item.syncId, + ); + + queue = await mutateQueue(readQueue); + const reviewSnapshot = queue.reviews; + await signedPost('/amazon/reviews', serializeReviews(reviewSnapshot)); + await removeSynced( + 'reviews', + new Set(reviewSnapshot.map((item) => item.id)), + (item: AmazonReview) => item.id, + ); + + return mutateQueue(async () => { + const current = await readQueue(); + return writeState({ + status: pendingCount(current) === 0 ? 'synced' : 'pending', + pendingCount: pendingCount(current), + lastSuccessAt: new Date().toISOString(), + }); + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await mutateQueue(async () => { + const current = await readQueue(); + await writeState({ + status: 'failed', + pendingCount: pendingCount(current), + lastError: message, + }); + }); + throw error; + } +} + +function flushQueue() { + if (flushOperation) { + flushAgain = true; + } else { + flushOperation = flushQueueInternal().finally(() => { + flushOperation = undefined; + if (flushAgain) { + flushAgain = false; + void flushQueue().catch(() => undefined); + } + }); + } + return flushOperation; +} + +async function enqueue(payload: AmazonSyncPayload) { + const state = await mutateQueue(async () => { + const queue = await readQueue(); + queue.searchItems = mergeUnique( + queue.searchItems, + payload.searchItems || [], + (item) => item.syncId, + ); + queue.detailItems = mergeUnique( + queue.detailItems, + payload.detailItems || [], + (item) => item.syncId, + ); + queue.reviews = mergeUnique(queue.reviews, payload.reviews || [], (item) => item.id); + await writeQueue(queue); + return writeState({ status: 'pending', pendingCount: pendingCount(queue) }); + }); + void flushQueue().catch(() => undefined); + return state; +} + +onMessage('amazon-sync-enqueue', ({ data }) => enqueue(data)); +onMessage('amazon-sync-flush', () => flushQueue()); + +browser.alarms.create(AMAZON_SYNC_ALARM, { periodInMinutes: 1 }); +browser.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === AMAZON_SYNC_ALARM) void flushQueue().catch(() => undefined); +}); + +void flushQueue().catch(() => undefined); diff --git a/src/background/main.ts b/src/background/main.ts index c5422b7..19f2103 100644 --- a/src/background/main.ts +++ b/src/background/main.ts @@ -2,6 +2,7 @@ import { isFirefox } from '~/env'; // https://github.com/serversideup/webext-bridge/issues/67#issuecomment-2676094094 import('webext-bridge/background'); +import('./amazon-sync'); // only on dev mode if (import.meta.hot) { diff --git a/src/composables/useLongTask.ts b/src/composables/useLongTask.ts index 5fc1797..34bfb07 100644 --- a/src/composables/useLongTask.ts +++ b/src/composables/useLongTask.ts @@ -1,17 +1,18 @@ export function useLongTask() { const isRunning = ref(false); + const lastError = shallowRef(); const startTask = async (task: () => Promise) => { if (isRunning.value) { throw Error('Task is still running.'); } isRunning.value = true; + lastError.value = undefined; let result = undefined; try { result = await task(); } catch (error) { - console.log(error); - + lastError.value = error; console.error('Task failed:', error); } isRunning.value = false; @@ -20,6 +21,7 @@ export function useLongTask() { return { isRunning, + lastError, startTask, }; } diff --git a/src/env.ts b/src/env.ts index 11eeafc..ae171ac 100644 --- a/src/env.ts +++ b/src/env.ts @@ -14,9 +14,6 @@ export function isForbiddenUrl(url: string): boolean { export const isFirefox = navigator.userAgent.includes('Firefox'); -// export const remoteHost = __DEV__ ? '127.0.0.1:8000' : '47.251.4.191:8000'; -// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'vm8nc3zr-18000.usw2.devtunnels.ms'; -// export const remoteHost = __DEV__ ? 'ogsm-dev.rekeymed.com:8888' : 'amzplugin.rekeymed.com'; -export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'; -// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'; -// export const remoteHost = '47.251.4.191:8000'; +export const remoteHost = + import.meta.env.VITE_REMOTE_HOST || + (__DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'); diff --git a/src/logic/amazon-sync-client.ts b/src/logic/amazon-sync-client.ts new file mode 100644 index 0000000..44f8c02 --- /dev/null +++ b/src/logic/amazon-sync-client.ts @@ -0,0 +1,9 @@ +import { sendMessage } from 'webext-bridge/sidepanel'; + +export async function enqueueAmazonSync(payload: AmazonSyncPayload) { + return sendMessage('amazon-sync-enqueue', payload, 'background'); +} + +export async function flushAmazonSyncQueue() { + return sendMessage('amazon-sync-flush', undefined, 'background'); +} diff --git a/src/logic/amazon-sync-shared.ts b/src/logic/amazon-sync-shared.ts new file mode 100644 index 0000000..fbc4d32 --- /dev/null +++ b/src/logic/amazon-sync-shared.ts @@ -0,0 +1,12 @@ +export const AMAZON_SYNC_QUEUE_KEY = 'amazon-sync-queue-v1'; +export const AMAZON_SYNC_STATE_KEY = 'amazon-sync-state-v1'; +export const AMAZON_SYNC_ALARM = 'amazon-sync-retry'; + +export const EMPTY_AMAZON_SYNC_STATE: AmazonSyncState = { + status: 'idle', + pendingCount: 0, +}; + +export function createAmazonSyncId(prefix: 'search' | 'detail') { + return `${prefix}-${crypto.randomUUID()}`; +} diff --git a/src/logic/execute-script.ts b/src/logic/execute-script.ts index 21b030f..6e970d7 100644 --- a/src/logic/execute-script.ts +++ b/src/logic/execute-script.ts @@ -47,45 +47,45 @@ export async function exec>( options: ExecOptions = {}, ): Promise { const { timeout = 30000 } = options; - return new Promise(async (resolve, reject) => { - // 基本刷新 - // await browser.tabs.reload(tabId); - // console.log('exec', browser.tabs.get(tabId), tabId); - - // console.log('exec', func, (await browser.tabs.get(tabId)).url); - for (let i = 0; i < 50; i++) { - await new Promise((r) => setTimeout(r, 200)); - const tab = await browser.tabs.get(tabId); - if (tab.status === 'complete') { - break; - } - } - + const deadline = Date.now() + timeout; + let pageLoaded = false; + while (Date.now() < deadline) { const tab = await browser.tabs.get(tabId); - if (tab.status !== 'complete') { - await browser.tabs.reload(tabId); - await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始 + if (tab.status === 'complete') { + pageLoaded = true; + break; } + const remaining = deadline - Date.now(); + await new Promise((resolve) => setTimeout(resolve, Math.min(200, remaining))); + } + if (!pageLoaded) throw new Error(`页面加载超时（${timeout}ms），未执行采集脚本`); - for (let i = 0; i < 50; i++) { - await new Promise((r) => setTimeout(r, 200)); - const tab = await browser.tabs.get(tabId); - if (tab.status === 'complete') { - break; - } - } + let timeoutId: ReturnType | undefined; + const remainingTimeout = Math.max(1, deadline - Date.now()); + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`脚本运行超时（总预算 ${timeout}ms）`)), + remainingTimeout, + ); + }); - setTimeout(() => reject(`脚本运行超时, foucs on ${func}`), timeout); - try { - const injectResults = await browser.scripting.executeScript({ + try { + const injectResults = await Promise.race([ + browser.scripting.executeScript({ target: { tabId }, func, args: payload ? [payload] : undefined, - }); - const ret = injectResults.pop(); - resolve(ret!.result as T); - } catch (e) { - throw new Error(`脚本运行失败: ${e}`); - } - }); + }), + timeoutPromise, + ]); + const ret = injectResults.at(-1); + if (!ret) throw new Error('脚本未返回执行结果'); + return ret.result as T; + } catch (error) { + const cause = error instanceof Error ? error : new Error(String(error)); + if (cause.message.startsWith('脚本运行超时')) throw cause; + throw new Error(`脚本运行失败: ${cause.message}`); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } } diff --git a/src/options/views/AmazonResultTable.vue b/src/options/views/AmazonResultTable.vue index e21732b..aa685e3 100644 --- a/src/options/views/AmazonResultTable.vue +++ b/src/options/views/AmazonResultTable.vue @@ -8,10 +8,41 @@ import { allDetailItems, allItems, itemColumnSettings, reviewItems } from '~/sto const modal = useModal(); const excelHelper = useExcelHelper(); +const adTypeLabels: Record = { + organic: '自然排位', + sponsoredProducts: 'Sponsored Products', + sponsoredBrands: 'Sponsored Brands', + sponsoredDisplay: 'Sponsored Display', + unknownSponsored: '未知广告', +}; + +const placementLabels: Record = { + topOfSearch: '顶部', + middleOfSearch: '中部', + bottomOfSearch: '底部', + restOfSearch: '其余位置（旧数据）', + sidebar: '侧边栏', + productPage: '商品详情页', + unknown: '未知位置', +}; + +const renderPlacement = (row: AmazonItem) => { + if (!row.placement) return '-'; + const label = placementLabels[row.placement]; + if (row.placement === 'productPage') return label; + return row.page ? `第 ${row.page} 页${label}` : label; +}; + +const renderOptionalValue = (value: unknown) => value ?? '-'; + +const isDetailOnlyRow = (row: AmazonItem) => + !row.keywords && row.page == null && row.rank == null && row.pageRank == null; + const filter = ref<{ keywords?: string; search?: string; detailOnly?: boolean; + adType?: AmazonAdType | 'unclassified'; searchDateRange?: [number, number]; detailDateRange?: [number, number]; }>({}); @@ -34,18 +65,55 @@ const columns = computed(() => { key: 'keywords', minWidth: 120, hidden: !itemColumnSettings.value.has('keywords'), + render: (row) => renderOptionalValue(row.keywords), }, { title: '页码', key: 'page', minWidth: 60, hidden: !itemColumnSettings.value.has('page'), + render: (row) => renderOptionalValue(row.page), }, { title: '排位', key: 'rank', minWidth: 60, hidden: !itemColumnSettings.value.has('rank'), + render: (row) => renderOptionalValue(row.rank), + }, + { + title: '页内排位', + key: 'pageRank', + minWidth: 90, + hidden: !itemColumnSettings.value.has('pageRank'), + render: (row) => renderOptionalValue(row.pageRank), + }, + { + title: '广告类型', + key: 'adType', + minWidth: 150, + render: (row) => + isDetailOnlyRow(row) ? '-' : row.adType ? adTypeLabels[row.adType] : '历史数据/未识别', + }, + { + title: '广告排位', + key: 'adRank', + minWidth: 90, + hidden: !itemColumnSettings.value.has('adRank'), + render: (row) => renderOptionalValue(row.adRank), + }, + { + title: '广告位置', + key: 'placement', + minWidth: 140, + render: renderPlacement, + }, + { + title: '广告位顺序', + key: 'placementRank', + minWidth: 110, + hidden: !itemColumnSettings.value.has('placementRank'), + render: (row) => renderOptionalValue(row.placementRank), }, { title: 'ASIN', @@ -61,6 +129,11 @@ const columns = computed(() => { key: 'price', minWidth: 100, }, + { + title: '划线价格', + key: 'strikethroughPrice', + minWidth: 100, + }, { title: '封面图', key: 'imageSrc', @@ -181,7 +254,7 @@ const getItemHeaders = () => { }; const filteredData = computed(() => { - const { search, detailOnly, keywords, searchDateRange, detailDateRange } = filter.value; + const { search, detailOnly, keywords, adType, searchDateRange, detailDateRange } = filter.value; let data = toRaw(detailOnly ? allDetailItems.value : allItems.value); if (search && search.trim() !== '') { data = data.filter((r) => { @@ -193,6 +266,9 @@ const filteredData = computed(() => { if (keywords) { data = data.filter((r) => r.keywords === keywords); } + if (adType) { + data = data.filter((r) => (adType === 'unclassified' ? !r.adType : r.adType === adType)); + } if (searchDateRange) { const start = dayjs(searchDateRange[0]); const end = dayjs(searchDateRange[1]); @@ -279,6 +355,17 @@ const handleClearData = async () => { }))" /> + + + { + + + diff --git a/src/options/views/help/guide.md b/src/options/views/help/guide.md index fb0aab9..145e9a3 100644 --- a/src/options/views/help/guide.md +++ b/src/options/views/help/guide.md @@ -44,7 +44,7 @@ 位于搜索页面板，可以根据输入的关键词采集搜索页中呈现的商品信息，主要为**关键词排名**。 输入：关键词（最多十个） -输出：商品搜索页信息，包含关键词、页码、排位、 ASIN、标题、 价格、 封面图、获取日期。 +输出：商品搜索页信息，包含关键词、页码、排位、广告类型、广告排位、广告位置、ASIN、标题、当前价格、划线价格、封面图、获取日期。 ### 详情页 @@ -52,7 +52,7 @@ 位于详情页面板，可以根据输入的多个ASIN采集商品详情页的商品信息。 输入：ASIN（每行输入一个，可以输入多个） -输出：商品详情页信息，包含ASIN、标题、 价格、 封面图、获取日期、评分、评论数、 大类、大类排行、小类、 小类排行、 商品图片链接、A+截图等等。 +输出：商品详情页信息，包含ASIN、标题、当前价格、划线价格、封面图、获取日期、评分、评论数、大类、大类排行、小类、小类排行、商品图片链接、A+截图等等。 ### 评论页 @@ -86,9 +86,15 @@ Items表记录商品信息，表头信息如下。 | 关键词 | 产品的关键词 | | 页码 | 产品在关键词搜索列表中的页码 | | 排位 | 产品的排名 | +| 页内排位 | 产品在当前搜索页商品流中的排名 | +| 广告类型 | 自然排位、Sponsored Products、Sponsored Brands、Sponsored Display 或未知广告 | +| 广告排位 | 同一关键词、同一广告类型中的累计排名 | +| 广告位置 | 搜索顶部、搜索其余位置或侧边栏 | +| 广告位顺序 | 同一关键词采集过程中广告模块的累计顺序 | | ASIN | 亚马逊标准识别号 | | 标题 | 产品的标题 | -| 价格 | 产品的价格 | +| 价格 | 产品的当前价格 | +| 划线价格 | 页面中以划线形式展示的参考价格 | | 封面图 | 产品的封面图片链接 | | 获取日期 | 数据获取的日期 | | 商品链接 | 产品的链接 | diff --git a/src/page-worker/composables/amazon.ts b/src/page-worker/composables/amazon.ts index 32f27be..6826d1a 100644 --- a/src/page-worker/composables/amazon.ts +++ b/src/page-worker/composables/amazon.ts @@ -3,7 +3,8 @@ import amazon from '../impls/amazon'; import { uploadImage } from '~/logic/upload-image'; import { detailItems, reviewItems, searchItems } from '~/storages/amazon'; import { createGlobalState } from '@vueuse/core'; -import { useAmazonService } from '~/page-worker/services/amazon'; +import { enqueueAmazonSync } from '~/logic/amazon-sync-client'; +import { createAmazonSyncId } from '~/logic/amazon-sync-shared'; import { taskOptionBase, WorkerComposable } from '../interfaces/common'; import { AmazonPageWorker } from '../interfaces/amazon'; @@ -18,10 +19,7 @@ function buildAmazonPageWorker(): WorkerComposable({}); // Long task management - const { isRunning, startTask } = useLongTask(); - // Amazon service instance - const service = useAmazonService(); - + const { isRunning, lastError, startTask } = useLongTask(); // Get the worker instance from implementation const worker = amazon.getAmazonPageWorker(); @@ -29,10 +27,13 @@ function buildAmazonPageWorker(): WorkerComposable(); const reviewCache = new Map(); + let commitOperation: Promise = Promise.resolve(); // Add search items to cache const updateSearchCache = (data: AmazonSearchItem[]) => { - searchCache.push(...data); + searchCache.push( + ...data.map((item) => ({ ...item, syncId: item.syncId || createAmazonSyncId('search') })), + ); }; // Update or add detail item in cache @@ -42,7 +43,10 @@ function buildAmazonPageWorker(): WorkerComposable item.id)); for (const review of reviews) { - ids.has(review.id) || values.push(review); + if (!ids.has(review.id)) { + values.push(review); + ids.add(review.id); + } } reviewCache.set(asin, values); }; @@ -64,44 +71,90 @@ function buildAmazonPageWorker(): WorkerComposable { + // Persist locally first, then atomically enqueue for background synchronization. + const commitChangeInternal = async () => { const { objects } = settings.value; - // Commit search items - if (objects?.includes('search')) { - searchItems.value = searchItems.value.concat(searchCache); - await service.commitSearchItems(searchCache); - searchCache.splice(0, searchCache.length); + const searchSnapshot = objects?.includes('search') ? searchCache.slice() : []; + const detailSnapshot = objects?.includes('detail') ? Array.from(detailCache.values()) : []; + const reviewSnapshot = objects?.includes('review') + ? Array.from(reviewCache.entries()).flatMap(([asin, reviews]) => + reviews.map((review) => ({ asin, ...review })), + ) + : []; + + if (searchSnapshot.length === 0 && detailSnapshot.length === 0 && reviewSnapshot.length === 0) { + return; } - // Commit detail items - if (objects?.includes('detail')) { - for (const [k, v] of detailCache.entries()) { - if (detailItems.value.has(k)) { - const item = detailItems.value.get(k)!; - detailItems.value.set(k, { ...item, ...v }); + + if (searchSnapshot.length > 0) { + const existingIds = new Set(searchItems.value.map((item) => item.syncId).filter(Boolean)); + searchItems.value = searchItems.value.concat( + searchSnapshot.filter((item) => !item.syncId || !existingIds.has(item.syncId)), + ); + } + if (detailSnapshot.length > 0) { + const nextDetailItems = new Map(detailItems.value); + for (const v of detailSnapshot) { + const k = v.asin; + if (nextDetailItems.has(k)) { + const item = nextDetailItems.get(k)!; + nextDetailItems.set(k, { ...item, ...v }); } else { - detailItems.value.set(k, v); + nextDetailItems.set(k, v); } } - await service.commitDetailItems(detailCache); - detailCache.clear(); + // Replacing the Map gives every extension page a deterministic reactive + // change to persist and observe; in-place Map.set could leave the open + // results page showing stale data during long batches. + detailItems.value = nextDetailItems; } - // Commit reviews - if (objects?.includes('review')) { - for (const [asin, reviews] of reviewCache.entries()) { - if (reviewItems.value.has(asin)) { + if (reviewSnapshot.length > 0) { + const groupedReviews = reviewSnapshot.reduce>((grouped, item) => { + const values = grouped.get(item.asin) || []; + const { asin: _, ...review } = item; + values.push(review); + grouped.set(item.asin, values); + return grouped; + }, new Map()); + const nextReviewItems = new Map(reviewItems.value); + for (const [asin, reviews] of groupedReviews.entries()) { + if (nextReviewItems.has(asin)) { const addedIds = new Set(reviews.map((x) => x.id)); - const origin = reviewItems.value.get(asin)!; + const origin = nextReviewItems.get(asin)!; const newReviews = origin.filter((x) => !addedIds.has(x.id)).concat(reviews); newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo))); - reviewItems.value.set(asin, newReviews); + nextReviewItems.set(asin, newReviews); } else { - reviewItems.value.set(asin, reviews); + nextReviewItems.set(asin, reviews); } } - await service.commitReviews(reviewCache); - reviewCache.clear(); + reviewItems.value = nextReviewItems; } + + // The background acknowledges only after the queue is durably written to extension storage. + await enqueueAmazonSync({ + searchItems: searchSnapshot, + detailItems: detailSnapshot, + reviews: reviewSnapshot, + }); + + const sentSearchIds = new Set(searchSnapshot.map((item) => item.syncId)); + const remainingSearch = searchCache.filter((item) => !sentSearchIds.has(item.syncId)); + searchCache.splice(0, searchCache.length, ...remainingSearch); + for (const item of detailSnapshot) { + if (detailCache.get(item.asin)?.syncId === item.syncId) detailCache.delete(item.asin); + } + const sentReviewIds = new Set(reviewSnapshot.map((item) => item.id)); + for (const [asin, reviews] of reviewCache.entries()) { + const remaining = reviews.filter((item) => !sentReviewIds.has(item.id)); + remaining.length > 0 ? reviewCache.set(asin, remaining) : reviewCache.delete(asin); + } + }; + + const commitChange = () => { + const result = commitOperation.then(commitChangeInternal, commitChangeInternal); + commitOperation = result.catch(() => undefined); + return result; }; // Register all relevant worker event handlers @@ -125,7 +178,7 @@ function buildAmazonPageWorker(): WorkerComposable { - updateDetailCache(ev); + if (ev.imageUrls?.length) updateDetailCache(ev); }), // Collect A+ content screenshots and upload worker.on('item-aplus-screenshot-collect', async (ev) => { @@ -161,15 +214,25 @@ function buildAmazonPageWorker(): WorkerComposable Promise>(func: T) => { - const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value; - clearAllCaches(); + const taskWrapper1 = Promise>( + func: T, + objectType: 'search' | 'review', + ) => { return (...params: Parameters) => startTask(async () => { - const interval = setInterval(() => commitChange(), commitChangeIngerval); - await func(...params); - clearInterval(interval); - await commitChange(); + settings.value = { ...settings.value, objects: [objectType] }; + clearAllCaches(); + const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value; + const interval = setInterval( + () => void commitChange().catch(console.error), + commitChangeIngerval, + ); + try { + await func(...params); + } finally { + clearInterval(interval); + await commitChange(); + } }); }; @@ -179,9 +242,10 @@ function buildAmazonPageWorker(): WorkerComposable Promise>( func: T, ) => { - clearAllCaches(); return (...params: Parameters) => startTask(async () => { + settings.value = { ...settings.value, objects: ['detail'] }; + clearAllCaches(); if (!params?.[1]) { params[1] = {}; } @@ -196,20 +260,24 @@ function buildAmazonPageWorker(): WorkerComposable { + const previousTab = await browser.tabs.get(tabId); + const previousUrl = previousTab.url; + const updatedTab = await browser.tabs.update(tabId, { url, active: true }); + let navigationObserved = + updatedTab.status === 'loading' || + (Boolean(updatedTab.url) && updatedTab.url !== previousUrl); + let lastUrl = updatedTab.url; + for (let i = 0; i < 150; i++) { + const currentTab = await browser.tabs.get(tabId); + lastUrl = currentTab.url; + navigationObserved ||= + currentTab.status === 'loading' || + (Boolean(currentTab.url) && currentTab.url !== previousUrl); + const reachedTarget = hasReachedAmazonNavigationTarget(currentTab.url, url); + const reachedRedirectedDetail = + options.allowDetailRedirect && hasReachedAmazonDetailPage(currentTab.url, url); + if (navigationObserved && currentTab.status === 'complete' && (reachedTarget || reachedRedirectedDetail)) { + return currentTab.url!; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`等待目标页面加载超时: ${url}；当前页面: ${lastUrl || '未知'}`); + } + private async wanderSearchSinglePage(tab: Tabs.Tab) { const injector = new AmazonSearchPageInjector(tab); + const runStage = async (stage: string, operation: () => Promise): Promise => { + try { + return await operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`搜索页${stage}失败: ${message}`); + } + }; // Wait for the Next button to appear, indicating that the product items have finished loading - await injector.waitForPageLoaded(); + await runStage('加载等待', () => injector.waitForPageLoaded()); // Determine the type of product search page https://github.com/primedigitaltech/azon_seeker/issues/1 - const pagePattern = await injector.getPagePattern(); + const pagePattern = await runStage('布局识别', () => injector.getPagePattern()); // Retrieve key nodes and their information from the critical product search page - const data = await injector.getPageData(pagePattern); + const data = await runStage('数据解析', () => injector.getPageData(pagePattern)); + const explicitNoResults = + data?.length === 0 + ? await runStage('空结果识别', () => injector.isExplicitNoResultsPage()) + : false; // get current page - const page = await injector.getCurrentPage(); - // Determine if it is the last page, otherwise navigate to the next page - const hasNextPage = await injector.determineHasNextPage(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - if (data === null || typeof hasNextPage !== 'boolean') { + const page = await runStage('页码识别', () => injector.getCurrentPage()); + // Read the next URL without navigating inside the injected script. Clicking here would move + // the current Amazon frame into BFCache before executeScript can return its result. + const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl()); + if (data === null) { this.emit('error', { message: '爬取单页信息失败', url: tab.url }); throw new Error('爬取单页信息失败'); } - return { data, hasNextPage, page }; + if (data.length === 0 && !explicitNoResults) { + throw new Error( + 'Amazon 搜索页未发现可采集商品，且页面没有明确显示“无结果”；请检查是否出现临时错误页、验证码或页面结构变化', + ); + } + return { data, nextPageUrl, page }; } @withErrorHandling @@ -69,14 +121,27 @@ class AmazonPageWorkerImpl url.searchParams.append('k', keywords); let tab = await this.getCurrentTab(); if (!tab.url?.startsWith('http')) { + // A newly created tab may not have a URL in the returned Tab object yet, so never pass its + // transient `tab.url` to `new URL()`. Navigate by tab id and wait for the target explicitly. tab = await this.createNewTab('https://www.amazon.com/'); - tab.url = 'https://www.amazon.com/'; + await this.navigateTabAndWait(tab.id!, url.toString()); + return url.toString(); } const currentUrl = new URL(tab.url!); - if (currentUrl.hostname !== url.hostname || currentUrl.searchParams.get('k') !== keywords) { - tab = await browser.tabs.update(tab.id, { url: url.toString(), active: true }); - await new Promise((resolve) => setTimeout(resolve, 1000)); + const sameSearch = + currentUrl.hostname === url.hostname && currentUrl.searchParams.get('k') === keywords; + if (sameSearch) { + try { + // Preserve the old behavior: a healthy same-keyword page continues from its current page. + await new AmazonSearchPageInjector(tab).waitForPageLoaded(); + return currentUrl.toString(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes('临时错误页')) throw error; + // Only a known Amazon transient error page is recovered by returning to page one. + } } + await this.navigateTabAndWait(tab.id!, url.toString()); return url.toString(); } @@ -84,22 +149,46 @@ class AmazonPageWorkerImpl public async wanderSearchPage(): Promise { const tab = await this.getCurrentTab(); let offset = 0; + let placementOffset = 0; + const adOffsets = new Map(); while (true) { - const { hasNextPage, data, page } = await this.wanderSearchSinglePage(tab); + const { nextPageUrl, data, page } = await this.wanderSearchSinglePage(tab); const keywords = new URL(tab.url!).searchParams.get('k')!; - const objs = data.map((r, i) => ({ - ...r, - keywords, - page, - rank: offset + 1 + i, - createTime: dayjs().format('YYYY/M/D HH:mm:ss'), - asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0], - })); + const pageAdCounts = new Map(); + const objs = data.map((r, i) => { + const localAdRank = r.adRank; + const localPlacementRank = r.placementRank; + const adRank = + r.adType && r.adType !== 'organic' && localAdRank + ? (adOffsets.get(r.adType) || 0) + localAdRank + : undefined; + if (r.adType && r.adType !== 'organic') { + pageAdCounts.set(r.adType, Math.max(pageAdCounts.get(r.adType) || 0, localAdRank || 0)); + } + const placementRank = localPlacementRank ? placementOffset + localPlacementRank : undefined; + return { + ...r, + keywords, + page, + rank: offset + 1 + i, + pageRank: r.pageRank || i + 1, + adRank, + placementRank, + placementId: placementRank + ? `${encodeURIComponent(keywords)}:${page}:${placementRank}` + : undefined, + createTime: dayjs().format('YYYY/M/D HH:mm:ss'), + }; + }); this.emit('item-links-collected', { objs }); offset += data.length; - if (!hasNextPage) { - break; - } + placementOffset += Math.max(0, ...data.map((item) => item.placementRank || 0)); + pageAdCounts.forEach((count, adType) => { + adOffsets.set(adType, (adOffsets.get(adType) || 0) + count); + }); + if (!nextPageUrl) break; + await this.navigateTabAndWait(tab.id!, nextPageUrl); + await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); } return new Promise((resolve) => setTimeout(resolve, 1000)); } @@ -113,8 +202,7 @@ class AmazonPageWorkerImpl const { image = false, aplus = false, extra = false } = options; const params = { asin: '', url: '' }; if (entry.match(/^https?:\/\/www\.amazon\.com.*\/dp\/[A-Z0-9]{10}/)) { - const [asin] = /\/\/dp\/[A-Z0-9]{10}/.exec(entry)!; - params.asin = asin; + params.asin = extractAmazonDetailAsin(entry) || ''; params.url = entry; } else if (entry.match(/^[A-Z0-9]{10}$/)) { params.asin = entry; @@ -124,9 +212,8 @@ class AmazonPageWorkerImpl if (!tab.url || isForbiddenUrl(tab.url)) { tab = await this.createNewTab(params.url); } else { - tab = await browser.tabs.update(tab.id, { - url: params.url, - }); + await this.navigateTabAndWait(tab.id!, params.url, { allowDetailRedirect: true }); + tab = await browser.tabs.get(tab.id!); } const injector = new AmazonDetailPageInjector(tab); @@ -142,6 +229,16 @@ class AmazonPageWorkerImpl //#region Await Production Introduction Element Loaded await injector.waitForPageLoaded(); await new Promise((resolve) => setTimeout(resolve, 3000)); // Wait 3 seconds. + tab = await browser.tabs.get(tab.id!); + const requestedAsin = params.asin; + const resolvedAsin = extractAmazonDetailAsin(tab.url); + if (!resolvedAsin) { + throw new Error(`Amazon 详情页未能识别实际 ASIN：${tab.url || '未知页面'}`); + } + params.asin = resolvedAsin; + if (requestedAsin !== resolvedAsin) { + await this.emit('item-detail-redirected', { requestedAsin, resolvedAsin }); + } //#endregion //#region Fetch Base Info const baseInfo = await injector.getBaseInfo(); @@ -183,11 +280,10 @@ class AmazonPageWorkerImpl //#region Fetch Goods' Images if (image) { const imageUrls = await injector.getImageUrls(); - imageUrls.length > 0 && - (await this.emit('item-images-collected', { - asin: params.asin, - imageUrls: Array.from(new Set(imageUrls)), - })); + await this.emit('item-images-collected', { + asin: params.asin, + imageUrls: Array.from(new Set(imageUrls)), + }); await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds. } //#endregion @@ -217,24 +313,72 @@ class AmazonPageWorkerImpl @withErrorHandling public async wanderReviewPage(asin: string, options: { recent?: boolean } = {}) { const { recent } = options; - const url = new URL( + const baseUrl = new URL( `https://www.amazon.com/product-reviews/${asin}/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews`, ); - recent && url.searchParams.set('sortBy', 'recent'); - const tab = await this.createNewTab(url.toString()); + recent && baseUrl.searchParams.set('sortBy', 'recent'); + const tab = await this.createNewTab(baseUrl.toString()); const injector = new AmazonReviewPageInjector(tab); - await injector.waitForPageLoad(); - for (let star = 1; star <= 5; star++) { - await injector.showStarsDropDownMenu(); - await injector.selectStar(star); + const runStage = async (stage: string, operation: () => Promise): Promise => { + try { + return await operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`评论页${stage}失败: ${message}`); + } + }; + await runStage('初始加载等待', () => injector.waitForPageLoad()); + const starFilters = [ + { name: 'one_star', star: 1 }, + { name: 'two_star', star: 2 }, + { name: 'three_star', star: 3 }, + { name: 'four_star', star: 4 }, + { name: 'five_star', star: 5 }, + ] as const; + const collectedReviewIds = new Set(); + + for (const { name: filterByStar, star } of starFilters) { + // The original collector gathered every rating bucket. Build the same Amazon filter URL + // directly instead of requiring a particular dropdown DOM structure just to discover it. + const starUrl = new URL(baseUrl); + starUrl.searchParams.set('filterByStar', filterByStar); + starUrl.searchParams.delete('pageNumber'); + await runStage(`${star} 星筛选导航`, () => + this.navigateTabAndWait(tab.id!, starUrl.toString()), + ); + const visitedPageUrls = new Set(); + let page = 1; + let filterIgnored = false; while (true) { - await injector.waitForPageLoad(); - const reviews = await injector.getSinglePageReviews(); - reviews.length > 0 && this.emit('item-review-collected', { asin, reviews }); - const hasNextPage = await injector.jumpToNextPageIfExist(); - if (!hasNextPage) { - break; + const currentTab = await browser.tabs.get(tab.id!); + if (currentTab.url) visitedPageUrls.add(currentTab.url); + // Match v0.7.1.12-beta semantics: collect the rating parsed from each review card. The + // requested filter is only a navigation aid and must not overwrite or reject real data. + await runStage(`${star} 星第 ${page} 页加载等待`, () => injector.waitForPageLoad()); + const reviews = await runStage('评论解析', () => injector.getSinglePageReviews()); + if (page === 1 && isAmazonReviewRatingFilterIgnored(reviews, star)) { + // Amazon currently experiments with review pages that preserve filterByStar in the URL + // but return the same unfiltered review cards. Keep the cards' real ratings, then stop + // revisiting the same eight reviews for the remaining rating buckets. + filterIgnored = true; } + const uniqueReviews = reviews.filter((review) => { + if (collectedReviewIds.has(review.id)) return false; + collectedReviewIds.add(review.id); + return true; + }); + uniqueReviews.length > 0 && + this.emit('item-review-collected', { asin, reviews: uniqueReviews }); + const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl()); + if (!nextPageUrl) break; + if (visitedPageUrls.has(nextPageUrl)) { + throw new Error(`评论页分页出现循环，已停止重复采集: ${nextPageUrl}`); + } + await runStage('分页导航', () => this.navigateTabAndWait(tab.id!, nextPageUrl)); + page += 1; + } + if (filterIgnored) { + break; } } setTimeout(() => browser.tabs.remove(tab.id!), 1000); @@ -254,7 +398,7 @@ class AmazonPageWorkerImpl const kw = remains.shift()!; await this.doSearch(kw); await this.wanderSearchPage(); - progress && progress(remains); + if (progress) await progress(remains); } unsubscribe(); } @@ -272,7 +416,10 @@ class AmazonPageWorkerImpl while (remains.length > 0 && !interrupt) { const asin = remains.shift()!; await this.wanderDetailPage(asin, options); - progress && progress(remains); + // The progress callback persists the completed item locally and enqueues + // it for server synchronization. Do not start the next ASIN until that + // handoff has completed, otherwise a batch can outrun its result writes. + if (progress) await progress(remains); } unsubscribe(); } @@ -290,7 +437,7 @@ class AmazonPageWorkerImpl while (remains.length > 0 && !interrupt) { const asin = remains.shift()!; await this.wanderReviewPage(asin, options); - progress && progress(remains); + if (progress) await progress(remains); } unsubscribe(); } diff --git a/src/page-worker/interfaces/amazon.ts b/src/page-worker/interfaces/amazon.ts index 9d88b14..5fb9a0e 100644 --- a/src/page-worker/interfaces/amazon.ts +++ b/src/page-worker/interfaces/amazon.ts @@ -4,6 +4,9 @@ export interface AmazonPageWorkerEvents { /** The event is fired when worker collected links to items on the Amazon search page. */ ['item-links-collected']: { objs: AmazonSearchItem[] }; + /** Amazon resolved a requested detail ASIN to a canonical or variation ASIN. */ + ['item-detail-redirected']: { requestedAsin: string; resolvedAsin: string }; + /** The event is fired when worker collected goods' base info on the Amazon detail page.*/ ['item-base-info-collected']: Pick< AmazonDetailItem, @@ -11,6 +14,7 @@ export interface AmazonPageWorkerEvents { | 'title' | 'boughtInfo' | 'price' + | 'strikethroughPrice' | 'rating' | 'ratingCount' | 'categories' @@ -41,7 +45,10 @@ export interface AmazonPageWorkerEvents { }; /** The event is fired when reviews collected in all review page */ - ['item-review-collected']: { asin: string; reviews: AmazonReview[] }; + ['item-review-collected']: { + asin: string; + reviews: AmazonReview[]; + }; /** Error event that occurs when there is an issue with the Amazon page worker*/ ['error']: { message: string; url?: string }; diff --git a/src/page-worker/interfaces/common.ts b/src/page-worker/interfaces/common.ts index 34de5ac..3c0b2e1 100644 --- a/src/page-worker/interfaces/common.ts +++ b/src/page-worker/interfaces/common.ts @@ -9,4 +9,5 @@ export interface taskOptionBase { export type WorkerComposable = Base & { settings: Ref; isRunning: Ref; + lastError?: Ref; }; diff --git a/src/page-worker/page-navigation.ts b/src/page-worker/page-navigation.ts new file mode 100644 index 0000000..ac8bc0a --- /dev/null +++ b/src/page-worker/page-navigation.ts @@ -0,0 +1,58 @@ +const navigationQueryKeys = ['k', 'page', 'pageNumber', 'filterByStar'] as const; + +function routeIdentity(url: URL) { + const reviewMatch = url.pathname.match(/^\/product-reviews\/([A-Z0-9]{10})/i); + if (reviewMatch) return `/product-reviews/${reviewMatch[1].toUpperCase()}`; + const detailMatch = url.pathname.match(/\/dp\/([A-Z0-9]{10})/i); + if (detailMatch) return `/dp/${detailMatch[1].toUpperCase()}`; + return url.pathname.replace(/\/ref=.*$/, '') || '/'; +} + +export function extractAmazonDetailAsin(url: string | undefined) { + if (!url) return null; + try { + return new URL(url).pathname.match(/\/dp\/([A-Z0-9]{10})/i)?.[1]?.toUpperCase() || null; + } catch { + return null; + } +} + +/** Accepts an Amazon canonical/variation redirect only when both URLs are detail pages. */ +export function hasReachedAmazonDetailPage( + currentUrl: string | undefined, + targetUrl: string, +) { + if (!currentUrl) return false; + try { + const current = new URL(currentUrl); + const target = new URL(targetUrl); + return ( + current.origin === target.origin && + Boolean(extractAmazonDetailAsin(currentUrl)) && + Boolean(extractAmazonDetailAsin(targetUrl)) + ); + } catch { + return false; + } +} + +/** + * Amazon may rewrite tracking parameters while navigating. Compare the stable route and the + * query parameters which identify the requested search/review page instead of the whole URL. + */ +export function hasReachedAmazonNavigationTarget(currentUrl: string | undefined, targetUrl: string) { + if (!currentUrl) return false; + try { + const current = new URL(currentUrl); + const target = new URL(targetUrl); + if (current.origin !== target.origin || routeIdentity(current) !== routeIdentity(target)) { + return false; + } + return navigationQueryKeys.every((key) => { + const expected = target.searchParams.get(key); + return expected === null || current.searchParams.get(key) === expected; + }); + } catch { + return false; + } +} diff --git a/src/page-worker/web-injectors/amazon-detail-images.ts b/src/page-worker/web-injectors/amazon-detail-images.ts new file mode 100644 index 0000000..0b9541c --- /dev/null +++ b/src/page-worker/web-injectors/amazon-detail-images.ts @@ -0,0 +1,103 @@ +/** + * Collect product image URLs from an Amazon detail page. + * + * Keep this function self-contained: browser.scripting.executeScript serializes + * it before running it in the target tab, so it must not depend on closures. + */ +export async function collectAmazonDetailImageUrls(): Promise { + const delay = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + const isProductImage = (value: string) => + /^https:\/\/(?:m\.media-amazon\.com|images-na\.ssl-images-amazon\.com)\/images\/I\//.test( + value, + ); + const unique = (values: string[]) => + Array.from(new Set(values.map((value) => value.replaceAll('\\/', '/')))).filter( + isProductImage, + ); + + const highResolutionUrls: string[] = []; + const largeUrls: string[] = []; + + // Amazon has changed the surrounding P.when(...) wrapper several times. + // Scan every inline script for the same hiRes/large metadata used by the old parser. + for (const scriptNode of Array.from(document.scripts)) { + const script = (scriptNode.textContent || '').replaceAll('\\/', '/'); + for (const match of script.matchAll( + /"hiRes"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g, + )) { + highResolutionUrls.push(match[1]); + } + for (const match of script.matchAll( + /"large"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g, + )) { + largeUrls.push(match[1]); + } + } + + let urls = unique(highResolutionUrls.length > 0 ? highResolutionUrls : largeUrls); + + // Current detail pages expose the main high-resolution image directly. + if (urls.length === 0) { + const landingImage = document.querySelector('#landingImage'); + const oldHighResolutionUrl = landingImage?.getAttribute('data-old-hires'); + if (oldHighResolutionUrl) urls.push(oldHighResolutionUrl); + + const dynamicImages = landingImage?.getAttribute('data-a-dynamic-image'); + if (dynamicImages) { + try { + const candidates = Object.entries(JSON.parse(dynamicImages) as Record).sort( + ([, firstSize], [, secondSize]) => + (secondSize[0] || 0) * (secondSize[1] || 0) - + (firstSize[0] || 0) * (firstSize[1] || 0), + ); + if (candidates[0]?.[0]) urls.push(candidates[0][0]); + } catch { + // Continue to the legacy gallery fallback below. + } + } + urls = unique(urls); + } + + // Restore the proven v0.7.2 fallback (commit b062d96): open the image + // viewer and read each full-size image. Current selectors are included too. + if (urls.length === 0) { + const viewerTrigger = document.querySelector( + '.overlayRestOfImages, #canvasCaption > a', + ); + viewerTrigger?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + await delay(1000); + + const thumbnailNodes = Array.from( + document.querySelectorAll( + '.ivRow div.ivThumbImage, #ivThumbs .ivThumb, #ivThumbs img', + ), + ); + for (const thumbnailNode of thumbnailNodes) { + thumbnailNode.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + await delay(500); + + for (let retryCount = 0; retryCount < 5; retryCount++) { + const fullImage = document.querySelector('#ivLargeImage > img'); + const source = fullImage?.src; + if ( + source && + !source.includes('/ui/loadIndicators/loading-large_labeled') && + isProductImage(source) + ) { + urls.push(source); + break; + } + await delay(500); + } + } + urls = unique(urls); + } + + document + .querySelector( + 'header > [data-action="a-popover-close"], body > div.a-modal-scroller.a-declarative', + ) + ?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + return urls; +} diff --git a/src/page-worker/web-injectors/amazon-detail-page-ready.ts b/src/page-worker/web-injectors/amazon-detail-page-ready.ts new file mode 100644 index 0000000..b7a0ef8 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-detail-page-ready.ts @@ -0,0 +1,32 @@ +/** Wait until the fields required by the detail parser are available. */ +export async function waitForAmazonDetailPageReady() { + const deadline = Date.now() + 20000; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 详情页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 详情页触发了验证码'); + } + + const unavailable = document.querySelector( + '#availability .a-color-price, #outOfStock, #exports_desktop_qualifiedBuybox', + ); + const title = document.querySelector('#title, #productTitle'); + const titleReady = Boolean(title?.textContent?.trim()); + if (document.readyState !== 'loading' && titleReady) { + title?.scrollIntoView({ behavior: 'auto', block: 'center' }); + return true; + } + + if (document.readyState !== 'loading' && unavailable) { + throw new Error('Amazon 商品当前不可用，详情标题未加载'); + } + await sleep(300); + } + + throw new Error('Amazon 详情标题在 20 秒内未加载完成'); +} diff --git a/src/page-worker/web-injectors/amazon-review-page.ts b/src/page-worker/web-injectors/amazon-review-page.ts new file mode 100644 index 0000000..dfaeacb --- /dev/null +++ b/src/page-worker/web-injectors/amazon-review-page.ts @@ -0,0 +1,73 @@ +type ReviewReadyOptions = { timeoutMs?: number }; + +/** Detects Amazon pages that keep filterByStar in the URL but return other ratings. */ +export function isAmazonReviewRatingFilterIgnored( + reviews: Pick[], + expectedStar: number, +) { + return ( + reviews.length > 0 && + reviews.some((review) => Number.parseInt(review.rating, 10) !== expectedStar) + ); +} + +/** Waits until an Amazon review page, including its requested star filter, is ready. */ +export async function waitForAmazonReviewPageReady(options: ReviewReadyOptions = {}) { + const { timeoutMs = 20000 } = options; + const deadline = Date.now() + timeoutMs; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 评论页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 评论页触发了验证码'); + } + + const reviewList = document.querySelector('#cm_cr-review_list'); + const reviewsContent = document.querySelector('.reviews-content'); + const loadingNode = document.querySelector('.reviews-loading'); + const contentReady = document.readyState !== 'loading' && Boolean(reviewList || reviewsContent); + const loadingFinished = !loadingNode || loadingNode.getClientRects().length === 0; + if (contentReady && loadingFinished) { + (reviewList || reviewsContent)?.scrollIntoView({ behavior: 'auto', block: 'center' }); + return; + } + await sleep(300); + } + throw new Error(`Amazon 评论内容在 ${Math.ceil(timeoutMs / 1000)} 秒内未加载完成`); +} + +/** Reads the next review-page URL without navigating from the injected script. */ +export async function collectAmazonNextReviewPageUrl() { + // Amazon may not render the pagination controls until the last review has entered the + // viewport. The original collector deliberately scrolled here; keep that behavior while + // returning the URL instead of clicking it inside the injected script (which can close the + // extension port when Chrome moves the old document into BFCache). + const reviewCards = Array.from( + document.querySelectorAll( + '#cm_cr-review_list [data-hook="review"], #cm_cr-review_list div[id*="review-card"]', + ), + ); + reviewCards.at(-1)?.scrollIntoView({ behavior: 'auto', block: 'center' }); + await new Promise((resolve) => setTimeout(resolve, 1000)); + + const pagination = + document.querySelector('[data-hook="pagination-bar"] .a-pagination') || + document.querySelector('[data-hook="pagination-bar"]') || + document.querySelector('ul.a-pagination'); + if (!pagination) return null; + + const nextItem = + pagination.querySelector('li.a-last') || + pagination.querySelector(':scope > *:nth-last-child(1)'); + if (!nextItem || nextItem.classList.contains('a-disabled')) return null; + + const nextAnchor = + nextItem.matches('a[href]') + ? (nextItem as HTMLAnchorElement) + : nextItem.querySelector('a[href]'); + const href = nextAnchor?.getAttribute('href'); + return href ? new URL(href, window.location.href).toString() : null; +} diff --git a/src/page-worker/web-injectors/amazon-search-page-ready.ts b/src/page-worker/web-injectors/amazon-search-page-ready.ts new file mode 100644 index 0000000..4ecd2a4 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-page-ready.ts @@ -0,0 +1,46 @@ +/** + * Waits for the useful search-page DOM rather than every spinner on the page. + * Amazon ad, carousel and recommendation widgets can keep a visible `.a-spinner` + * indefinitely even after the search results are ready to collect. + */ +export async function waitForAmazonSearchPageReady() { + const deadline = Date.now() + 20000; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 搜索页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 搜索页触发了验证码'); + } + const bodyText = document.body?.innerText || document.body?.textContent || ''; + if ( + /sorry[,.\s]+something went wrong on our end/i.test(bodyText) || + /please go back and try again/i.test(bodyText) + ) { + throw new Error('Amazon 返回临时错误页，可能触发了限流或风控，请稍后重试'); + } + + const resultCount = document.querySelectorAll( + '[data-component-type="s-search-result"][data-asin], .puis-card-container', + ).length; + const nextButton = document.querySelector('.s-pagination-next'); + const noResults = /no results for/i.test(bodyText); + const pageReady = + document.readyState !== 'loading' && + (resultCount > 0 || nextButton !== null || noResults); + + if (pageReady) { + nextButton?.scrollIntoView({ behavior: 'auto', block: 'center' }); + await sleep(750); + return; + } + + window.scrollBy(0, 500 + ~~(Math.random() * 500)); + await sleep(300); + } + + throw new Error('Amazon 搜索结果在 20 秒内未加载完成'); +} diff --git a/src/page-worker/web-injectors/amazon-search-pagination.ts b/src/page-worker/web-injectors/amazon-search-pagination.ts new file mode 100644 index 0000000..20d140b --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-pagination.ts @@ -0,0 +1,12 @@ +/** + * Reads the next search-page URL without navigating from the injected script. + * Navigation must happen only after browser.scripting.executeScript has returned, + * otherwise Amazon can move the frame into BFCache and destroy the script context. + */ +export async function collectAmazonNextPageUrl() { + const nextButton = document.querySelector('.s-pagination-next'); + if (!nextButton || nextButton.classList.contains('s-pagination-disabled')) return null; + + const href = nextButton.getAttribute('href'); + return href ? new URL(href, window.location.href).toString() : null; +} diff --git a/src/page-worker/web-injectors/amazon-search-parser.ts b/src/page-worker/web-injectors/amazon-search-parser.ts new file mode 100644 index 0000000..73050c3 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-parser.ts @@ -0,0 +1,340 @@ +export type AmazonSearchPagePattern = 'pattern-1' | 'pattern-2'; + +type SearchPageCollectorPayload = { + pattern: AmazonSearchPagePattern; +}; + +/** + * Collects product exposures from an Amazon search page. + * + * Keep this function self-contained: browser.scripting.executeScript serializes + * the function and cannot provide imported closures in the page context. + */ +export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) { + type CollectedItem = Pick< + AmazonSearchItem, + | 'asin' + | 'link' + | 'title' + | 'imageSrc' + | 'price' + | 'strikethroughPrice' + | 'adType' + | 'adFormat' + | 'adRank' + | 'pageRank' + | 'placement' + | 'placementRank' + | 'placementId' + | 'adProductIndex' + >; + + const text = (node: Element | null | undefined) => + (node?.textContent || '').replace(/\s+/g, ' ').trim(); + + const isVisible = (node: HTMLElement) => + !node.hidden && + node.getAttribute('aria-hidden') !== 'true' && + node.style.display !== 'none' && + node.style.visibility !== 'hidden'; + + const decodeRepeatedly = (value: string) => { + let decoded = value; + for (let i = 0; i < 3; i++) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) break; + decoded = next; + } catch { + break; + } + } + return decoded; + }; + + const parseAsin = (node: HTMLElement, href?: string) => { + const dataAsin = node.closest('[data-asin]')?.dataset.asin || node.dataset.asin; + if (dataAsin && /^[A-Z0-9]{10}$/.test(dataAsin)) return dataAsin; + + const candidates = [href || '', decodeRepeatedly(href || '')]; + for (const candidate of candidates) { + const match = /\/(?:dp|gp\/product)\/([A-Z0-9]{10})(?:[/?]|$)/i.exec(candidate); + if (match) return match[1].toUpperCase(); + } + return undefined; + }; + + const hasSponsoredDisclosure = (node: Element) => { + const candidates = [ + node, + ...Array.from(node.querySelectorAll('[aria-label], [title], span, a')), + ]; + return candidates.some((candidate) => { + const accessibleLabels = [ + candidate.getAttribute('aria-label'), + candidate.getAttribute('title'), + ]; + if ( + accessibleLabels.some((value) => /(?:^|\b)Sponsored(?:\b|\s|$)|赞助/i.test(value || '')) + ) { + return true; + } + return /^(Sponsored|赞助)(?:\b|\s|$)/i.test(text(candidate)); + }); + }; + + const hasSponsoredLink = (node: Element) => + Array.from(node.querySelectorAll('a[href]')).some((anchor) => { + const href = decodeRepeatedly(anchor.href).toLowerCase(); + return ( + href.includes('/sspa/click') || + href.includes('/gp/slredirect') || + href.includes('adid=') || + href.includes('qualifier=') + ); + }); + + const metadata = (node: Element) => + [ + node.getAttribute('data-component-type'), + node.getAttribute('data-cel-widget'), + node.getAttribute('data-ad-details'), + node.id, + node.className, + ] + .filter((value) => typeof value === 'string') + .join(' ') + .toLowerCase(); + + const isMainResultsBoundary = (node: Element) => + node.id === 'search' || + node.matches('.s-main-slot, [data-component-type="s-search-results"], body'); + + const uniqueAsinsIn = (node: Element) => { + const asins = new Set(); + node.querySelectorAll('[data-asin]').forEach((candidate) => { + const asin = candidate.dataset.asin; + if (asin && /^[A-Z0-9]{10}$/.test(asin)) asins.add(asin); + }); + return asins.size; + }; + + const hasExplicitAdMetadata = (node: Element) => + /sponsored.?brand|brand.?ad|display.?ad|sponsored.?display|adholder/.test(metadata(node)); + + const hasBrandCreative = (node: Element) => + /sponsored.?brand|brand.?ad/.test(metadata(node)) || + Boolean(node.querySelector('video')) || + (uniqueAsinsIn(node) > 1 && + Boolean( + node.querySelector( + 'a[href*="/stores/"], a[href*="/brands/"], [class*="brand-logo"], [class*="brandLogo"]', + ), + )); + + const findAdModule = (card: HTMLElement) => { + let current: HTMLElement | null = card; + for (let depth = 0; current && depth < 6; depth++, current = current.parentElement) { + if (isMainResultsBoundary(current)) break; + const explicitModule = hasExplicitAdMetadata(current); + const brandedSponsoredModule = + current !== card && hasSponsoredDisclosure(current) && hasBrandCreative(current); + const singleProductSponsored = + current !== card && hasSponsoredDisclosure(current) && uniqueAsinsIn(current) === 1; + if (explicitModule || brandedSponsoredModule || singleProductSponsored) + return current; + } + return card; + }; + + const classifyAd = (card: HTMLElement, module: HTMLElement) => { + const cardMeta = metadata(card); + const moduleMeta = metadata(module); + const combinedMeta = `${cardMeta} ${moduleMeta}`; + const sidebar = Boolean( + card.closest( + 'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"]', + ), + ); + const sponsored = + hasSponsoredDisclosure(card) || + hasSponsoredDisclosure(module) || + hasSponsoredLink(card) || + hasSponsoredLink(module) || + hasExplicitAdMetadata(card) || + hasExplicitAdMetadata(module); + + if (!sponsored) { + return { + adType: 'organic' as const, + placement: undefined, + adFormat: undefined, + }; + } + if (sidebar || /display.?ad|sponsored.?display/.test(combinedMeta)) { + return { + adType: 'sponsoredDisplay' as const, + placement: 'sidebar' as const, + adFormat: 'standard' as const, + }; + } + if (hasBrandCreative(module)) { + return { + adType: 'sponsoredBrands' as const, + placement: 'middleOfSearch' as const, + adFormat: module.querySelector('video') ? ('video' as const) : ('standard' as const), + }; + } + if (hasSponsoredDisclosure(card) || hasSponsoredLink(card)) { + return { + adType: 'sponsoredProducts' as const, + placement: 'middleOfSearch' as const, + adFormat: 'standard' as const, + }; + } + return { + adType: 'unknownSponsored' as const, + placement: 'unknown' as const, + adFormat: 'unknown' as const, + }; + }; + + const candidateSet = new Set(); + document + .querySelectorAll( + '[data-component-type="s-search-result"][data-asin], .puis-card-container, .puisg-row', + ) + .forEach((node) => { + const stableCard = + node.closest('[data-component-type="s-search-result"]') || node; + if ( + isVisible(stableCard) && + (stableCard.dataset.asin || + stableCard.querySelector('a[href*="/dp/"], a[href*="%2Fdp%2F"]')) + ) { + candidateSet.add(stableCard); + } + }); + + document.querySelectorAll('[data-asin]').forEach((node) => { + if ( + isVisible(node) && + /^[A-Z0-9]{10}$/.test(node.dataset.asin || '') && + node.querySelector('h2, h3, img') + ) { + const stableCard = + node.closest('[data-component-type="s-search-result"]') || node; + candidateSet.add(stableCard); + } + }); + + const cards = Array.from(candidateSet) + .filter( + (card, index, all) => + !all.some( + (other, otherIndex) => + index !== otherIndex && + card.contains(other) && + card.dataset.asin === other.dataset.asin, + ), + ) + .sort((left, right) => { + if (left === right) return 0; + return left.compareDocumentPosition(right) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1; + }); + + const organicCandidates = cards.filter((card) => { + const module = findAdModule(card); + return classifyAd(card, module).adType === 'organic'; + }); + const firstOrganicCandidate = organicCandidates[0]; + const lastOrganicCandidate = organicCandidates[organicCandidates.length - 1]; + const placementRanks = new Map(); + const adRanks = new Map(); + const moduleProductIndexes = new Map(); + let nextPlacementRank = 1; + + return cards.reduce((items, card) => { + const anchors = Array.from(card.querySelectorAll('a[href]')); + const productAnchor = anchors.find((anchor) => parseAsin(card, anchor.href)); + const asin = parseAsin(card, productAnchor?.href); + if (!asin) return items; + + const titleNode = card.querySelector('h2[aria-label], h2, h3[aria-label], h3'); + const imageNode = card.querySelector('img.s-image, img[src]'); + const title = + titleNode?.getAttribute('aria-label')?.trim() || + text(titleNode) || + imageNode?.alt?.trim() || + asin; + const imageSrc = imageNode?.src; + if (!imageSrc) return items; + + const currentPriceNode = card.querySelector( + '.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen', + ); + const strikethroughPriceNode = card.querySelector( + '[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString', + ); + const price = text(currentPriceNode) || undefined; + const rawStrikethroughPrice = text(strikethroughPriceNode) || undefined; + const strikethroughPrice = + rawStrikethroughPrice && rawStrikethroughPrice !== price ? rawStrikethroughPrice : undefined; + + const module = findAdModule(card); + const classification = classifyAd(card, module); + let placementRank: number | undefined; + let placementId: string | undefined; + let adRank: number | undefined; + let adProductIndex: number | undefined; + let placement: AmazonAdPlacement | undefined = classification.placement; + + if (classification.adType !== 'organic') { + if (!placementRanks.has(module)) placementRanks.set(module, nextPlacementRank++); + placementRank = placementRanks.get(module)!; + placementId = `search-page-placement-${placementRank}`; + adRank = (adRanks.get(classification.adType) || 0) + 1; + adRanks.set(classification.adType, adRank); + adProductIndex = (moduleProductIndexes.get(module) || 0) + 1; + moduleProductIndexes.set(module, adProductIndex); + + if (placement !== 'sidebar') { + const beforeFirstOrganic = + firstOrganicCandidate && + Boolean( + module.compareDocumentPosition(firstOrganicCandidate) & + Node.DOCUMENT_POSITION_FOLLOWING, + ); + const afterLastOrganic = + lastOrganicCandidate && + Boolean( + module.compareDocumentPosition(lastOrganicCandidate) & + Node.DOCUMENT_POSITION_PRECEDING, + ); + placement = beforeFirstOrganic + ? 'topOfSearch' + : afterLastOrganic + ? 'bottomOfSearch' + : 'middleOfSearch'; + } + } + + items.push({ + asin, + link: `https://www.amazon.com/dp/${asin}`, + title, + imageSrc, + price, + strikethroughPrice, + adType: classification.adType, + adFormat: classification.adFormat, + adRank, + pageRank: items.length + 1, + placement, + placementRank, + placementId, + adProductIndex, + }); + return items; + }, []); +} diff --git a/src/page-worker/web-injectors/amazon.ts b/src/page-worker/web-injectors/amazon.ts index b28e12a..e398553 100644 --- a/src/page-worker/web-injectors/amazon.ts +++ b/src/page-worker/web-injectors/amazon.ts @@ -1,182 +1,18 @@ import { BaseInjector } from './base'; +import { collectAmazonSearchPageData, type AmazonSearchPagePattern } from './amazon-search-parser'; +import { collectAmazonDetailImageUrls } from './amazon-detail-images'; +import { waitForAmazonDetailPageReady } from './amazon-detail-page-ready'; +import { collectAmazonNextPageUrl } from './amazon-search-pagination'; +import { waitForAmazonSearchPageReady } from './amazon-search-page-ready'; +import { + collectAmazonNextReviewPageUrl, + waitForAmazonReviewPageReady, +} from './amazon-review-page'; export class AmazonSearchPageInjector extends BaseInjector { public waitForPageLoaded() { - return this.run(async () => { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - while (true) { - const targetNode = document.querySelector('.s-pagination-next'); - - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500)); - - const h = Math.max( - document.documentElement.scrollHeight, - document.body.scrollHeight, - document.documentElement.offsetHeight, - document.body.offsetHeight, - document.documentElement.clientHeight, - ); - - // await modernScrollTo({ top: h * (0.4 + Math.random() * 0.2), behavior: 'smooth' }); - - if (targetNode || document.readyState === 'complete') { - // await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - // 第一段：滚到中下部 - // window.scrollTo({ - // top: h * (1 - ~~(Math.random() * 50)), - // behavior: 'smooth' - // }); - await modernScrollTo({ top: h * (0.9 + Math.random() * 0.08), behavior: 'smooth' }); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - // targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await scrollIntoViewAndWait(targetNode as HTMLElement, { - behavior: 'smooth', - block: 'center', - }); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - break; - } - } - while (true) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // const spins = Array.from(document.querySelectorAll('.a-spinner')).filter( - // (e) => e.getClientRects().length > 0, - // ); - // if (spins.length === 0) { - // break; - // } - - // const spins = Array.from(document.querySelectorAll('.a-carousel-card-empty')) - // if (spins.length === 0) { - // break; - // } - - const pagination = document.querySelectorAll('.rhf-sign-in-button'); - if (pagination.length > 0) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - break; - } - } - - /** - * 平滑滚动并等待滚动结束 - * @param options 原生 ScrollToOptions 接口 (包含 top, left, behavior) - */ - async function modernScrollTo(options: ScrollToOptions): Promise { - return new Promise((resolve) => { - // 1. 针对已经处于目标位置的情况做兜底处理 - // 如果目标位置与当前位置一致，部分浏览器可能不会触发 scrollend - const targetTop = options.top ?? window.scrollY; - const targetLeft = options.left ?? window.scrollX; - - if (targetTop === window.scrollY && targetLeft === window.scrollX) { - resolve(); - return; - } - - // 2. 监听原生滚动结束事件 - // 注意：scrollend 是 2023-2024 年起普及的标准，2026 年已是主流 - document.addEventListener( - 'scrollend', - () => { - resolve(); - }, - { once: true }, - ); - - // 3. 执行滚动 - window.scrollTo(options); - }); - } - - /** - * 将元素滚动到视野中心并等待动画结束 - * @param element 目标 DOM 元素 - * @param options 滚动配置 - */ - async function scrollIntoViewAndWait( - element: HTMLElement, - options: ScrollIntoViewOptions = { behavior: 'smooth', block: 'center' }, - ): Promise { - return new Promise((resolve) => { - // 监听滚动结束事件 - // scrollend 会在滚动圆满完成或被中断时触发 - document.addEventListener( - 'scrollend', - () => { - resolve(); - }, - { once: true }, - ); - - // 触发滚动 - element.scrollIntoView(options); - - // 兼容性兜底：如果元素已经在视野内，浏览器可能不触发滚动 - // 可以在此处检查位置，若无位移则直接 resolve - }); - } - }); + return this.run(waitForAmazonSearchPageReady); } - // public waitForPageLoaded() { - // return this.run(async () => { - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - - // // 第一个 while 循环：添加 12 秒超时刷新机制 - // const timeoutMs = 12000; // 12 秒 - // let startTime = Date.now(); - // let refreshCount = 0; - // const maxRefresh = 1; // 最多刷新 1 次 - // console.log('waitForPageLoaded'); - - // while (true) { - // const targetNode = document.querySelector('.s-pagination-next'); - // window.scrollBy(0, ~~(Math.random() * 500) + 500); - // await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500)); - - // if (targetNode || document.readyState === 'complete') { - // targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - // break; - // } - - // // 检查是否超时 - // if (Date.now() - startTime > timeoutMs) { - // if (refreshCount < maxRefresh) { - // // 刷新页面 - // console.log('检查是否超时，刷新页面'); - - // browser.tabs.query({ active: true, currentWindow: true }).then(tabs => { - // browser.tabs.reload(tabs[0].id); - // }); - - // await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始 - - // // 重置计时器和刷新计数 - // refreshCount++; - // startTime = Date.now(); - // continue; // 重新从 while 循环开始 - // } else { - // // 已刷新过一次，仍然超时，抛出错误 - // throw new Error('等待页面加载超时（已尝试刷新）'); - // } - // } - // } - - // // 第二个 while 循环：等待 spinner 消失 - // while (true) { - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // const spins = Array.from(document.querySelectorAll('.a-spinner')).filter( - // (e) => e.getClientRects().length > 0 - // ); - // if (spins.length === 0) { - // break; - // } - // } - // }); - // } public async getPagePattern() { return this.run(async () => { @@ -190,78 +26,15 @@ export class AmazonSearchPageInjector extends BaseInjector { }); } - public async getPageData(pattern: 'pattern-1' | 'pattern-2') { - let data: Pick[] | null = null; - switch (pattern) { - // 处理商品以列表形式展示的情况 - case 'pattern-1': - data = await this.run(async () => { - const items = Array.from( - document.querySelectorAll( - '.puisg-row:has(.a-section.a-spacing-small.a-spacing-top-small:not(.a-text-right))', - ), - ).filter((e) => e.getClientRects().length > 0); - const linkObjs = items.reduce< - Pick[] - >((objs, el) => { - const link = el.querySelector('a')?.href; - const title = el - .querySelector('h2.a-color-base')! - .getAttribute('aria-label')!; - const imageSrc = el.querySelector('img.s-image')!.src!; - const price = - el.querySelector('.a-price:not(.a-text-price) .a-offscreen') - ?.innerText || - ( - document.evaluate( - `.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`, - el, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLSpanElement | null - )?.innerText; - link && objs.push({ link, title, imageSrc, price }); - return objs; - }, []); - return linkObjs; - }); - break; - // 处理商品以二维图片格展示的情况 - case 'pattern-2': - data = await this.run(async () => { - const items = Array.from( - document.querySelectorAll( - '.puis-card-container', - ) as unknown as HTMLElement[], - ).filter((e) => e.getClientRects().length > 0); - const linkObjs = items.reduce< - Pick[] - >((objs, el) => { - const link = el.querySelector('a.a-link-normal')?.href; - const title = el.querySelector('h2.a-color-base')!.innerText; - const imageSrc = el.querySelector('img.s-image')!.src!; - const price = - el.querySelector('.a-price:not(.a-text-price) .a-offscreen') - ?.innerText || - ( - document.evaluate( - `.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`, - el, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLSpanElement | null - )?.innerText; - link && objs.push({ link, title, imageSrc, price }); - return objs; - }, []); - return linkObjs; - }); - break; - default: - break; - } - data = data && data.filter((r) => new URL(r.link).pathname.includes('/dp/')); // No advertisement only - return data; + public async getPageData(pattern: AmazonSearchPagePattern) { + return this.run(collectAmazonSearchPageData, { pattern }); + } + + public async isExplicitNoResultsPage() { + return this.run(async () => { + const bodyText = document.body?.innerText || document.body?.textContent || ''; + return /no results for/i.test(bodyText); + }); } public async getCurrentPage() { @@ -271,111 +44,9 @@ export class AmazonSearchPageInjector extends BaseInjector { }); } - /** - * 检测当前亚马逊搜索页面是否有下一页，并自动点击翻页按钮。 - * - * 该方法在页面上下文中执行，查找亚马逊标准分页按钮（'.s-pagination-next'）， - * 检查按钮是否未被禁用（'s-pagination-disabled' 类），然后模拟用户点击。 - * 点击前会随机等待 500-1000 毫秒以避免被识别为机器人。 - * - * @returns `true` 表示有下一页且已点击翻页按钮，页面正在加载下一页内容； - * `false` 表示没有下一页（按钮不存在或被禁用）。 - */ - public async determineHasNextPage() { - return this.run(async () => { - const nextButton = document.querySelector('.s-pagination-next'); - if (nextButton) { - if (!nextButton.classList.contains('s-pagination-disabled')) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - nextButton.click(); - return true; - } else { - return false; - } - } else { - return false; - } - }); + public async getNextPageUrl() { + return this.run(collectAmazonNextPageUrl); } - // /** - // * 检测并执行亚马逊搜索页面翻页，等待页面刷新完成。 - // * @returns 能否翻页（true=翻页成功，false=没有下一页或翻页失败） - // */ - // public async determineHasNextPage(): Promise { - // return this.run(async () => { - // const nextButton = document.querySelector('.s-pagination-next'); - - // if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) { - // // 记录当前页码 - // const initialPage = await this.getCurrentPage(); - - // // 随机等待后点击 - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // nextButton.click(); - - // // 等待页面刷新完成 - // await this.waitForPageLoaded(); - - // // 验证翻页是否成功 - // const newPage = await this.getCurrentPage(); - - // // 只有页码真正变化才算翻页成功 - // return newPage !== null && newPage !== initialPage; - // } - - // return false; - // }); - // } - - /** - * 检测并执行亚马逊搜索页面翻页，通过 URL 变化确认刷新完成。 - * @returns 能否翻页（true=已翻页，false=已是最后一页） - */ - // public async determineHasNextPage(): Promise { - // return this.run(async () => { - // const nextButton = document.querySelector('.s-pagination-next'); - - // if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) { - // // 1. 记录当前 URL - // const initialUrl = window.location.href; - - // // 2. 随机等待后点击 - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // nextButton.click(); - - // // 3. 等待 URL 变化（表示页面已开始导航） - // await new Promise((resolve) => { - // const checkUrl = () => { - // if (window.location.href !== initialUrl) { - // resolve(); - // } else { - // setTimeout(checkUrl, 100); - // } - // }; - // checkUrl(); - // }); - - // // 4. 等待页面稳定（document.readyState === 'complete'） - // await new Promise((resolve) => { - // const checkReadyState = () => { - // if (document.readyState === 'complete') { - // resolve(); - // } else { - // setTimeout(checkReadyState, 100); - // } - // }; - // checkReadyState(); - // }); - - // // 5. 额外等待确保内容加载 - // await new Promise((resolve) => setTimeout(resolve, 500)); - - // return true; - // } - - // return false; - // }); - // } } export class AmazonDetailPageInjector extends BaseInjector { @@ -389,35 +60,27 @@ export class AmazonDetailPageInjector extends BaseInjector { /**等待页面加载完成 */ public async waitForPageLoaded() { - return this.run(async () => { - while (true) { - window.scrollBy(0, ~~(Math.random() * 500) + 500); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 100) + 200)); - const targetNode = document.querySelector( - '#prodDetails:has(td), #detailBulletsWrapper_feature_div:has(li), .av-page-desktop, #productDescription_feature_div', - ); - const exceptionalNodeSelectors = ['.music-detail-header', '.avu-retail-page']; - for (const selector of exceptionalNodeSelectors) { - if (document.querySelector(selector)) { - return false; - } - } - if (targetNode && document.readyState !== 'loading') { - targetNode.scrollIntoView({ behavior: 'smooth', block: 'center' }); - break; - } - } - return true; - }); + return this.run(waitForAmazonDetailPageReady); } /**获取基本信息 */ public async getBaseInfo() { return this.run(async () => { const title = document.querySelector('#title')!.innerText; - const price = document.querySelector( - '.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen', + const priceRoot = + document.querySelector( + '#corePrice_feature_div, #corePriceDisplay_desktop_feature_div, #apex_desktop', + ) || document.body; + const price = priceRoot.querySelector( + '.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen', )?.innerText; + const rawStrikethroughPrice = priceRoot.querySelector( + '[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString', + )?.innerText; + const strikethroughPrice = + rawStrikethroughPrice && rawStrikethroughPrice !== price + ? rawStrikethroughPrice + : undefined; const boughtInfo = document.querySelector( `#social-proofing-faceout-title-tk_bought`, )?.innerText; @@ -436,7 +99,16 @@ export class AmazonDetailPageInjector extends BaseInjector { '#fulfillerInfoFeature_feature_div > *:last-of-type', )?.innerText; const soldBy = document.querySelector(`#sellerProfileTriggerId`)?.innerText; - return { title, price, boughtInfo, availableDate, categories, shipFrom, soldBy }; + return { + title, + price, + strikethroughPrice, + boughtInfo, + availableDate, + categories, + shipFrom, + soldBy, + }; }); } @@ -488,78 +160,7 @@ export class AmazonDetailPageInjector extends BaseInjector { /**获取图像链接 */ public async getImageUrls() { - return this.run(async () => { - const overlay = document.querySelector('.overlayRestOfImages'); - - if (overlay) { - if (document.querySelector('#ivThumbs')!.getClientRects().length === 0) { - // Click to see full view - document.querySelector('#canvasCaption > a')?.click(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - } else { - // finally click to see full view if not clicked before - document.querySelector('#canvasCaption > a')?.click(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - let script = document.evaluate( - `//script[starts-with(text(), "\nP.when(\'A\').register") or contains(text(), "\nP.when('A').register")]`, - document, - null, - XPathResult.STRING_TYPE, - ).stringValue; - const extractUrls = (pattern: RegExp) => - Array.from(script.matchAll(pattern)).map((e) => e[0]); - let urls = extractUrls( - /(?<="hiRes":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g, - ); - if (urls.length === 0) { - urls = extractUrls( - /(?<="large":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g, - ); - } - - if (urls.length === 0) { - const divs: NodeListOf = - document.querySelectorAll('.ivRow div.ivThumbImage'); - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - for (const div of divs) { - // click div com - div.click(); - - // wait for image to load - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - - // if img src is not loading gif, then get url and break, otherwise continue to wait, with a max retry count to avoid infinite loop - let retryCount = 0; - const maxRetries = 5; - - while (true) { - const loadingGifUrl = - 'https://m.media-amazon.com/images/G/01/ui/loadIndicators/loading-large_labeled._CB485921664_.gif'; - // get img dom - const imgDom = document.querySelector('#ivLargeImage > img'); - - // get url from img dom src attribute - if (imgDom && 'src' in imgDom && imgDom.src !== loadingGifUrl) { - urls.push(imgDom.src as string); - break; - } else { - // wait for 500ms and try again - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - retryCount++; - if (retryCount >= maxRetries) { - console.error('在detail页面下载图片，已达最大重试次数:'); - break; - } - } - } - } - } - - document.querySelector('body > div.a-modal-scroller.a-declarative')?.click(); - return urls; - }); + return this.run(collectAmazonDetailImageUrls); } /**获取精选评论 */ @@ -697,30 +298,7 @@ export class AmazonDetailPageInjector extends BaseInjector { export class AmazonReviewPageInjector extends BaseInjector { public async waitForPageLoad() { - return this.run(async () => { - await new Promise((resolve) => setTimeout(resolve, 1000)); - while (true) { - const targetNode = document.querySelector( - '.reviews-content, #cm_cr-review_list ul[role="list"]:not(.histogram)', - ); - targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - if ( - targetNode && - targetNode.getClientRects().length > 0 && - document.readyState !== 'loading' - ) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - while (true) { - const loadingNode = document.querySelector('.reviews-loading'); - if (loadingNode && loadingNode.getClientRects().length === 0) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - }); + return this.run(waitForAmazonReviewPageReady); } public async getSinglePageReviews() { @@ -773,54 +351,7 @@ export class AmazonReviewPageInjector extends BaseInjector { }); } - public jumpToNextPageIfExist() { - return this.run(async () => { - const latestReview = document.evaluate( - `//*[@id='cm_cr-review_list']//li[@data-hook='review'][last()]`, - document.body, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLElement | null; - latestReview?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await new Promise((resolve) => setTimeout(resolve, 1000)); - const nextPageNode = document.querySelector( - '[data-hook="pagination-bar"] .a-pagination > *:nth-of-type(2)', - ); - nextPageNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await new Promise((resolve) => setTimeout(resolve, 1000)); - const ret = nextPageNode && !nextPageNode.classList.contains('a-disabled'); - ret && nextPageNode?.querySelector('a')?.click(); - return ret; - }); - } - - public async showStarsDropDownMenu() { - return this.run(async () => { - while (true) { - const dropdown = document.querySelector('#star-count-dropdown')!; - dropdown.scrollIntoView({ behavior: 'smooth', block: 'center' }); - dropdown.click(); - if (dropdown.getAttribute('aria-expanded') === 'true') { - break; - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - }); - } - - public async selectStar(star: number) { - return this.run( - async ({ star }) => { - const starNode = document.evaluate( - `//ul[@role='listbox']/li/a[text()="${star} star only"]`, - document.body, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLElement; - starNode.click(); - await new Promise((resolve) => setTimeout(resolve, 100)); - }, - { star }, - ); + public getNextPageUrl() { + return this.run(collectAmazonNextReviewPageUrl); } } diff --git a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue index a0dccff..a3aa59f 100644 --- a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue +++ b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue @@ -20,6 +20,14 @@ worker.on('error', ({ message: msg }) => { }); message.error(msg); }); +worker.on('item-detail-redirected', (ev) => { + timelines.value.push({ + type: 'warning', + title: '商品发生重定向', + time: new Date().toLocaleString(), + content: `请求 ASIN：${ev.requestedAsin}\n实际 ASIN：${ev.resolvedAsin}\n数据将按实际 ASIN 保存`, + }); +}); worker.on('item-base-info-collected', (ev) => { timelines.value.push({ type: 'success', @@ -40,11 +48,15 @@ worker.on('item-category-rank-collected', (ev) => { }); }); worker.on('item-images-collected', (ev) => { + const imageCount = ev.imageUrls?.length || 0; timelines.value.push({ - type: 'success', + type: imageCount > 0 ? 'success' : 'warning', title: `商品${ev.asin}图像`, time: new Date().toLocaleString(), - content: `图片数： ${ev.imageUrls!.length}`, + content: + imageCount > 0 + ? `图片数： ${imageCount}` + : '未找到商品图片；Amazon 页面未提供可识别的图片数据', }); }); // worker.on('item-top-reviews-collected', (ev) => { @@ -98,11 +110,12 @@ const launch = async () => { }, ...detailWorkerSettings.value, }); + const failed = Boolean(worker.lastError?.value); timelines.value.push({ - type: 'info', + type: failed ? 'error' : 'info', title: '结束', time: new Date().toLocaleString(), - content: '数据采集完成', + content: failed ? '数据采集失败，已保留此前成功采集的数据' : '数据采集完成', }); }; diff --git a/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue b/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue index 66e7d5f..4bb17f0 100644 --- a/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue +++ b/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue @@ -4,7 +4,10 @@ import { usePageWorker } from '~/page-worker'; import { reviewAsinInput, reviewWorkerSettings } from '~/storages/amazon'; const worker = usePageWorker('amazon', { objects: ['review'] }); +const collectedReviewIds = new Map>(); +const hadFatalError = ref(false); worker.on('error', ({ message: msg }) => { + hadFatalError.value = true; timelines.value.push({ type: 'error', title: '错误发生', @@ -13,14 +16,16 @@ worker.on('error', ({ message: msg }) => { }); }); worker.on('item-review-collected', (ev) => { + const ids = collectedReviewIds.get(ev.asin) || new Set(); + ev.reviews.forEach((review) => ids.add(review.id)); + collectedReviewIds.set(ev.asin, ids); timelines.value.push({ type: 'success', title: `商品${ev.asin}评价`, time: new Date().toLocaleString(), - content: `获取到 ${ev.reviews.length} 条评价`, + content: `已累计获取 ${ids.size} 条评价`, }); }); - const asinInputRef = useTemplateRef('asin-input'); const message = useMessage(); @@ -29,6 +34,8 @@ const timelines = ref([]); const launch = async () => { const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0); + collectedReviewIds.clear(); + hadFatalError.value = false; timelines.value = [ { type: 'info', @@ -43,12 +50,21 @@ const launch = async () => { }, recent: reviewWorkerSettings.value.recent, }); - timelines.value.push({ - type: 'info', - title: '结束', - time: new Date().toLocaleString(), - content: '数据采集完成', - }); + if (hadFatalError.value || worker.lastError?.value) { + timelines.value.push({ + type: 'error', + title: '结束', + time: new Date().toLocaleString(), + content: '数据采集失败，已保留此前成功采集的数据', + }); + } else { + timelines.value.push({ + type: 'info', + title: '结束', + time: new Date().toLocaleString(), + content: '数据采集完成', + }); + } }; const handleStart = () => { diff --git a/src/storages/amazon-sync.ts b/src/storages/amazon-sync.ts new file mode 100644 index 0000000..dcba737 --- /dev/null +++ b/src/storages/amazon-sync.ts @@ -0,0 +1,8 @@ +import { useWebExtensionStorage } from '~/composables/useWebExtensionStorage'; +import { AMAZON_SYNC_STATE_KEY, EMPTY_AMAZON_SYNC_STATE } from '~/logic/amazon-sync-shared'; + +export const amazonSyncState = useWebExtensionStorage( + AMAZON_SYNC_STATE_KEY, + EMPTY_AMAZON_SYNC_STATE, + { flush: 'sync' }, +); diff --git a/src/storages/amazon.ts b/src/storages/amazon.ts index bc323e2..9736d7b 100644 --- a/src/storages/amazon.ts +++ b/src/storages/amazon.ts @@ -7,8 +7,20 @@ export const detailAsinInput = useWebExtensionStorage('detailAsinInputTe export const reviewAsinInput = useWebExtensionStorage('reviewAsinInputText', ''); export const itemColumnSettings = useWebExtensionStorage< - Set> ->('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'createTime'])); + Set< + keyof Pick< + AmazonItem, + | 'keywords' + | 'page' + | 'rank' + | 'pageRank' + | 'adRank' + | 'placementRank' + | 'createTime' + | 'timestamp' + > + > +>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'adRank', 'createTime'])); export const detailWorkerSettings = useWebExtensionStorage('amazon-detail-worker-settings', { image: true, @@ -55,8 +67,17 @@ export const allItems = computed({ 'title', 'imageSrc', 'price', + 'strikethroughPrice', 'link', 'rank', + 'pageRank', + 'adType', + 'adRank', + 'placement', + 'placementRank', + 'placementId', + 'adProductIndex', + 'adFormat', 'createTime', ]; searchItems.value = newValue diff --git a/src/types/amazon-sync.d.ts b/src/types/amazon-sync.d.ts new file mode 100644 index 0000000..cb1a375 --- /dev/null +++ b/src/types/amazon-sync.d.ts @@ -0,0 +1,25 @@ +import type { ProtocolWithReturn } from 'webext-bridge'; + +declare global { + type AmazonSyncPayload = { + searchItems?: AmazonSearchItem[]; + detailItems?: AmazonDetailItem[]; + reviews?: ({ asin: string } & AmazonReview)[]; + }; + + type AmazonSyncStatus = 'idle' | 'pending' | 'syncing' | 'synced' | 'failed'; + + type AmazonSyncState = { + status: AmazonSyncStatus; + pendingCount: number; + lastSuccessAt?: string; + lastError?: string; + }; +} + +declare module 'webext-bridge' { + export interface ProtocolMap { + 'amazon-sync-enqueue': ProtocolWithReturn; + 'amazon-sync-flush': ProtocolWithReturn; + } +} diff --git a/src/types/amazon.d.ts b/src/types/amazon.d.ts index f0faf7a..ccb1b62 100644 --- a/src/types/amazon.d.ts +++ b/src/types/amazon.d.ts @@ -1,4 +1,25 @@ +declare type AmazonAdType = + | 'organic' + | 'sponsoredProducts' + | 'sponsoredBrands' + | 'sponsoredDisplay' + | 'unknownSponsored'; + +declare type AmazonAdPlacement = + | 'topOfSearch' + | 'middleOfSearch' + | 'bottomOfSearch' + /** 兼容旧版采集数据 */ + | 'restOfSearch' + | 'sidebar' + | 'productPage' + | 'unknown'; + +declare type AmazonAdFormat = 'standard' | 'video' | 'unknown'; + declare type AmazonSearchItem = { + /** 客户端生成的幂等同步标识 */ + syncId?: string; /** 搜索关键词 */ keywords: string; /** 商品排名 */ @@ -13,6 +34,24 @@ declare type AmazonSearchItem = { asin: string; /** 商品价格（可选） */ price?: string; + /** 页面展示的划线参考价格 */ + strikethroughPrice?: string; + /** 商品在当前页商品流中的排名 */ + pageRank?: number; + /** 广告类型；历史数据可能为空 */ + adType?: AmazonAdType; + /** 同一关键词、同一广告类型中的累计广告排名 */ + adRank?: number; + /** 广告在页面中的位置 */ + placement?: AmazonAdPlacement; + /** 同一关键词采集过程中广告模块的累计顺序 */ + placementRank?: number; + /** 同一广告模块内商品的关联标识 */ + placementId?: string; + /** 商品在广告模块内的顺序 */ + adProductIndex?: number; + /** 广告素材形式 */ + adFormat?: AmazonAdFormat; /** 商品图片链接 */ imageSrc: string; /** 创建时间 */ @@ -20,6 +59,8 @@ declare type AmazonSearchItem = { }; declare type AmazonDetailItem = { + /** 客户端生成的幂等同步标识 */ + syncId?: string; /** 商品的 ASIN（亚马逊标准识别号） */ asin: string; /** 商品标题 */ @@ -30,6 +71,8 @@ declare type AmazonDetailItem = { boughtInfo?: string; /** 商品价格 */ price?: string; + /** 页面展示的划线参考价格 */ + strikethroughPrice?: string; /** 商品评分 */ rating?: number; /** 评分数量 */