From 90110a48f0cd85f64dc39dc29209a961798ed967 Mon Sep 17 00:00:00 2001 From: Ktight <1032771964@qq.com> Date: Thu, 10 Sep 2026 18:03:11 +0800 Subject: [PATCH] Feat/advertisement (#26) * feat: add Amazon collection metadata and reliable sync * fix: route extension sync through configured server * chore: release extension v0.7.13 * fix: handle extension ports across BFCache navigation * fix: classify bottom Sponsored Brands video placements * fix: exclude hidden Amazon ad cards * fix: derive Amazon result page from URL * fix: ignore hidden Amazon sponsored products * fix: preserve base Amazon product collection * fix: bind Amazon results to visible page * fix: classify visible Amazon ad formats * fix: wait for lazy Amazon ad modules * perf: reduce Amazon lazy-load scroll stops * fix: classify Amazon sponsored brand and video search ads Co-authored-by: Cursor * fix: collect Shop by type thematic SP ads via redirect chains without data-asin Co-authored-by: Cursor * fix: classify sponsored ads across new Amazon templates --------- Co-authored-by: Cursor Co-authored-by: PetrichorFun <127877748+PetrichorFun@users.noreply.github.com> --- .gitignore | 1 + README.md | 16 +- package.json | 22 +- scripts/manifest.ts | 10 +- scripts/pack.ts | 25 +- scripts/test-amazon-detail-images.ts | 51 + scripts/test-amazon-detail-page-ready.ts | 49 + scripts/test-amazon-page-navigation.ts | 81 ++ scripts/test-amazon-review-page.ts | 82 ++ scripts/test-amazon-search-page-ready.ts | 76 ++ scripts/test-amazon-search-pagination.ts | 93 ++ scripts/test-amazon-search-parser.ts | 1210 +++++++++++++++++ scripts/test-execute-script.ts | 57 + scripts/test-port-disconnect.ts | 13 + scripts/test-remote-endpoint.ts | 34 + scripts/verify-extension-runtime.mjs | 71 + scripts/verify-extension-sync-e2e.mjs | 115 ++ scripts/verify-port-disconnect-build.mjs | 16 + scripts/verify-production-endpoint.mjs | 29 + src/background/amazon-sync.ts | 214 +++ src/background/main.ts | 1 + src/composables/useCloudExporter.ts | 12 +- src/composables/useLongTask.ts | 6 +- src/env.ts | 17 +- src/logic/amazon-sync-client.ts | 9 + src/logic/amazon-sync-shared.ts | 12 + src/logic/execute-script.ts | 68 +- src/logic/remote-endpoint.ts | 28 + src/logic/upload-image.ts | 7 +- src/options/views/AmazonResultTable.vue | 92 +- src/options/views/help/guide.md | 12 +- src/page-worker/composables/amazon.ts | 160 ++- src/page-worker/impls/amazon.ts | 249 +++- src/page-worker/interfaces/amazon.ts | 9 +- src/page-worker/interfaces/common.ts | 1 + src/page-worker/page-navigation.ts | 58 + src/page-worker/services/base.ts | 4 +- .../web-injectors/amazon-detail-images.ts | 103 ++ .../web-injectors/amazon-detail-page-ready.ts | 32 + .../web-injectors/amazon-review-page.ts | 73 + .../web-injectors/amazon-search-page-ready.ts | 107 ++ .../web-injectors/amazon-search-pagination.ts | 64 + .../web-injectors/amazon-search-parser.ts | 685 ++++++++++ src/page-worker/web-injectors/amazon.ts | 569 +------- .../views/AmazonEntries/DetailPageEntry.vue | 21 +- .../views/AmazonEntries/ReviewPageEntry.vue | 32 +- src/storages/amazon-sync.ts | 8 + src/storages/amazon.ts | 25 +- src/types/amazon-sync.d.ts | 25 + src/types/amazon.d.ts | 43 + webext-bridge/src/background.ts | 4 + webext-bridge/src/internal/persistent-port.ts | 30 +- webext-bridge/src/internal/port-disconnect.ts | 3 + 53 files changed, 4124 insertions(+), 710 deletions(-) create mode 100644 scripts/test-amazon-detail-images.ts create mode 100644 scripts/test-amazon-detail-page-ready.ts create mode 100644 scripts/test-amazon-page-navigation.ts create mode 100644 scripts/test-amazon-review-page.ts create mode 100644 scripts/test-amazon-search-page-ready.ts create mode 100644 scripts/test-amazon-search-pagination.ts create mode 100644 scripts/test-amazon-search-parser.ts create mode 100644 scripts/test-execute-script.ts create mode 100644 scripts/test-port-disconnect.ts create mode 100644 scripts/test-remote-endpoint.ts create mode 100644 scripts/verify-extension-runtime.mjs create mode 100644 scripts/verify-extension-sync-e2e.mjs create mode 100644 scripts/verify-port-disconnect-build.mjs create mode 100644 scripts/verify-production-endpoint.mjs create mode 100644 src/background/amazon-sync.ts create mode 100644 src/logic/amazon-sync-client.ts create mode 100644 src/logic/amazon-sync-shared.ts create mode 100644 src/logic/remote-endpoint.ts create mode 100644 src/page-worker/page-navigation.ts create mode 100644 src/page-worker/web-injectors/amazon-detail-images.ts create mode 100644 src/page-worker/web-injectors/amazon-detail-page-ready.ts create mode 100644 src/page-worker/web-injectors/amazon-review-page.ts create mode 100644 src/page-worker/web-injectors/amazon-search-page-ready.ts create mode 100644 src/page-worker/web-injectors/amazon-search-pagination.ts create mode 100644 src/page-worker/web-injectors/amazon-search-parser.ts create mode 100644 src/storages/amazon-sync.ts create mode 100644 src/types/amazon-sync.d.ts create mode 100644 webext-bridge/src/internal/port-disconnect.ts diff --git a/.gitignore b/.gitignore index 5fc4637..063bb37 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ dist-ssr extension/manifest.json extension-firefox/manifest.json node_modules +.runtime src/auto-imports.d.ts src/components.d.ts .eslintcache diff --git a/README.md b/README.md index 8aef4a2..ae74eea 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Azon Seeker is a browser extension designed to collect and manage product inform - 🛒 **Collect Product Data:** Easily gather product details and reviews from Amazon and other websites. - 📦 **Import/Export:** Supports importing and exporting data in Excel format for further analysis. - ☁️ **Cloud Export:** Export your data to the cloud for backup or sharing. +- 🔄 **Reliable Server Sync:** Collected Amazon data is durably queued, retried in the background, and removed from the queue only after the server confirms persistence. - 📊 **Data Table:** View, filter, and manage collected data in a user-friendly table. - 🖼️ **Image Preview:** Hover to preview product images. - 🌐 **Multi-browser Support:** Works with Chromium-based browsers and Firefox. @@ -80,4 +81,17 @@ pnpm run build-firefox # For Firefox pnpm run pack:7z ``` -After the build is completed, the extension package will be generated in `build/extension.7z` +After the build is completed, the Chromium package will be generated in +`build/extension-v.7z`. + +## Amazon Server Synchronization + +- Development builds use `127.0.0.1:8001` by default. +- Production builds use `amzplugin.rekeymed.com` over HTTPS/WSS by default. +- Override either target at build time with `VITE_REMOTE_HOST=host:port`. +- The extension persists unsent batches in `browser.storage.local` and retries once per minute. +- Search and detail records include a client-generated `syncId`; the server uses it as a unique idempotency key. +- The side panel displays pending, syncing, synchronized, and failed states and provides a manual retry action. + +Before publishing a production package, deploy the backend migrations and verify that the +production `/openapi.json` contains `syncId`, `adType`, and `strikethroughPrice`. diff --git a/package.json b/package.json index e629ecb..4dd14ca 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "azon-seeker", - "displayName": "Azon Seeker v0.7.1.12-beta", - "version": "0.7.2", + "displayName": "Azon Seeker v0.7.13", + "version": "0.7.13", "private": true, "description": "Starter modify by honestfox101 and PetrichorFun", "scripts": { @@ -11,8 +11,9 @@ "dev:web": "vite", "dev:background": "npm run build:background -- --mode development", "dev:js": "npm run build:js -- --mode development", - "build": "cross-env NODE_ENV=production run-s clear build:web build:prepare build:background build:js", - "build-firefox": "cross-env NODE_ENV=production EXTENSION=firefox run-s clear-firefox build:web build:prepare build:background build:js", + "build": "cross-env NODE_ENV=production run-s clear build:bridge build:web build:prepare build:background build:js", + "build-firefox": "cross-env NODE_ENV=production EXTENSION=firefox run-s clear-firefox build:bridge build:web build:prepare build:background build:js", + "build:bridge": "npm --prefix webext-bridge run build", "build:prepare": "esno scripts/prepare.ts", "build:background": "vite build --config vite.config.background.mts", "build:web": "vite build", @@ -23,6 +24,19 @@ "clear": "rimraf --glob extension/dist extension/manifest.json extension.* ", "clear-firefox": "rimraf --glob extension-firefox/dist extension-firefox/manifest.json extension.*", "typecheck": "tsc --noEmit", + "test:amazon-parser": "esno scripts/test-amazon-search-parser.ts", + "test:amazon-page-ready": "esno scripts/test-amazon-search-page-ready.ts", + "test:amazon-pagination": "esno scripts/test-amazon-search-pagination.ts", + "test:amazon-detail-images": "esno scripts/test-amazon-detail-images.ts", + "test:amazon-detail-page-ready": "esno scripts/test-amazon-detail-page-ready.ts", + "test:amazon-review-page": "esno scripts/test-amazon-review-page.ts", + "test:amazon-page-navigation": "esno scripts/test-amazon-page-navigation.ts", + "test:execute-script": "esno scripts/test-execute-script.ts", + "test:extension-runtime": "node scripts/verify-extension-runtime.mjs", + "test:remote-endpoint": "esno scripts/test-remote-endpoint.ts", + "test:production-endpoint": "node scripts/verify-production-endpoint.mjs", + "test:port-disconnect": "esno scripts/test-port-disconnect.ts", + "test:port-disconnect-build": "node scripts/verify-port-disconnect-build.mjs", "prepare": "husky" }, "devDependencies": { diff --git a/scripts/manifest.ts b/scripts/manifest.ts index 06fba1d..9b2ceca 100644 --- a/scripts/manifest.ts +++ b/scripts/manifest.ts @@ -30,7 +30,15 @@ async function getManifest() { 48: './assets/icon-512.png', 128: './assets/icon-512.png', }, - permissions: ['tabs', 'storage', 'activeTab', 'scripting', 'unlimitedStorage', 'contextMenus'], + permissions: [ + 'tabs', + 'storage', + 'activeTab', + 'scripting', + 'unlimitedStorage', + 'contextMenus', + 'alarms', + ], content_security_policy: { extension_pages: isDev ? `script-src \'self\' http://localhost:${port}; object-src \'self\';` diff --git a/scripts/pack.ts b/scripts/pack.ts index ef65341..329fc56 100644 --- a/scripts/pack.ts +++ b/scripts/pack.ts @@ -1,17 +1,24 @@ import _7z from '7zip-min'; import { basename } from 'path'; +import fs from 'fs-extra'; import { r, log } from './utils'; import packageJson from '../package.json'; const scriptPath = ['extension', 'extension-firefox']; -scriptPath.forEach(async (src) => { - const srcPath = r(src); - const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`); - try { - const output = await _7z.pack(srcPath, distPath); - log('Info', `stdout of the 7za command execution ${output}`); - } catch (err) { - log('Error', `${err}`); +async function main() { + for (const src of scriptPath) { + const srcPath = r(src); + if (!(await fs.pathExists(r(src, 'manifest.json')))) continue; + const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`); + try { + const output = await _7z.pack(srcPath, distPath); + log('Info', `stdout of the 7za command execution ${output}`); + } catch (err) { + log('Error', `${err}`); + process.exitCode = 1; + } } -}); +} + +void main(); diff --git a/scripts/test-amazon-detail-images.ts b/scripts/test-amazon-detail-images.ts new file mode 100644 index 0000000..1cf42bc --- /dev/null +++ b/scripts/test-amazon-detail-images.ts @@ -0,0 +1,51 @@ +import assert from 'node:assert/strict'; +import { JSDOM } from 'jsdom'; +import { collectAmazonDetailImageUrls } from '../src/page-worker/web-injectors/amazon-detail-images'; + +async function runWithHtml(html: string) { + const dom = new JSDOM(html, { url: 'https://www.amazon.com/dp/B0DMNDL5NH' }); + const previousDocument = globalThis.document; + const previousMouseEvent = globalThis.MouseEvent; + Object.assign(globalThis, { + document: dom.window.document, + MouseEvent: dom.window.MouseEvent, + }); + try { + return await collectAmazonDetailImageUrls(); + } finally { + Object.assign(globalThis, { + document: previousDocument, + MouseEvent: previousMouseEvent, + }); + dom.window.close(); + } +} + +async function main() { + const scriptUrls = await runWithHtml(` + +`); + assert.deepEqual(scriptUrls, [ + 'https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg', + 'https://m.media-amazon.com/images/I/second._AC_SL1500_.jpg', + ]); + + const landingUrl = 'https://m.media-amazon.com/images/I/landing._AC_SL1500_.jpg'; + const landingUrls = await runWithHtml( + ``, + ); + assert.deepEqual(landingUrls, [landingUrl]); + + console.log('Amazon detail image parser tests passed.'); +} + +void main(); diff --git a/scripts/test-amazon-detail-page-ready.ts b/scripts/test-amazon-detail-page-ready.ts new file mode 100644 index 0000000..663d118 --- /dev/null +++ b/scripts/test-amazon-detail-page-ready.ts @@ -0,0 +1,49 @@ +import { JSDOM } from 'jsdom'; +import { waitForAmazonDetailPageReady } from '../src/page-worker/web-injectors/amazon-detail-page-ready'; + +const installDom = (html: string, url = 'https://www.amazon.com/dp/B0DMNDL5NH') => { + const dom = new JSDOM(html, { url }); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + return dom; +}; + +const main = async () => { + const modernDetail = installDom('Ready product'); + await waitForAmazonDetailPageReady(); + modernDetail.window.close(); + + const legacyDetail = installDom('Legacy product'); + await waitForAmazonDetailPageReady(); + legacyDetail.window.close(); + + const signIn = installDom('
Sign in
', 'https://www.amazon.com/ap/signin'); + await waitForAmazonDetailPageReady().then( + () => { + throw new Error('Sign-in page should fail immediately'); + }, + (error) => { + if (!String(error).includes('需要重新登录')) throw error; + }, + ); + signIn.window.close(); + + const captcha = installDom(''); + await waitForAmazonDetailPageReady().then( + () => { + throw new Error('Captcha page should fail immediately'); + }, + (error) => { + if (!String(error).includes('验证码')) throw error; + }, + ); + captcha.window.close(); + + console.log('Amazon detail page readiness tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-page-navigation.ts b/scripts/test-amazon-page-navigation.ts new file mode 100644 index 0000000..a06ba24 --- /dev/null +++ b/scripts/test-amazon-page-navigation.ts @@ -0,0 +1,81 @@ +import { + extractAmazonDetailAsin, + hasReachedAmazonDetailPage, + hasReachedAmazonNavigationTarget, +} from '../src/page-worker/page-navigation'; + +const assert = (condition: boolean, message: string) => { + if (!condition) throw new Error(message); +}; + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews&filterByStar=one_star', + ), + 'An unfiltered completed page must not satisfy a star-filter navigation.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_viewopt_sr?filterByStar=one_star&pageNumber=2', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_paging_btm_next_2?filterByStar=one_star&pageNumber=2', + ), + 'Tracking ref rewrites must not prevent matching the requested review page.', +); + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=1', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=2', + ), + 'A previous review page must not satisfy the next-page navigation.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/s?k=security+camera&page=17&ref=sr_pg_17', + 'https://www.amazon.com/s?ref=sr_pg_17&page=17&k=security+camera', + ), + 'Search navigation should tolerate query ordering differences.', +); + +assert( + hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH/ref=sr_1_1', + 'https://www.amazon.com/dp/B0DMNDL5NH', + ), + 'A canonical product slug must still match the requested detail ASIN.', +); + +assert( + !hasReachedAmazonNavigationTarget( + 'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH', + 'https://www.amazon.com/dp/B0BKPNRLC8', + ), + 'The previous detail ASIN must not satisfy the next-ASIN navigation.', +); + +assert( + hasReachedAmazonDetailPage( + 'https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1', + 'https://www.amazon.com/dp/B0D2MC2865', + ), + 'A canonical Amazon detail redirect must be accepted as a loaded detail page.', +); + +assert( + !hasReachedAmazonDetailPage( + 'https://www.amazon.com/', + 'https://www.amazon.com/dp/B0D2MC2865', + ), + 'The Amazon home page must not be accepted as a detail redirect.', +); + +assert( + extractAmazonDetailAsin('https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1') === + 'B0CCXB29LF', + 'The resolved ASIN must be extracted from canonical detail URLs.', +); + +console.log('Amazon page navigation tests passed.'); diff --git a/scripts/test-amazon-review-page.ts b/scripts/test-amazon-review-page.ts new file mode 100644 index 0000000..4c44405 --- /dev/null +++ b/scripts/test-amazon-review-page.ts @@ -0,0 +1,82 @@ +import { JSDOM } from 'jsdom'; +import { + collectAmazonNextReviewPageUrl, + isAmazonReviewRatingFilterIgnored, + waitForAmazonReviewPageReady, +} from '../src/page-worker/web-injectors/amazon-review-page'; + +const installDom = (html: string, url = 'https://www.amazon.com/product-reviews/B08XM6S6ZL') => { + const dom = new JSDOM(html, { url }); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + return dom; +}; + +const main = async () => { + if ( + !isAmazonReviewRatingFilterIgnored( + [{ rating: '5.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }], + 3, + ) + ) { + throw new Error('Mixed ratings must be detected when Amazon ignores filterByStar'); + } + if ( + isAmazonReviewRatingFilterIgnored( + [{ rating: '3.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }], + 3, + ) + ) { + throw new Error('A correctly filtered review page must not be treated as limited'); + } + + const readyDom = installDom('
'); + await waitForAmazonReviewPageReady({}); + readyDom.window.close(); + + const paginationDom = installDom( + '
', + ); + const nextUrl = await collectAmazonNextReviewPageUrl(); + if (nextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2') { + throw new Error(`Unexpected next review URL: ${nextUrl}`); + } + paginationDom.window.close(); + + const standalonePaginationDom = installDom( + '
', + 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2', + ); + const standaloneNextUrl = await collectAmazonNextReviewPageUrl(); + if (standaloneNextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=3') { + throw new Error(`Unexpected standalone review pagination URL: ${standaloneNextUrl}`); + } + standalonePaginationDom.window.close(); + + const lastPageDom = installDom( + '
  • Next
', + ); + if ((await collectAmazonNextReviewPageUrl()) !== null) { + throw new Error('Disabled review pagination must not return a URL'); + } + lastPageDom.window.close(); + + const signInDom = installDom('
Sign in
', 'https://www.amazon.com/ap/signin'); + await waitForAmazonReviewPageReady({}).then( + () => { + throw new Error('Sign-in page should fail immediately'); + }, + (error) => { + if (!String(error).includes('需要重新登录')) throw error; + }, + ); + signInDom.window.close(); + + console.log('Amazon review page tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-page-ready.ts b/scripts/test-amazon-search-page-ready.ts new file mode 100644 index 0000000..daec039 --- /dev/null +++ b/scripts/test-amazon-search-page-ready.ts @@ -0,0 +1,76 @@ +import { JSDOM } from 'jsdom'; +import { waitForAmazonSearchPageReady } from '../src/page-worker/web-injectors/amazon-search-page-ready'; + +const main = async () => { + const dom = new JSDOM( + '
Next
', + { url: 'https://www.amazon.com/s?k=security+camera' }, + ); + Object.defineProperty(dom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined }); + Object.assign(dom.window, { scrollTo: () => undefined }); + + dom.window.setTimeout(() => { + const sponsoredModule = dom.window.document.createElement('section'); + sponsoredModule.innerHTML = + 'Sponsored

Lazy sponsored product

'; + dom.window.document.querySelector('main')?.append(sponsoredModule); + }, 500); + + const started = Date.now(); + await waitForAmazonSearchPageReady(); + const elapsed = Date.now() - started; + + if (!dom.window.document.querySelector('[data-asin="B000000002"]')) { + throw new Error('Lazy sponsored DOM was not loaded before page readiness completed'); + } + if (elapsed >= 5000) { + throw new Error(`Ready search results were delayed by a persistent spinner (${elapsed}ms)`); + } + dom.window.close(); + + const errorDom = new JSDOM( + '

SORRY

Something went wrong on our end.

Please go back and try again.

', + { url: 'https://www.amazon.com/s?k=protein+powder' }, + ); + Object.defineProperty(errorDom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: errorDom.window.document, window: errorDom.window }); + await waitForAmazonSearchPageReady().then( + () => { + throw new Error('Amazon transient error page must not be accepted as an empty result page'); + }, + (error) => { + if (!String(error).includes('临时错误页')) throw error; + }, + ); + errorDom.window.close(); + + const captchaDom = new JSDOM('', { + url: 'https://www.amazon.com/s?k=protein+powder', + }); + Object.defineProperty(captchaDom.window.document, 'readyState', { + configurable: true, + value: 'complete', + }); + Object.assign(globalThis, { document: captchaDom.window.document, window: captchaDom.window }); + await waitForAmazonSearchPageReady().then( + () => { + throw new Error('Amazon captcha page must fail immediately'); + }, + (error) => { + if (!String(error).includes('验证码')) throw error; + }, + ); + captchaDom.window.close(); + + console.log('Amazon search page-ready tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-pagination.ts b/scripts/test-amazon-search-pagination.ts new file mode 100644 index 0000000..925718b --- /dev/null +++ b/scripts/test-amazon-search-pagination.ts @@ -0,0 +1,93 @@ +import { JSDOM } from 'jsdom'; +import { + collectAmazonCurrentPage, + collectAmazonNextPageUrl, +} from '../src/page-worker/web-injectors/amazon-search-pagination'; + +const withDom = async (url: string, html: string, callback: () => T | Promise) => { + const dom = new JSDOM(html, { url }); + const previousDocument = globalThis.document; + const previousWindow = globalThis.window; + Object.assign(globalThis, { document: dom.window.document, window: dom.window }); + try { + return await callback(); + } finally { + Object.assign(globalThis, { document: previousDocument, window: previousWindow }); + dom.window.close(); + } +}; + +const testNextPage = async (name: string, html: string, expected: string | null) => { + const actual = await withDom( + 'https://www.amazon.com/s?k=security+camera&page=1', + html, + collectAmazonNextPageUrl, + ); + if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`); +}; + +const testCurrentPage = async (name: string, url: string, html: string, expected: number) => { + const actual = await withDom(url, html, collectAmazonCurrentPage); + if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`); +}; + +const main = async () => { + await testNextPage( + 'active next page', + 'Next', + 'https://www.amazon.com/s?k=security+camera&page=2', + ); + await testNextPage( + 'disabled next page', + 'Next', + null, + ); + await testNextPage('missing next page', '
No pagination
', null); + await testCurrentPage( + 'visible page wins over hidden stale pagination selection', + 'https://www.amazon.com/s?k=home+security&page=11&ref=sr_pg_11', + '
12
11', + 11, + ); + for (const page of [1, 2, 5, 10, 20]) { + await testCurrentPage( + `URL page ${page}`, + `https://www.amazon.com/s?k=home+security&page=${page}&ref=sr_pg_${page}`, + `${page}`, + page, + ); + } + await testCurrentPage( + 'visible page wins while URL has advanced', + 'https://www.amazon.com/s?k=home+security&page=3&ref=sr_pg_3', + '2', + 2, + ); + await testCurrentPage( + 'invalid URL page uses ref fallback', + 'https://www.amazon.com/s?k=home+security&page=invalid&ref=sr_pg_9', + '
No selected pagination item
', + 9, + ); + await testCurrentPage( + 'ref page fallback', + 'https://www.amazon.com/s?k=home+security&ref=sr_pg_7', + '
No selected pagination item
', + 7, + ); + await testCurrentPage( + 'selected pagination fallback', + 'https://www.amazon.com/s?k=home+security', + '5', + 5, + ); + await testCurrentPage( + 'first page fallback', + 'https://www.amazon.com/s?k=home+security', + '
No pagination
', + 1, + ); + console.log('Amazon search pagination tests passed.'); +}; + +void main(); diff --git a/scripts/test-amazon-search-parser.ts b/scripts/test-amazon-search-parser.ts new file mode 100644 index 0000000..e58c9b9 --- /dev/null +++ b/scripts/test-amazon-search-parser.ts @@ -0,0 +1,1210 @@ +import assert from 'node:assert/strict'; +import { JSDOM } from 'jsdom'; +import { collectAmazonSearchPageData } from '../src/page-worker/web-injectors/amazon-search-parser'; + +const html = ` +
+
+ Sponsored + +

Example brand headline

+
+

Brand product one

+ +
+
+

Brand product two

+ +
+
+ +
+
+
+ Amazon's Choice +

Organic product

+ + $29.99 +
+ +
+ +

Sponsored product

+ + $33.93 + $39.99 + $0.28 +
+ +
+

Second organic product

+ +
+
+ +
+ Sponsored +
+

Unknown sponsored product

+ +
+
+ +
+ + Sponsored + +

Video brand headline

+
+
+
+

Sponsored Brands video product

+ +
+
+
+
+ +
+ Sponsored +
+

Hidden preloaded product

+ + + +
+
+ + +
+ + +
+`; + +const dom = new JSDOM(html, { url: 'https://www.amazon.com/s?k=mirror' }); +Object.assign(globalThis, { + document: dom.window.document, + Node: dom.window.Node, + Element: dom.window.Element, + HTMLElement: dom.window.HTMLElement, + HTMLAnchorElement: dom.window.HTMLAnchorElement, + HTMLImageElement: dom.window.HTMLImageElement, +}); + +async function main() { + const items = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + assert.equal(items.length, 8); + + const byAsin = new Map(items.map((item) => [item.asin, item])); + assert.equal(byAsin.get('B000000003')?.adType, 'organic'); + assert.equal(byAsin.get('B000000004')?.adType, 'sponsoredProducts'); + assert.equal(byAsin.get('B000000004')?.price, '$33.93'); + assert.equal(byAsin.get('B000000004')?.strikethroughPrice, '$39.99'); + assert.equal(byAsin.get('B000000004')?.link, 'https://www.amazon.com/dp/B000000004'); + assert.equal(byAsin.get('B000000004')?.placement, 'middleOfSearch'); + assert.equal(byAsin.get('B000000001')?.adType, 'sponsoredBrands'); + assert.equal(byAsin.get('B000000002')?.adType, 'sponsoredBrands'); + assert.equal(byAsin.get('B000000001')?.placement, 'topOfSearch'); + assert.equal(byAsin.get('B000000001')?.placementId, byAsin.get('B000000002')?.placementId); + assert.equal(byAsin.get('B000000001')?.adProductIndex, 1); + assert.equal(byAsin.get('B000000002')?.adProductIndex, 2); + assert.equal(byAsin.get('B000000005')?.adType, 'sponsoredProducts'); + assert.equal(byAsin.get('B000000005')?.placement, 'bottomOfSearch'); + assert.equal(byAsin.get('B000000006')?.adType, 'sponsoredDisplay'); + assert.equal(byAsin.get('B000000006')?.placement, 'sidebar'); + assert.equal(byAsin.get('B000000008')?.adType, 'sponsoredBrands'); + assert.equal(byAsin.get('B000000008')?.adFormat, 'video'); + assert.equal(byAsin.get('B000000008')?.placement, 'bottomOfSearch'); + assert.equal(items.filter((item) => item.asin === 'B000000008').length, 1); + assert.equal(byAsin.has('B000000009'), false); + assert.equal(byAsin.has('B000000010'), false); + assert.deepEqual( + items.map((item) => item.pageRank), + [1, 2, 3, 4, 5, 6, 7, 8], + ); + + const hiddenDisclosureDom = new JSDOM( + ` +
+
+
+

Visible organic product

+ Sponsored + + + +
+
+
+ `, + { url: 'https://www.amazon.com/s?k=mirror' }, + ); + Object.assign(globalThis, { + document: hiddenDisclosureDom.window.document, + Node: hiddenDisclosureDom.window.Node, + Element: hiddenDisclosureDom.window.Element, + HTMLElement: hiddenDisclosureDom.window.HTMLElement, + HTMLAnchorElement: hiddenDisclosureDom.window.HTMLAnchorElement, + HTMLImageElement: hiddenDisclosureDom.window.HTMLImageElement, + }); + const hiddenDisclosureItems = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + assert.equal(hiddenDisclosureItems.length, 1); + assert.equal(hiddenDisclosureItems[0].adType, 'organic'); + + const groupedAdsDom = new JSDOM( + ` +
+
+
+ Sponsored + Hidden brand destination +
+

Grouped sponsored product one

+ +
+
+

Grouped sponsored product two

+ +
+
+
+ +
+ `, + { url: 'https://www.amazon.com/s?k=mirror&page=8' }, + ); + Object.assign(globalThis, { + document: groupedAdsDom.window.document, + Node: groupedAdsDom.window.Node, + Element: groupedAdsDom.window.Element, + HTMLElement: groupedAdsDom.window.HTMLElement, + HTMLAnchorElement: groupedAdsDom.window.HTMLAnchorElement, + HTMLImageElement: groupedAdsDom.window.HTMLImageElement, + }); + const groupedAds = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const groupedAdsByAsin = new Map(groupedAds.map((item) => [item.asin, item])); + assert.equal(groupedAdsByAsin.get('B000000012')?.adType, 'sponsoredProducts'); + assert.equal(groupedAdsByAsin.get('B000000013')?.adType, 'sponsoredProducts'); + assert.equal( + groupedAdsByAsin.get('B000000012')?.placementId, + groupedAdsByAsin.get('B000000013')?.placementId, + ); + assert.equal(groupedAdsByAsin.get('B000000012')?.adProductIndex, 1); + assert.equal(groupedAdsByAsin.get('B000000013')?.adProductIndex, 2); + assert.equal(groupedAdsByAsin.get('B000000014')?.adType, 'sponsoredDisplay'); + assert.equal(groupedAdsByAsin.get('B000000014')?.placement, 'sidebar'); + + const liveSponsoredBrandsDom = new JSDOM( + ` +
+
+
+
Arlo Security for any Home
+ Sponsored +
+ Video Doorbell + + + +
+
+ Outdoor Camera + + + +
+
+ Bundle + + + +
+
+
+ Sponsored +

Ring Spotlight Cam Plus, Battery

+ +
+
+

Organic camera

+ +
+
+ Sponsored + +
PROTECT YOUR EQUIPMENT & FACILITIES
+
+

Vosker VKX Solar Powered 4G-LTE Outdoor Cellular Security Camera

+ + $379.99 +
+
+
+

Brands related to your search

+ Sponsored +
+ VOSKER + VOSKER +
+
+ Reolink + Reolink +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: liveSponsoredBrandsDom.window.document, + Node: liveSponsoredBrandsDom.window.Node, + Element: liveSponsoredBrandsDom.window.Element, + HTMLElement: liveSponsoredBrandsDom.window.HTMLElement, + HTMLAnchorElement: liveSponsoredBrandsDom.window.HTMLAnchorElement, + HTMLImageElement: liveSponsoredBrandsDom.window.HTMLImageElement, + }); + const liveSponsoredBrands = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const liveByAsin = new Map(liveSponsoredBrands.map((item) => [item.asin, item])); + assert.equal(liveByAsin.get('B08AR2S6MC')?.adType, 'sponsoredBrands'); + assert.equal(liveByAsin.get('B0ARLO0002')?.adType, 'sponsoredBrands'); + assert.equal(liveByAsin.get('B0ARLO0003')?.adType, 'sponsoredBrands'); + assert.equal(liveByAsin.get('B08AR2S6MC')?.placement, 'topOfSearch'); + assert.equal(liveByAsin.get('B09RING001')?.adType, 'sponsoredProducts'); + assert.equal(liveByAsin.get('B09ORG0001')?.adType, 'organic'); + assert.equal(liveByAsin.get('B0VOSKER01')?.adType, 'sponsoredBrands'); + assert.equal(liveByAsin.get('B0VOSKER01')?.adFormat, 'video'); + assert.equal(liveByAsin.get('B0VOSKER01')?.placement, 'bottomOfSearch'); + assert.equal(liveByAsin.get('B0BRAND001')?.adType, 'sponsoredBrands'); + assert.equal(liveByAsin.get('B0BRAND002')?.adType, 'sponsoredBrands'); + + const lazyImageDom = new JSDOM( + ` +
+
+
+
Arlo Security for any Home
+ Sponsored +
+

Video Doorbell

+ + + +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: lazyImageDom.window.document, + Node: lazyImageDom.window.Node, + Element: lazyImageDom.window.Element, + HTMLElement: lazyImageDom.window.HTMLElement, + HTMLAnchorElement: lazyImageDom.window.HTMLAnchorElement, + HTMLImageElement: lazyImageDom.window.HTMLImageElement, + }); + const lazyImageItems = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + assert.equal(lazyImageItems.length, 1); + assert.equal(lazyImageItems[0]?.asin, 'B08AR2LAZY'); + assert.equal(lazyImageItems[0]?.adType, 'sponsoredBrands'); + assert.equal(lazyImageItems[0]?.imageSrc, 'https://example.com/arlo-lazy.jpg'); + + const classificationBoundaryDom = new JSDOM( + ` +
+
+
+
Arlo Security for any Home
+ Sponsored +
+ Video Doorbell + + + +
+
+ Outdoor Camera + + + +
+
+ +
+
+ Sponsored +

Arlo Video Doorbell Wired

+ + + +
+
+ Sponsored +

Ring Spotlight Cam Plus, Battery

+ + + +
+
+

Organic camera

+ +
+
+ + + +
+ Sponsored +

Ring Indoor Cam with demo

+ + + + +
+ +
+ Sponsored + +
PROTECT YOUR EQUIPMENT & FACILITIES
+
+

Vosker VKX Solar Powered 4G-LTE Outdoor Cellular Security Camera

+ + + +
+
+ +
+

Organic with product video

+ + + + +
+ +
+

Brands related to your search

+ Sponsored +
+ VOSKER + VOSKER +
+
+
+ +
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: classificationBoundaryDom.window.document, + Node: classificationBoundaryDom.window.Node, + Element: classificationBoundaryDom.window.Element, + HTMLElement: classificationBoundaryDom.window.HTMLElement, + HTMLAnchorElement: classificationBoundaryDom.window.HTMLAnchorElement, + HTMLImageElement: classificationBoundaryDom.window.HTMLImageElement, + }); + const classificationBoundary = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const boundaryByAsin = new Map(classificationBoundary.map((item) => [item.asin, item])); + assert.equal(boundaryByAsin.get('B08ARLOSB1')?.adType, 'sponsoredBrands'); + assert.equal(boundaryByAsin.get('B08ARLOSB2')?.adType, 'sponsoredBrands'); + assert.equal(boundaryByAsin.get('B0ARLOGRID')?.adType, 'sponsoredProducts'); + assert.equal(boundaryByAsin.get('B09RINGSP1')?.adType, 'sponsoredProducts'); + assert.equal(boundaryByAsin.get('B09ORGKEEP')?.adType, 'organic'); + assert.equal(boundaryByAsin.get('B0TRACKSP1')?.adType, 'sponsoredProducts'); + assert.equal(boundaryByAsin.has('B0TRACKXX2'), false); + assert.equal(boundaryByAsin.get('B0DEMOSP01')?.adType, 'sponsoredProducts'); + assert.equal(boundaryByAsin.get('B0VOSKER02')?.adType, 'sponsoredBrands'); + assert.equal(boundaryByAsin.get('B0VOSKER02')?.adFormat, 'video'); + assert.equal(boundaryByAsin.get('B0NOADVID1')?.adType, 'organic'); + assert.equal(boundaryByAsin.get('B0BRANDWL1')?.adType, 'sponsoredBrands'); + assert.equal(boundaryByAsin.get('B0SIDEBAR1')?.adType, 'sponsoredDisplay'); + assert.equal(boundaryByAsin.get('B0SIDEBAR1')?.placement, 'sidebar'); + + const thematicLoomDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+
+

Trending now

+ + Sponsored + + +
    +
  1. +
    +
    +

    iulock Keypad Smart Door Lock

    + + + +
    +
    +
    2. +
  2. +
    +
    +

    Richelock fingerprint lock

    + + + +
    +
    +
    3. +
  3. +
    +
    +

    Swann MaxRanger4K

    + + + +
    +
    +
    4. + + +
+ +
+
+ +
+ Sponsored + +
PROTECT YOUR EQUIPMENT & FACILITIES
+
+

Vosker VKX Solar Powered camera

+ +
+
+ +
+
Arlo Security for any Home
+ Sponsored +
+ Video Doorbell + + + +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: thematicLoomDom.window.document, + Node: thematicLoomDom.window.Node, + Element: thematicLoomDom.window.Element, + HTMLElement: thematicLoomDom.window.HTMLElement, + HTMLAnchorElement: thematicLoomDom.window.HTMLAnchorElement, + HTMLImageElement: thematicLoomDom.window.HTMLImageElement, + }); + const thematicLoom = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const thematicByAsin = new Map(thematicLoom.map((item) => [item.asin, item])); + assert.equal(thematicByAsin.get('B0ORGLOOM1')?.adType, 'organic'); + assert.equal(thematicByAsin.get('B0B2MPYRKP')?.adType, 'sponsoredProducts'); + assert.equal(thematicByAsin.get('B0F8HS3RRT')?.adType, 'sponsoredProducts'); + assert.equal(thematicByAsin.get('B0GQX6GHK2')?.adType, 'sponsoredProducts'); + assert.equal( + thematicByAsin.get('B0B2MPYRKP')?.placementId, + thematicByAsin.get('B0F8HS3RRT')?.placementId, + ); + assert.equal(thematicByAsin.get('B0B2MPYRKP')?.adProductIndex, 1); + assert.equal(thematicByAsin.get('B0F8HS3RRT')?.adProductIndex, 2); + assert.equal(thematicByAsin.get('B0GQX6GHK2')?.adProductIndex, 3); + assert.equal(thematicByAsin.has('B0CN95HR9V'), false); + assert.equal(thematicByAsin.has('B07M68S376'), false); + assert.equal(thematicByAsin.get('B0VOSKERLM')?.adType, 'sponsoredBrands'); + assert.equal(thematicByAsin.get('B0VOSKERLM')?.adFormat, 'video'); + assert.equal(thematicByAsin.get('B08ARLOLM1')?.adType, 'sponsoredBrands'); + + const organicLoomDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+
+

Customers frequently viewed

+ + Sponsored + +
    +
  1. +
    +
    +

    ANNKE security camera

    + + + +
    +
    +
    2. +
  2. +
    +
    +

    Night Owl camera

    + + + +
    +
    +
    3. +
+ +
+
+ +
+

Recently bought and rated

+ +
    +
  1. +
    +
    +

    Rated doorbell

    + + + +
    +
    +
    2. +
+ +
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: organicLoomDom.window.document, + Node: organicLoomDom.window.Node, + Element: organicLoomDom.window.Element, + HTMLElement: organicLoomDom.window.HTMLElement, + HTMLAnchorElement: organicLoomDom.window.HTMLAnchorElement, + HTMLImageElement: organicLoomDom.window.HTMLImageElement, + }); + const organicLoom = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const organicLoomByAsin = new Map(organicLoom.map((item) => [item.asin, item])); + assert.equal(organicLoomByAsin.get('B0ORGCFV01')?.adType, 'organic'); + assert.equal(organicLoomByAsin.get('B0CFV00001')?.adType, 'organic'); + assert.equal(organicLoomByAsin.get('B0CFV00002')?.adType, 'organic'); + assert.equal(organicLoomByAsin.get('B0RBR00001')?.adType, 'organic'); + assert.equal( + organicLoom.some((item) => item.adType === 'sponsoredProducts'), + false, + ); + + const organicVideoStripDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+ + +
Chamberlain myQ
+
Control your garage from anywhere
+
+

Chamberlain garage opener

+ +
+
+ +
+ Sponsored + + +
PROTECT YOUR EQUIPMENT
+
+

Vosker solar camera

+ +
+
+ +
+ Sponsored + + +
PROTECT YOUR EQUIPMENT
+
+

Vosker with visible sponsored

+ +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: organicVideoStripDom.window.document, + Node: organicVideoStripDom.window.Node, + Element: organicVideoStripDom.window.Element, + HTMLElement: organicVideoStripDom.window.HTMLElement, + HTMLAnchorElement: organicVideoStripDom.window.HTMLAnchorElement, + HTMLImageElement: organicVideoStripDom.window.HTMLImageElement, + }); + const organicVideoStrip = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const videoStripByAsin = new Map(organicVideoStrip.map((item) => [item.asin, item])); + assert.equal(videoStripByAsin.get('B0ORGVID01')?.adType, 'organic'); + assert.equal(videoStripByAsin.get('B0NOSBV001')?.adType, 'organic'); + assert.equal(videoStripByAsin.get('B0HIDDENSB')?.adType, 'organic'); + assert.equal(videoStripByAsin.get('B0YESBV001')?.adType, 'sponsoredBrands'); + assert.equal(videoStripByAsin.get('B0YESBV001')?.adFormat, 'video'); + assert.deepEqual( + organicVideoStrip.filter((item) => item.adType === 'sponsoredBrands').map((item) => item.asin), + ['B0YESBV001'], + ); + + const liveBrandWallDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+ Brands related to your search + Sponsored +
+ + Vosker + +
+
+ + REOLINK + +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: liveBrandWallDom.window.document, + Node: liveBrandWallDom.window.Node, + Element: liveBrandWallDom.window.Element, + HTMLElement: liveBrandWallDom.window.HTMLElement, + HTMLAnchorElement: liveBrandWallDom.window.HTMLAnchorElement, + HTMLImageElement: liveBrandWallDom.window.HTMLImageElement, + }); + const liveBrandWall = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const liveBrandWallByAsin = new Map(liveBrandWall.map((item) => [item.asin, item])); + assert.equal(liveBrandWallByAsin.get('B0ORGGRID1')?.adType, 'organic'); + assert.equal(liveBrandWallByAsin.get('B0D5Z3V464')?.adType, 'sponsoredBrands'); + assert.equal(liveBrandWallByAsin.get('B0C2PY2KTJ')?.adType, 'sponsoredBrands'); + assert.equal(liveBrandWallByAsin.get('B0D5Z3V464')?.title, 'Vosker'); + assert.equal( + liveBrandWallByAsin.get('B0D5Z3V464')?.imageSrc, + 'https://example.com/vosker-wall.jpg', + ); + assert.equal( + liveBrandWallByAsin.get('B0C2PY2KTJ')?.imageSrc, + 'https://example.com/reolink-wall.jpg', + ); + assert.equal(liveBrandWallByAsin.has('B0FY2QCXRJ'), false); + + const influencerPicksDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+

Picks from Amazon Influencers

+ Sponsored + From Amazon Influencer storefronts +
+

ANNKE 4K PoE Security Camera System

+ +
+
+

Night Owl DVR

+ storefront + +
+
+

Solar Street Lights

+ +
+
+ +
+
+ jobsite + camera + + packaging +
+
+ Sponsored +

Vosker VKX Solar Powered 4G-LTE Outdoor Cellular Security Camera

+ + + + $379.99 +
+
+ +
+
+ PROTECT YOUR INVESTMENT + CELLULAR SECURITY CAMERA + +
+
+ Sponsored +

Vosker VKX Solar Powered camera

+ + + +
+
+ +
+ Sponsored + Previously viewed +

Vosker VKX Solar Powered 4G-LTE Outdoor Cellular Security Camera

+ + + + $149.99 +
+
+ Sponsored +

Owl Labs Meeting Owl 3

+ + + + $1,099.00 +
+ +
+

Related searches

+ home security camera + outdoor camera +
+
+
+ `, + { url: 'https://www.amazon.com/s?k=home+security' }, + ); + Object.assign(globalThis, { + document: influencerPicksDom.window.document, + Node: influencerPicksDom.window.Node, + Element: influencerPicksDom.window.Element, + HTMLElement: influencerPicksDom.window.HTMLElement, + HTMLAnchorElement: influencerPicksDom.window.HTMLAnchorElement, + HTMLImageElement: influencerPicksDom.window.HTMLImageElement, + }); + const influencerPicks = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const influencerByAsin = new Map(influencerPicks.map((item) => [item.asin, item])); + assert.equal(influencerByAsin.get('B0ORGINF01')?.adType, 'organic'); + assert.equal(influencerByAsin.get('B0ANNKE001')?.adType, 'sponsoredProducts'); + assert.equal(influencerByAsin.get('B0NIGHTO01')?.adType, 'sponsoredProducts'); + assert.equal(influencerByAsin.get('B0SOLAR001')?.adType, 'sponsoredProducts'); + assert.equal( + influencerByAsin.get('B0ANNKE001')?.placementId, + influencerByAsin.get('B0NIGHTO01')?.placementId, + ); + assert.equal(influencerByAsin.get('B0VOSKERVK')?.adType, 'sponsoredBrands'); + assert.equal(influencerByAsin.get('B0VOSKERVK')?.adFormat, 'standard'); + assert.equal(influencerByAsin.get('B0VOSKERSB')?.adType, 'sponsoredBrands'); + assert.equal(influencerByAsin.get('B0VIEWVOSK')?.adType, 'sponsoredProducts'); + assert.equal(influencerByAsin.get('B0OWLLABS1')?.adType, 'sponsoredProducts'); + assert.equal( + influencerPicks.some((item) => /related searches/i.test(item.title)), + false, + ); + + const shopByTypeDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+ Sponsored + + +
STEALTH CAM
+
+

Stealth Cam trail camera

+ + + +
+
+ +
+

Shop cameras by type

+ Sponsored +
+

security system

+
+

Night Owl 2-Way Audio

+ + + + $549.99 +
+
+
+

webcam

+
+

Prisual PTZ Camera

+ + + + $494.09 +
+
+
+

instant print

+
+

Kids Instant Print Camera

+ + + +
+
+
+

point and shoot

+
+

Digital Camera 4K

+ + + +
+
+
+

film camera

+
+

Kodak EKTAR H35N

+ + + +
+
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=camera' }, + ); + Object.assign(globalThis, { + document: shopByTypeDom.window.document, + Node: shopByTypeDom.window.Node, + Element: shopByTypeDom.window.Element, + HTMLElement: shopByTypeDom.window.HTMLElement, + HTMLAnchorElement: shopByTypeDom.window.HTMLAnchorElement, + HTMLImageElement: shopByTypeDom.window.HTMLImageElement, + }); + const shopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const shopByTypeByAsin = new Map(shopByType.map((item) => [item.asin, item])); + const shopByTypeAsins = ['B0NIGHTOWL', 'B0PRISUAL1', 'B0KIDINST1', 'B0POINTSH1', 'B0KODAKH35']; + const shopByTypeItems = shopByTypeAsins.map((asin) => shopByTypeByAsin.get(asin)); + assert.ok(shopByTypeItems.filter(Boolean).length > 0, 'Shop by type cards must be collected'); + assert.equal(shopByTypeItems.filter(Boolean).length, 5); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.asin, 'B0NIGHTOWL'); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.title, 'Night Owl 2-Way Audio'); + assert.equal( + shopByTypeByAsin.get('B0NIGHTOWL')?.imageSrc, + 'https://example.com/nightowl-type.jpg', + ); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.price, '$549.99'); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.asin, 'B0PRISUAL1'); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.title, 'Prisual PTZ Camera'); + assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.asin, 'B0KIDINST1'); + assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.title, 'Kids Instant Print Camera'); + assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.asin, 'B0POINTSH1'); + assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.title, 'Digital Camera 4K'); + assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.asin, 'B0KODAKH35'); + assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.title, 'Kodak EKTAR H35N'); + assert.ok(shopByTypeItems.every((item) => item?.asin && item.title && item.imageSrc)); + assert.equal(shopByTypeByAsin.get('B0ORGCAM01')?.adType, 'organic'); + assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adType, 'sponsoredBrands'); + assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adFormat, 'video'); + assert.ok(shopByTypeItems.every((item) => item?.adType === 'sponsoredProducts')); + assert.equal( + shopByTypeByAsin.get('B0NIGHTOWL')?.placementId, + shopByTypeByAsin.get('B0PRISUAL1')?.placementId, + ); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.adProductIndex, 1); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.adProductIndex, 2); + assert.equal( + shopByType.some((item) => + /^(security system|webcam|instant print|point and shoot|film camera)$/i.test(item.title), + ), + false, + ); + + const organicShopByTypeDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+

Shop cameras by type

+
+

webcam

+
+

Organic type webcam

+ + + +
+
+
+

film camera

+
+

Organic type film camera

+ + + +
+
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=camera' }, + ); + Object.assign(globalThis, { + document: organicShopByTypeDom.window.document, + Node: organicShopByTypeDom.window.Node, + Element: organicShopByTypeDom.window.Element, + HTMLElement: organicShopByTypeDom.window.HTMLElement, + HTMLAnchorElement: organicShopByTypeDom.window.HTMLAnchorElement, + HTMLImageElement: organicShopByTypeDom.window.HTMLImageElement, + }); + const organicShopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const organicShopByTypeByAsin = new Map(organicShopByType.map((item) => [item.asin, item])); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.asin, 'B0ORGTYPE1'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.title, 'Organic type webcam'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.asin, 'B0ORGTYPE2'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.title, 'Organic type film camera'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE0')?.adType, 'organic'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.adType, 'organic'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.adType, 'organic'); + assert.equal( + organicShopByType.some( + (item) => item.adType === 'sponsoredProducts' || item.adType === 'sponsoredBrands', + ), + false, + ); + + const unknownTemplatesDom = new JSDOM( + ` +
+
+
+

Generic organic product

+ +
+ +
+

A heading that is not Shop by type

+ Sponsored +
+

Future SP one

+ + + +
+
+

Future SP two

+ + + +
+
+ +
+ Sponsored + +

Future Brand collection

+
+

Future SB product

+ +
+
+ +
+ Sponsored +
+

Future SD product

+ +
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=future-layout' }, + ); + Object.assign(globalThis, { + document: unknownTemplatesDom.window.document, + Node: unknownTemplatesDom.window.Node, + Element: unknownTemplatesDom.window.Element, + HTMLElement: unknownTemplatesDom.window.HTMLElement, + HTMLAnchorElement: unknownTemplatesDom.window.HTMLAnchorElement, + HTMLImageElement: unknownTemplatesDom.window.HTMLImageElement, + }); + const unknownTemplates = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const unknownTemplatesByAsin = new Map(unknownTemplates.map((item) => [item.asin, item])); + assert.equal(unknownTemplatesByAsin.get('B0GENORG01')?.adType, 'organic'); + assert.equal(unknownTemplatesByAsin.get('B0GENSP001')?.adType, 'sponsoredProducts'); + assert.equal(unknownTemplatesByAsin.get('B0GENSP002')?.adType, 'sponsoredProducts'); + assert.equal( + unknownTemplatesByAsin.get('B0GENSP001')?.placementId, + unknownTemplatesByAsin.get('B0GENSP002')?.placementId, + ); + assert.equal(unknownTemplatesByAsin.get('B0GENSB001')?.adType, 'sponsoredBrands'); + assert.equal(unknownTemplatesByAsin.get('B0GENSD001')?.adType, 'sponsoredDisplay'); + assert.ok( + ['B0GENSP001', 'B0GENSP002', 'B0GENSB001', 'B0GENSD001'].every((asin) => + unknownTemplatesByAsin.has(asin), + ), + 'Visible SP/SB/SD products must be collected even when their wrapper classes are unknown', + ); + + console.log('Amazon search parser fixtures passed.'); +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/scripts/test-execute-script.ts b/scripts/test-execute-script.ts new file mode 100644 index 0000000..80fde65 --- /dev/null +++ b/scripts/test-execute-script.ts @@ -0,0 +1,57 @@ +import { exec } from '../src/logic/execute-script'; + +const setBrowser = ( + executeScript: () => Promise, + getTab: () => Promise<{ status: string }> = async () => ({ status: 'complete' }), +) => { + Object.assign(globalThis, { + browser: { + tabs: { get: getTab }, + scripting: { executeScript }, + }, + }); +}; + +const main = async () => { + setBrowser(async () => { + throw new Error('Frame was removed'); + }); + const rejectionStarted = Date.now(); + await exec(1, async () => true, undefined, { timeout: 1000 }).then( + () => { + throw new Error('Expected executeScript to reject'); + }, + (error) => { + if (!String(error).includes('Frame was removed')) throw error; + if (Date.now() - rejectionStarted >= 1000) { + throw new Error('The original executeScript error was delayed until the timeout'); + } + }, + ); + + setBrowser(async () => [{ result: 42 }]); + const result = await exec(1, async () => 42, undefined, { timeout: 1000 }); + if (result !== 42) throw new Error(`Expected 42, received ${result}`); + + let injectionAttempted = false; + setBrowser( + async () => { + injectionAttempted = true; + return [{ result: true }]; + }, + async () => ({ status: 'loading' }), + ); + await exec(1, async () => true, undefined, { timeout: 100 }).then( + () => { + throw new Error('A perpetually loading page should time out before injection'); + }, + (error) => { + if (!String(error).includes('页面加载超时')) throw error; + }, + ); + if (injectionAttempted) throw new Error('Script was injected into a still-loading page'); + + console.log('Execute-script error propagation tests passed.'); +}; + +void main(); diff --git a/scripts/test-port-disconnect.ts b/scripts/test-port-disconnect.ts new file mode 100644 index 0000000..c02c80a --- /dev/null +++ b/scripts/test-port-disconnect.ts @@ -0,0 +1,13 @@ +import assert from 'node:assert/strict'; +import { isBackForwardCachePortError } from '../webext-bridge/src/internal/port-disconnect'; + +assert.equal( + isBackForwardCachePortError( + 'The page keeping the extension port is moved into back/forward cache, so the message channel is closed.', + ), + true, +); +assert.equal(isBackForwardCachePortError('Extension context invalidated.'), false); +assert.equal(isBackForwardCachePortError(), false); + +console.log('Port disconnect tests passed.'); diff --git a/scripts/test-remote-endpoint.ts b/scripts/test-remote-endpoint.ts new file mode 100644 index 0000000..36a8b15 --- /dev/null +++ b/scripts/test-remote-endpoint.ts @@ -0,0 +1,34 @@ +import assert from 'node:assert/strict'; +import { resolveRemoteEndpoint } from '../src/logic/remote-endpoint'; + +assert.deepEqual(resolveRemoteEndpoint({ isDev: true }), { + host: '127.0.0.1:18000', + httpBaseUrl: 'http://127.0.0.1:18000', + webSocketBaseUrl: 'ws://127.0.0.1:18000', +}); + +assert.deepEqual(resolveRemoteEndpoint({ isDev: false }), { + host: 'amzplugin.rekeymed.com', + httpBaseUrl: 'https://amzplugin.rekeymed.com', + webSocketBaseUrl: 'wss://amzplugin.rekeymed.com', +}); + +assert.deepEqual( + resolveRemoteEndpoint({ + isDev: true, + host: 'staging.example.test:8443', + protocol: 'https', + }), + { + host: 'staging.example.test:8443', + httpBaseUrl: 'https://staging.example.test:8443', + webSocketBaseUrl: 'wss://staging.example.test:8443', + }, +); + +assert.throws( + () => resolveRemoteEndpoint({ isDev: true, protocol: 'ftp' }), + /Unsupported remote protocol/, +); + +console.log('Remote endpoint tests passed.'); diff --git a/scripts/verify-extension-runtime.mjs b/scripts/verify-extension-runtime.mjs new file mode 100644 index 0000000..b0071cc --- /dev/null +++ b/scripts/verify-extension-runtime.mjs @@ -0,0 +1,71 @@ +const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9225'; +const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) => + response.json(), +); +const worker = targets.find( + (target) => target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'), +); +if (!worker) throw new Error('Azon Seeker background service worker was not found.'); + +const socket = new WebSocket(worker.webSocketDebuggerUrl); +const pending = new Map(); +let nextId = 1; + +socket.addEventListener('message', (event) => { + const message = JSON.parse(event.data); + if (!message.id || !pending.has(message.id)) return; + const { resolve, reject } = pending.get(message.id); + pending.delete(message.id); + message.error ? reject(new Error(message.error.message)) : resolve(message.result); +}); + +await new Promise((resolve, reject) => { + socket.addEventListener('open', resolve, { once: true }); + socket.addEventListener('error', reject, { once: true }); +}); + +function request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); +} + +const evaluation = await request('Runtime.evaluate', { + expression: `(async () => ({ + manifest: chrome.runtime.getManifest(), + alarm: await chrome.alarms.get('amazon-sync-retry'), + storage: await chrome.storage.local.get([ + 'amazon-sync-queue-v1', + 'amazon-sync-state-v1' + ]) + }))()`, + awaitPromise: true, + returnByValue: true, +}); +socket.close(); + +if (evaluation.exceptionDetails) { + throw new Error(evaluation.exceptionDetails.text || 'Extension evaluation failed.'); +} +const runtime = evaluation.result.value; +if (runtime.manifest.version !== '0.7.13') throw new Error('Unexpected extension version.'); +if (!runtime.manifest.permissions.includes('alarms')) throw new Error('Missing alarms permission.'); +if (runtime.alarm?.name !== 'amazon-sync-retry') throw new Error('Retry alarm was not created.'); +const state = JSON.parse(runtime.storage['amazon-sync-state-v1']); +if (state.pendingCount !== 0 || state.status !== 'synced') { + throw new Error(`Unexpected initial sync state: ${JSON.stringify(state)}`); +} +console.log( + JSON.stringify( + { + extensionId: new URL(worker.url).host, + version: runtime.manifest.version, + retryAlarm: runtime.alarm.name, + syncState: state, + }, + null, + 2, + ), +); diff --git a/scripts/verify-extension-sync-e2e.mjs b/scripts/verify-extension-sync-e2e.mjs new file mode 100644 index 0000000..a02dba3 --- /dev/null +++ b/scripts/verify-extension-sync-e2e.mjs @@ -0,0 +1,115 @@ +const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9226'; +const syncId = process.env.AZON_E2E_SYNC_ID; +const expectFailure = process.env.AZON_EXPECT_SYNC_FAILURE === '1'; +if (!syncId) throw new Error('AZON_E2E_SYNC_ID is required.'); + +let worker; +for (let attempt = 0; attempt < 60 && !worker; attempt++) { + const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) => + response.json(), + ); + worker = targets.find( + (target) => + target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'), + ); + if (!worker) await new Promise((resolve) => setTimeout(resolve, 250)); +} +if (!worker) throw new Error('Azon Seeker background service worker was not found.'); + +const socket = new WebSocket(worker.webSocketDebuggerUrl); +const pending = new Map(); +let nextId = 1; +socket.addEventListener('message', (event) => { + const message = JSON.parse(event.data); + if (!message.id || !pending.has(message.id)) return; + const { resolve, reject } = pending.get(message.id); + pending.delete(message.id); + message.error ? reject(new Error(message.error.message)) : resolve(message.result); +}); +await new Promise((resolve, reject) => { + socket.addEventListener('open', resolve, { once: true }); + socket.addEventListener('error', reject, { once: true }); +}); +function request(method, params = {}) { + return new Promise((resolve, reject) => { + const id = nextId++; + pending.set(id, { resolve, reject }); + socket.send(JSON.stringify({ id, method, params })); + }); +} +async function evaluate(expression) { + const result = await request('Runtime.evaluate', { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (result.exceptionDetails) throw new Error(result.exceptionDetails.text); + return result.result.value; +} + +const searchItem = { + syncId, + asin: 'B000000098', + keywords: 'extension queue e2e', + rank: 1, + page: 1, + pageRank: 1, + link: 'https://www.amazon.com/dp/B000000098', + title: 'Extension queue end-to-end fixture', + price: '$33.93', + strikethroughPrice: '$39.99', + adType: 'sponsoredDisplay', + adRank: 1, + placement: 'sidebar', + placementRank: 1, + placementId: 'extension-e2e-placement', + adProductIndex: 1, + adFormat: 'standard', + imageSrc: 'https://example.com/extension-e2e.jpg', + createTime: '2026/08/31 11:45:00', +}; +await evaluate(`(async () => { + await chrome.storage.local.set({ + 'amazon-sync-queue-v1': ${JSON.stringify({ + searchItems: [searchItem], + detailItems: [], + reviews: [], + })}, + 'amazon-sync-state-v1': JSON.stringify({status: 'pending', pendingCount: 1}) + }); + await chrome.alarms.create('amazon-sync-retry', { when: Date.now() + 200 }); + return true; +})()`); + +let state; +for (let attempt = 0; attempt < 30; attempt++) { + await new Promise((resolve) => setTimeout(resolve, 500)); + state = await evaluate(`(async () => { + const stored = await chrome.storage.local.get(['amazon-sync-queue-v1', 'amazon-sync-state-v1']); + return { + queue: stored['amazon-sync-queue-v1'], + state: JSON.parse(stored['amazon-sync-state-v1']) + }; + })()`); + if (state.state.status === 'failed') { + if (expectFailure) break; + throw new Error(state.state.lastError); + } + if (state.state.status === 'synced' && state.state.pendingCount === 0) break; +} +socket.close(); +if (expectFailure) { + if ( + state.state.status !== 'failed' || + state.state.pendingCount !== 1 || + state.queue.searchItems.length !== 1 + ) { + throw new Error(`Failed synchronization was not retained: ${JSON.stringify(state)}`); + } + console.log(JSON.stringify({ syncId, retained: true, ...state.state }, null, 2)); + process.exit(0); +} +if (state.state.status !== 'synced' || state.queue.searchItems.length !== 0) { + throw new Error(`Queue did not drain: ${JSON.stringify(state)}`); +} +console.log(JSON.stringify({ syncId, ...state.state }, null, 2)); diff --git a/scripts/verify-port-disconnect-build.mjs b/scripts/verify-port-disconnect-build.mjs new file mode 100644 index 0000000..8f24501 --- /dev/null +++ b/scripts/verify-port-disconnect-build.mjs @@ -0,0 +1,16 @@ +import { readFile } from 'node:fs/promises'; + +const background = await readFile('extension/dist/background/index.mjs', 'utf8'); +const contentScript = await readFile('extension/dist/contentScripts/index.global.js', 'utf8'); + +if (!background.includes('lastError')) { + throw new Error('Background bundle does not consume runtime.lastError on port disconnect.'); +} +if (!contentScript.includes('back/forward cache')) { + throw new Error('Content script bundle does not recognize BFCache port closures.'); +} +if (!contentScript.includes('pagehide') || !contentScript.includes('pageshow')) { + throw new Error('Content script bundle does not suspend and resume the BFCache port lifecycle.'); +} + +console.log('Production BFCache port handling verification passed.'); diff --git a/scripts/verify-production-endpoint.mjs b/scripts/verify-production-endpoint.mjs new file mode 100644 index 0000000..083f9ef --- /dev/null +++ b/scripts/verify-production-endpoint.mjs @@ -0,0 +1,29 @@ +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +const background = await readFile('extension/dist/background/index.mjs', 'utf8'); +async function readJavaScriptFiles(directory) { + const entries = await readdir(directory, { withFileTypes: true }); + const contents = await Promise.all( + entries.map(async (entry) => { + const path = join(directory, entry.name); + if (entry.isDirectory()) return readJavaScriptFiles(path); + if (!/\.(?:m?js)$/.test(entry.name)) return ''; + return readFile(path, 'utf8'); + }), + ); + return contents.flat().join('\n'); +} +const allJavaScript = await readJavaScriptFiles('extension/dist'); + +if (!background.includes('amzplugin.rekeymed.com')) { + throw new Error('Production background bundle does not target the public server.'); +} +if (background.includes('127.0.0.1:18000')) { + throw new Error('Production background bundle still contains the local API endpoint.'); +} +if (!allJavaScript.includes('wss://') || !background.includes('https://')) { + throw new Error('Production bundle is missing secure HTTP/WebSocket protocols.'); +} + +console.log('Production endpoint verification passed.'); diff --git a/src/background/amazon-sync.ts b/src/background/amazon-sync.ts new file mode 100644 index 0000000..1f51e80 --- /dev/null +++ b/src/background/amazon-sync.ts @@ -0,0 +1,214 @@ +import { onMessage } from 'webext-bridge/background'; +import { flattenObject } from '~/logic/convert'; +import { exportKey, generateKeyPair, signData } from '~/logic/crypto'; +import { remoteBaseUrl } from '~/env'; +import { + AMAZON_SYNC_ALARM, + AMAZON_SYNC_QUEUE_KEY, + AMAZON_SYNC_STATE_KEY, + EMPTY_AMAZON_SYNC_STATE, +} from '~/logic/amazon-sync-shared'; + +type AmazonSyncQueue = Required; + +const emptyQueue = (): AmazonSyncQueue => ({ searchItems: [], detailItems: [], reviews: [] }); +const keyPairPromise = generateKeyPair(); +const publicKeyPromise = keyPairPromise.then(({ publicKey }) => exportKey(publicKey)); +let queueMutation: Promise = Promise.resolve(); +let flushOperation: Promise | undefined; +let flushAgain = false; + +function mutateQueue(operation: () => Promise): Promise { + const result = queueMutation.then(operation, operation); + queueMutation = result.then( + () => undefined, + () => undefined, + ); + return result; +} + +async function readQueue(): Promise { + const stored = await browser.storage.local.get(AMAZON_SYNC_QUEUE_KEY); + return { ...emptyQueue(), ...(stored[AMAZON_SYNC_QUEUE_KEY] as AmazonSyncQueue | undefined) }; +} + +function pendingCount(queue: AmazonSyncQueue) { + return queue.searchItems.length + queue.detailItems.length + queue.reviews.length; +} + +async function writeQueue(queue: AmazonSyncQueue) { + await browser.storage.local.set({ [AMAZON_SYNC_QUEUE_KEY]: queue }); +} + +async function writeState(state: AmazonSyncState) { + await browser.storage.local.set({ [AMAZON_SYNC_STATE_KEY]: JSON.stringify(state) }); + return state; +} + +function mergeUnique(current: T[], incoming: T[], key: (item: T) => string | undefined) { + const values = new Map(current.map((item) => [key(item), item])); + for (const item of incoming) values.set(key(item), item); + values.delete(undefined); + return Array.from(values.values()); +} + +async function signedPost(path: string, data: unknown[]) { + if (data.length === 0) return; + const body = JSON.stringify(data); + const { privateKey } = await keyPairPromise; + const response = await fetch(`${remoteBaseUrl}${path}`, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-signature': await signData(privateKey, body), + 'x-public-key': await publicKeyPromise, + }, + body, + signal: AbortSignal.timeout(10000), + }); + if (!response.ok) throw new Error(`${path} HTTP ${response.status}`); + const result = (await response.json()) as { success?: boolean; count?: number }; + if (result.success !== true || result.count !== data.length) { + throw new Error(`${path} 返回的确认数量不匹配`); + } +} + +function serializeDetailItems(items: AmazonDetailItem[]) { + return items.map((item) => + flattenObject( + item.imageUrls + ? { + ...item, + imageUrls: item.imageUrls.join(';'), + } + : item, + ), + ); +} + +function serializeReviews(items: ({ asin: string } & AmazonReview)[]) { + return items.map((item) => ({ ...item, imageSrc: item.imageSrc?.join(';') })); +} + +async function removeSynced( + field: keyof AmazonSyncQueue, + sentKeys: Set, + key: (item: any) => string | undefined, +) { + await mutateQueue(async () => { + const current = await readQueue(); + (current[field] as any[]) = (current[field] as any[]).filter((item) => { + const value = key(item); + return !value || !sentKeys.has(value); + }); + await writeQueue(current); + }); +} + +async function flushQueueInternal(): Promise { + let queue = await mutateQueue(readQueue); + if (pendingCount(queue) === 0) { + return mutateQueue(async () => { + const current = await readQueue(); + return writeState({ + ...EMPTY_AMAZON_SYNC_STATE, + status: pendingCount(current) === 0 ? 'synced' : 'pending', + pendingCount: pendingCount(current), + }); + }); + } + await writeState({ status: 'syncing', pendingCount: pendingCount(queue) }); + try { + const searchSnapshot = queue.searchItems; + await signedPost('/amazon/search_items', searchSnapshot); + await removeSynced( + 'searchItems', + new Set(searchSnapshot.map((item) => item.syncId).filter(Boolean) as string[]), + (item: AmazonSearchItem) => item.syncId, + ); + + queue = await mutateQueue(readQueue); + const detailSnapshot = queue.detailItems; + await signedPost('/amazon/detail_items', serializeDetailItems(detailSnapshot)); + await removeSynced( + 'detailItems', + new Set(detailSnapshot.map((item) => item.syncId).filter(Boolean) as string[]), + (item: AmazonDetailItem) => item.syncId, + ); + + queue = await mutateQueue(readQueue); + const reviewSnapshot = queue.reviews; + await signedPost('/amazon/reviews', serializeReviews(reviewSnapshot)); + await removeSynced( + 'reviews', + new Set(reviewSnapshot.map((item) => item.id)), + (item: AmazonReview) => item.id, + ); + + return mutateQueue(async () => { + const current = await readQueue(); + return writeState({ + status: pendingCount(current) === 0 ? 'synced' : 'pending', + pendingCount: pendingCount(current), + lastSuccessAt: new Date().toISOString(), + }); + }); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + await mutateQueue(async () => { + const current = await readQueue(); + await writeState({ + status: 'failed', + pendingCount: pendingCount(current), + lastError: message, + }); + }); + throw error; + } +} + +function flushQueue() { + if (flushOperation) { + flushAgain = true; + } else { + flushOperation = flushQueueInternal().finally(() => { + flushOperation = undefined; + if (flushAgain) { + flushAgain = false; + void flushQueue().catch(() => undefined); + } + }); + } + return flushOperation; +} + +async function enqueue(payload: AmazonSyncPayload) { + const state = await mutateQueue(async () => { + const queue = await readQueue(); + queue.searchItems = mergeUnique( + queue.searchItems, + payload.searchItems || [], + (item) => item.syncId, + ); + queue.detailItems = mergeUnique( + queue.detailItems, + payload.detailItems || [], + (item) => item.syncId, + ); + queue.reviews = mergeUnique(queue.reviews, payload.reviews || [], (item) => item.id); + await writeQueue(queue); + return writeState({ status: 'pending', pendingCount: pendingCount(queue) }); + }); + void flushQueue().catch(() => undefined); + return state; +} + +onMessage('amazon-sync-enqueue', ({ data }) => enqueue(data)); +onMessage('amazon-sync-flush', () => flushQueue()); + +browser.alarms.create(AMAZON_SYNC_ALARM, { periodInMinutes: 1 }); +browser.alarms.onAlarm.addListener((alarm) => { + if (alarm.name === AMAZON_SYNC_ALARM) void flushQueue().catch(() => undefined); +}); + +void flushQueue().catch(() => undefined); diff --git a/src/background/main.ts b/src/background/main.ts index c5422b7..19f2103 100644 --- a/src/background/main.ts +++ b/src/background/main.ts @@ -2,6 +2,7 @@ import { isFirefox } from '~/env'; // https://github.com/serversideup/webext-bridge/issues/67#issuecomment-2676094094 import('webext-bridge/background'); +import('./amazon-sync'); // only on dev mode if (import.meta.hot) { diff --git a/src/composables/useCloudExporter.ts b/src/composables/useCloudExporter.ts index 2e6f985..dd18f3d 100644 --- a/src/composables/useCloudExporter.ts +++ b/src/composables/useCloudExporter.ts @@ -1,4 +1,4 @@ -import { remoteHost } from '~/env'; +import { remoteBaseUrl, remoteWebSocketBaseUrl } from '~/env'; import { useLongTask } from './useLongTask'; type AddDataFragmentsCommand = { @@ -16,16 +16,12 @@ type WebSocketResponse = | { type: 'progress'; current: number; total: number } | { type: 'result'; result: string }; -// 使用 wss:// 和 https:// -const protocol = __DEV__ ? 'ws' : 'wss'; -const httpProtocol = __DEV__ ? 'http' : 'https'; - class ExportExcelPipeline { private socket: WebSocket; constructor() { this.socket = new WebSocket( - `${protocol}://${remoteHost}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`, + `${remoteWebSocketBaseUrl}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`, ); } @@ -39,7 +35,7 @@ class ExportExcelPipeline { case WebSocket.CLOSED: case WebSocket.CLOSING: this.socket = new WebSocket( - `${protocol}://${remoteHost}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`, + `${remoteWebSocketBaseUrl}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`, ); case WebSocket.CONNECTING: return new Promise((resolve) => { @@ -126,7 +122,7 @@ export const useCloudExporter = () => { await pipeline.close(); if (file) { - const url = `${httpProtocol}://${remoteHost}${file}`; + const url = `${remoteBaseUrl}${file}`; const link = document.createElement('a'); link.href = url; link.download = `${dayjs().format('YYYY-MM-DD')}.xlsx`; diff --git a/src/composables/useLongTask.ts b/src/composables/useLongTask.ts index 5fc1797..34bfb07 100644 --- a/src/composables/useLongTask.ts +++ b/src/composables/useLongTask.ts @@ -1,17 +1,18 @@ export function useLongTask() { const isRunning = ref(false); + const lastError = shallowRef(); const startTask = async (task: () => Promise) => { if (isRunning.value) { throw Error('Task is still running.'); } isRunning.value = true; + lastError.value = undefined; let result = undefined; try { result = await task(); } catch (error) { - console.log(error); - + lastError.value = error; console.error('Task failed:', error); } isRunning.value = false; @@ -20,6 +21,7 @@ export function useLongTask() { return { isRunning, + lastError, startTask, }; } diff --git a/src/env.ts b/src/env.ts index 11eeafc..66eeb81 100644 --- a/src/env.ts +++ b/src/env.ts @@ -1,3 +1,5 @@ +import { resolveRemoteEndpoint } from '~/logic/remote-endpoint'; + const forbiddenProtocols = [ 'chrome-extension://', 'chrome-search://', @@ -14,9 +16,12 @@ export function isForbiddenUrl(url: string): boolean { export const isFirefox = navigator.userAgent.includes('Firefox'); -// export const remoteHost = __DEV__ ? '127.0.0.1:8000' : '47.251.4.191:8000'; -// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'vm8nc3zr-18000.usw2.devtunnels.ms'; -// export const remoteHost = __DEV__ ? 'ogsm-dev.rekeymed.com:8888' : 'amzplugin.rekeymed.com'; -export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'; -// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'; -// export const remoteHost = '47.251.4.191:8000'; +const remoteEndpoint = resolveRemoteEndpoint({ + isDev: __DEV__, + host: import.meta.env.VITE_REMOTE_HOST, + protocol: import.meta.env.VITE_REMOTE_PROTOCOL, +}); + +export const remoteHost = remoteEndpoint.host; +export const remoteBaseUrl = remoteEndpoint.httpBaseUrl; +export const remoteWebSocketBaseUrl = remoteEndpoint.webSocketBaseUrl; diff --git a/src/logic/amazon-sync-client.ts b/src/logic/amazon-sync-client.ts new file mode 100644 index 0000000..44f8c02 --- /dev/null +++ b/src/logic/amazon-sync-client.ts @@ -0,0 +1,9 @@ +import { sendMessage } from 'webext-bridge/sidepanel'; + +export async function enqueueAmazonSync(payload: AmazonSyncPayload) { + return sendMessage('amazon-sync-enqueue', payload, 'background'); +} + +export async function flushAmazonSyncQueue() { + return sendMessage('amazon-sync-flush', undefined, 'background'); +} diff --git a/src/logic/amazon-sync-shared.ts b/src/logic/amazon-sync-shared.ts new file mode 100644 index 0000000..fbc4d32 --- /dev/null +++ b/src/logic/amazon-sync-shared.ts @@ -0,0 +1,12 @@ +export const AMAZON_SYNC_QUEUE_KEY = 'amazon-sync-queue-v1'; +export const AMAZON_SYNC_STATE_KEY = 'amazon-sync-state-v1'; +export const AMAZON_SYNC_ALARM = 'amazon-sync-retry'; + +export const EMPTY_AMAZON_SYNC_STATE: AmazonSyncState = { + status: 'idle', + pendingCount: 0, +}; + +export function createAmazonSyncId(prefix: 'search' | 'detail') { + return `${prefix}-${crypto.randomUUID()}`; +} diff --git a/src/logic/execute-script.ts b/src/logic/execute-script.ts index 21b030f..6e970d7 100644 --- a/src/logic/execute-script.ts +++ b/src/logic/execute-script.ts @@ -47,45 +47,45 @@ export async function exec>( options: ExecOptions = {}, ): Promise { const { timeout = 30000 } = options; - return new Promise(async (resolve, reject) => { - // 基本刷新 - // await browser.tabs.reload(tabId); - // console.log('exec', browser.tabs.get(tabId), tabId); - - // console.log('exec', func, (await browser.tabs.get(tabId)).url); - for (let i = 0; i < 50; i++) { - await new Promise((r) => setTimeout(r, 200)); - const tab = await browser.tabs.get(tabId); - if (tab.status === 'complete') { - break; - } - } - + const deadline = Date.now() + timeout; + let pageLoaded = false; + while (Date.now() < deadline) { const tab = await browser.tabs.get(tabId); - if (tab.status !== 'complete') { - await browser.tabs.reload(tabId); - await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始 + if (tab.status === 'complete') { + pageLoaded = true; + break; } + const remaining = deadline - Date.now(); + await new Promise((resolve) => setTimeout(resolve, Math.min(200, remaining))); + } + if (!pageLoaded) throw new Error(`页面加载超时（${timeout}ms），未执行采集脚本`); - for (let i = 0; i < 50; i++) { - await new Promise((r) => setTimeout(r, 200)); - const tab = await browser.tabs.get(tabId); - if (tab.status === 'complete') { - break; - } - } + let timeoutId: ReturnType | undefined; + const remainingTimeout = Math.max(1, deadline - Date.now()); + const timeoutPromise = new Promise((_, reject) => { + timeoutId = setTimeout( + () => reject(new Error(`脚本运行超时（总预算 ${timeout}ms）`)), + remainingTimeout, + ); + }); - setTimeout(() => reject(`脚本运行超时, foucs on ${func}`), timeout); - try { - const injectResults = await browser.scripting.executeScript({ + try { + const injectResults = await Promise.race([ + browser.scripting.executeScript({ target: { tabId }, func, args: payload ? [payload] : undefined, - }); - const ret = injectResults.pop(); - resolve(ret!.result as T); - } catch (e) { - throw new Error(`脚本运行失败: ${e}`); - } - }); + }), + timeoutPromise, + ]); + const ret = injectResults.at(-1); + if (!ret) throw new Error('脚本未返回执行结果'); + return ret.result as T; + } catch (error) { + const cause = error instanceof Error ? error : new Error(String(error)); + if (cause.message.startsWith('脚本运行超时')) throw cause; + throw new Error(`脚本运行失败: ${cause.message}`); + } finally { + if (timeoutId !== undefined) clearTimeout(timeoutId); + } } diff --git a/src/logic/remote-endpoint.ts b/src/logic/remote-endpoint.ts new file mode 100644 index 0000000..b5f59c2 --- /dev/null +++ b/src/logic/remote-endpoint.ts @@ -0,0 +1,28 @@ +type RemoteEndpointOptions = { + isDev: boolean; + host?: string; + protocol?: string; +}; + +export type RemoteEndpoint = { + host: string; + httpBaseUrl: string; + webSocketBaseUrl: string; +}; + +export function resolveRemoteEndpoint({ + isDev, + host, + protocol, +}: RemoteEndpointOptions): RemoteEndpoint { + const resolvedHost = host || (isDev ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com'); + const resolvedProtocol = protocol || (isDev ? 'http' : 'https'); + if (resolvedProtocol !== 'http' && resolvedProtocol !== 'https') { + throw new Error(`Unsupported remote protocol: ${resolvedProtocol}`); + } + return { + host: resolvedHost, + httpBaseUrl: `${resolvedProtocol}://${resolvedHost}`, + webSocketBaseUrl: `${resolvedProtocol === 'https' ? 'wss' : 'ws'}://${resolvedHost}`, + }; +} diff --git a/src/logic/upload-image.ts b/src/logic/upload-image.ts index 8ede900..eea9e73 100644 --- a/src/logic/upload-image.ts +++ b/src/logic/upload-image.ts @@ -1,5 +1,5 @@ /** 上传图片工具类 */ -import { remoteHost } from '~/env'; +import { remoteBaseUrl } from '~/env'; export async function uploadImage( base64String: string, @@ -23,8 +23,7 @@ export async function uploadImage( const formData = new FormData(); formData.append('file', blob, filename); - const httpProtocol = __DEV__ ? 'http' : 'https'; - const url = `${httpProtocol}://${remoteHost}/upload/image/${encodeURIComponent(filename)}`; + const url = `${remoteBaseUrl}/upload/image/${encodeURIComponent(filename)}`; const resp = (await fetch(url, { method: 'POST', body: formData, @@ -33,5 +32,5 @@ export async function uploadImage( return undefined; } const data = await resp.json(); - return `${httpProtocol}://${remoteHost}${data.file}`; + return `${remoteBaseUrl}${data.file}`; } diff --git a/src/options/views/AmazonResultTable.vue b/src/options/views/AmazonResultTable.vue index 426c34d..6164cac 100644 --- a/src/options/views/AmazonResultTable.vue +++ b/src/options/views/AmazonResultTable.vue @@ -8,10 +8,41 @@ import { allDetailItems, allItems, itemColumnSettings, reviewItems } from '~/sto const modal = useModal(); const excelHelper = useExcelHelper(); +const adTypeLabels: Record = { + organic: '自然排位', + sponsoredProducts: 'Sponsored Products', + sponsoredBrands: 'Sponsored Brands', + sponsoredDisplay: 'Sponsored Display', + unknownSponsored: '未知广告', +}; + +const placementLabels: Record = { + topOfSearch: '顶部', + middleOfSearch: '中部', + bottomOfSearch: '底部', + restOfSearch: '其余位置（旧数据）', + sidebar: '侧边栏', + productPage: '商品详情页', + unknown: '未知位置', +}; + +const renderPlacement = (row: AmazonItem) => { + if (!row.placement) return '-'; + const label = placementLabels[row.placement]; + if (row.placement === 'productPage') return label; + return row.page ? `第 ${row.page} 页${label}` : label; +}; + +const renderOptionalValue = (value: unknown) => value ?? '-'; + +const isDetailOnlyRow = (row: AmazonItem) => + !row.keywords && row.page == null && row.rank == null && row.pageRank == null; + const filter = ref<{ keywords?: string; search?: string; detailOnly?: boolean; + adType?: AmazonAdType | 'unclassified'; searchDateRange?: [number, number]; detailDateRange?: [number, number]; }>({}); @@ -34,18 +65,55 @@ const columns = computed(() => { key: 'keywords', minWidth: 120, hidden: !itemColumnSettings.value.has('keywords'), + render: (row) => renderOptionalValue(row.keywords), }, { title: '页码', key: 'page', minWidth: 60, hidden: !itemColumnSettings.value.has('page'), + render: (row) => renderOptionalValue(row.page), }, { title: '排位', key: 'rank', minWidth: 60, hidden: !itemColumnSettings.value.has('rank'), + render: (row) => renderOptionalValue(row.rank), + }, + { + title: '页内排位', + key: 'pageRank', + minWidth: 90, + hidden: !itemColumnSettings.value.has('pageRank'), + render: (row) => renderOptionalValue(row.pageRank), + }, + { + title: '广告类型', + key: 'adType', + minWidth: 150, + render: (row) => + isDetailOnlyRow(row) ? '-' : row.adType ? adTypeLabels[row.adType] : '历史数据/未识别', + }, + { + title: '广告排位', + key: 'adRank', + minWidth: 90, + hidden: !itemColumnSettings.value.has('adRank'), + render: (row) => renderOptionalValue(row.adRank), + }, + { + title: '广告位置', + key: 'placement', + minWidth: 140, + render: renderPlacement, + }, + { + title: '广告位顺序', + key: 'placementRank', + minWidth: 110, + hidden: !itemColumnSettings.value.has('placementRank'), + render: (row) => renderOptionalValue(row.placementRank), }, { title: 'ASIN', @@ -61,6 +129,11 @@ const columns = computed(() => { key: 'price', minWidth: 100, }, + { + title: '划线价格', + key: 'strikethroughPrice', + minWidth: 100, + }, { title: '封面图', key: 'imageSrc', @@ -181,7 +254,7 @@ const getItemHeaders = () => { }; const filteredData = computed(() => { - const { search, detailOnly, keywords, searchDateRange, detailDateRange } = filter.value; + const { search, detailOnly, keywords, adType, searchDateRange, detailDateRange } = filter.value; let data = toRaw(detailOnly ? allDetailItems.value : allItems.value); if (search && search.trim() !== '') { data = data.filter((r) => { @@ -193,6 +266,9 @@ const filteredData = computed(() => { if (keywords) { data = data.filter((r) => r.keywords === keywords); } + if (adType) { + data = data.filter((r) => (adType === 'unclassified' ? !r.adType : r.adType === adType)); + } if (searchDateRange) { const start = dayjs(searchDateRange[0]); const end = dayjs(searchDateRange[1]); @@ -321,6 +397,17 @@ const handleClearData = async () => { }))" /> + + + { + + + diff --git a/src/options/views/help/guide.md b/src/options/views/help/guide.md index fb0aab9..145e9a3 100644 --- a/src/options/views/help/guide.md +++ b/src/options/views/help/guide.md @@ -44,7 +44,7 @@ 位于搜索页面板，可以根据输入的关键词采集搜索页中呈现的商品信息，主要为**关键词排名**。 输入：关键词（最多十个） -输出：商品搜索页信息，包含关键词、页码、排位、 ASIN、标题、 价格、 封面图、获取日期。 +输出：商品搜索页信息，包含关键词、页码、排位、广告类型、广告排位、广告位置、ASIN、标题、当前价格、划线价格、封面图、获取日期。 ### 详情页 @@ -52,7 +52,7 @@ 位于详情页面板，可以根据输入的多个ASIN采集商品详情页的商品信息。 输入：ASIN（每行输入一个，可以输入多个） -输出：商品详情页信息，包含ASIN、标题、 价格、 封面图、获取日期、评分、评论数、 大类、大类排行、小类、 小类排行、 商品图片链接、A+截图等等。 +输出：商品详情页信息，包含ASIN、标题、当前价格、划线价格、封面图、获取日期、评分、评论数、大类、大类排行、小类、小类排行、商品图片链接、A+截图等等。 ### 评论页 @@ -86,9 +86,15 @@ Items表记录商品信息，表头信息如下。 | 关键词 | 产品的关键词 | | 页码 | 产品在关键词搜索列表中的页码 | | 排位 | 产品的排名 | +| 页内排位 | 产品在当前搜索页商品流中的排名 | +| 广告类型 | 自然排位、Sponsored Products、Sponsored Brands、Sponsored Display 或未知广告 | +| 广告排位 | 同一关键词、同一广告类型中的累计排名 | +| 广告位置 | 搜索顶部、搜索其余位置或侧边栏 | +| 广告位顺序 | 同一关键词采集过程中广告模块的累计顺序 | | ASIN | 亚马逊标准识别号 | | 标题 | 产品的标题 | -| 价格 | 产品的价格 | +| 价格 | 产品的当前价格 | +| 划线价格 | 页面中以划线形式展示的参考价格 | | 封面图 | 产品的封面图片链接 | | 获取日期 | 数据获取的日期 | | 商品链接 | 产品的链接 | diff --git a/src/page-worker/composables/amazon.ts b/src/page-worker/composables/amazon.ts index 32f27be..6826d1a 100644 --- a/src/page-worker/composables/amazon.ts +++ b/src/page-worker/composables/amazon.ts @@ -3,7 +3,8 @@ import amazon from '../impls/amazon'; import { uploadImage } from '~/logic/upload-image'; import { detailItems, reviewItems, searchItems } from '~/storages/amazon'; import { createGlobalState } from '@vueuse/core'; -import { useAmazonService } from '~/page-worker/services/amazon'; +import { enqueueAmazonSync } from '~/logic/amazon-sync-client'; +import { createAmazonSyncId } from '~/logic/amazon-sync-shared'; import { taskOptionBase, WorkerComposable } from '../interfaces/common'; import { AmazonPageWorker } from '../interfaces/amazon'; @@ -18,10 +19,7 @@ function buildAmazonPageWorker(): WorkerComposable({}); // Long task management - const { isRunning, startTask } = useLongTask(); - // Amazon service instance - const service = useAmazonService(); - + const { isRunning, lastError, startTask } = useLongTask(); // Get the worker instance from implementation const worker = amazon.getAmazonPageWorker(); @@ -29,10 +27,13 @@ function buildAmazonPageWorker(): WorkerComposable(); const reviewCache = new Map(); + let commitOperation: Promise = Promise.resolve(); // Add search items to cache const updateSearchCache = (data: AmazonSearchItem[]) => { - searchCache.push(...data); + searchCache.push( + ...data.map((item) => ({ ...item, syncId: item.syncId || createAmazonSyncId('search') })), + ); }; // Update or add detail item in cache @@ -42,7 +43,10 @@ function buildAmazonPageWorker(): WorkerComposable item.id)); for (const review of reviews) { - ids.has(review.id) || values.push(review); + if (!ids.has(review.id)) { + values.push(review); + ids.add(review.id); + } } reviewCache.set(asin, values); }; @@ -64,44 +71,90 @@ function buildAmazonPageWorker(): WorkerComposable { + // Persist locally first, then atomically enqueue for background synchronization. + const commitChangeInternal = async () => { const { objects } = settings.value; - // Commit search items - if (objects?.includes('search')) { - searchItems.value = searchItems.value.concat(searchCache); - await service.commitSearchItems(searchCache); - searchCache.splice(0, searchCache.length); + const searchSnapshot = objects?.includes('search') ? searchCache.slice() : []; + const detailSnapshot = objects?.includes('detail') ? Array.from(detailCache.values()) : []; + const reviewSnapshot = objects?.includes('review') + ? Array.from(reviewCache.entries()).flatMap(([asin, reviews]) => + reviews.map((review) => ({ asin, ...review })), + ) + : []; + + if (searchSnapshot.length === 0 && detailSnapshot.length === 0 && reviewSnapshot.length === 0) { + return; } - // Commit detail items - if (objects?.includes('detail')) { - for (const [k, v] of detailCache.entries()) { - if (detailItems.value.has(k)) { - const item = detailItems.value.get(k)!; - detailItems.value.set(k, { ...item, ...v }); + + if (searchSnapshot.length > 0) { + const existingIds = new Set(searchItems.value.map((item) => item.syncId).filter(Boolean)); + searchItems.value = searchItems.value.concat( + searchSnapshot.filter((item) => !item.syncId || !existingIds.has(item.syncId)), + ); + } + if (detailSnapshot.length > 0) { + const nextDetailItems = new Map(detailItems.value); + for (const v of detailSnapshot) { + const k = v.asin; + if (nextDetailItems.has(k)) { + const item = nextDetailItems.get(k)!; + nextDetailItems.set(k, { ...item, ...v }); } else { - detailItems.value.set(k, v); + nextDetailItems.set(k, v); } } - await service.commitDetailItems(detailCache); - detailCache.clear(); + // Replacing the Map gives every extension page a deterministic reactive + // change to persist and observe; in-place Map.set could leave the open + // results page showing stale data during long batches. + detailItems.value = nextDetailItems; } - // Commit reviews - if (objects?.includes('review')) { - for (const [asin, reviews] of reviewCache.entries()) { - if (reviewItems.value.has(asin)) { + if (reviewSnapshot.length > 0) { + const groupedReviews = reviewSnapshot.reduce>((grouped, item) => { + const values = grouped.get(item.asin) || []; + const { asin: _, ...review } = item; + values.push(review); + grouped.set(item.asin, values); + return grouped; + }, new Map()); + const nextReviewItems = new Map(reviewItems.value); + for (const [asin, reviews] of groupedReviews.entries()) { + if (nextReviewItems.has(asin)) { const addedIds = new Set(reviews.map((x) => x.id)); - const origin = reviewItems.value.get(asin)!; + const origin = nextReviewItems.get(asin)!; const newReviews = origin.filter((x) => !addedIds.has(x.id)).concat(reviews); newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo))); - reviewItems.value.set(asin, newReviews); + nextReviewItems.set(asin, newReviews); } else { - reviewItems.value.set(asin, reviews); + nextReviewItems.set(asin, reviews); } } - await service.commitReviews(reviewCache); - reviewCache.clear(); + reviewItems.value = nextReviewItems; } + + // The background acknowledges only after the queue is durably written to extension storage. + await enqueueAmazonSync({ + searchItems: searchSnapshot, + detailItems: detailSnapshot, + reviews: reviewSnapshot, + }); + + const sentSearchIds = new Set(searchSnapshot.map((item) => item.syncId)); + const remainingSearch = searchCache.filter((item) => !sentSearchIds.has(item.syncId)); + searchCache.splice(0, searchCache.length, ...remainingSearch); + for (const item of detailSnapshot) { + if (detailCache.get(item.asin)?.syncId === item.syncId) detailCache.delete(item.asin); + } + const sentReviewIds = new Set(reviewSnapshot.map((item) => item.id)); + for (const [asin, reviews] of reviewCache.entries()) { + const remaining = reviews.filter((item) => !sentReviewIds.has(item.id)); + remaining.length > 0 ? reviewCache.set(asin, remaining) : reviewCache.delete(asin); + } + }; + + const commitChange = () => { + const result = commitOperation.then(commitChangeInternal, commitChangeInternal); + commitOperation = result.catch(() => undefined); + return result; }; // Register all relevant worker event handlers @@ -125,7 +178,7 @@ function buildAmazonPageWorker(): WorkerComposable { - updateDetailCache(ev); + if (ev.imageUrls?.length) updateDetailCache(ev); }), // Collect A+ content screenshots and upload worker.on('item-aplus-screenshot-collect', async (ev) => { @@ -161,15 +214,25 @@ function buildAmazonPageWorker(): WorkerComposable Promise>(func: T) => { - const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value; - clearAllCaches(); + const taskWrapper1 = Promise>( + func: T, + objectType: 'search' | 'review', + ) => { return (...params: Parameters) => startTask(async () => { - const interval = setInterval(() => commitChange(), commitChangeIngerval); - await func(...params); - clearInterval(interval); - await commitChange(); + settings.value = { ...settings.value, objects: [objectType] }; + clearAllCaches(); + const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value; + const interval = setInterval( + () => void commitChange().catch(console.error), + commitChangeIngerval, + ); + try { + await func(...params); + } finally { + clearInterval(interval); + await commitChange(); + } }); }; @@ -179,9 +242,10 @@ function buildAmazonPageWorker(): WorkerComposable Promise>( func: T, ) => { - clearAllCaches(); return (...params: Parameters) => startTask(async () => { + settings.value = { ...settings.value, objects: ['detail'] }; + clearAllCaches(); if (!params?.[1]) { params[1] = {}; } @@ -196,20 +260,24 @@ function buildAmazonPageWorker(): WorkerComposable { + const previousTab = await browser.tabs.get(tabId); + const previousUrl = previousTab.url; + const updatedTab = await browser.tabs.update(tabId, { url, active: true }); + let navigationObserved = + updatedTab.status === 'loading' || + (Boolean(updatedTab.url) && updatedTab.url !== previousUrl); + let lastUrl = updatedTab.url; + for (let i = 0; i < 150; i++) { + const currentTab = await browser.tabs.get(tabId); + lastUrl = currentTab.url; + navigationObserved ||= + currentTab.status === 'loading' || + (Boolean(currentTab.url) && currentTab.url !== previousUrl); + const reachedTarget = hasReachedAmazonNavigationTarget(currentTab.url, url); + const reachedRedirectedDetail = + options.allowDetailRedirect && hasReachedAmazonDetailPage(currentTab.url, url); + if (navigationObserved && currentTab.status === 'complete' && (reachedTarget || reachedRedirectedDetail)) { + return currentTab.url!; + } + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`等待目标页面加载超时: ${url}；当前页面: ${lastUrl || '未知'}`); + } + private async wanderSearchSinglePage(tab: Tabs.Tab) { const injector = new AmazonSearchPageInjector(tab); + const runStage = async (stage: string, operation: () => Promise): Promise => { + try { + return await operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`搜索页${stage}失败: ${message}`); + } + }; // Wait for the Next button to appear, indicating that the product items have finished loading - await injector.waitForPageLoaded(); + await runStage('加载等待', () => injector.waitForPageLoaded()); // Determine the type of product search page https://github.com/primedigitaltech/azon_seeker/issues/1 - const pagePattern = await injector.getPagePattern(); + const pagePattern = await runStage('布局识别', () => injector.getPagePattern()); // Retrieve key nodes and their information from the critical product search page - const data = await injector.getPageData(pagePattern); + const data = await runStage('数据解析', () => injector.getPageData(pagePattern)); + const explicitNoResults = + data?.length === 0 + ? await runStage('空结果识别', () => injector.isExplicitNoResultsPage()) + : false; // get current page - const page = await injector.getCurrentPage(); - // Determine if it is the last page, otherwise navigate to the next page - const hasNextPage = await injector.determineHasNextPage(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - if (data === null || typeof hasNextPage !== 'boolean') { + const page = await runStage('页码识别', () => injector.getCurrentPage()); + // Read the next URL without navigating inside the injected script. Clicking here would move + // the current Amazon frame into BFCache before executeScript can return its result. + const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl()); + if (data === null) { this.emit('error', { message: '爬取单页信息失败', url: tab.url }); throw new Error('爬取单页信息失败'); } - return { data, hasNextPage, page }; + if (data.length === 0 && !explicitNoResults) { + throw new Error( + 'Amazon 搜索页未发现可采集商品，且页面没有明确显示“无结果”；请检查是否出现临时错误页、验证码或页面结构变化', + ); + } + return { data, nextPageUrl, page }; } @withErrorHandling @@ -69,14 +121,27 @@ class AmazonPageWorkerImpl url.searchParams.append('k', keywords); let tab = await this.getCurrentTab(); if (!tab.url?.startsWith('http')) { + // A newly created tab may not have a URL in the returned Tab object yet, so never pass its + // transient `tab.url` to `new URL()`. Navigate by tab id and wait for the target explicitly. tab = await this.createNewTab('https://www.amazon.com/'); - tab.url = 'https://www.amazon.com/'; + await this.navigateTabAndWait(tab.id!, url.toString()); + return url.toString(); } const currentUrl = new URL(tab.url!); - if (currentUrl.hostname !== url.hostname || currentUrl.searchParams.get('k') !== keywords) { - tab = await browser.tabs.update(tab.id, { url: url.toString(), active: true }); - await new Promise((resolve) => setTimeout(resolve, 1000)); + const sameSearch = + currentUrl.hostname === url.hostname && currentUrl.searchParams.get('k') === keywords; + if (sameSearch) { + try { + // Preserve the old behavior: a healthy same-keyword page continues from its current page. + await new AmazonSearchPageInjector(tab).waitForPageLoaded(); + return currentUrl.toString(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (!message.includes('临时错误页')) throw error; + // Only a known Amazon transient error page is recovered by returning to page one. + } } + await this.navigateTabAndWait(tab.id!, url.toString()); return url.toString(); } @@ -84,22 +149,46 @@ class AmazonPageWorkerImpl public async wanderSearchPage(): Promise { const tab = await this.getCurrentTab(); let offset = 0; + let placementOffset = 0; + const adOffsets = new Map(); while (true) { - const { hasNextPage, data, page } = await this.wanderSearchSinglePage(tab); + const { nextPageUrl, data, page } = await this.wanderSearchSinglePage(tab); const keywords = new URL(tab.url!).searchParams.get('k')!; - const objs = data.map((r, i) => ({ - ...r, - keywords, - page, - rank: offset + 1 + i, - createTime: dayjs().format('YYYY/M/D HH:mm:ss'), - asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0], - })); + const pageAdCounts = new Map(); + const objs = data.map((r, i) => { + const localAdRank = r.adRank; + const localPlacementRank = r.placementRank; + const adRank = + r.adType && r.adType !== 'organic' && localAdRank + ? (adOffsets.get(r.adType) || 0) + localAdRank + : undefined; + if (r.adType && r.adType !== 'organic') { + pageAdCounts.set(r.adType, Math.max(pageAdCounts.get(r.adType) || 0, localAdRank || 0)); + } + const placementRank = localPlacementRank ? placementOffset + localPlacementRank : undefined; + return { + ...r, + keywords, + page, + rank: offset + 1 + i, + pageRank: r.pageRank || i + 1, + adRank, + placementRank, + placementId: placementRank + ? `${encodeURIComponent(keywords)}:${page}:${placementRank}` + : undefined, + createTime: dayjs().format('YYYY/M/D HH:mm:ss'), + }; + }); this.emit('item-links-collected', { objs }); offset += data.length; - if (!hasNextPage) { - break; - } + placementOffset += Math.max(0, ...data.map((item) => item.placementRank || 0)); + pageAdCounts.forEach((count, adType) => { + adOffsets.set(adType, (adOffsets.get(adType) || 0) + count); + }); + if (!nextPageUrl) break; + await this.navigateTabAndWait(tab.id!, nextPageUrl); + await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); } return new Promise((resolve) => setTimeout(resolve, 1000)); } @@ -113,8 +202,7 @@ class AmazonPageWorkerImpl const { image = false, aplus = false, extra = false } = options; const params = { asin: '', url: '' }; if (entry.match(/^https?:\/\/www\.amazon\.com.*\/dp\/[A-Z0-9]{10}/)) { - const [asin] = /\/\/dp\/[A-Z0-9]{10}/.exec(entry)!; - params.asin = asin; + params.asin = extractAmazonDetailAsin(entry) || ''; params.url = entry; } else if (entry.match(/^[A-Z0-9]{10}$/)) { params.asin = entry; @@ -124,9 +212,8 @@ class AmazonPageWorkerImpl if (!tab.url || isForbiddenUrl(tab.url)) { tab = await this.createNewTab(params.url); } else { - tab = await browser.tabs.update(tab.id, { - url: params.url, - }); + await this.navigateTabAndWait(tab.id!, params.url, { allowDetailRedirect: true }); + tab = await browser.tabs.get(tab.id!); } const injector = new AmazonDetailPageInjector(tab); @@ -142,6 +229,16 @@ class AmazonPageWorkerImpl //#region Await Production Introduction Element Loaded await injector.waitForPageLoaded(); await new Promise((resolve) => setTimeout(resolve, 3000)); // Wait 3 seconds. + tab = await browser.tabs.get(tab.id!); + const requestedAsin = params.asin; + const resolvedAsin = extractAmazonDetailAsin(tab.url); + if (!resolvedAsin) { + throw new Error(`Amazon 详情页未能识别实际 ASIN：${tab.url || '未知页面'}`); + } + params.asin = resolvedAsin; + if (requestedAsin !== resolvedAsin) { + await this.emit('item-detail-redirected', { requestedAsin, resolvedAsin }); + } //#endregion //#region Fetch Base Info const baseInfo = await injector.getBaseInfo(); @@ -183,11 +280,10 @@ class AmazonPageWorkerImpl //#region Fetch Goods' Images if (image) { const imageUrls = await injector.getImageUrls(); - imageUrls.length > 0 && - (await this.emit('item-images-collected', { - asin: params.asin, - imageUrls: Array.from(new Set(imageUrls)), - })); + await this.emit('item-images-collected', { + asin: params.asin, + imageUrls: Array.from(new Set(imageUrls)), + }); await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds. } //#endregion @@ -217,24 +313,72 @@ class AmazonPageWorkerImpl @withErrorHandling public async wanderReviewPage(asin: string, options: { recent?: boolean } = {}) { const { recent } = options; - const url = new URL( + const baseUrl = new URL( `https://www.amazon.com/product-reviews/${asin}/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews`, ); - recent && url.searchParams.set('sortBy', 'recent'); - const tab = await this.createNewTab(url.toString()); + recent && baseUrl.searchParams.set('sortBy', 'recent'); + const tab = await this.createNewTab(baseUrl.toString()); const injector = new AmazonReviewPageInjector(tab); - await injector.waitForPageLoad(); - for (let star = 1; star <= 5; star++) { - await injector.showStarsDropDownMenu(); - await injector.selectStar(star); + const runStage = async (stage: string, operation: () => Promise): Promise => { + try { + return await operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`评论页${stage}失败: ${message}`); + } + }; + await runStage('初始加载等待', () => injector.waitForPageLoad()); + const starFilters = [ + { name: 'one_star', star: 1 }, + { name: 'two_star', star: 2 }, + { name: 'three_star', star: 3 }, + { name: 'four_star', star: 4 }, + { name: 'five_star', star: 5 }, + ] as const; + const collectedReviewIds = new Set(); + + for (const { name: filterByStar, star } of starFilters) { + // The original collector gathered every rating bucket. Build the same Amazon filter URL + // directly instead of requiring a particular dropdown DOM structure just to discover it. + const starUrl = new URL(baseUrl); + starUrl.searchParams.set('filterByStar', filterByStar); + starUrl.searchParams.delete('pageNumber'); + await runStage(`${star} 星筛选导航`, () => + this.navigateTabAndWait(tab.id!, starUrl.toString()), + ); + const visitedPageUrls = new Set(); + let page = 1; + let filterIgnored = false; while (true) { - await injector.waitForPageLoad(); - const reviews = await injector.getSinglePageReviews(); - reviews.length > 0 && this.emit('item-review-collected', { asin, reviews }); - const hasNextPage = await injector.jumpToNextPageIfExist(); - if (!hasNextPage) { - break; + const currentTab = await browser.tabs.get(tab.id!); + if (currentTab.url) visitedPageUrls.add(currentTab.url); + // Match v0.7.1.12-beta semantics: collect the rating parsed from each review card. The + // requested filter is only a navigation aid and must not overwrite or reject real data. + await runStage(`${star} 星第 ${page} 页加载等待`, () => injector.waitForPageLoad()); + const reviews = await runStage('评论解析', () => injector.getSinglePageReviews()); + if (page === 1 && isAmazonReviewRatingFilterIgnored(reviews, star)) { + // Amazon currently experiments with review pages that preserve filterByStar in the URL + // but return the same unfiltered review cards. Keep the cards' real ratings, then stop + // revisiting the same eight reviews for the remaining rating buckets. + filterIgnored = true; } + const uniqueReviews = reviews.filter((review) => { + if (collectedReviewIds.has(review.id)) return false; + collectedReviewIds.add(review.id); + return true; + }); + uniqueReviews.length > 0 && + this.emit('item-review-collected', { asin, reviews: uniqueReviews }); + const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl()); + if (!nextPageUrl) break; + if (visitedPageUrls.has(nextPageUrl)) { + throw new Error(`评论页分页出现循环，已停止重复采集: ${nextPageUrl}`); + } + await runStage('分页导航', () => this.navigateTabAndWait(tab.id!, nextPageUrl)); + page += 1; + } + if (filterIgnored) { + break; } } setTimeout(() => browser.tabs.remove(tab.id!), 1000); @@ -254,7 +398,7 @@ class AmazonPageWorkerImpl const kw = remains.shift()!; await this.doSearch(kw); await this.wanderSearchPage(); - progress && progress(remains); + if (progress) await progress(remains); } unsubscribe(); } @@ -272,7 +416,10 @@ class AmazonPageWorkerImpl while (remains.length > 0 && !interrupt) { const asin = remains.shift()!; await this.wanderDetailPage(asin, options); - progress && progress(remains); + // The progress callback persists the completed item locally and enqueues + // it for server synchronization. Do not start the next ASIN until that + // handoff has completed, otherwise a batch can outrun its result writes. + if (progress) await progress(remains); } unsubscribe(); } @@ -290,7 +437,7 @@ class AmazonPageWorkerImpl while (remains.length > 0 && !interrupt) { const asin = remains.shift()!; await this.wanderReviewPage(asin, options); - progress && progress(remains); + if (progress) await progress(remains); } unsubscribe(); } diff --git a/src/page-worker/interfaces/amazon.ts b/src/page-worker/interfaces/amazon.ts index 9d88b14..5fb9a0e 100644 --- a/src/page-worker/interfaces/amazon.ts +++ b/src/page-worker/interfaces/amazon.ts @@ -4,6 +4,9 @@ export interface AmazonPageWorkerEvents { /** The event is fired when worker collected links to items on the Amazon search page. */ ['item-links-collected']: { objs: AmazonSearchItem[] }; + /** Amazon resolved a requested detail ASIN to a canonical or variation ASIN. */ + ['item-detail-redirected']: { requestedAsin: string; resolvedAsin: string }; + /** The event is fired when worker collected goods' base info on the Amazon detail page.*/ ['item-base-info-collected']: Pick< AmazonDetailItem, @@ -11,6 +14,7 @@ export interface AmazonPageWorkerEvents { | 'title' | 'boughtInfo' | 'price' + | 'strikethroughPrice' | 'rating' | 'ratingCount' | 'categories' @@ -41,7 +45,10 @@ export interface AmazonPageWorkerEvents { }; /** The event is fired when reviews collected in all review page */ - ['item-review-collected']: { asin: string; reviews: AmazonReview[] }; + ['item-review-collected']: { + asin: string; + reviews: AmazonReview[]; + }; /** Error event that occurs when there is an issue with the Amazon page worker*/ ['error']: { message: string; url?: string }; diff --git a/src/page-worker/interfaces/common.ts b/src/page-worker/interfaces/common.ts index 34de5ac..3c0b2e1 100644 --- a/src/page-worker/interfaces/common.ts +++ b/src/page-worker/interfaces/common.ts @@ -9,4 +9,5 @@ export interface taskOptionBase { export type WorkerComposable = Base & { settings: Ref; isRunning: Ref; + lastError?: Ref; }; diff --git a/src/page-worker/page-navigation.ts b/src/page-worker/page-navigation.ts new file mode 100644 index 0000000..ac8bc0a --- /dev/null +++ b/src/page-worker/page-navigation.ts @@ -0,0 +1,58 @@ +const navigationQueryKeys = ['k', 'page', 'pageNumber', 'filterByStar'] as const; + +function routeIdentity(url: URL) { + const reviewMatch = url.pathname.match(/^\/product-reviews\/([A-Z0-9]{10})/i); + if (reviewMatch) return `/product-reviews/${reviewMatch[1].toUpperCase()}`; + const detailMatch = url.pathname.match(/\/dp\/([A-Z0-9]{10})/i); + if (detailMatch) return `/dp/${detailMatch[1].toUpperCase()}`; + return url.pathname.replace(/\/ref=.*$/, '') || '/'; +} + +export function extractAmazonDetailAsin(url: string | undefined) { + if (!url) return null; + try { + return new URL(url).pathname.match(/\/dp\/([A-Z0-9]{10})/i)?.[1]?.toUpperCase() || null; + } catch { + return null; + } +} + +/** Accepts an Amazon canonical/variation redirect only when both URLs are detail pages. */ +export function hasReachedAmazonDetailPage( + currentUrl: string | undefined, + targetUrl: string, +) { + if (!currentUrl) return false; + try { + const current = new URL(currentUrl); + const target = new URL(targetUrl); + return ( + current.origin === target.origin && + Boolean(extractAmazonDetailAsin(currentUrl)) && + Boolean(extractAmazonDetailAsin(targetUrl)) + ); + } catch { + return false; + } +} + +/** + * Amazon may rewrite tracking parameters while navigating. Compare the stable route and the + * query parameters which identify the requested search/review page instead of the whole URL. + */ +export function hasReachedAmazonNavigationTarget(currentUrl: string | undefined, targetUrl: string) { + if (!currentUrl) return false; + try { + const current = new URL(currentUrl); + const target = new URL(targetUrl); + if (current.origin !== target.origin || routeIdentity(current) !== routeIdentity(target)) { + return false; + } + return navigationQueryKeys.every((key) => { + const expected = target.searchParams.get(key); + return expected === null || current.searchParams.get(key) === expected; + }); + } catch { + return false; + } +} diff --git a/src/page-worker/services/base.ts b/src/page-worker/services/base.ts index 0ead32c..9e487b8 100644 --- a/src/page-worker/services/base.ts +++ b/src/page-worker/services/base.ts @@ -1,6 +1,6 @@ import { createAlova } from 'alova'; import adapterFetch from 'alova/fetch'; -import { remoteHost } from '~/env'; +import { remoteBaseUrl } from '~/env'; import VueHook from 'alova/vue'; import { exportKey, generateKeyPair, signData } from '~/logic/crypto'; @@ -8,7 +8,7 @@ const keyPairPromise = generateKeyPair(); const publicKeyBase64Promise = keyPairPromise.then(async ({ publicKey }) => exportKey(publicKey)); const httpClient = createAlova({ - baseURL: `${__DEV__ ? 'http' : 'https'}://${remoteHost}`, + baseURL: remoteBaseUrl, requestAdapter: adapterFetch(), timeout: 10000, statesHook: VueHook, diff --git a/src/page-worker/web-injectors/amazon-detail-images.ts b/src/page-worker/web-injectors/amazon-detail-images.ts new file mode 100644 index 0000000..0b9541c --- /dev/null +++ b/src/page-worker/web-injectors/amazon-detail-images.ts @@ -0,0 +1,103 @@ +/** + * Collect product image URLs from an Amazon detail page. + * + * Keep this function self-contained: browser.scripting.executeScript serializes + * it before running it in the target tab, so it must not depend on closures. + */ +export async function collectAmazonDetailImageUrls(): Promise { + const delay = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + const isProductImage = (value: string) => + /^https:\/\/(?:m\.media-amazon\.com|images-na\.ssl-images-amazon\.com)\/images\/I\//.test( + value, + ); + const unique = (values: string[]) => + Array.from(new Set(values.map((value) => value.replaceAll('\\/', '/')))).filter( + isProductImage, + ); + + const highResolutionUrls: string[] = []; + const largeUrls: string[] = []; + + // Amazon has changed the surrounding P.when(...) wrapper several times. + // Scan every inline script for the same hiRes/large metadata used by the old parser. + for (const scriptNode of Array.from(document.scripts)) { + const script = (scriptNode.textContent || '').replaceAll('\\/', '/'); + for (const match of script.matchAll( + /"hiRes"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g, + )) { + highResolutionUrls.push(match[1]); + } + for (const match of script.matchAll( + /"large"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g, + )) { + largeUrls.push(match[1]); + } + } + + let urls = unique(highResolutionUrls.length > 0 ? highResolutionUrls : largeUrls); + + // Current detail pages expose the main high-resolution image directly. + if (urls.length === 0) { + const landingImage = document.querySelector('#landingImage'); + const oldHighResolutionUrl = landingImage?.getAttribute('data-old-hires'); + if (oldHighResolutionUrl) urls.push(oldHighResolutionUrl); + + const dynamicImages = landingImage?.getAttribute('data-a-dynamic-image'); + if (dynamicImages) { + try { + const candidates = Object.entries(JSON.parse(dynamicImages) as Record).sort( + ([, firstSize], [, secondSize]) => + (secondSize[0] || 0) * (secondSize[1] || 0) - + (firstSize[0] || 0) * (firstSize[1] || 0), + ); + if (candidates[0]?.[0]) urls.push(candidates[0][0]); + } catch { + // Continue to the legacy gallery fallback below. + } + } + urls = unique(urls); + } + + // Restore the proven v0.7.2 fallback (commit b062d96): open the image + // viewer and read each full-size image. Current selectors are included too. + if (urls.length === 0) { + const viewerTrigger = document.querySelector( + '.overlayRestOfImages, #canvasCaption > a', + ); + viewerTrigger?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + await delay(1000); + + const thumbnailNodes = Array.from( + document.querySelectorAll( + '.ivRow div.ivThumbImage, #ivThumbs .ivThumb, #ivThumbs img', + ), + ); + for (const thumbnailNode of thumbnailNodes) { + thumbnailNode.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + await delay(500); + + for (let retryCount = 0; retryCount < 5; retryCount++) { + const fullImage = document.querySelector('#ivLargeImage > img'); + const source = fullImage?.src; + if ( + source && + !source.includes('/ui/loadIndicators/loading-large_labeled') && + isProductImage(source) + ) { + urls.push(source); + break; + } + await delay(500); + } + } + urls = unique(urls); + } + + document + .querySelector( + 'header > [data-action="a-popover-close"], body > div.a-modal-scroller.a-declarative', + ) + ?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + return urls; +} diff --git a/src/page-worker/web-injectors/amazon-detail-page-ready.ts b/src/page-worker/web-injectors/amazon-detail-page-ready.ts new file mode 100644 index 0000000..b7a0ef8 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-detail-page-ready.ts @@ -0,0 +1,32 @@ +/** Wait until the fields required by the detail parser are available. */ +export async function waitForAmazonDetailPageReady() { + const deadline = Date.now() + 20000; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 详情页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 详情页触发了验证码'); + } + + const unavailable = document.querySelector( + '#availability .a-color-price, #outOfStock, #exports_desktop_qualifiedBuybox', + ); + const title = document.querySelector('#title, #productTitle'); + const titleReady = Boolean(title?.textContent?.trim()); + if (document.readyState !== 'loading' && titleReady) { + title?.scrollIntoView({ behavior: 'auto', block: 'center' }); + return true; + } + + if (document.readyState !== 'loading' && unavailable) { + throw new Error('Amazon 商品当前不可用，详情标题未加载'); + } + await sleep(300); + } + + throw new Error('Amazon 详情标题在 20 秒内未加载完成'); +} diff --git a/src/page-worker/web-injectors/amazon-review-page.ts b/src/page-worker/web-injectors/amazon-review-page.ts new file mode 100644 index 0000000..dfaeacb --- /dev/null +++ b/src/page-worker/web-injectors/amazon-review-page.ts @@ -0,0 +1,73 @@ +type ReviewReadyOptions = { timeoutMs?: number }; + +/** Detects Amazon pages that keep filterByStar in the URL but return other ratings. */ +export function isAmazonReviewRatingFilterIgnored( + reviews: Pick[], + expectedStar: number, +) { + return ( + reviews.length > 0 && + reviews.some((review) => Number.parseInt(review.rating, 10) !== expectedStar) + ); +} + +/** Waits until an Amazon review page, including its requested star filter, is ready. */ +export async function waitForAmazonReviewPageReady(options: ReviewReadyOptions = {}) { + const { timeoutMs = 20000 } = options; + const deadline = Date.now() + timeoutMs; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 评论页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 评论页触发了验证码'); + } + + const reviewList = document.querySelector('#cm_cr-review_list'); + const reviewsContent = document.querySelector('.reviews-content'); + const loadingNode = document.querySelector('.reviews-loading'); + const contentReady = document.readyState !== 'loading' && Boolean(reviewList || reviewsContent); + const loadingFinished = !loadingNode || loadingNode.getClientRects().length === 0; + if (contentReady && loadingFinished) { + (reviewList || reviewsContent)?.scrollIntoView({ behavior: 'auto', block: 'center' }); + return; + } + await sleep(300); + } + throw new Error(`Amazon 评论内容在 ${Math.ceil(timeoutMs / 1000)} 秒内未加载完成`); +} + +/** Reads the next review-page URL without navigating from the injected script. */ +export async function collectAmazonNextReviewPageUrl() { + // Amazon may not render the pagination controls until the last review has entered the + // viewport. The original collector deliberately scrolled here; keep that behavior while + // returning the URL instead of clicking it inside the injected script (which can close the + // extension port when Chrome moves the old document into BFCache). + const reviewCards = Array.from( + document.querySelectorAll( + '#cm_cr-review_list [data-hook="review"], #cm_cr-review_list div[id*="review-card"]', + ), + ); + reviewCards.at(-1)?.scrollIntoView({ behavior: 'auto', block: 'center' }); + await new Promise((resolve) => setTimeout(resolve, 1000)); + + const pagination = + document.querySelector('[data-hook="pagination-bar"] .a-pagination') || + document.querySelector('[data-hook="pagination-bar"]') || + document.querySelector('ul.a-pagination'); + if (!pagination) return null; + + const nextItem = + pagination.querySelector('li.a-last') || + pagination.querySelector(':scope > *:nth-last-child(1)'); + if (!nextItem || nextItem.classList.contains('a-disabled')) return null; + + const nextAnchor = + nextItem.matches('a[href]') + ? (nextItem as HTMLAnchorElement) + : nextItem.querySelector('a[href]'); + const href = nextAnchor?.getAttribute('href'); + return href ? new URL(href, window.location.href).toString() : null; +} diff --git a/src/page-worker/web-injectors/amazon-search-page-ready.ts b/src/page-worker/web-injectors/amazon-search-page-ready.ts new file mode 100644 index 0000000..e322c88 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-page-ready.ts @@ -0,0 +1,107 @@ +/** + * Waits for the useful search-page DOM rather than every spinner on the page. + * Amazon ad, carousel and recommendation widgets can keep a visible `.a-spinner` + * indefinitely even after the search results are ready to collect. + */ +export async function waitForAmazonSearchPageReady() { + const deadline = Date.now() + 20000; + const sleep = (milliseconds: number) => + new Promise((resolve) => setTimeout(resolve, milliseconds)); + let nextButton: HTMLElement | null = null; + let pageBecameReady = false; + + while (Date.now() < deadline) { + if (window.location.pathname.startsWith('/ap/signin')) { + throw new Error('Amazon 搜索页需要重新登录'); + } + if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) { + throw new Error('Amazon 搜索页触发了验证码'); + } + const bodyText = document.body?.innerText || document.body?.textContent || ''; + if ( + /sorry[,.\s]+something went wrong on our end/i.test(bodyText) || + /please go back and try again/i.test(bodyText) + ) { + throw new Error('Amazon 返回临时错误页，可能触发了限流或风控，请稍后重试'); + } + + const resultCount = document.querySelectorAll( + '[data-component-type="s-search-result"][data-asin], .puis-card-container', + ).length; + nextButton = document.querySelector('.s-pagination-next'); + const noResults = /no results for/i.test(bodyText); + const pageReady = + document.readyState !== 'loading' && (resultCount > 0 || nextButton !== null || noResults); + + if (pageReady) { + pageBecameReady = true; + break; + } + + window.scrollBy(0, 500 + ~~(Math.random() * 500)); + await sleep(300); + } + + if (!pageBecameReady) { + throw new Error('Amazon 搜索结果在 20 秒内未加载完成'); + } + + // Visit the full result stream instead of jumping straight to pagination. Amazon loads + // recommendation and sponsored modules when their region approaches the viewport. + for (const fraction of [0.15, 0.35, 0.55, 0.75, 0.92]) { + const documentHeight = Math.max( + document.documentElement.scrollHeight, + document.body?.scrollHeight || 0, + document.documentElement.offsetHeight, + document.body?.offsetHeight || 0, + ); + const top = Math.max(0, (documentHeight - window.innerHeight) * fraction); + window.scrollTo({ top, behavior: 'auto' }); + await sleep(180); + } + nextButton?.scrollIntoView({ behavior: 'auto', block: 'center' }); + + const relevantDomFingerprint = () => { + const root = document.querySelector('#search') || document.body; + if (!root) return ''; + const asins = Array.from(root.querySelectorAll('[data-asin]')) + .map((node) => node.dataset.asin || '') + .filter((asin) => /^[A-Z0-9]{10}$/.test(asin)); + const productLinks = Array.from( + root.querySelectorAll( + 'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', + ), + ).map((anchor) => anchor.href); + const sponsoredLabels = root.querySelectorAll( + '[aria-label*="Sponsored" i], [title*="Sponsored" i]', + ).length; + const cards = root.querySelectorAll( + '[data-component-type="s-search-result"], .puis-card-container, .puisg-row, [class*="shop-by-type"]', + ).length; + return [ + Array.from(new Set(asins)).sort().join(','), + Array.from(new Set(productLinks)).sort().join(','), + sponsoredLabels, + cards, + root.querySelectorAll('video').length, + root.querySelectorAll('img[src]').length, + ].join('|'); + }; + + const stabilityStartedAt = Date.now(); + const minimumStabilityWait = 1500; + const stableForRequired = 1000; + const maximumStabilityWait = 3500; + let fingerprint = relevantDomFingerprint(); + let stableSince = Date.now(); + while (Date.now() - stabilityStartedAt < maximumStabilityWait) { + await sleep(250); + const nextFingerprint = relevantDomFingerprint(); + if (nextFingerprint !== fingerprint) { + fingerprint = nextFingerprint; + stableSince = Date.now(); + } + const elapsed = Date.now() - stabilityStartedAt; + if (elapsed >= minimumStabilityWait && Date.now() - stableSince >= stableForRequired) return; + } +} diff --git a/src/page-worker/web-injectors/amazon-search-pagination.ts b/src/page-worker/web-injectors/amazon-search-pagination.ts new file mode 100644 index 0000000..eab3c92 --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-pagination.ts @@ -0,0 +1,64 @@ +/** + * Resolves the page represented by the currently rendered search results. + * Amazon can update the URL before replacing the visible result DOM, so the + * rendered selected-page control is authoritative and the URL is only a fallback. + */ +export async function collectAmazonCurrentPage(): Promise { + const currentUrl = new URL(window.location.href); + const parsePage = (value: string | null | undefined) => { + const page = Number.parseInt(value || '', 10); + return Number.isInteger(page) && page > 0 ? page : null; + }; + + const isRendered = (node: HTMLElement) => { + for (let current: HTMLElement | null = node; current; current = current.parentElement) { + if (current.hidden || current.getAttribute('aria-hidden') === 'true') return false; + const style = current.ownerDocument.defaultView?.getComputedStyle(current); + if ( + style && + (style.display === 'none' || + style.visibility === 'hidden' || + style.visibility === 'collapse' || + style.opacity === '0') + ) { + return false; + } + } + if (document.documentElement.getClientRects().length === 0) return true; + return Array.from(node.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0); + }; + + const renderedSelectedPages = Array.from( + document.querySelectorAll('.s-pagination-item.s-pagination-selected'), + ) + .filter(isRendered) + .map((node) => parsePage(node.textContent?.trim())) + .filter((page): page is number => page !== null); + if (renderedSelectedPages.length > 0) return renderedSelectedPages[0]; + + const urlPage = parsePage(currentUrl.searchParams.get('page')); + if (urlPage) return urlPage; + + const refPage = parsePage( + /(?:^|_)pg_(\d+)(?:_|$)/.exec(currentUrl.searchParams.get('ref') || '')?.[1], + ); + if (refPage) return refPage; + + const anySelected = document.querySelector( + '.s-pagination-item.s-pagination-selected', + ); + return parsePage(anySelected?.textContent?.trim()) || 1; +} + +/** + * Reads the next search-page URL without navigating from the injected script. + * Navigation must happen only after browser.scripting.executeScript has returned, + * otherwise Amazon can move the frame into BFCache and destroy the script context. + */ +export async function collectAmazonNextPageUrl() { + const nextButton = document.querySelector('.s-pagination-next'); + if (!nextButton || nextButton.classList.contains('s-pagination-disabled')) return null; + + const href = nextButton.getAttribute('href'); + return href ? new URL(href, window.location.href).toString() : null; +} diff --git a/src/page-worker/web-injectors/amazon-search-parser.ts b/src/page-worker/web-injectors/amazon-search-parser.ts new file mode 100644 index 0000000..8ea067c --- /dev/null +++ b/src/page-worker/web-injectors/amazon-search-parser.ts @@ -0,0 +1,685 @@ +export type AmazonSearchPagePattern = 'pattern-1' | 'pattern-2'; + +type SearchPageCollectorPayload = { + pattern: AmazonSearchPagePattern; +}; + +/** + * Collects product exposures from an Amazon search page. + * + * Keep this function self-contained: browser.scripting.executeScript serializes + * the function and cannot provide imported closures in the page context. + */ +export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) { + type CollectedItem = Pick< + AmazonSearchItem, + | 'asin' + | 'link' + | 'title' + | 'imageSrc' + | 'price' + | 'strikethroughPrice' + | 'adType' + | 'adFormat' + | 'adRank' + | 'pageRank' + | 'placement' + | 'placementRank' + | 'placementId' + | 'adProductIndex' + >; + + const text = (node: Element | null | undefined) => + (node?.textContent || '').replace(/\s+/g, ' ').trim(); + + const isVisible = (node: HTMLElement) => { + for (let current: HTMLElement | null = node; current; current = current.parentElement) { + if (current.hidden || current.getAttribute('aria-hidden') === 'true') return false; + const style = current.ownerDocument.defaultView?.getComputedStyle(current); + if ( + style && + (style.display === 'none' || + style.visibility === 'hidden' || + style.visibility === 'collapse' || + style.opacity === '0' || + style.getPropertyValue('content-visibility') === 'hidden') + ) { + return false; + } + } + + // JSDOM has no layout engine, so computed-style checks above are the authoritative test path. + // In Chromium, require the card or one of its key visual descendants to own a real render box. + if (node.ownerDocument.documentElement.getClientRects().length === 0) return true; + return [node, ...Array.from(node.querySelectorAll('img, h2, h3'))].some( + (candidate) => + Array.from(candidate.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0), + ); + }; + + const isVisuallyHiddenText = (node: HTMLElement) => { + if (!isVisible(node)) return true; + const style = node.ownerDocument.defaultView?.getComputedStyle(node); + if (!style) return false; + + const width = Number.parseFloat(style.width); + const height = Number.parseFloat(style.height); + const screenReaderOnlyBox = + (style.position === 'absolute' || style.position === 'fixed') && + style.overflow === 'hidden' && + Number.isFinite(width) && + Number.isFinite(height) && + width <= 1 && + height <= 1; + const zeroClip = /rect\(\s*0(?:px)?[, ]+\s*0(?:px)?[, ]+\s*0(?:px)?[, ]+\s*0(?:px)?\s*\)/i.test( + style.clip || '', + ); + const insetClip = /inset\(\s*(?:50%|100%)/i.test(style.clipPath || ''); + return screenReaderOnlyBox || zeroClip || insetClip; + }; + + const decodeRepeatedly = (value: string) => { + let decoded = value; + for (let i = 0; i < 3; i++) { + try { + const next = decodeURIComponent(decoded); + if (next === decoded) break; + decoded = next; + } catch { + break; + } + } + return decoded; + }; + + const parseAsinFromText = (value: string) => { + const decoded = decodeRepeatedly(value); + const patterns = [ + /\/(?:dp|gp\/product|gp\/aw\/d)\/([A-Z0-9]{10})(?:[/?]|$)/i, + /[?&#]asin=([A-Z0-9]{10})(?:[&]|$)/i, + /[?&#]lp_asins=([A-Z0-9]{10})/i, + /\/dp\/([A-Z0-9]{10})/i, + ]; + for (const pattern of patterns) { + const match = pattern.exec(decoded); + if (match) return match[1].toUpperCase(); + } + return undefined; + }; + + const parseAsin = (node: HTMLElement, href?: string) => { + const dataAsin = node.closest('[data-asin]')?.dataset.asin || node.dataset.asin; + if (dataAsin && /^[A-Z0-9]{10}$/.test(dataAsin)) return dataAsin; + + const candidates = [href || '', decodeRepeatedly(href || '')]; + for (const candidate of candidates) { + const asin = parseAsinFromText(candidate); + if (asin) return asin; + } + return undefined; + }; + + const sponsoredDisclosureCandidates = (node: Element) => + [ + ...(node.matches('[aria-label], [title], span') ? [node] : []), + ...Array.from(node.querySelectorAll('[aria-label], [title], span')), + ].filter((candidate): candidate is HTMLElement => candidate instanceof HTMLElement); + + const isSponsoredDisclosure = (candidate: HTMLElement) => { + if (isVisuallyHiddenText(candidate)) return false; + const accessibleLabels = [ + candidate.getAttribute('aria-label'), + candidate.getAttribute('title'), + ]; + if (accessibleLabels.some((value) => /(?:^|\b)Sponsored(?:\b|\s|$)|赞助/i.test(value || ''))) { + return true; + } + return candidate.matches('span') && /^(Sponsored|赞助)(?:\b|\s|$)/i.test(text(candidate)); + }; + + const hasSponsoredDisclosure = (node: Element) => + sponsoredDisclosureCandidates(node).some(isSponsoredDisclosure); + + const closestAsinHost = (node: Element) => { + const host = node.closest('[data-asin]'); + return host && /^[A-Z0-9]{10}$/.test(host.dataset.asin || '') ? host : null; + }; + + const hasModuleSponsoredDisclosure = (node: Element) => + sponsoredDisclosureCandidates(node).some( + (candidate) => !closestAsinHost(candidate) && isSponsoredDisclosure(candidate), + ); + + const metadata = (node: Element) => + [ + node.getAttribute('data-component-type'), + node.getAttribute('data-cel-widget'), + node.getAttribute('data-ad-details'), + node.id, + node.className, + ] + .filter((value) => typeof value === 'string') + .join(' ') + .toLowerCase(); + + const isMainResultsBoundary = (node: Element) => + node.id === 'search' || + node.matches('.s-main-slot, [data-component-type="s-search-results"], body'); + + const uniqueAsinsIn = (node: Element) => { + const asins = new Set(); + node.querySelectorAll('[data-asin]').forEach((candidate) => { + const asin = candidate.dataset.asin; + if (asin && /^[A-Z0-9]{10}$/.test(asin)) asins.add(asin); + }); + node.querySelectorAll('a[href]').forEach((anchor) => { + const asin = parseAsinFromText(anchor.href); + if (asin) asins.add(asin); + }); + return asins.size; + }; + + const hasExplicitAdMetadata = (node: Element) => + /sponsored.?brand|brand.?ad|display.?ad|sponsored.?display|adholder/.test(metadata(node)); + + const hasVisibleElement = (node: Element, selector: string) => + Array.from(node.querySelectorAll(selector)).some(isVisible); + + const isSearchResultCard = (node: Element) => + node.matches('[data-component-type="s-search-result"]'); + + const hasModuleHeadline = (node: Element) => + Array.from( + node.querySelectorAll( + 'h1, h2, h3, [class*="headline"], [class*="sb-headline"], [class*="sbv-"]', + ), + ).some( + (heading) => + !heading.closest('[data-asin]') && isVisible(heading) && text(heading).length > 0, + ); + + const isOutsideProductCard = (node: Element, root: Element) => { + if (node.closest('[data-component-type="s-search-result"]')) return false; + const asinHost = node.closest('[data-asin]'); + return !asinHost || asinHost === root; + }; + + const hasModuleLevelVideo = (node: Element) => + Array.from(node.querySelectorAll('video')).some((video) => isOutsideProductCard(video, node)); + + const hasModuleLevelBrandVisual = (node: Element) => { + if (hasModuleLevelVideo(node)) return true; + const playPause = Array.from( + node.querySelectorAll( + '[aria-label*="Pause" i], [aria-label*="Play" i], [title*="Pause" i], [title*="Play" i]', + ), + ).some((candidate) => isOutsideProductCard(candidate, node) && isVisible(candidate)); + if (playPause) return true; + const outsideImages = Array.from(node.querySelectorAll('img')).filter( + (image) => isOutsideProductCard(image, node) && isVisible(image), + ); + return outsideImages.length >= 2; + }; + + const hasInfluencerThematicCopy = (node: Element) => + Array.from(node.querySelectorAll('h1, h2, h3, span, a, [class*="headline"]')).some( + (candidate) => { + if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false; + return /picks from amazon influencers|influencer storefronts|amazon influencer/i.test( + text(candidate), + ); + }, + ); + + const hasShopByTypeCopy = (node: Element) => + Array.from(node.querySelectorAll('h1, h2, h3, span, a, [class*="headline"]')).some( + (candidate) => { + if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false; + return /\bshop\b.+\bby type\b/i.test(text(candidate)); + }, + ); + + const thematicMetadata = (node: Element) => + [ + metadata(node), + node.getAttribute('cel_widget_id') || '', + node.getAttribute('data-multi-ad-feedback-form-trigger') || '', + node.getAttribute('data-component-props') || '', + ] + .join(' ') + .toLowerCase(); + + const hasDirectThematicSpSignal = (node: Element) => { + const blob = thematicMetadata(node); + if (/brand-footer|multi-brand-creative|brands-related|desktop-hsa-3psl/.test(blob)) { + return false; + } + return /featured.?asins.?list|sponsored.?products|sp_search_thematic|loom-desktop|influencer|shop.?by.?type|shop_by_type/.test( + blob, + ); + }; + + const hasThematicSpSignal = (node: Element) => { + if (hasDirectThematicSpSignal(node)) return true; + return hasInfluencerThematicCopy(node) || hasShopByTypeCopy(node); + }; + + const isSponsoredProductsThematic = (node: Element) => { + if (hasThematicSpSignal(node)) return true; + if ( + Array.from( + node.querySelectorAll( + '[cel_widget_id], [data-multi-ad-feedback-form-trigger], [id*="loom"], [class*="loom"]', + ), + ).some(hasThematicSpSignal) + ) { + return true; + } + for ( + let current = node.parentElement; + current && !isMainResultsBoundary(current); + current = current.parentElement + ) { + if (hasThematicSpSignal(current)) return true; + } + return false; + }; + + const hasVisibleBrandCreative = (node: Element) => { + if (isSearchResultCard(node)) return false; + if (isSponsoredProductsThematic(node)) return false; + + const visibleBrandDestination = hasVisibleElement( + node, + 'a[href*="/stores/"], a[href*="/brands/"], [class*="brand-logo"], [class*="brandLogo"]', + ); + const explicitBrandModule = /sponsored.?brand|brand.?ad|brands-related|brand-card|\bsbv-/.test( + metadata(node), + ); + const brandsRelatedCopy = /brands related to your search/i.test(text(node)); + const moduleSponsored = hasModuleSponsoredDisclosure(node) || hasSponsoredDisclosure(node); + const videoStrip = hasModuleLevelBrandVisual(node) && moduleSponsored; + const headlineUnit = hasModuleHeadline(node) && moduleSponsored; + return ( + visibleBrandDestination || + explicitBrandModule || + brandsRelatedCopy || + videoStrip || + headlineUnit + ); + }; + + const findAdModule = (card: HTMLElement) => { + let current: HTMLElement | null = card; + let singleProductModule: HTMLElement | undefined; + for (; current; current = current.parentElement) { + if (isMainResultsBoundary(current)) break; + const explicitModule = hasExplicitAdMetadata(current); + const moduleDisclosure = current !== card && hasModuleSponsoredDisclosure(current); + const brandedSponsoredModule = moduleDisclosure && hasVisibleBrandCreative(current); + const sharedSponsoredProductsModule = moduleDisclosure && uniqueAsinsIn(current) > 1; + const singleProductSponsored = moduleDisclosure && uniqueAsinsIn(current) === 1; + if (explicitModule || brandedSponsoredModule || sharedSponsoredProductsModule) return current; + if (singleProductSponsored && !singleProductModule) singleProductModule = current; + } + return singleProductModule || card; + }; + + const classifyAd = (card: HTMLElement, module: HTMLElement) => { + const cardMeta = metadata(card); + const moduleMeta = metadata(module); + const combinedMeta = `${cardMeta} ${moduleMeta}`; + const sidebar = Boolean( + card.closest( + 'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"]', + ), + ); + const sponsored = + hasSponsoredDisclosure(card) || (module !== card && hasModuleSponsoredDisclosure(module)); + + if (!sponsored) { + return { + adType: 'organic' as const, + placement: undefined, + adFormat: undefined, + }; + } + if (sidebar || /display.?ad|sponsored.?display/.test(combinedMeta)) { + return { + adType: 'sponsoredDisplay' as const, + placement: 'sidebar' as const, + adFormat: 'standard' as const, + }; + } + if (isSearchResultCard(card) || isSponsoredProductsThematic(module)) { + return { + adType: 'sponsoredProducts' as const, + placement: 'middleOfSearch' as const, + adFormat: hasVisibleElement(card, 'video') ? ('video' as const) : ('standard' as const), + }; + } + if (hasVisibleBrandCreative(module)) { + return { + adType: 'sponsoredBrands' as const, + placement: 'middleOfSearch' as const, + adFormat: hasModuleLevelVideo(module) ? ('video' as const) : ('standard' as const), + }; + } + return { + adType: 'sponsoredProducts' as const, + placement: 'middleOfSearch' as const, + adFormat: hasVisibleElement(module, 'video') ? ('video' as const) : ('standard' as const), + }; + }; + + const hasCollectableProductLink = (node: Element) => + Boolean( + node.querySelector( + 'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', + ), + ); + + const addCandidate = (node: HTMLElement) => { + if ( + isVisible(node) && + (node.dataset.asin || + hasCollectableProductLink(node) || + parseAsin(node, node.querySelector('a[href]')?.href)) + ) { + candidateSet.add(node); + } + }; + + const findProductCardFromAnchor = (anchor: HTMLAnchorElement, module: HTMLElement) => { + const asinHost = closestAsinHost(anchor); + if (asinHost && module.contains(asinHost) && asinHost !== module) return asinHost; + + let current: HTMLElement | null = anchor.parentElement; + while (current && current !== module) { + if (current.querySelector('img') && current.querySelector('h2, h3')) return current; + current = current.parentElement; + } + return anchor.parentElement instanceof HTMLElement && anchor.parentElement !== module + ? anchor.parentElement + : null; + }; + + const candidateSet = new Set(); + document + .querySelectorAll( + '[data-component-type="s-search-result"][data-asin], .puis-card-container, .puisg-row', + ) + .forEach((node) => { + const stableCard = + node.closest('[data-component-type="s-search-result"]') || node; + addCandidate(stableCard); + }); + + document.querySelectorAll('[data-asin]').forEach((node) => { + if ( + isVisible(node) && + /^[A-Z0-9]{10}$/.test(node.dataset.asin || '') && + node.querySelector('h2, h3, img') + ) { + const stableCard = + node.closest('[data-component-type="s-search-result"]') || node; + candidateSet.add(stableCard); + } + }); + + document + .querySelectorAll( + 'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"], [data-component-type*="sponsored-display"]', + ) + .forEach((sidebarAd) => { + if ( + isVisible(sidebarAd) && + hasSponsoredDisclosure(sidebarAd) && + sidebarAd.querySelector('h2, h3') && + sidebarAd.querySelector('img[src], img[data-src]') && + hasCollectableProductLink(sidebarAd) + ) { + candidateSet.add(sidebarAd); + } + }); + + document + .querySelectorAll( + '.AdHolder, [class*="brands-related"], [class*="brand-card"], [data-component-type="sponsored-brand"]', + ) + .forEach((module) => { + if (!isVisible(module)) return; + const nestedCards = module.querySelectorAll( + '[data-asin], [class*="brand-card"], .sbv-product, [data-slot*="3psl"], [data-slot*="hsa"]', + ); + if (nestedCards.length > 0) { + nestedCards.forEach(addCandidate); + return; + } + addCandidate(module); + }); + + const collectThematicModuleCards = (module: HTMLElement) => { + module.querySelectorAll('[data-asin]').forEach(addCandidate); + module.querySelectorAll('a[href]').forEach((anchor) => { + const href = anchor.getAttribute('href') || anchor.href; + if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return; + const card = findProductCardFromAnchor(anchor, module); + if (card) addCandidate(card); + }); + }; + + /** + * Amazon frequently changes the wrapper/template names used for ads. Discover a + * sponsored unit from its visible disclosure first, then find the closest + * ancestor that actually owns one or more product links. This keeps template + * names and marketing headings out of the SP/SB/SD classification contract. + */ + const findSponsoredProductScope = (disclosure: HTMLElement) => { + for ( + let current = disclosure.parentElement; + current && !isMainResultsBoundary(current); + current = current.parentElement + ) { + if ( + isVisible(current) && + (hasExplicitAdMetadata(current) || hasDirectThematicSpSignal(current)) && + hasCollectableProductLink(current) && + current.querySelector('img[src], img[data-src], img[srcset]') + ) { + return current; + } + } + + const searchResult = disclosure.closest('[data-component-type="s-search-result"]'); + if (searchResult) return searchResult; + + const asinHost = closestAsinHost(disclosure); + if (asinHost) return asinHost; + + for ( + let current = disclosure.parentElement; + current && !isMainResultsBoundary(current); + current = current.parentElement + ) { + if ( + isVisible(current) && + hasCollectableProductLink(current) && + current.querySelector('img[src], img[data-src], img[srcset]') + ) { + return current; + } + } + return null; + }; + + sponsoredDisclosureCandidates(document.body).forEach((disclosure) => { + if (!isSponsoredDisclosure(disclosure)) return; + const scope = findSponsoredProductScope(disclosure); + if (!scope) return; + collectThematicModuleCards(scope); + addCandidate(scope); + }); + + document + .querySelectorAll( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"], [cel_widget_id*="SHOP_BY_TYPE" i], [cel_widget_id*="shop-by-type" i]', + ) + .forEach((module) => { + if (!isVisible(module) || !isSponsoredProductsThematic(module)) return; + collectThematicModuleCards(module); + }); + + document.querySelectorAll('h1, h2, h3, [class*="headline"]').forEach((heading) => { + if (closestAsinHost(heading) || !/\bshop\b.+\bby type\b/i.test(text(heading))) return; + const module = + heading.closest( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], section', + ) || heading.parentElement; + if (module instanceof HTMLElement && isVisible(module)) collectThematicModuleCards(module); + }); + + document + .querySelectorAll( + 'a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', + ) + .forEach((anchor) => { + if (!isVisible(anchor)) return; + const href = anchor.getAttribute('href') || anchor.href; + if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return; + const module = + anchor.closest( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"]', + ) || document.body; + const card = findProductCardFromAnchor(anchor, module); + if (card) addCandidate(card); + }); + + const cards = Array.from(candidateSet) + .filter( + (card, index, all) => + !all.some( + (other, otherIndex) => + index !== otherIndex && + card.contains(other) && + parseAsin(card, card.querySelector('a[href]')?.href) === + parseAsin(other, other.querySelector('a[href]')?.href), + ), + ) + .sort((left, right) => { + if (left === right) return 0; + return left.compareDocumentPosition(right) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1; + }); + + const organicCandidates = cards.filter((card) => { + const module = findAdModule(card); + return classifyAd(card, module).adType === 'organic'; + }); + const firstOrganicCandidate = organicCandidates[0]; + const lastOrganicCandidate = organicCandidates[organicCandidates.length - 1]; + const placementRanks = new Map(); + const adRanks = new Map(); + const moduleProductIndexes = new Map(); + const moduleAsins = new Map>(); + let nextPlacementRank = 1; + + return cards.reduce((items, card) => { + const anchors = Array.from(card.querySelectorAll('a[href]')); + const productAnchor = anchors.find((anchor) => parseAsin(card, anchor.href)); + const asin = parseAsin(card, productAnchor?.href); + if (!asin) return items; + + const titleNode = + card.querySelector('h2[aria-label], h2') || + card.querySelector('h3[aria-label], h3'); + const imageNode = card.querySelector( + 'img.s-image, img[src], img[data-src], img[srcset]', + ); + const title = + titleNode?.getAttribute('aria-label')?.trim() || + text(titleNode) || + imageNode?.alt?.trim() || + asin; + const dataSrc = imageNode?.getAttribute('data-src') || imageNode?.dataset.src; + const srcsetUrl = (imageNode?.getAttribute('srcset') || '') + .split(',')[0] + ?.trim() + .split(/\s+/)[0]; + const rawSrc = imageNode?.src && !imageNode.src.startsWith('data:') ? imageNode.src : undefined; + const placeholderSrc = /grey-pixel|transparent-pixel|\/1x1(?:\.|$)/i.test(rawSrc || ''); + const imageSrc = (placeholderSrc ? undefined : rawSrc) || dataSrc || srcsetUrl; + if (!imageSrc) return items; + + const currentPriceNode = card.querySelector( + '.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen', + ); + const strikethroughPriceNode = card.querySelector( + '[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString', + ); + const price = text(currentPriceNode) || undefined; + const rawStrikethroughPrice = text(strikethroughPriceNode) || undefined; + const strikethroughPrice = + rawStrikethroughPrice && rawStrikethroughPrice !== price ? rawStrikethroughPrice : undefined; + + const module = findAdModule(card); + const classification = classifyAd(card, module); + if (classification.adType !== 'organic') { + const seenAsins = moduleAsins.get(module) || new Set(); + if (seenAsins.has(asin)) return items; + seenAsins.add(asin); + moduleAsins.set(module, seenAsins); + } + let placementRank: number | undefined; + let placementId: string | undefined; + let adRank: number | undefined; + let adProductIndex: number | undefined; + let placement: AmazonAdPlacement | undefined = classification.placement; + + if (classification.adType !== 'organic') { + if (!placementRanks.has(module)) placementRanks.set(module, nextPlacementRank++); + placementRank = placementRanks.get(module)!; + placementId = `search-page-placement-${placementRank}`; + adRank = (adRanks.get(classification.adType) || 0) + 1; + adRanks.set(classification.adType, adRank); + adProductIndex = (moduleProductIndexes.get(module) || 0) + 1; + moduleProductIndexes.set(module, adProductIndex); + + if (placement !== 'sidebar') { + const beforeFirstOrganic = + firstOrganicCandidate && + Boolean( + module.compareDocumentPosition(firstOrganicCandidate) & + Node.DOCUMENT_POSITION_FOLLOWING, + ); + const afterLastOrganic = + lastOrganicCandidate && + Boolean( + module.compareDocumentPosition(lastOrganicCandidate) & Node.DOCUMENT_POSITION_PRECEDING, + ); + placement = beforeFirstOrganic + ? 'topOfSearch' + : afterLastOrganic + ? 'bottomOfSearch' + : 'middleOfSearch'; + } + } + + items.push({ + asin, + link: `https://www.amazon.com/dp/${asin}`, + title, + imageSrc, + price, + strikethroughPrice, + adType: classification.adType, + adFormat: classification.adFormat, + adRank, + pageRank: items.length + 1, + placement, + placementRank, + placementId, + adProductIndex, + }); + return items; + }, []); +} diff --git a/src/page-worker/web-injectors/amazon.ts b/src/page-worker/web-injectors/amazon.ts index b28e12a..a50322c 100644 --- a/src/page-worker/web-injectors/amazon.ts +++ b/src/page-worker/web-injectors/amazon.ts @@ -1,182 +1,15 @@ import { BaseInjector } from './base'; +import { collectAmazonSearchPageData, type AmazonSearchPagePattern } from './amazon-search-parser'; +import { collectAmazonDetailImageUrls } from './amazon-detail-images'; +import { waitForAmazonDetailPageReady } from './amazon-detail-page-ready'; +import { collectAmazonCurrentPage, collectAmazonNextPageUrl } from './amazon-search-pagination'; +import { waitForAmazonSearchPageReady } from './amazon-search-page-ready'; +import { collectAmazonNextReviewPageUrl, waitForAmazonReviewPageReady } from './amazon-review-page'; export class AmazonSearchPageInjector extends BaseInjector { public waitForPageLoaded() { - return this.run(async () => { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - while (true) { - const targetNode = document.querySelector('.s-pagination-next'); - - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500)); - - const h = Math.max( - document.documentElement.scrollHeight, - document.body.scrollHeight, - document.documentElement.offsetHeight, - document.body.offsetHeight, - document.documentElement.clientHeight, - ); - - // await modernScrollTo({ top: h * (0.4 + Math.random() * 0.2), behavior: 'smooth' }); - - if (targetNode || document.readyState === 'complete') { - // await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - // 第一段：滚到中下部 - // window.scrollTo({ - // top: h * (1 - ~~(Math.random() * 50)), - // behavior: 'smooth' - // }); - await modernScrollTo({ top: h * (0.9 + Math.random() * 0.08), behavior: 'smooth' }); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - // targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await scrollIntoViewAndWait(targetNode as HTMLElement, { - behavior: 'smooth', - block: 'center', - }); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250)); - - break; - } - } - while (true) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // const spins = Array.from(document.querySelectorAll('.a-spinner')).filter( - // (e) => e.getClientRects().length > 0, - // ); - // if (spins.length === 0) { - // break; - // } - - // const spins = Array.from(document.querySelectorAll('.a-carousel-card-empty')) - // if (spins.length === 0) { - // break; - // } - - const pagination = document.querySelectorAll('.rhf-sign-in-button'); - if (pagination.length > 0) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - break; - } - } - - /** - * 平滑滚动并等待滚动结束 - * @param options 原生 ScrollToOptions 接口 (包含 top, left, behavior) - */ - async function modernScrollTo(options: ScrollToOptions): Promise { - return new Promise((resolve) => { - // 1. 针对已经处于目标位置的情况做兜底处理 - // 如果目标位置与当前位置一致，部分浏览器可能不会触发 scrollend - const targetTop = options.top ?? window.scrollY; - const targetLeft = options.left ?? window.scrollX; - - if (targetTop === window.scrollY && targetLeft === window.scrollX) { - resolve(); - return; - } - - // 2. 监听原生滚动结束事件 - // 注意：scrollend 是 2023-2024 年起普及的标准，2026 年已是主流 - document.addEventListener( - 'scrollend', - () => { - resolve(); - }, - { once: true }, - ); - - // 3. 执行滚动 - window.scrollTo(options); - }); - } - - /** - * 将元素滚动到视野中心并等待动画结束 - * @param element 目标 DOM 元素 - * @param options 滚动配置 - */ - async function scrollIntoViewAndWait( - element: HTMLElement, - options: ScrollIntoViewOptions = { behavior: 'smooth', block: 'center' }, - ): Promise { - return new Promise((resolve) => { - // 监听滚动结束事件 - // scrollend 会在滚动圆满完成或被中断时触发 - document.addEventListener( - 'scrollend', - () => { - resolve(); - }, - { once: true }, - ); - - // 触发滚动 - element.scrollIntoView(options); - - // 兼容性兜底：如果元素已经在视野内，浏览器可能不触发滚动 - // 可以在此处检查位置，若无位移则直接 resolve - }); - } - }); + return this.run(waitForAmazonSearchPageReady); } - // public waitForPageLoaded() { - // return this.run(async () => { - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - - // // 第一个 while 循环：添加 12 秒超时刷新机制 - // const timeoutMs = 12000; // 12 秒 - // let startTime = Date.now(); - // let refreshCount = 0; - // const maxRefresh = 1; // 最多刷新 1 次 - // console.log('waitForPageLoaded'); - - // while (true) { - // const targetNode = document.querySelector('.s-pagination-next'); - // window.scrollBy(0, ~~(Math.random() * 500) + 500); - // await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500)); - - // if (targetNode || document.readyState === 'complete') { - // targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - // break; - // } - - // // 检查是否超时 - // if (Date.now() - startTime > timeoutMs) { - // if (refreshCount < maxRefresh) { - // // 刷新页面 - // console.log('检查是否超时，刷新页面'); - - // browser.tabs.query({ active: true, currentWindow: true }).then(tabs => { - // browser.tabs.reload(tabs[0].id); - // }); - - // await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始 - - // // 重置计时器和刷新计数 - // refreshCount++; - // startTime = Date.now(); - // continue; // 重新从 while 循环开始 - // } else { - // // 已刷新过一次，仍然超时，抛出错误 - // throw new Error('等待页面加载超时（已尝试刷新）'); - // } - // } - // } - - // // 第二个 while 循环：等待 spinner 消失 - // while (true) { - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // const spins = Array.from(document.querySelectorAll('.a-spinner')).filter( - // (e) => e.getClientRects().length > 0 - // ); - // if (spins.length === 0) { - // break; - // } - // } - // }); - // } public async getPagePattern() { return this.run(async () => { @@ -190,192 +23,24 @@ export class AmazonSearchPageInjector extends BaseInjector { }); } - public async getPageData(pattern: 'pattern-1' | 'pattern-2') { - let data: Pick[] | null = null; - switch (pattern) { - // 处理商品以列表形式展示的情况 - case 'pattern-1': - data = await this.run(async () => { - const items = Array.from( - document.querySelectorAll( - '.puisg-row:has(.a-section.a-spacing-small.a-spacing-top-small:not(.a-text-right))', - ), - ).filter((e) => e.getClientRects().length > 0); - const linkObjs = items.reduce< - Pick[] - >((objs, el) => { - const link = el.querySelector('a')?.href; - const title = el - .querySelector('h2.a-color-base')! - .getAttribute('aria-label')!; - const imageSrc = el.querySelector('img.s-image')!.src!; - const price = - el.querySelector('.a-price:not(.a-text-price) .a-offscreen') - ?.innerText || - ( - document.evaluate( - `.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`, - el, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLSpanElement | null - )?.innerText; - link && objs.push({ link, title, imageSrc, price }); - return objs; - }, []); - return linkObjs; - }); - break; - // 处理商品以二维图片格展示的情况 - case 'pattern-2': - data = await this.run(async () => { - const items = Array.from( - document.querySelectorAll( - '.puis-card-container', - ) as unknown as HTMLElement[], - ).filter((e) => e.getClientRects().length > 0); - const linkObjs = items.reduce< - Pick[] - >((objs, el) => { - const link = el.querySelector('a.a-link-normal')?.href; - const title = el.querySelector('h2.a-color-base')!.innerText; - const imageSrc = el.querySelector('img.s-image')!.src!; - const price = - el.querySelector('.a-price:not(.a-text-price) .a-offscreen') - ?.innerText || - ( - document.evaluate( - `.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`, - el, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLSpanElement | null - )?.innerText; - link && objs.push({ link, title, imageSrc, price }); - return objs; - }, []); - return linkObjs; - }); - break; - default: - break; - } - data = data && data.filter((r) => new URL(r.link).pathname.includes('/dp/')); // No advertisement only - return data; + public async getPageData(pattern: AmazonSearchPagePattern) { + return this.run(collectAmazonSearchPageData, { pattern }); + } + + public async isExplicitNoResultsPage() { + return this.run(async () => { + const bodyText = document.body?.innerText || document.body?.textContent || ''; + return /no results for/i.test(bodyText); + }); } public async getCurrentPage() { - return this.run(async () => { - const node = document.querySelector('.s-pagination-item.s-pagination-selected'); - return node ? Number(node.innerText) : 1; - }); + return this.run(collectAmazonCurrentPage); } - /** - * 检测当前亚马逊搜索页面是否有下一页，并自动点击翻页按钮。 - * - * 该方法在页面上下文中执行，查找亚马逊标准分页按钮（'.s-pagination-next'）， - * 检查按钮是否未被禁用（'s-pagination-disabled' 类），然后模拟用户点击。 - * 点击前会随机等待 500-1000 毫秒以避免被识别为机器人。 - * - * @returns `true` 表示有下一页且已点击翻页按钮，页面正在加载下一页内容； - * `false` 表示没有下一页（按钮不存在或被禁用）。 - */ - public async determineHasNextPage() { - return this.run(async () => { - const nextButton = document.querySelector('.s-pagination-next'); - if (nextButton) { - if (!nextButton.classList.contains('s-pagination-disabled')) { - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - nextButton.click(); - return true; - } else { - return false; - } - } else { - return false; - } - }); + public async getNextPageUrl() { + return this.run(collectAmazonNextPageUrl); } - // /** - // * 检测并执行亚马逊搜索页面翻页，等待页面刷新完成。 - // * @returns 能否翻页（true=翻页成功，false=没有下一页或翻页失败） - // */ - // public async determineHasNextPage(): Promise { - // return this.run(async () => { - // const nextButton = document.querySelector('.s-pagination-next'); - - // if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) { - // // 记录当前页码 - // const initialPage = await this.getCurrentPage(); - - // // 随机等待后点击 - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // nextButton.click(); - - // // 等待页面刷新完成 - // await this.waitForPageLoaded(); - - // // 验证翻页是否成功 - // const newPage = await this.getCurrentPage(); - - // // 只有页码真正变化才算翻页成功 - // return newPage !== null && newPage !== initialPage; - // } - - // return false; - // }); - // } - - /** - * 检测并执行亚马逊搜索页面翻页，通过 URL 变化确认刷新完成。 - * @returns 能否翻页（true=已翻页，false=已是最后一页） - */ - // public async determineHasNextPage(): Promise { - // return this.run(async () => { - // const nextButton = document.querySelector('.s-pagination-next'); - - // if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) { - // // 1. 记录当前 URL - // const initialUrl = window.location.href; - - // // 2. 随机等待后点击 - // await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - // nextButton.click(); - - // // 3. 等待 URL 变化（表示页面已开始导航） - // await new Promise((resolve) => { - // const checkUrl = () => { - // if (window.location.href !== initialUrl) { - // resolve(); - // } else { - // setTimeout(checkUrl, 100); - // } - // }; - // checkUrl(); - // }); - - // // 4. 等待页面稳定（document.readyState === 'complete'） - // await new Promise((resolve) => { - // const checkReadyState = () => { - // if (document.readyState === 'complete') { - // resolve(); - // } else { - // setTimeout(checkReadyState, 100); - // } - // }; - // checkReadyState(); - // }); - - // // 5. 额外等待确保内容加载 - // await new Promise((resolve) => setTimeout(resolve, 500)); - - // return true; - // } - - // return false; - // }); - // } } export class AmazonDetailPageInjector extends BaseInjector { @@ -389,35 +54,27 @@ export class AmazonDetailPageInjector extends BaseInjector { /**等待页面加载完成 */ public async waitForPageLoaded() { - return this.run(async () => { - while (true) { - window.scrollBy(0, ~~(Math.random() * 500) + 500); - await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 100) + 200)); - const targetNode = document.querySelector( - '#prodDetails:has(td), #detailBulletsWrapper_feature_div:has(li), .av-page-desktop, #productDescription_feature_div', - ); - const exceptionalNodeSelectors = ['.music-detail-header', '.avu-retail-page']; - for (const selector of exceptionalNodeSelectors) { - if (document.querySelector(selector)) { - return false; - } - } - if (targetNode && document.readyState !== 'loading') { - targetNode.scrollIntoView({ behavior: 'smooth', block: 'center' }); - break; - } - } - return true; - }); + return this.run(waitForAmazonDetailPageReady); } /**获取基本信息 */ public async getBaseInfo() { return this.run(async () => { const title = document.querySelector('#title')!.innerText; - const price = document.querySelector( - '.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen', + const priceRoot = + document.querySelector( + '#corePrice_feature_div, #corePriceDisplay_desktop_feature_div, #apex_desktop', + ) || document.body; + const price = priceRoot.querySelector( + '.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen', )?.innerText; + const rawStrikethroughPrice = priceRoot.querySelector( + '[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString', + )?.innerText; + const strikethroughPrice = + rawStrikethroughPrice && rawStrikethroughPrice !== price + ? rawStrikethroughPrice + : undefined; const boughtInfo = document.querySelector( `#social-proofing-faceout-title-tk_bought`, )?.innerText; @@ -436,7 +93,16 @@ export class AmazonDetailPageInjector extends BaseInjector { '#fulfillerInfoFeature_feature_div > *:last-of-type', )?.innerText; const soldBy = document.querySelector(`#sellerProfileTriggerId`)?.innerText; - return { title, price, boughtInfo, availableDate, categories, shipFrom, soldBy }; + return { + title, + price, + strikethroughPrice, + boughtInfo, + availableDate, + categories, + shipFrom, + soldBy, + }; }); } @@ -488,78 +154,7 @@ export class AmazonDetailPageInjector extends BaseInjector { /**获取图像链接 */ public async getImageUrls() { - return this.run(async () => { - const overlay = document.querySelector('.overlayRestOfImages'); - - if (overlay) { - if (document.querySelector('#ivThumbs')!.getClientRects().length === 0) { - // Click to see full view - document.querySelector('#canvasCaption > a')?.click(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - } else { - // finally click to see full view if not clicked before - document.querySelector('#canvasCaption > a')?.click(); - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - let script = document.evaluate( - `//script[starts-with(text(), "\nP.when(\'A\').register") or contains(text(), "\nP.when('A').register")]`, - document, - null, - XPathResult.STRING_TYPE, - ).stringValue; - const extractUrls = (pattern: RegExp) => - Array.from(script.matchAll(pattern)).map((e) => e[0]); - let urls = extractUrls( - /(?<="hiRes":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g, - ); - if (urls.length === 0) { - urls = extractUrls( - /(?<="large":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g, - ); - } - - if (urls.length === 0) { - const divs: NodeListOf = - document.querySelectorAll('.ivRow div.ivThumbImage'); - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - for (const div of divs) { - // click div com - div.click(); - - // wait for image to load - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - - // if img src is not loading gif, then get url and break, otherwise continue to wait, with a max retry count to avoid infinite loop - let retryCount = 0; - const maxRetries = 5; - - while (true) { - const loadingGifUrl = - 'https://m.media-amazon.com/images/G/01/ui/loadIndicators/loading-large_labeled._CB485921664_.gif'; - // get img dom - const imgDom = document.querySelector('#ivLargeImage > img'); - - // get url from img dom src attribute - if (imgDom && 'src' in imgDom && imgDom.src !== loadingGifUrl) { - urls.push(imgDom.src as string); - break; - } else { - // wait for 500ms and try again - await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random()))); - retryCount++; - if (retryCount >= maxRetries) { - console.error('在detail页面下载图片，已达最大重试次数:'); - break; - } - } - } - } - } - - document.querySelector('body > div.a-modal-scroller.a-declarative')?.click(); - return urls; - }); + return this.run(collectAmazonDetailImageUrls); } /**获取精选评论 */ @@ -697,30 +292,7 @@ export class AmazonDetailPageInjector extends BaseInjector { export class AmazonReviewPageInjector extends BaseInjector { public async waitForPageLoad() { - return this.run(async () => { - await new Promise((resolve) => setTimeout(resolve, 1000)); - while (true) { - const targetNode = document.querySelector( - '.reviews-content, #cm_cr-review_list ul[role="list"]:not(.histogram)', - ); - targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - if ( - targetNode && - targetNode.getClientRects().length > 0 && - document.readyState !== 'loading' - ) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - while (true) { - const loadingNode = document.querySelector('.reviews-loading'); - if (loadingNode && loadingNode.getClientRects().length === 0) { - break; - } - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - }); + return this.run(waitForAmazonReviewPageReady); } public async getSinglePageReviews() { @@ -773,54 +345,7 @@ export class AmazonReviewPageInjector extends BaseInjector { }); } - public jumpToNextPageIfExist() { - return this.run(async () => { - const latestReview = document.evaluate( - `//*[@id='cm_cr-review_list']//li[@data-hook='review'][last()]`, - document.body, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLElement | null; - latestReview?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await new Promise((resolve) => setTimeout(resolve, 1000)); - const nextPageNode = document.querySelector( - '[data-hook="pagination-bar"] .a-pagination > *:nth-of-type(2)', - ); - nextPageNode?.scrollIntoView({ behavior: 'smooth', block: 'center' }); - await new Promise((resolve) => setTimeout(resolve, 1000)); - const ret = nextPageNode && !nextPageNode.classList.contains('a-disabled'); - ret && nextPageNode?.querySelector('a')?.click(); - return ret; - }); - } - - public async showStarsDropDownMenu() { - return this.run(async () => { - while (true) { - const dropdown = document.querySelector('#star-count-dropdown')!; - dropdown.scrollIntoView({ behavior: 'smooth', block: 'center' }); - dropdown.click(); - if (dropdown.getAttribute('aria-expanded') === 'true') { - break; - } - await new Promise((resolve) => setTimeout(resolve, 100)); - } - }); - } - - public async selectStar(star: number) { - return this.run( - async ({ star }) => { - const starNode = document.evaluate( - `//ul[@role='listbox']/li/a[text()="${star} star only"]`, - document.body, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - ).singleNodeValue as HTMLElement; - starNode.click(); - await new Promise((resolve) => setTimeout(resolve, 100)); - }, - { star }, - ); + public getNextPageUrl() { + return this.run(collectAmazonNextReviewPageUrl); } } diff --git a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue index a0dccff..a3aa59f 100644 --- a/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue +++ b/src/sidepanel/views/AmazonEntries/DetailPageEntry.vue @@ -20,6 +20,14 @@ worker.on('error', ({ message: msg }) => { }); message.error(msg); }); +worker.on('item-detail-redirected', (ev) => { + timelines.value.push({ + type: 'warning', + title: '商品发生重定向', + time: new Date().toLocaleString(), + content: `请求 ASIN：${ev.requestedAsin}\n实际 ASIN：${ev.resolvedAsin}\n数据将按实际 ASIN 保存`, + }); +}); worker.on('item-base-info-collected', (ev) => { timelines.value.push({ type: 'success', @@ -40,11 +48,15 @@ worker.on('item-category-rank-collected', (ev) => { }); }); worker.on('item-images-collected', (ev) => { + const imageCount = ev.imageUrls?.length || 0; timelines.value.push({ - type: 'success', + type: imageCount > 0 ? 'success' : 'warning', title: `商品${ev.asin}图像`, time: new Date().toLocaleString(), - content: `图片数： ${ev.imageUrls!.length}`, + content: + imageCount > 0 + ? `图片数： ${imageCount}` + : '未找到商品图片；Amazon 页面未提供可识别的图片数据', }); }); // worker.on('item-top-reviews-collected', (ev) => { @@ -98,11 +110,12 @@ const launch = async () => { }, ...detailWorkerSettings.value, }); + const failed = Boolean(worker.lastError?.value); timelines.value.push({ - type: 'info', + type: failed ? 'error' : 'info', title: '结束', time: new Date().toLocaleString(), - content: '数据采集完成', + content: failed ? '数据采集失败，已保留此前成功采集的数据' : '数据采集完成', }); }; diff --git a/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue b/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue index 66e7d5f..4bb17f0 100644 --- a/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue +++ b/src/sidepanel/views/AmazonEntries/ReviewPageEntry.vue @@ -4,7 +4,10 @@ import { usePageWorker } from '~/page-worker'; import { reviewAsinInput, reviewWorkerSettings } from '~/storages/amazon'; const worker = usePageWorker('amazon', { objects: ['review'] }); +const collectedReviewIds = new Map>(); +const hadFatalError = ref(false); worker.on('error', ({ message: msg }) => { + hadFatalError.value = true; timelines.value.push({ type: 'error', title: '错误发生', @@ -13,14 +16,16 @@ worker.on('error', ({ message: msg }) => { }); }); worker.on('item-review-collected', (ev) => { + const ids = collectedReviewIds.get(ev.asin) || new Set(); + ev.reviews.forEach((review) => ids.add(review.id)); + collectedReviewIds.set(ev.asin, ids); timelines.value.push({ type: 'success', title: `商品${ev.asin}评价`, time: new Date().toLocaleString(), - content: `获取到 ${ev.reviews.length} 条评价`, + content: `已累计获取 ${ids.size} 条评价`, }); }); - const asinInputRef = useTemplateRef('asin-input'); const message = useMessage(); @@ -29,6 +34,8 @@ const timelines = ref([]); const launch = async () => { const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0); + collectedReviewIds.clear(); + hadFatalError.value = false; timelines.value = [ { type: 'info', @@ -43,12 +50,21 @@ const launch = async () => { }, recent: reviewWorkerSettings.value.recent, }); - timelines.value.push({ - type: 'info', - title: '结束', - time: new Date().toLocaleString(), - content: '数据采集完成', - }); + if (hadFatalError.value || worker.lastError?.value) { + timelines.value.push({ + type: 'error', + title: '结束', + time: new Date().toLocaleString(), + content: '数据采集失败，已保留此前成功采集的数据', + }); + } else { + timelines.value.push({ + type: 'info', + title: '结束', + time: new Date().toLocaleString(), + content: '数据采集完成', + }); + } }; const handleStart = () => { diff --git a/src/storages/amazon-sync.ts b/src/storages/amazon-sync.ts new file mode 100644 index 0000000..dcba737 --- /dev/null +++ b/src/storages/amazon-sync.ts @@ -0,0 +1,8 @@ +import { useWebExtensionStorage } from '~/composables/useWebExtensionStorage'; +import { AMAZON_SYNC_STATE_KEY, EMPTY_AMAZON_SYNC_STATE } from '~/logic/amazon-sync-shared'; + +export const amazonSyncState = useWebExtensionStorage( + AMAZON_SYNC_STATE_KEY, + EMPTY_AMAZON_SYNC_STATE, + { flush: 'sync' }, +); diff --git a/src/storages/amazon.ts b/src/storages/amazon.ts index bc323e2..9736d7b 100644 --- a/src/storages/amazon.ts +++ b/src/storages/amazon.ts @@ -7,8 +7,20 @@ export const detailAsinInput = useWebExtensionStorage('detailAsinInputTe export const reviewAsinInput = useWebExtensionStorage('reviewAsinInputText', ''); export const itemColumnSettings = useWebExtensionStorage< - Set> ->('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'createTime'])); + Set< + keyof Pick< + AmazonItem, + | 'keywords' + | 'page' + | 'rank' + | 'pageRank' + | 'adRank' + | 'placementRank' + | 'createTime' + | 'timestamp' + > + > +>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'adRank', 'createTime'])); export const detailWorkerSettings = useWebExtensionStorage('amazon-detail-worker-settings', { image: true, @@ -55,8 +67,17 @@ export const allItems = computed({ 'title', 'imageSrc', 'price', + 'strikethroughPrice', 'link', 'rank', + 'pageRank', + 'adType', + 'adRank', + 'placement', + 'placementRank', + 'placementId', + 'adProductIndex', + 'adFormat', 'createTime', ]; searchItems.value = newValue diff --git a/src/types/amazon-sync.d.ts b/src/types/amazon-sync.d.ts new file mode 100644 index 0000000..cb1a375 --- /dev/null +++ b/src/types/amazon-sync.d.ts @@ -0,0 +1,25 @@ +import type { ProtocolWithReturn } from 'webext-bridge'; + +declare global { + type AmazonSyncPayload = { + searchItems?: AmazonSearchItem[]; + detailItems?: AmazonDetailItem[]; + reviews?: ({ asin: string } & AmazonReview)[]; + }; + + type AmazonSyncStatus = 'idle' | 'pending' | 'syncing' | 'synced' | 'failed'; + + type AmazonSyncState = { + status: AmazonSyncStatus; + pendingCount: number; + lastSuccessAt?: string; + lastError?: string; + }; +} + +declare module 'webext-bridge' { + export interface ProtocolMap { + 'amazon-sync-enqueue': ProtocolWithReturn; + 'amazon-sync-flush': ProtocolWithReturn; + } +} diff --git a/src/types/amazon.d.ts b/src/types/amazon.d.ts index f0faf7a..ccb1b62 100644 --- a/src/types/amazon.d.ts +++ b/src/types/amazon.d.ts @@ -1,4 +1,25 @@ +declare type AmazonAdType = + | 'organic' + | 'sponsoredProducts' + | 'sponsoredBrands' + | 'sponsoredDisplay' + | 'unknownSponsored'; + +declare type AmazonAdPlacement = + | 'topOfSearch' + | 'middleOfSearch' + | 'bottomOfSearch' + /** 兼容旧版采集数据 */ + | 'restOfSearch' + | 'sidebar' + | 'productPage' + | 'unknown'; + +declare type AmazonAdFormat = 'standard' | 'video' | 'unknown'; + declare type AmazonSearchItem = { + /** 客户端生成的幂等同步标识 */ + syncId?: string; /** 搜索关键词 */ keywords: string; /** 商品排名 */ @@ -13,6 +34,24 @@ declare type AmazonSearchItem = { asin: string; /** 商品价格（可选） */ price?: string; + /** 页面展示的划线参考价格 */ + strikethroughPrice?: string; + /** 商品在当前页商品流中的排名 */ + pageRank?: number; + /** 广告类型；历史数据可能为空 */ + adType?: AmazonAdType; + /** 同一关键词、同一广告类型中的累计广告排名 */ + adRank?: number; + /** 广告在页面中的位置 */ + placement?: AmazonAdPlacement; + /** 同一关键词采集过程中广告模块的累计顺序 */ + placementRank?: number; + /** 同一广告模块内商品的关联标识 */ + placementId?: string; + /** 商品在广告模块内的顺序 */ + adProductIndex?: number; + /** 广告素材形式 */ + adFormat?: AmazonAdFormat; /** 商品图片链接 */ imageSrc: string; /** 创建时间 */ @@ -20,6 +59,8 @@ declare type AmazonSearchItem = { }; declare type AmazonDetailItem = { + /** 客户端生成的幂等同步标识 */ + syncId?: string; /** 商品的 ASIN（亚马逊标准识别号） */ asin: string; /** 商品标题 */ @@ -30,6 +71,8 @@ declare type AmazonDetailItem = { boughtInfo?: string; /** 商品价格 */ price?: string; + /** 页面展示的划线参考价格 */ + strikethroughPrice?: string; /** 商品评分 */ rating?: number; /** 评分数量 */ diff --git a/webext-bridge/src/background.ts b/webext-bridge/src/background.ts index f05f53f..3548ad8 100644 --- a/webext-bridge/src/background.ts +++ b/webext-bridge/src/background.ts @@ -279,6 +279,10 @@ browser.runtime.onConnect.addListener((incomingPort) => { }); incomingPort.onDisconnect.addListener(() => { + // Reading lastError prevents Chrome from reporting an expected BFCache port closure as + // an unchecked runtime error in the extension service worker console. + void browser.runtime.lastError; + // sometimes previous content script's onDisconnect is called *after* the fresh content-script's // onConnect. So without this fingerprint equality check, we would remove the new port from map if (connMap.get(connArgs.endpointName)?.fingerprint === connArgs.fingerprint) diff --git a/webext-bridge/src/internal/persistent-port.ts b/webext-bridge/src/internal/persistent-port.ts index 29ae09e..28070d5 100644 --- a/webext-bridge/src/internal/persistent-port.ts +++ b/webext-bridge/src/internal/persistent-port.ts @@ -7,6 +7,7 @@ import { encodeConnectionArgs } from './connection-args'; import { createDeliveryLogger } from './delivery-logger'; import type { StatusMessage } from './port-message'; import { PortMessage } from './port-message'; +import { isBackForwardCachePortError } from './port-disconnect'; /** * Manfiest V3 extensions can have their service worker terminated at any point @@ -18,6 +19,8 @@ import { PortMessage } from './port-message'; export const createPersistentPort = (name = '') => { const fingerprint = createFingerprint(); let port: Runtime.Port; + let connected = false; + let inBackForwardCache = false; let undeliveredQueue: ReadonlyArray = []; const pendingResponses = createDeliveryLogger(); const onMessageListeners = new Set<(message: InternalMessage, port: Runtime.Port) => void>(); @@ -77,14 +80,28 @@ export const createPersistentPort = (name = '') => { }; const connect = () => { + if (connected || inBackForwardCache) return; + port = browser.runtime.connect({ name: encodeConnectionArgs({ endpointName: name, fingerprint, }), }); + connected = true; + const connectedPort = port; port.onMessage.addListener(handleMessage); - port.onDisconnect.addListener(connect); + port.onDisconnect.addListener(() => { + const disconnectMessage = browser.runtime.lastError?.message; + if (port !== connectedPort) return; + + connected = false; + if (isBackForwardCachePortError(disconnectMessage)) { + inBackForwardCache = true; + return; + } + connect(); + }); PortMessage.toBackground(port, { type: 'sync', @@ -95,6 +112,17 @@ export const createPersistentPort = (name = '') => { }); }; + if (typeof window !== 'undefined') { + window.addEventListener('pagehide', (event) => { + if (event.persisted) inBackForwardCache = true; + }); + window.addEventListener('pageshow', (event) => { + if (!event.persisted) return; + inBackForwardCache = false; + connect(); + }); + } + connect(); return { diff --git a/webext-bridge/src/internal/port-disconnect.ts b/webext-bridge/src/internal/port-disconnect.ts new file mode 100644 index 0000000..0ae1bbd --- /dev/null +++ b/webext-bridge/src/internal/port-disconnect.ts @@ -0,0 +1,3 @@ +export function isBackForwardCachePortError(message?: string): boolean { + return message?.includes('back/forward cache') ?? false; +}