mirror of
https://github.com/primedigitaltech/azon_seeker.git
synced 2026-10-03 22:28:01 +08:00
Feat/advertisement (#26)
* feat: add Amazon collection metadata and reliable sync * fix: route extension sync through configured server * chore: release extension v0.7.13 * fix: handle extension ports across BFCache navigation * fix: classify bottom Sponsored Brands video placements * fix: exclude hidden Amazon ad cards * fix: derive Amazon result page from URL * fix: ignore hidden Amazon sponsored products * fix: preserve base Amazon product collection * fix: bind Amazon results to visible page * fix: classify visible Amazon ad formats * fix: wait for lazy Amazon ad modules * perf: reduce Amazon lazy-load scroll stops * fix: classify Amazon sponsored brand and video search ads Co-authored-by: Cursor <cursoragent@cursor.com> * fix: collect Shop by type thematic SP ads via redirect chains without data-asin Co-authored-by: Cursor <cursoragent@cursor.com> * fix: classify sponsored ads across new Amazon templates --------- Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: PetrichorFun <127877748+PetrichorFun@users.noreply.github.com>
This commit is contained in:
Ktight
GitHub
parent
0f50620a12
commit
90110a48f0
1
.gitignore
vendored
1
.gitignore
vendored
@ -14,6 +14,7 @@ dist-ssr
|
||||
extension/manifest.json
|
||||
extension-firefox/manifest.json
|
||||
node_modules
|
||||
.runtime
|
||||
src/auto-imports.d.ts
|
||||
src/components.d.ts
|
||||
.eslintcache
|
||||
|
||||
16
README.md
16
README.md
@ -11,6 +11,7 @@ Azon Seeker is a browser extension designed to collect and manage product inform
|
||||
- 🛒 **Collect Product Data:** Easily gather product details and reviews from Amazon and other websites.
|
||||
- 📦 **Import/Export:** Supports importing and exporting data in Excel format for further analysis.
|
||||
- ☁️ **Cloud Export:** Export your data to the cloud for backup or sharing.
|
||||
- 🔄 **Reliable Server Sync:** Collected Amazon data is durably queued, retried in the background, and removed from the queue only after the server confirms persistence.
|
||||
- 📊 **Data Table:** View, filter, and manage collected data in a user-friendly table.
|
||||
- 🖼️ **Image Preview:** Hover to preview product images.
|
||||
- 🌐 **Multi-browser Support:** Works with Chromium-based browsers and Firefox.
|
||||
@ -80,4 +81,17 @@ pnpm run build-firefox # For Firefox
|
||||
pnpm run pack:7z
|
||||
```
|
||||
|
||||
After the build is completed, the extension package will be generated in `build/extension.7z`
|
||||
After the build is completed, the Chromium package will be generated in
|
||||
`build/extension-v<version>.7z`.
|
||||
|
||||
## Amazon Server Synchronization
|
||||
|
||||
- Development builds use `127.0.0.1:8001` by default.
|
||||
- Production builds use `amzplugin.rekeymed.com` over HTTPS/WSS by default.
|
||||
- Override either target at build time with `VITE_REMOTE_HOST=host:port`.
|
||||
- The extension persists unsent batches in `browser.storage.local` and retries once per minute.
|
||||
- Search and detail records include a client-generated `syncId`; the server uses it as a unique idempotency key.
|
||||
- The side panel displays pending, syncing, synchronized, and failed states and provides a manual retry action.
|
||||
|
||||
Before publishing a production package, deploy the backend migrations and verify that the
|
||||
production `/openapi.json` contains `syncId`, `adType`, and `strikethroughPrice`.
|
||||
|
||||
22
package.json
22
package.json
@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "azon-seeker",
|
||||
"displayName": "Azon Seeker v0.7.1.12-beta",
|
||||
"version": "0.7.2",
|
||||
"displayName": "Azon Seeker v0.7.13",
|
||||
"version": "0.7.13",
|
||||
"private": true,
|
||||
"description": "Starter modify by honestfox101 and PetrichorFun",
|
||||
"scripts": {
|
||||
@ -11,8 +11,9 @@
|
||||
"dev:web": "vite",
|
||||
"dev:background": "npm run build:background -- --mode development",
|
||||
"dev:js": "npm run build:js -- --mode development",
|
||||
"build": "cross-env NODE_ENV=production run-s clear build:web build:prepare build:background build:js",
|
||||
"build-firefox": "cross-env NODE_ENV=production EXTENSION=firefox run-s clear-firefox build:web build:prepare build:background build:js",
|
||||
"build": "cross-env NODE_ENV=production run-s clear build:bridge build:web build:prepare build:background build:js",
|
||||
"build-firefox": "cross-env NODE_ENV=production EXTENSION=firefox run-s clear-firefox build:bridge build:web build:prepare build:background build:js",
|
||||
"build:bridge": "npm --prefix webext-bridge run build",
|
||||
"build:prepare": "esno scripts/prepare.ts",
|
||||
"build:background": "vite build --config vite.config.background.mts",
|
||||
"build:web": "vite build",
|
||||
@ -23,6 +24,19 @@
|
||||
"clear": "rimraf --glob extension/dist extension/manifest.json extension.* ",
|
||||
"clear-firefox": "rimraf --glob extension-firefox/dist extension-firefox/manifest.json extension.*",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test:amazon-parser": "esno scripts/test-amazon-search-parser.ts",
|
||||
"test:amazon-page-ready": "esno scripts/test-amazon-search-page-ready.ts",
|
||||
"test:amazon-pagination": "esno scripts/test-amazon-search-pagination.ts",
|
||||
"test:amazon-detail-images": "esno scripts/test-amazon-detail-images.ts",
|
||||
"test:amazon-detail-page-ready": "esno scripts/test-amazon-detail-page-ready.ts",
|
||||
"test:amazon-review-page": "esno scripts/test-amazon-review-page.ts",
|
||||
"test:amazon-page-navigation": "esno scripts/test-amazon-page-navigation.ts",
|
||||
"test:execute-script": "esno scripts/test-execute-script.ts",
|
||||
"test:extension-runtime": "node scripts/verify-extension-runtime.mjs",
|
||||
"test:remote-endpoint": "esno scripts/test-remote-endpoint.ts",
|
||||
"test:production-endpoint": "node scripts/verify-production-endpoint.mjs",
|
||||
"test:port-disconnect": "esno scripts/test-port-disconnect.ts",
|
||||
"test:port-disconnect-build": "node scripts/verify-port-disconnect-build.mjs",
|
||||
"prepare": "husky"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@ -30,7 +30,15 @@ async function getManifest() {
|
||||
48: './assets/icon-512.png',
|
||||
128: './assets/icon-512.png',
|
||||
},
|
||||
permissions: ['tabs', 'storage', 'activeTab', 'scripting', 'unlimitedStorage', 'contextMenus'],
|
||||
permissions: [
|
||||
'tabs',
|
||||
'storage',
|
||||
'activeTab',
|
||||
'scripting',
|
||||
'unlimitedStorage',
|
||||
'contextMenus',
|
||||
'alarms',
|
||||
],
|
||||
content_security_policy: {
|
||||
extension_pages: isDev
|
||||
? `script-src \'self\' http://localhost:${port}; object-src \'self\';`
|
||||
|
||||
@ -1,17 +1,24 @@
|
||||
import _7z from '7zip-min';
|
||||
import { basename } from 'path';
|
||||
import fs from 'fs-extra';
|
||||
import { r, log } from './utils';
|
||||
import packageJson from '../package.json';
|
||||
|
||||
const scriptPath = ['extension', 'extension-firefox'];
|
||||
|
||||
scriptPath.forEach(async (src) => {
|
||||
const srcPath = r(src);
|
||||
const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`);
|
||||
try {
|
||||
const output = await _7z.pack(srcPath, distPath);
|
||||
log('Info', `stdout of the 7za command execution ${output}`);
|
||||
} catch (err) {
|
||||
log('Error', `${err}`);
|
||||
async function main() {
|
||||
for (const src of scriptPath) {
|
||||
const srcPath = r(src);
|
||||
if (!(await fs.pathExists(r(src, 'manifest.json')))) continue;
|
||||
const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`);
|
||||
try {
|
||||
const output = await _7z.pack(srcPath, distPath);
|
||||
log('Info', `stdout of the 7za command execution ${output}`);
|
||||
} catch (err) {
|
||||
log('Error', `${err}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void main();
|
||||
|
||||
51
scripts/test-amazon-detail-images.ts
Normal file
51
scripts/test-amazon-detail-images.ts
Normal file
@ -0,0 +1,51 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { collectAmazonDetailImageUrls } from '../src/page-worker/web-injectors/amazon-detail-images';
|
||||
|
||||
async function runWithHtml(html: string) {
|
||||
const dom = new JSDOM(html, { url: 'https://www.amazon.com/dp/B0DMNDL5NH' });
|
||||
const previousDocument = globalThis.document;
|
||||
const previousMouseEvent = globalThis.MouseEvent;
|
||||
Object.assign(globalThis, {
|
||||
document: dom.window.document,
|
||||
MouseEvent: dom.window.MouseEvent,
|
||||
});
|
||||
try {
|
||||
return await collectAmazonDetailImageUrls();
|
||||
} finally {
|
||||
Object.assign(globalThis, {
|
||||
document: previousDocument,
|
||||
MouseEvent: previousMouseEvent,
|
||||
});
|
||||
dom.window.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const scriptUrls = await runWithHtml(`
|
||||
<script>
|
||||
window.imageData = {
|
||||
"colorImages": {
|
||||
"initial": [
|
||||
{"hiRes":"https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg","large":"https://m.media-amazon.com/images/I/first-large.jpg"},
|
||||
{"hiRes":"https:\\/\\/m.media-amazon.com\\/images\\/I\\/second._AC_SL1500_.jpg","large":"https://m.media-amazon.com/images/I/second-large.jpg"}
|
||||
]
|
||||
}
|
||||
};
|
||||
</script>
|
||||
`);
|
||||
assert.deepEqual(scriptUrls, [
|
||||
'https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg',
|
||||
'https://m.media-amazon.com/images/I/second._AC_SL1500_.jpg',
|
||||
]);
|
||||
|
||||
const landingUrl = 'https://m.media-amazon.com/images/I/landing._AC_SL1500_.jpg';
|
||||
const landingUrls = await runWithHtml(
|
||||
`<img id="landingImage" data-old-hires="${landingUrl}" data-a-dynamic-image="{}">`,
|
||||
);
|
||||
assert.deepEqual(landingUrls, [landingUrl]);
|
||||
|
||||
console.log('Amazon detail image parser tests passed.');
|
||||
}
|
||||
|
||||
void main();
|
||||
49
scripts/test-amazon-detail-page-ready.ts
Normal file
49
scripts/test-amazon-detail-page-ready.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { waitForAmazonDetailPageReady } from '../src/page-worker/web-injectors/amazon-detail-page-ready';
|
||||
|
||||
const installDom = (html: string, url = 'https://www.amazon.com/dp/B0DMNDL5NH') => {
|
||||
const dom = new JSDOM(html, { url });
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
return dom;
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const modernDetail = installDom('<span id="productTitle">Ready product</span>');
|
||||
await waitForAmazonDetailPageReady();
|
||||
modernDetail.window.close();
|
||||
|
||||
const legacyDetail = installDom('<span id="title">Legacy product</span>');
|
||||
await waitForAmazonDetailPageReady();
|
||||
legacyDetail.window.close();
|
||||
|
||||
const signIn = installDom('<main>Sign in</main>', 'https://www.amazon.com/ap/signin');
|
||||
await waitForAmazonDetailPageReady().then(
|
||||
() => {
|
||||
throw new Error('Sign-in page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('需要重新登录')) throw error;
|
||||
},
|
||||
);
|
||||
signIn.window.close();
|
||||
|
||||
const captcha = installDom('<input id="captchacharacters">');
|
||||
await waitForAmazonDetailPageReady().then(
|
||||
() => {
|
||||
throw new Error('Captcha page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('验证码')) throw error;
|
||||
},
|
||||
);
|
||||
captcha.window.close();
|
||||
|
||||
console.log('Amazon detail page readiness tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
81
scripts/test-amazon-page-navigation.ts
Normal file
81
scripts/test-amazon-page-navigation.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import {
|
||||
extractAmazonDetailAsin,
|
||||
hasReachedAmazonDetailPage,
|
||||
hasReachedAmazonNavigationTarget,
|
||||
} from '../src/page-worker/page-navigation';
|
||||
|
||||
const assert = (condition: boolean, message: string) => {
|
||||
if (!condition) throw new Error(message);
|
||||
};
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews&filterByStar=one_star',
|
||||
),
|
||||
'An unfiltered completed page must not satisfy a star-filter navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_viewopt_sr?filterByStar=one_star&pageNumber=2',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_paging_btm_next_2?filterByStar=one_star&pageNumber=2',
|
||||
),
|
||||
'Tracking ref rewrites must not prevent matching the requested review page.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=1',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=2',
|
||||
),
|
||||
'A previous review page must not satisfy the next-page navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/s?k=security+camera&page=17&ref=sr_pg_17',
|
||||
'https://www.amazon.com/s?ref=sr_pg_17&page=17&k=security+camera',
|
||||
),
|
||||
'Search navigation should tolerate query ordering differences.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH/ref=sr_1_1',
|
||||
'https://www.amazon.com/dp/B0DMNDL5NH',
|
||||
),
|
||||
'A canonical product slug must still match the requested detail ASIN.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH',
|
||||
'https://www.amazon.com/dp/B0BKPNRLC8',
|
||||
),
|
||||
'The previous detail ASIN must not satisfy the next-ASIN navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonDetailPage(
|
||||
'https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1',
|
||||
'https://www.amazon.com/dp/B0D2MC2865',
|
||||
),
|
||||
'A canonical Amazon detail redirect must be accepted as a loaded detail page.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonDetailPage(
|
||||
'https://www.amazon.com/',
|
||||
'https://www.amazon.com/dp/B0D2MC2865',
|
||||
),
|
||||
'The Amazon home page must not be accepted as a detail redirect.',
|
||||
);
|
||||
|
||||
assert(
|
||||
extractAmazonDetailAsin('https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1') ===
|
||||
'B0CCXB29LF',
|
||||
'The resolved ASIN must be extracted from canonical detail URLs.',
|
||||
);
|
||||
|
||||
console.log('Amazon page navigation tests passed.');
|
||||
82
scripts/test-amazon-review-page.ts
Normal file
82
scripts/test-amazon-review-page.ts
Normal file
@ -0,0 +1,82 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import {
|
||||
collectAmazonNextReviewPageUrl,
|
||||
isAmazonReviewRatingFilterIgnored,
|
||||
waitForAmazonReviewPageReady,
|
||||
} from '../src/page-worker/web-injectors/amazon-review-page';
|
||||
|
||||
const installDom = (html: string, url = 'https://www.amazon.com/product-reviews/B08XM6S6ZL') => {
|
||||
const dom = new JSDOM(html, { url });
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
return dom;
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
if (
|
||||
!isAmazonReviewRatingFilterIgnored(
|
||||
[{ rating: '5.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }],
|
||||
3,
|
||||
)
|
||||
) {
|
||||
throw new Error('Mixed ratings must be detected when Amazon ignores filterByStar');
|
||||
}
|
||||
if (
|
||||
isAmazonReviewRatingFilterIgnored(
|
||||
[{ rating: '3.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }],
|
||||
3,
|
||||
)
|
||||
) {
|
||||
throw new Error('A correctly filtered review page must not be treated as limited');
|
||||
}
|
||||
|
||||
const readyDom = installDom('<div id="cm_cr-review_list"></div>');
|
||||
await waitForAmazonReviewPageReady({});
|
||||
readyDom.window.close();
|
||||
|
||||
const paginationDom = installDom(
|
||||
'<div data-hook="pagination-bar"><ul class="a-pagination"><li>Previous</li><li class="a-last"><a href="/product-reviews/B08XM6S6ZL?pageNumber=2">Next</a></li></ul></div>',
|
||||
);
|
||||
const nextUrl = await collectAmazonNextReviewPageUrl();
|
||||
if (nextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2') {
|
||||
throw new Error(`Unexpected next review URL: ${nextUrl}`);
|
||||
}
|
||||
paginationDom.window.close();
|
||||
|
||||
const standalonePaginationDom = installDom(
|
||||
'<div id="cm_cr-review_list"><div id="R1-review-card" data-hook="review"></div></div><ul class="a-pagination"><li>Previous</li><li class="a-last"><a href="/product-reviews/B08XM6S6ZL?pageNumber=3">Next</a></li></ul>',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2',
|
||||
);
|
||||
const standaloneNextUrl = await collectAmazonNextReviewPageUrl();
|
||||
if (standaloneNextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=3') {
|
||||
throw new Error(`Unexpected standalone review pagination URL: ${standaloneNextUrl}`);
|
||||
}
|
||||
standalonePaginationDom.window.close();
|
||||
|
||||
const lastPageDom = installDom(
|
||||
'<div data-hook="pagination-bar"><ul class="a-pagination"><li class="a-last a-disabled">Next</li></ul></div>',
|
||||
);
|
||||
if ((await collectAmazonNextReviewPageUrl()) !== null) {
|
||||
throw new Error('Disabled review pagination must not return a URL');
|
||||
}
|
||||
lastPageDom.window.close();
|
||||
|
||||
const signInDom = installDom('<main>Sign in</main>', 'https://www.amazon.com/ap/signin');
|
||||
await waitForAmazonReviewPageReady({}).then(
|
||||
() => {
|
||||
throw new Error('Sign-in page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('需要重新登录')) throw error;
|
||||
},
|
||||
);
|
||||
signInDom.window.close();
|
||||
|
||||
console.log('Amazon review page tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
76
scripts/test-amazon-search-page-ready.ts
Normal file
76
scripts/test-amazon-search-page-ready.ts
Normal file
@ -0,0 +1,76 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { waitForAmazonSearchPageReady } from '../src/page-worker/web-injectors/amazon-search-page-ready';
|
||||
|
||||
const main = async () => {
|
||||
const dom = new JSDOM(
|
||||
'<main><div class="a-spinner">Persistent widget spinner</div><div data-component-type="s-search-result" data-asin="B000000001"></div><a class="s-pagination-next" href="/s?page=2">Next</a></main>',
|
||||
{ url: 'https://www.amazon.com/s?k=security+camera' },
|
||||
);
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
Object.assign(dom.window, { scrollTo: () => undefined });
|
||||
|
||||
dom.window.setTimeout(() => {
|
||||
const sponsoredModule = dom.window.document.createElement('section');
|
||||
sponsoredModule.innerHTML =
|
||||
'<span aria-label="Sponsored">Sponsored</span><article data-asin="B000000002"><h3>Lazy sponsored product</h3><a href="/dp/B000000002"><img src="https://example.com/lazy.jpg"></a></article>';
|
||||
dom.window.document.querySelector('main')?.append(sponsoredModule);
|
||||
}, 500);
|
||||
|
||||
const started = Date.now();
|
||||
await waitForAmazonSearchPageReady();
|
||||
const elapsed = Date.now() - started;
|
||||
|
||||
if (!dom.window.document.querySelector('[data-asin="B000000002"]')) {
|
||||
throw new Error('Lazy sponsored DOM was not loaded before page readiness completed');
|
||||
}
|
||||
if (elapsed >= 5000) {
|
||||
throw new Error(`Ready search results were delayed by a persistent spinner (${elapsed}ms)`);
|
||||
}
|
||||
dom.window.close();
|
||||
|
||||
const errorDom = new JSDOM(
|
||||
'<main><h1>SORRY</h1><p>Something went wrong on our end.</p><p>Please go back and try again.</p></main>',
|
||||
{ url: 'https://www.amazon.com/s?k=protein+powder' },
|
||||
);
|
||||
Object.defineProperty(errorDom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: errorDom.window.document, window: errorDom.window });
|
||||
await waitForAmazonSearchPageReady().then(
|
||||
() => {
|
||||
throw new Error('Amazon transient error page must not be accepted as an empty result page');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('临时错误页')) throw error;
|
||||
},
|
||||
);
|
||||
errorDom.window.close();
|
||||
|
||||
const captchaDom = new JSDOM('<input id="captchacharacters">', {
|
||||
url: 'https://www.amazon.com/s?k=protein+powder',
|
||||
});
|
||||
Object.defineProperty(captchaDom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: captchaDom.window.document, window: captchaDom.window });
|
||||
await waitForAmazonSearchPageReady().then(
|
||||
() => {
|
||||
throw new Error('Amazon captcha page must fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('验证码')) throw error;
|
||||
},
|
||||
);
|
||||
captchaDom.window.close();
|
||||
|
||||
console.log('Amazon search page-ready tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
93
scripts/test-amazon-search-pagination.ts
Normal file
93
scripts/test-amazon-search-pagination.ts
Normal file
@ -0,0 +1,93 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import {
|
||||
collectAmazonCurrentPage,
|
||||
collectAmazonNextPageUrl,
|
||||
} from '../src/page-worker/web-injectors/amazon-search-pagination';
|
||||
|
||||
const withDom = async <T>(url: string, html: string, callback: () => T | Promise<T>) => {
|
||||
const dom = new JSDOM(html, { url });
|
||||
const previousDocument = globalThis.document;
|
||||
const previousWindow = globalThis.window;
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
try {
|
||||
return await callback();
|
||||
} finally {
|
||||
Object.assign(globalThis, { document: previousDocument, window: previousWindow });
|
||||
dom.window.close();
|
||||
}
|
||||
};
|
||||
|
||||
const testNextPage = async (name: string, html: string, expected: string | null) => {
|
||||
const actual = await withDom(
|
||||
'https://www.amazon.com/s?k=security+camera&page=1',
|
||||
html,
|
||||
collectAmazonNextPageUrl,
|
||||
);
|
||||
if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`);
|
||||
};
|
||||
|
||||
const testCurrentPage = async (name: string, url: string, html: string, expected: number) => {
|
||||
const actual = await withDom(url, html, collectAmazonCurrentPage);
|
||||
if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`);
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
await testNextPage(
|
||||
'active next page',
|
||||
'<a class="s-pagination-next" href="/s?k=security+camera&page=2">Next</a>',
|
||||
'https://www.amazon.com/s?k=security+camera&page=2',
|
||||
);
|
||||
await testNextPage(
|
||||
'disabled next page',
|
||||
'<span class="s-pagination-next s-pagination-disabled">Next</span>',
|
||||
null,
|
||||
);
|
||||
await testNextPage('missing next page', '<main>No pagination</main>', null);
|
||||
await testCurrentPage(
|
||||
'visible page wins over hidden stale pagination selection',
|
||||
'https://www.amazon.com/s?k=home+security&page=11&ref=sr_pg_11',
|
||||
'<div style="display:none"><span class="s-pagination-item s-pagination-selected">12</span></div><span class="s-pagination-item s-pagination-selected">11</span>',
|
||||
11,
|
||||
);
|
||||
for (const page of [1, 2, 5, 10, 20]) {
|
||||
await testCurrentPage(
|
||||
`URL page ${page}`,
|
||||
`https://www.amazon.com/s?k=home+security&page=${page}&ref=sr_pg_${page}`,
|
||||
`<span class="s-pagination-item s-pagination-selected">${page}</span>`,
|
||||
page,
|
||||
);
|
||||
}
|
||||
await testCurrentPage(
|
||||
'visible page wins while URL has advanced',
|
||||
'https://www.amazon.com/s?k=home+security&page=3&ref=sr_pg_3',
|
||||
'<span class="s-pagination-item s-pagination-selected">2</span>',
|
||||
2,
|
||||
);
|
||||
await testCurrentPage(
|
||||
'invalid URL page uses ref fallback',
|
||||
'https://www.amazon.com/s?k=home+security&page=invalid&ref=sr_pg_9',
|
||||
'<main>No selected pagination item</main>',
|
||||
9,
|
||||
);
|
||||
await testCurrentPage(
|
||||
'ref page fallback',
|
||||
'https://www.amazon.com/s?k=home+security&ref=sr_pg_7',
|
||||
'<main>No selected pagination item</main>',
|
||||
7,
|
||||
);
|
||||
await testCurrentPage(
|
||||
'selected pagination fallback',
|
||||
'https://www.amazon.com/s?k=home+security',
|
||||
'<span class="s-pagination-item s-pagination-selected">5</span>',
|
||||
5,
|
||||
);
|
||||
await testCurrentPage(
|
||||
'first page fallback',
|
||||
'https://www.amazon.com/s?k=home+security',
|
||||
'<main>No pagination</main>',
|
||||
1,
|
||||
);
|
||||
console.log('Amazon search pagination tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
1210
scripts/test-amazon-search-parser.ts
Normal file
1210
scripts/test-amazon-search-parser.ts
Normal file
File diff suppressed because it is too large
Load Diff
57
scripts/test-execute-script.ts
Normal file
57
scripts/test-execute-script.ts
Normal file
@ -0,0 +1,57 @@
|
||||
import { exec } from '../src/logic/execute-script';
|
||||
|
||||
const setBrowser = (
|
||||
executeScript: () => Promise<unknown>,
|
||||
getTab: () => Promise<{ status: string }> = async () => ({ status: 'complete' }),
|
||||
) => {
|
||||
Object.assign(globalThis, {
|
||||
browser: {
|
||||
tabs: { get: getTab },
|
||||
scripting: { executeScript },
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
setBrowser(async () => {
|
||||
throw new Error('Frame was removed');
|
||||
});
|
||||
const rejectionStarted = Date.now();
|
||||
await exec(1, async () => true, undefined, { timeout: 1000 }).then(
|
||||
() => {
|
||||
throw new Error('Expected executeScript to reject');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('Frame was removed')) throw error;
|
||||
if (Date.now() - rejectionStarted >= 1000) {
|
||||
throw new Error('The original executeScript error was delayed until the timeout');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
setBrowser(async () => [{ result: 42 }]);
|
||||
const result = await exec(1, async () => 42, undefined, { timeout: 1000 });
|
||||
if (result !== 42) throw new Error(`Expected 42, received ${result}`);
|
||||
|
||||
let injectionAttempted = false;
|
||||
setBrowser(
|
||||
async () => {
|
||||
injectionAttempted = true;
|
||||
return [{ result: true }];
|
||||
},
|
||||
async () => ({ status: 'loading' }),
|
||||
);
|
||||
await exec(1, async () => true, undefined, { timeout: 100 }).then(
|
||||
() => {
|
||||
throw new Error('A perpetually loading page should time out before injection');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('页面加载超时')) throw error;
|
||||
},
|
||||
);
|
||||
if (injectionAttempted) throw new Error('Script was injected into a still-loading page');
|
||||
|
||||
console.log('Execute-script error propagation tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
13
scripts/test-port-disconnect.ts
Normal file
13
scripts/test-port-disconnect.ts
Normal file
@ -0,0 +1,13 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { isBackForwardCachePortError } from '../webext-bridge/src/internal/port-disconnect';
|
||||
|
||||
assert.equal(
|
||||
isBackForwardCachePortError(
|
||||
'The page keeping the extension port is moved into back/forward cache, so the message channel is closed.',
|
||||
),
|
||||
true,
|
||||
);
|
||||
assert.equal(isBackForwardCachePortError('Extension context invalidated.'), false);
|
||||
assert.equal(isBackForwardCachePortError(), false);
|
||||
|
||||
console.log('Port disconnect tests passed.');
|
||||
34
scripts/test-remote-endpoint.ts
Normal file
34
scripts/test-remote-endpoint.ts
Normal file
@ -0,0 +1,34 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { resolveRemoteEndpoint } from '../src/logic/remote-endpoint';
|
||||
|
||||
assert.deepEqual(resolveRemoteEndpoint({ isDev: true }), {
|
||||
host: '127.0.0.1:18000',
|
||||
httpBaseUrl: 'http://127.0.0.1:18000',
|
||||
webSocketBaseUrl: 'ws://127.0.0.1:18000',
|
||||
});
|
||||
|
||||
assert.deepEqual(resolveRemoteEndpoint({ isDev: false }), {
|
||||
host: 'amzplugin.rekeymed.com',
|
||||
httpBaseUrl: 'https://amzplugin.rekeymed.com',
|
||||
webSocketBaseUrl: 'wss://amzplugin.rekeymed.com',
|
||||
});
|
||||
|
||||
assert.deepEqual(
|
||||
resolveRemoteEndpoint({
|
||||
isDev: true,
|
||||
host: 'staging.example.test:8443',
|
||||
protocol: 'https',
|
||||
}),
|
||||
{
|
||||
host: 'staging.example.test:8443',
|
||||
httpBaseUrl: 'https://staging.example.test:8443',
|
||||
webSocketBaseUrl: 'wss://staging.example.test:8443',
|
||||
},
|
||||
);
|
||||
|
||||
assert.throws(
|
||||
() => resolveRemoteEndpoint({ isDev: true, protocol: 'ftp' }),
|
||||
/Unsupported remote protocol/,
|
||||
);
|
||||
|
||||
console.log('Remote endpoint tests passed.');
|
||||
71
scripts/verify-extension-runtime.mjs
Normal file
71
scripts/verify-extension-runtime.mjs
Normal file
@ -0,0 +1,71 @@
|
||||
const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9225';
|
||||
const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
const worker = targets.find(
|
||||
(target) => target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'),
|
||||
);
|
||||
if (!worker) throw new Error('Azon Seeker background service worker was not found.');
|
||||
|
||||
const socket = new WebSocket(worker.webSocketDebuggerUrl);
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (!message.id || !pending.has(message.id)) return;
|
||||
const { resolve, reject } = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
message.error ? reject(new Error(message.error.message)) : resolve(message.result);
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
socket.addEventListener('open', resolve, { once: true });
|
||||
socket.addEventListener('error', reject, { once: true });
|
||||
});
|
||||
|
||||
function request(method, params = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
socket.send(JSON.stringify({ id, method, params }));
|
||||
});
|
||||
}
|
||||
|
||||
const evaluation = await request('Runtime.evaluate', {
|
||||
expression: `(async () => ({
|
||||
manifest: chrome.runtime.getManifest(),
|
||||
alarm: await chrome.alarms.get('amazon-sync-retry'),
|
||||
storage: await chrome.storage.local.get([
|
||||
'amazon-sync-queue-v1',
|
||||
'amazon-sync-state-v1'
|
||||
])
|
||||
}))()`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
socket.close();
|
||||
|
||||
if (evaluation.exceptionDetails) {
|
||||
throw new Error(evaluation.exceptionDetails.text || 'Extension evaluation failed.');
|
||||
}
|
||||
const runtime = evaluation.result.value;
|
||||
if (runtime.manifest.version !== '0.7.13') throw new Error('Unexpected extension version.');
|
||||
if (!runtime.manifest.permissions.includes('alarms')) throw new Error('Missing alarms permission.');
|
||||
if (runtime.alarm?.name !== 'amazon-sync-retry') throw new Error('Retry alarm was not created.');
|
||||
const state = JSON.parse(runtime.storage['amazon-sync-state-v1']);
|
||||
if (state.pendingCount !== 0 || state.status !== 'synced') {
|
||||
throw new Error(`Unexpected initial sync state: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
extensionId: new URL(worker.url).host,
|
||||
version: runtime.manifest.version,
|
||||
retryAlarm: runtime.alarm.name,
|
||||
syncState: state,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
115
scripts/verify-extension-sync-e2e.mjs
Normal file
115
scripts/verify-extension-sync-e2e.mjs
Normal file
@ -0,0 +1,115 @@
|
||||
const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9226';
|
||||
const syncId = process.env.AZON_E2E_SYNC_ID;
|
||||
const expectFailure = process.env.AZON_EXPECT_SYNC_FAILURE === '1';
|
||||
if (!syncId) throw new Error('AZON_E2E_SYNC_ID is required.');
|
||||
|
||||
let worker;
|
||||
for (let attempt = 0; attempt < 60 && !worker; attempt++) {
|
||||
const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
worker = targets.find(
|
||||
(target) =>
|
||||
target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'),
|
||||
);
|
||||
if (!worker) await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
if (!worker) throw new Error('Azon Seeker background service worker was not found.');
|
||||
|
||||
const socket = new WebSocket(worker.webSocketDebuggerUrl);
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
socket.addEventListener('message', (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (!message.id || !pending.has(message.id)) return;
|
||||
const { resolve, reject } = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
message.error ? reject(new Error(message.error.message)) : resolve(message.result);
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
socket.addEventListener('open', resolve, { once: true });
|
||||
socket.addEventListener('error', reject, { once: true });
|
||||
});
|
||||
function request(method, params = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
socket.send(JSON.stringify({ id, method, params }));
|
||||
});
|
||||
}
|
||||
async function evaluate(expression) {
|
||||
const result = await request('Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (result.exceptionDetails) throw new Error(result.exceptionDetails.text);
|
||||
return result.result.value;
|
||||
}
|
||||
|
||||
const searchItem = {
|
||||
syncId,
|
||||
asin: 'B000000098',
|
||||
keywords: 'extension queue e2e',
|
||||
rank: 1,
|
||||
page: 1,
|
||||
pageRank: 1,
|
||||
link: 'https://www.amazon.com/dp/B000000098',
|
||||
title: 'Extension queue end-to-end fixture',
|
||||
price: '$33.93',
|
||||
strikethroughPrice: '$39.99',
|
||||
adType: 'sponsoredDisplay',
|
||||
adRank: 1,
|
||||
placement: 'sidebar',
|
||||
placementRank: 1,
|
||||
placementId: 'extension-e2e-placement',
|
||||
adProductIndex: 1,
|
||||
adFormat: 'standard',
|
||||
imageSrc: 'https://example.com/extension-e2e.jpg',
|
||||
createTime: '2026/08/31 11:45:00',
|
||||
};
|
||||
await evaluate(`(async () => {
|
||||
await chrome.storage.local.set({
|
||||
'amazon-sync-queue-v1': ${JSON.stringify({
|
||||
searchItems: [searchItem],
|
||||
detailItems: [],
|
||||
reviews: [],
|
||||
})},
|
||||
'amazon-sync-state-v1': JSON.stringify({status: 'pending', pendingCount: 1})
|
||||
});
|
||||
await chrome.alarms.create('amazon-sync-retry', { when: Date.now() + 200 });
|
||||
return true;
|
||||
})()`);
|
||||
|
||||
let state;
|
||||
for (let attempt = 0; attempt < 30; attempt++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
state = await evaluate(`(async () => {
|
||||
const stored = await chrome.storage.local.get(['amazon-sync-queue-v1', 'amazon-sync-state-v1']);
|
||||
return {
|
||||
queue: stored['amazon-sync-queue-v1'],
|
||||
state: JSON.parse(stored['amazon-sync-state-v1'])
|
||||
};
|
||||
})()`);
|
||||
if (state.state.status === 'failed') {
|
||||
if (expectFailure) break;
|
||||
throw new Error(state.state.lastError);
|
||||
}
|
||||
if (state.state.status === 'synced' && state.state.pendingCount === 0) break;
|
||||
}
|
||||
socket.close();
|
||||
if (expectFailure) {
|
||||
if (
|
||||
state.state.status !== 'failed' ||
|
||||
state.state.pendingCount !== 1 ||
|
||||
state.queue.searchItems.length !== 1
|
||||
) {
|
||||
throw new Error(`Failed synchronization was not retained: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(JSON.stringify({ syncId, retained: true, ...state.state }, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
if (state.state.status !== 'synced' || state.queue.searchItems.length !== 0) {
|
||||
throw new Error(`Queue did not drain: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(JSON.stringify({ syncId, ...state.state }, null, 2));
|
||||
16
scripts/verify-port-disconnect-build.mjs
Normal file
16
scripts/verify-port-disconnect-build.mjs
Normal file
@ -0,0 +1,16 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
|
||||
const background = await readFile('extension/dist/background/index.mjs', 'utf8');
|
||||
const contentScript = await readFile('extension/dist/contentScripts/index.global.js', 'utf8');
|
||||
|
||||
if (!background.includes('lastError')) {
|
||||
throw new Error('Background bundle does not consume runtime.lastError on port disconnect.');
|
||||
}
|
||||
if (!contentScript.includes('back/forward cache')) {
|
||||
throw new Error('Content script bundle does not recognize BFCache port closures.');
|
||||
}
|
||||
if (!contentScript.includes('pagehide') || !contentScript.includes('pageshow')) {
|
||||
throw new Error('Content script bundle does not suspend and resume the BFCache port lifecycle.');
|
||||
}
|
||||
|
||||
console.log('Production BFCache port handling verification passed.');
|
||||
29
scripts/verify-production-endpoint.mjs
Normal file
29
scripts/verify-production-endpoint.mjs
Normal file
@ -0,0 +1,29 @@
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
|
||||
const background = await readFile('extension/dist/background/index.mjs', 'utf8');
|
||||
async function readJavaScriptFiles(directory) {
|
||||
const entries = await readdir(directory, { withFileTypes: true });
|
||||
const contents = await Promise.all(
|
||||
entries.map(async (entry) => {
|
||||
const path = join(directory, entry.name);
|
||||
if (entry.isDirectory()) return readJavaScriptFiles(path);
|
||||
if (!/\.(?:m?js)$/.test(entry.name)) return '';
|
||||
return readFile(path, 'utf8');
|
||||
}),
|
||||
);
|
||||
return contents.flat().join('\n');
|
||||
}
|
||||
const allJavaScript = await readJavaScriptFiles('extension/dist');
|
||||
|
||||
if (!background.includes('amzplugin.rekeymed.com')) {
|
||||
throw new Error('Production background bundle does not target the public server.');
|
||||
}
|
||||
if (background.includes('127.0.0.1:18000')) {
|
||||
throw new Error('Production background bundle still contains the local API endpoint.');
|
||||
}
|
||||
if (!allJavaScript.includes('wss://') || !background.includes('https://')) {
|
||||
throw new Error('Production bundle is missing secure HTTP/WebSocket protocols.');
|
||||
}
|
||||
|
||||
console.log('Production endpoint verification passed.');
|
||||
214
src/background/amazon-sync.ts
Normal file
214
src/background/amazon-sync.ts
Normal file
@ -0,0 +1,214 @@
|
||||
import { onMessage } from 'webext-bridge/background';
|
||||
import { flattenObject } from '~/logic/convert';
|
||||
import { exportKey, generateKeyPair, signData } from '~/logic/crypto';
|
||||
import { remoteBaseUrl } from '~/env';
|
||||
import {
|
||||
AMAZON_SYNC_ALARM,
|
||||
AMAZON_SYNC_QUEUE_KEY,
|
||||
AMAZON_SYNC_STATE_KEY,
|
||||
EMPTY_AMAZON_SYNC_STATE,
|
||||
} from '~/logic/amazon-sync-shared';
|
||||
|
||||
type AmazonSyncQueue = Required<AmazonSyncPayload>;
|
||||
|
||||
const emptyQueue = (): AmazonSyncQueue => ({ searchItems: [], detailItems: [], reviews: [] });
|
||||
const keyPairPromise = generateKeyPair();
|
||||
const publicKeyPromise = keyPairPromise.then(({ publicKey }) => exportKey(publicKey));
|
||||
let queueMutation: Promise<unknown> = Promise.resolve();
|
||||
let flushOperation: Promise<AmazonSyncState> | undefined;
|
||||
let flushAgain = false;
|
||||
|
||||
function mutateQueue<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const result = queueMutation.then(operation, operation);
|
||||
queueMutation = result.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readQueue(): Promise<AmazonSyncQueue> {
|
||||
const stored = await browser.storage.local.get(AMAZON_SYNC_QUEUE_KEY);
|
||||
return { ...emptyQueue(), ...(stored[AMAZON_SYNC_QUEUE_KEY] as AmazonSyncQueue | undefined) };
|
||||
}
|
||||
|
||||
function pendingCount(queue: AmazonSyncQueue) {
|
||||
return queue.searchItems.length + queue.detailItems.length + queue.reviews.length;
|
||||
}
|
||||
|
||||
async function writeQueue(queue: AmazonSyncQueue) {
|
||||
await browser.storage.local.set({ [AMAZON_SYNC_QUEUE_KEY]: queue });
|
||||
}
|
||||
|
||||
async function writeState(state: AmazonSyncState) {
|
||||
await browser.storage.local.set({ [AMAZON_SYNC_STATE_KEY]: JSON.stringify(state) });
|
||||
return state;
|
||||
}
|
||||
|
||||
function mergeUnique<T>(current: T[], incoming: T[], key: (item: T) => string | undefined) {
|
||||
const values = new Map(current.map((item) => [key(item), item]));
|
||||
for (const item of incoming) values.set(key(item), item);
|
||||
values.delete(undefined);
|
||||
return Array.from(values.values());
|
||||
}
|
||||
|
||||
async function signedPost(path: string, data: unknown[]) {
|
||||
if (data.length === 0) return;
|
||||
const body = JSON.stringify(data);
|
||||
const { privateKey } = await keyPairPromise;
|
||||
const response = await fetch(`${remoteBaseUrl}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-signature': await signData(privateKey, body),
|
||||
'x-public-key': await publicKeyPromise,
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`${path} HTTP ${response.status}`);
|
||||
const result = (await response.json()) as { success?: boolean; count?: number };
|
||||
if (result.success !== true || result.count !== data.length) {
|
||||
throw new Error(`${path} 返回的确认数量不匹配`);
|
||||
}
|
||||
}
|
||||
|
||||
function serializeDetailItems(items: AmazonDetailItem[]) {
|
||||
return items.map((item) =>
|
||||
flattenObject(
|
||||
item.imageUrls
|
||||
? {
|
||||
...item,
|
||||
imageUrls: item.imageUrls.join(';'),
|
||||
}
|
||||
: item,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function serializeReviews(items: ({ asin: string } & AmazonReview)[]) {
|
||||
return items.map((item) => ({ ...item, imageSrc: item.imageSrc?.join(';') }));
|
||||
}
|
||||
|
||||
async function removeSynced(
|
||||
field: keyof AmazonSyncQueue,
|
||||
sentKeys: Set<string>,
|
||||
key: (item: any) => string | undefined,
|
||||
) {
|
||||
await mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
(current[field] as any[]) = (current[field] as any[]).filter((item) => {
|
||||
const value = key(item);
|
||||
return !value || !sentKeys.has(value);
|
||||
});
|
||||
await writeQueue(current);
|
||||
});
|
||||
}
|
||||
|
||||
async function flushQueueInternal(): Promise<AmazonSyncState> {
|
||||
let queue = await mutateQueue(readQueue);
|
||||
if (pendingCount(queue) === 0) {
|
||||
return mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
return writeState({
|
||||
...EMPTY_AMAZON_SYNC_STATE,
|
||||
status: pendingCount(current) === 0 ? 'synced' : 'pending',
|
||||
pendingCount: pendingCount(current),
|
||||
});
|
||||
});
|
||||
}
|
||||
await writeState({ status: 'syncing', pendingCount: pendingCount(queue) });
|
||||
try {
|
||||
const searchSnapshot = queue.searchItems;
|
||||
await signedPost('/amazon/search_items', searchSnapshot);
|
||||
await removeSynced(
|
||||
'searchItems',
|
||||
new Set(searchSnapshot.map((item) => item.syncId).filter(Boolean) as string[]),
|
||||
(item: AmazonSearchItem) => item.syncId,
|
||||
);
|
||||
|
||||
queue = await mutateQueue(readQueue);
|
||||
const detailSnapshot = queue.detailItems;
|
||||
await signedPost('/amazon/detail_items', serializeDetailItems(detailSnapshot));
|
||||
await removeSynced(
|
||||
'detailItems',
|
||||
new Set(detailSnapshot.map((item) => item.syncId).filter(Boolean) as string[]),
|
||||
(item: AmazonDetailItem) => item.syncId,
|
||||
);
|
||||
|
||||
queue = await mutateQueue(readQueue);
|
||||
const reviewSnapshot = queue.reviews;
|
||||
await signedPost('/amazon/reviews', serializeReviews(reviewSnapshot));
|
||||
await removeSynced(
|
||||
'reviews',
|
||||
new Set(reviewSnapshot.map((item) => item.id)),
|
||||
(item: AmazonReview) => item.id,
|
||||
);
|
||||
|
||||
return mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
return writeState({
|
||||
status: pendingCount(current) === 0 ? 'synced' : 'pending',
|
||||
pendingCount: pendingCount(current),
|
||||
lastSuccessAt: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
await writeState({
|
||||
status: 'failed',
|
||||
pendingCount: pendingCount(current),
|
||||
lastError: message,
|
||||
});
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function flushQueue() {
|
||||
if (flushOperation) {
|
||||
flushAgain = true;
|
||||
} else {
|
||||
flushOperation = flushQueueInternal().finally(() => {
|
||||
flushOperation = undefined;
|
||||
if (flushAgain) {
|
||||
flushAgain = false;
|
||||
void flushQueue().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
return flushOperation;
|
||||
}
|
||||
|
||||
async function enqueue(payload: AmazonSyncPayload) {
|
||||
const state = await mutateQueue(async () => {
|
||||
const queue = await readQueue();
|
||||
queue.searchItems = mergeUnique(
|
||||
queue.searchItems,
|
||||
payload.searchItems || [],
|
||||
(item) => item.syncId,
|
||||
);
|
||||
queue.detailItems = mergeUnique(
|
||||
queue.detailItems,
|
||||
payload.detailItems || [],
|
||||
(item) => item.syncId,
|
||||
);
|
||||
queue.reviews = mergeUnique(queue.reviews, payload.reviews || [], (item) => item.id);
|
||||
await writeQueue(queue);
|
||||
return writeState({ status: 'pending', pendingCount: pendingCount(queue) });
|
||||
});
|
||||
void flushQueue().catch(() => undefined);
|
||||
return state;
|
||||
}
|
||||
|
||||
onMessage('amazon-sync-enqueue', ({ data }) => enqueue(data));
|
||||
onMessage('amazon-sync-flush', () => flushQueue());
|
||||
|
||||
browser.alarms.create(AMAZON_SYNC_ALARM, { periodInMinutes: 1 });
|
||||
browser.alarms.onAlarm.addListener((alarm) => {
|
||||
if (alarm.name === AMAZON_SYNC_ALARM) void flushQueue().catch(() => undefined);
|
||||
});
|
||||
|
||||
void flushQueue().catch(() => undefined);
|
||||
@ -2,6 +2,7 @@ import { isFirefox } from '~/env';
|
||||
|
||||
// https://github.com/serversideup/webext-bridge/issues/67#issuecomment-2676094094
|
||||
import('webext-bridge/background');
|
||||
import('./amazon-sync');
|
||||
|
||||
// only on dev mode
|
||||
if (import.meta.hot) {
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { remoteHost } from '~/env';
|
||||
import { remoteBaseUrl, remoteWebSocketBaseUrl } from '~/env';
|
||||
import { useLongTask } from './useLongTask';
|
||||
|
||||
type AddDataFragmentsCommand = {
|
||||
@ -16,16 +16,12 @@ type WebSocketResponse =
|
||||
| { type: 'progress'; current: number; total: number }
|
||||
| { type: 'result'; result: string };
|
||||
|
||||
// 使用 wss:// 和 https://
|
||||
const protocol = __DEV__ ? 'ws' : 'wss';
|
||||
const httpProtocol = __DEV__ ? 'http' : 'https';
|
||||
|
||||
class ExportExcelPipeline {
|
||||
private socket: WebSocket;
|
||||
|
||||
constructor() {
|
||||
this.socket = new WebSocket(
|
||||
`${protocol}://${remoteHost}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`,
|
||||
`${remoteWebSocketBaseUrl}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`,
|
||||
);
|
||||
}
|
||||
|
||||
@ -39,7 +35,7 @@ class ExportExcelPipeline {
|
||||
case WebSocket.CLOSED:
|
||||
case WebSocket.CLOSING:
|
||||
this.socket = new WebSocket(
|
||||
`${protocol}://${remoteHost}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`,
|
||||
`${remoteWebSocketBaseUrl}/ws/daa0b9f1-4e4a-4e7c-9269-f5f0e86ae271`,
|
||||
);
|
||||
case WebSocket.CONNECTING:
|
||||
return new Promise<ExportExcelPipeline>((resolve) => {
|
||||
@ -126,7 +122,7 @@ export const useCloudExporter = () => {
|
||||
await pipeline.close();
|
||||
|
||||
if (file) {
|
||||
const url = `${httpProtocol}://${remoteHost}${file}`;
|
||||
const url = `${remoteBaseUrl}${file}`;
|
||||
const link = document.createElement('a');
|
||||
link.href = url;
|
||||
link.download = `${dayjs().format('YYYY-MM-DD')}.xlsx`;
|
||||
|
||||
@ -1,17 +1,18 @@
|
||||
export function useLongTask() {
|
||||
const isRunning = ref(false);
|
||||
const lastError = shallowRef<unknown>();
|
||||
|
||||
const startTask = async <T = undefined>(task: () => Promise<T>) => {
|
||||
if (isRunning.value) {
|
||||
throw Error('Task is still running.');
|
||||
}
|
||||
isRunning.value = true;
|
||||
lastError.value = undefined;
|
||||
let result = undefined;
|
||||
try {
|
||||
result = await task();
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
lastError.value = error;
|
||||
console.error('Task failed:', error);
|
||||
}
|
||||
isRunning.value = false;
|
||||
@ -20,6 +21,7 @@ export function useLongTask() {
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
lastError,
|
||||
startTask,
|
||||
};
|
||||
}
|
||||
|
||||
17
src/env.ts
17
src/env.ts
@ -1,3 +1,5 @@
|
||||
import { resolveRemoteEndpoint } from '~/logic/remote-endpoint';
|
||||
|
||||
const forbiddenProtocols = [
|
||||
'chrome-extension://',
|
||||
'chrome-search://',
|
||||
@ -14,9 +16,12 @@ export function isForbiddenUrl(url: string): boolean {
|
||||
|
||||
export const isFirefox = navigator.userAgent.includes('Firefox');
|
||||
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:8000' : '47.251.4.191:8000';
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'vm8nc3zr-18000.usw2.devtunnels.ms';
|
||||
// export const remoteHost = __DEV__ ? 'ogsm-dev.rekeymed.com:8888' : 'amzplugin.rekeymed.com';
|
||||
export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com';
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com';
|
||||
// export const remoteHost = '47.251.4.191:8000';
|
||||
const remoteEndpoint = resolveRemoteEndpoint({
|
||||
isDev: __DEV__,
|
||||
host: import.meta.env.VITE_REMOTE_HOST,
|
||||
protocol: import.meta.env.VITE_REMOTE_PROTOCOL,
|
||||
});
|
||||
|
||||
export const remoteHost = remoteEndpoint.host;
|
||||
export const remoteBaseUrl = remoteEndpoint.httpBaseUrl;
|
||||
export const remoteWebSocketBaseUrl = remoteEndpoint.webSocketBaseUrl;
|
||||
|
||||
9
src/logic/amazon-sync-client.ts
Normal file
9
src/logic/amazon-sync-client.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { sendMessage } from 'webext-bridge/sidepanel';
|
||||
|
||||
export async function enqueueAmazonSync(payload: AmazonSyncPayload) {
|
||||
return sendMessage('amazon-sync-enqueue', payload, 'background');
|
||||
}
|
||||
|
||||
export async function flushAmazonSyncQueue() {
|
||||
return sendMessage('amazon-sync-flush', undefined, 'background');
|
||||
}
|
||||
12
src/logic/amazon-sync-shared.ts
Normal file
12
src/logic/amazon-sync-shared.ts
Normal file
@ -0,0 +1,12 @@
|
||||
export const AMAZON_SYNC_QUEUE_KEY = 'amazon-sync-queue-v1';
|
||||
export const AMAZON_SYNC_STATE_KEY = 'amazon-sync-state-v1';
|
||||
export const AMAZON_SYNC_ALARM = 'amazon-sync-retry';
|
||||
|
||||
export const EMPTY_AMAZON_SYNC_STATE: AmazonSyncState = {
|
||||
status: 'idle',
|
||||
pendingCount: 0,
|
||||
};
|
||||
|
||||
export function createAmazonSyncId(prefix: 'search' | 'detail') {
|
||||
return `${prefix}-${crypto.randomUUID()}`;
|
||||
}
|
||||
@ -47,45 +47,45 @@ export async function exec<T, P extends Record<string, unknown>>(
|
||||
options: ExecOptions = {},
|
||||
): Promise<T> {
|
||||
const { timeout = 30000 } = options;
|
||||
return new Promise<T>(async (resolve, reject) => {
|
||||
// 基本刷新
|
||||
// await browser.tabs.reload(tabId);
|
||||
// console.log('exec', browser.tabs.get(tabId), tabId);
|
||||
|
||||
// console.log('exec', func, (await browser.tabs.get(tabId)).url);
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise<void>((r) => setTimeout(r, 200));
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status === 'complete') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeout;
|
||||
let pageLoaded = false;
|
||||
while (Date.now() < deadline) {
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status !== 'complete') {
|
||||
await browser.tabs.reload(tabId);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始
|
||||
if (tab.status === 'complete') {
|
||||
pageLoaded = true;
|
||||
break;
|
||||
}
|
||||
const remaining = deadline - Date.now();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, Math.min(200, remaining)));
|
||||
}
|
||||
if (!pageLoaded) throw new Error(`页面加载超时（${timeout}ms），未执行采集脚本`);
|
||||
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise<void>((r) => setTimeout(r, 200));
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status === 'complete') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
const remainingTimeout = Math.max(1, deadline - Date.now());
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(
|
||||
() => reject(new Error(`脚本运行超时（总预算 ${timeout}ms）`)),
|
||||
remainingTimeout,
|
||||
);
|
||||
});
|
||||
|
||||
setTimeout(() => reject(`脚本运行超时, foucs on ${func}`), timeout);
|
||||
try {
|
||||
const injectResults = await browser.scripting.executeScript({
|
||||
try {
|
||||
const injectResults = await Promise.race([
|
||||
browser.scripting.executeScript({
|
||||
target: { tabId },
|
||||
func,
|
||||
args: payload ? [payload] : undefined,
|
||||
});
|
||||
const ret = injectResults.pop();
|
||||
resolve(ret!.result as T);
|
||||
} catch (e) {
|
||||
throw new Error(`脚本运行失败: ${e}`);
|
||||
}
|
||||
});
|
||||
}),
|
||||
timeoutPromise,
|
||||
]);
|
||||
const ret = injectResults.at(-1);
|
||||
if (!ret) throw new Error('脚本未返回执行结果');
|
||||
return ret.result as T;
|
||||
} catch (error) {
|
||||
const cause = error instanceof Error ? error : new Error(String(error));
|
||||
if (cause.message.startsWith('脚本运行超时')) throw cause;
|
||||
throw new Error(`脚本运行失败: ${cause.message}`);
|
||||
} finally {
|
||||
if (timeoutId !== undefined) clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
28
src/logic/remote-endpoint.ts
Normal file
28
src/logic/remote-endpoint.ts
Normal file
@ -0,0 +1,28 @@
|
||||
type RemoteEndpointOptions = {
|
||||
isDev: boolean;
|
||||
host?: string;
|
||||
protocol?: string;
|
||||
};
|
||||
|
||||
export type RemoteEndpoint = {
|
||||
host: string;
|
||||
httpBaseUrl: string;
|
||||
webSocketBaseUrl: string;
|
||||
};
|
||||
|
||||
export function resolveRemoteEndpoint({
|
||||
isDev,
|
||||
host,
|
||||
protocol,
|
||||
}: RemoteEndpointOptions): RemoteEndpoint {
|
||||
const resolvedHost = host || (isDev ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com');
|
||||
const resolvedProtocol = protocol || (isDev ? 'http' : 'https');
|
||||
if (resolvedProtocol !== 'http' && resolvedProtocol !== 'https') {
|
||||
throw new Error(`Unsupported remote protocol: ${resolvedProtocol}`);
|
||||
}
|
||||
return {
|
||||
host: resolvedHost,
|
||||
httpBaseUrl: `${resolvedProtocol}://${resolvedHost}`,
|
||||
webSocketBaseUrl: `${resolvedProtocol === 'https' ? 'wss' : 'ws'}://${resolvedHost}`,
|
||||
};
|
||||
}
|
||||
@ -1,5 +1,5 @@
|
||||
/** 上传图片工具类 */
|
||||
import { remoteHost } from '~/env';
|
||||
import { remoteBaseUrl } from '~/env';
|
||||
|
||||
export async function uploadImage(
|
||||
base64String: string,
|
||||
@ -23,8 +23,7 @@ export async function uploadImage(
|
||||
const formData = new FormData();
|
||||
formData.append('file', blob, filename);
|
||||
|
||||
const httpProtocol = __DEV__ ? 'http' : 'https';
|
||||
const url = `${httpProtocol}://${remoteHost}/upload/image/${encodeURIComponent(filename)}`;
|
||||
const url = `${remoteBaseUrl}/upload/image/${encodeURIComponent(filename)}`;
|
||||
const resp = (await fetch(url, {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
@ -33,5 +32,5 @@ export async function uploadImage(
|
||||
return undefined;
|
||||
}
|
||||
const data = await resp.json();
|
||||
return `${httpProtocol}://${remoteHost}${data.file}`;
|
||||
return `${remoteBaseUrl}${data.file}`;
|
||||
}
|
||||
|
||||
@ -8,10 +8,41 @@ import { allDetailItems, allItems, itemColumnSettings, reviewItems } from '~/sto
|
||||
const modal = useModal();
|
||||
const excelHelper = useExcelHelper();
|
||||
|
||||
const adTypeLabels: Record<AmazonAdType, string> = {
|
||||
organic: '自然排位',
|
||||
sponsoredProducts: 'Sponsored Products',
|
||||
sponsoredBrands: 'Sponsored Brands',
|
||||
sponsoredDisplay: 'Sponsored Display',
|
||||
unknownSponsored: '未知广告',
|
||||
};
|
||||
|
||||
const placementLabels: Record<AmazonAdPlacement, string> = {
|
||||
topOfSearch: '顶部',
|
||||
middleOfSearch: '中部',
|
||||
bottomOfSearch: '底部',
|
||||
restOfSearch: '其余位置（旧数据）',
|
||||
sidebar: '侧边栏',
|
||||
productPage: '商品详情页',
|
||||
unknown: '未知位置',
|
||||
};
|
||||
|
||||
const renderPlacement = (row: AmazonItem) => {
|
||||
if (!row.placement) return '-';
|
||||
const label = placementLabels[row.placement];
|
||||
if (row.placement === 'productPage') return label;
|
||||
return row.page ? `第 ${row.page} 页${label}` : label;
|
||||
};
|
||||
|
||||
const renderOptionalValue = (value: unknown) => value ?? '-';
|
||||
|
||||
const isDetailOnlyRow = (row: AmazonItem) =>
|
||||
!row.keywords && row.page == null && row.rank == null && row.pageRank == null;
|
||||
|
||||
const filter = ref<{
|
||||
keywords?: string;
|
||||
search?: string;
|
||||
detailOnly?: boolean;
|
||||
adType?: AmazonAdType | 'unclassified';
|
||||
searchDateRange?: [number, number];
|
||||
detailDateRange?: [number, number];
|
||||
}>({});
|
||||
@ -34,18 +65,55 @@ const columns = computed<TableColumn[]>(() => {
|
||||
key: 'keywords',
|
||||
minWidth: 120,
|
||||
hidden: !itemColumnSettings.value.has('keywords'),
|
||||
render: (row) => renderOptionalValue(row.keywords),
|
||||
},
|
||||
{
|
||||
title: '页码',
|
||||
key: 'page',
|
||||
minWidth: 60,
|
||||
hidden: !itemColumnSettings.value.has('page'),
|
||||
render: (row) => renderOptionalValue(row.page),
|
||||
},
|
||||
{
|
||||
title: '排位',
|
||||
key: 'rank',
|
||||
minWidth: 60,
|
||||
hidden: !itemColumnSettings.value.has('rank'),
|
||||
render: (row) => renderOptionalValue(row.rank),
|
||||
},
|
||||
{
|
||||
title: '页内排位',
|
||||
key: 'pageRank',
|
||||
minWidth: 90,
|
||||
hidden: !itemColumnSettings.value.has('pageRank'),
|
||||
render: (row) => renderOptionalValue(row.pageRank),
|
||||
},
|
||||
{
|
||||
title: '广告类型',
|
||||
key: 'adType',
|
||||
minWidth: 150,
|
||||
render: (row) =>
|
||||
isDetailOnlyRow(row) ? '-' : row.adType ? adTypeLabels[row.adType] : '历史数据/未识别',
|
||||
},
|
||||
{
|
||||
title: '广告排位',
|
||||
key: 'adRank',
|
||||
minWidth: 90,
|
||||
hidden: !itemColumnSettings.value.has('adRank'),
|
||||
render: (row) => renderOptionalValue(row.adRank),
|
||||
},
|
||||
{
|
||||
title: '广告位置',
|
||||
key: 'placement',
|
||||
minWidth: 140,
|
||||
render: renderPlacement,
|
||||
},
|
||||
{
|
||||
title: '广告位顺序',
|
||||
key: 'placementRank',
|
||||
minWidth: 110,
|
||||
hidden: !itemColumnSettings.value.has('placementRank'),
|
||||
render: (row) => renderOptionalValue(row.placementRank),
|
||||
},
|
||||
{
|
||||
title: 'ASIN',
|
||||
@ -61,6 +129,11 @@ const columns = computed<TableColumn[]>(() => {
|
||||
key: 'price',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: '划线价格',
|
||||
key: 'strikethroughPrice',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: '封面图',
|
||||
key: 'imageSrc',
|
||||
@ -181,7 +254,7 @@ const getItemHeaders = () => {
|
||||
};
|
||||
|
||||
const filteredData = computed(() => {
|
||||
const { search, detailOnly, keywords, searchDateRange, detailDateRange } = filter.value;
|
||||
const { search, detailOnly, keywords, adType, searchDateRange, detailDateRange } = filter.value;
|
||||
let data = toRaw(detailOnly ? allDetailItems.value : allItems.value);
|
||||
if (search && search.trim() !== '') {
|
||||
data = data.filter((r) => {
|
||||
@ -193,6 +266,9 @@ const filteredData = computed(() => {
|
||||
if (keywords) {
|
||||
data = data.filter((r) => r.keywords === keywords);
|
||||
}
|
||||
if (adType) {
|
||||
data = data.filter((r) => (adType === 'unclassified' ? !r.adType : r.adType === adType));
|
||||
}
|
||||
if (searchDateRange) {
|
||||
const start = dayjs(searchDateRange[0]);
|
||||
const end = dayjs(searchDateRange[1]);
|
||||
@ -321,6 +397,17 @@ const handleClearData = async () => {
|
||||
}))"
|
||||
/>
|
||||
</n-form-item>
|
||||
<n-form-item label="广告类型">
|
||||
<n-select
|
||||
v-model:value="filter.adType"
|
||||
clearable
|
||||
placeholder="全部"
|
||||
:options="[
|
||||
...Object.entries(adTypeLabels).map(([value, label]) => ({ value, label })),
|
||||
{ value: 'unclassified', label: '历史数据/未识别' },
|
||||
]"
|
||||
/>
|
||||
</n-form-item>
|
||||
<n-form-item label="日期(搜索页)">
|
||||
<n-date-picker
|
||||
type="datetimerange"
|
||||
@ -344,6 +431,9 @@ const handleClearData = async () => {
|
||||
<n-checkbox value="keywords" label="关键词" />
|
||||
<n-checkbox value="page" label="页码" />
|
||||
<n-checkbox value="rank" label="排位" />
|
||||
<n-checkbox value="pageRank" label="页内排位" />
|
||||
<n-checkbox value="adRank" label="广告排位" />
|
||||
<n-checkbox value="placementRank" label="广告位顺序" />
|
||||
<n-checkbox value="createTime" label="获取日期" />
|
||||
<n-checkbox value="timestamp" label="获取日期（详情）" />
|
||||
</n-space>
|
||||
|
||||
@ -44,7 +44,7 @@
|
||||
|
||||
位于搜索页面板，可以根据输入的关键词采集搜索页中呈现的商品信息，主要为**关键词排名**。
|
||||
输入：关键词（最多十个）
|
||||
输出：商品搜索页信息，包含关键词、页码、排位、 ASIN、标题、 价格、 封面图、获取日期。
|
||||
输出：商品搜索页信息，包含关键词、页码、排位、广告类型、广告排位、广告位置、ASIN、标题、当前价格、划线价格、封面图、获取日期。
|
||||
|
||||
### 详情页
|
||||
|
||||
@ -52,7 +52,7 @@
|
||||
|
||||
位于详情页面板，可以根据输入的多个ASIN采集商品详情页的商品信息。
|
||||
输入：ASIN（每行输入一个，可以输入多个）
|
||||
输出：商品详情页信息，包含ASIN、标题、 价格、 封面图、获取日期、评分、评论数、 大类、大类排行、小类、 小类排行、 商品图片链接、A+截图等等。
|
||||
输出：商品详情页信息，包含ASIN、标题、当前价格、划线价格、封面图、获取日期、评分、评论数、大类、大类排行、小类、小类排行、商品图片链接、A+截图等等。
|
||||
|
||||
### 评论页
|
||||
|
||||
@ -86,9 +86,15 @@ Items表记录商品信息，表头信息如下。
|
||||
| 关键词 | 产品的关键词 |
|
||||
| 页码 | 产品在关键词搜索列表中的页码 |
|
||||
| 排位 | 产品的排名 |
|
||||
| 页内排位 | 产品在当前搜索页商品流中的排名 |
|
||||
| 广告类型 | 自然排位、Sponsored Products、Sponsored Brands、Sponsored Display 或未知广告 |
|
||||
| 广告排位 | 同一关键词、同一广告类型中的累计排名 |
|
||||
| 广告位置 | 搜索顶部、搜索其余位置或侧边栏 |
|
||||
| 广告位顺序 | 同一关键词采集过程中广告模块的累计顺序 |
|
||||
| ASIN | 亚马逊标准识别号 |
|
||||
| 标题 | 产品的标题 |
|
||||
| 价格 | 产品的价格 |
|
||||
| 价格 | 产品的当前价格 |
|
||||
| 划线价格 | 页面中以划线形式展示的参考价格 |
|
||||
| 封面图 | 产品的封面图片链接 |
|
||||
| 获取日期 | 数据获取的日期 |
|
||||
| 商品链接 | 产品的链接 |
|
||||
|
||||
@ -3,7 +3,8 @@ import amazon from '../impls/amazon';
|
||||
import { uploadImage } from '~/logic/upload-image';
|
||||
import { detailItems, reviewItems, searchItems } from '~/storages/amazon';
|
||||
import { createGlobalState } from '@vueuse/core';
|
||||
import { useAmazonService } from '~/page-worker/services/amazon';
|
||||
import { enqueueAmazonSync } from '~/logic/amazon-sync-client';
|
||||
import { createAmazonSyncId } from '~/logic/amazon-sync-shared';
|
||||
import { taskOptionBase, WorkerComposable } from '../interfaces/common';
|
||||
import { AmazonPageWorker } from '../interfaces/amazon';
|
||||
|
||||
@ -18,10 +19,7 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
// Reactive settings object
|
||||
const settings = shallowRef<AmazonPageWorkerSettings>({});
|
||||
// Long task management
|
||||
const { isRunning, startTask } = useLongTask();
|
||||
// Amazon service instance
|
||||
const service = useAmazonService();
|
||||
|
||||
const { isRunning, lastError, startTask } = useLongTask();
|
||||
// Get the worker instance from implementation
|
||||
const worker = amazon.getAmazonPageWorker();
|
||||
|
||||
@ -29,10 +27,13 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const searchCache = [] as AmazonSearchItem[];
|
||||
const detailCache = new Map<string, AmazonDetailItem>();
|
||||
const reviewCache = new Map<string, AmazonReview[]>();
|
||||
let commitOperation: Promise<void> = Promise.resolve();
|
||||
|
||||
// Add search items to cache
|
||||
const updateSearchCache = (data: AmazonSearchItem[]) => {
|
||||
searchCache.push(...data);
|
||||
searchCache.push(
|
||||
...data.map((item) => ({ ...item, syncId: item.syncId || createAmazonSyncId('search') })),
|
||||
);
|
||||
};
|
||||
|
||||
// Update or add detail item in cache
|
||||
@ -42,7 +43,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const origin = detailCache.get(data.asin);
|
||||
detailCache.set(asin, { ...origin, ...data } as AmazonDetailItem);
|
||||
} else {
|
||||
detailCache.set(asin, data as AmazonDetailItem);
|
||||
detailCache.set(asin, {
|
||||
...data,
|
||||
syncId: data.syncId || createAmazonSyncId('detail'),
|
||||
} as AmazonDetailItem);
|
||||
}
|
||||
};
|
||||
|
||||
@ -52,7 +56,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const values = reviewCache.get(asin) || [];
|
||||
const ids = new Set(values.map((item) => item.id));
|
||||
for (const review of reviews) {
|
||||
ids.has(review.id) || values.push(review);
|
||||
if (!ids.has(review.id)) {
|
||||
values.push(review);
|
||||
ids.add(review.id);
|
||||
}
|
||||
}
|
||||
reviewCache.set(asin, values);
|
||||
};
|
||||
@ -64,44 +71,90 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
reviewCache.clear();
|
||||
};
|
||||
|
||||
// Commit cached changes to storage and service
|
||||
const commitChange = async () => {
|
||||
// Persist locally first, then atomically enqueue for background synchronization.
|
||||
const commitChangeInternal = async () => {
|
||||
const { objects } = settings.value;
|
||||
// Commit search items
|
||||
if (objects?.includes('search')) {
|
||||
searchItems.value = searchItems.value.concat(searchCache);
|
||||
await service.commitSearchItems(searchCache);
|
||||
searchCache.splice(0, searchCache.length);
|
||||
const searchSnapshot = objects?.includes('search') ? searchCache.slice() : [];
|
||||
const detailSnapshot = objects?.includes('detail') ? Array.from(detailCache.values()) : [];
|
||||
const reviewSnapshot = objects?.includes('review')
|
||||
? Array.from(reviewCache.entries()).flatMap(([asin, reviews]) =>
|
||||
reviews.map((review) => ({ asin, ...review })),
|
||||
)
|
||||
: [];
|
||||
|
||||
if (searchSnapshot.length === 0 && detailSnapshot.length === 0 && reviewSnapshot.length === 0) {
|
||||
return;
|
||||
}
|
||||
// Commit detail items
|
||||
if (objects?.includes('detail')) {
|
||||
for (const [k, v] of detailCache.entries()) {
|
||||
if (detailItems.value.has(k)) {
|
||||
const item = detailItems.value.get(k)!;
|
||||
detailItems.value.set(k, { ...item, ...v });
|
||||
|
||||
if (searchSnapshot.length > 0) {
|
||||
const existingIds = new Set(searchItems.value.map((item) => item.syncId).filter(Boolean));
|
||||
searchItems.value = searchItems.value.concat(
|
||||
searchSnapshot.filter((item) => !item.syncId || !existingIds.has(item.syncId)),
|
||||
);
|
||||
}
|
||||
if (detailSnapshot.length > 0) {
|
||||
const nextDetailItems = new Map(detailItems.value);
|
||||
for (const v of detailSnapshot) {
|
||||
const k = v.asin;
|
||||
if (nextDetailItems.has(k)) {
|
||||
const item = nextDetailItems.get(k)!;
|
||||
nextDetailItems.set(k, { ...item, ...v });
|
||||
} else {
|
||||
detailItems.value.set(k, v);
|
||||
nextDetailItems.set(k, v);
|
||||
}
|
||||
}
|
||||
await service.commitDetailItems(detailCache);
|
||||
detailCache.clear();
|
||||
// Replacing the Map gives every extension page a deterministic reactive
|
||||
// change to persist and observe; in-place Map.set could leave the open
|
||||
// results page showing stale data during long batches.
|
||||
detailItems.value = nextDetailItems;
|
||||
}
|
||||
// Commit reviews
|
||||
if (objects?.includes('review')) {
|
||||
for (const [asin, reviews] of reviewCache.entries()) {
|
||||
if (reviewItems.value.has(asin)) {
|
||||
if (reviewSnapshot.length > 0) {
|
||||
const groupedReviews = reviewSnapshot.reduce<Map<string, AmazonReview[]>>((grouped, item) => {
|
||||
const values = grouped.get(item.asin) || [];
|
||||
const { asin: _, ...review } = item;
|
||||
values.push(review);
|
||||
grouped.set(item.asin, values);
|
||||
return grouped;
|
||||
}, new Map());
|
||||
const nextReviewItems = new Map(reviewItems.value);
|
||||
for (const [asin, reviews] of groupedReviews.entries()) {
|
||||
if (nextReviewItems.has(asin)) {
|
||||
const addedIds = new Set(reviews.map((x) => x.id));
|
||||
const origin = reviewItems.value.get(asin)!;
|
||||
const origin = nextReviewItems.get(asin)!;
|
||||
const newReviews = origin.filter((x) => !addedIds.has(x.id)).concat(reviews);
|
||||
newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo)));
|
||||
reviewItems.value.set(asin, newReviews);
|
||||
nextReviewItems.set(asin, newReviews);
|
||||
} else {
|
||||
reviewItems.value.set(asin, reviews);
|
||||
nextReviewItems.set(asin, reviews);
|
||||
}
|
||||
}
|
||||
await service.commitReviews(reviewCache);
|
||||
reviewCache.clear();
|
||||
reviewItems.value = nextReviewItems;
|
||||
}
|
||||
|
||||
// The background acknowledges only after the queue is durably written to extension storage.
|
||||
await enqueueAmazonSync({
|
||||
searchItems: searchSnapshot,
|
||||
detailItems: detailSnapshot,
|
||||
reviews: reviewSnapshot,
|
||||
});
|
||||
|
||||
const sentSearchIds = new Set(searchSnapshot.map((item) => item.syncId));
|
||||
const remainingSearch = searchCache.filter((item) => !sentSearchIds.has(item.syncId));
|
||||
searchCache.splice(0, searchCache.length, ...remainingSearch);
|
||||
for (const item of detailSnapshot) {
|
||||
if (detailCache.get(item.asin)?.syncId === item.syncId) detailCache.delete(item.asin);
|
||||
}
|
||||
const sentReviewIds = new Set(reviewSnapshot.map((item) => item.id));
|
||||
for (const [asin, reviews] of reviewCache.entries()) {
|
||||
const remaining = reviews.filter((item) => !sentReviewIds.has(item.id));
|
||||
remaining.length > 0 ? reviewCache.set(asin, remaining) : reviewCache.delete(asin);
|
||||
}
|
||||
};
|
||||
|
||||
const commitChange = () => {
|
||||
const result = commitOperation.then(commitChangeInternal, commitChangeInternal);
|
||||
commitOperation = result.catch(() => undefined);
|
||||
return result;
|
||||
};
|
||||
|
||||
// Register all relevant worker event handlers
|
||||
@ -125,7 +178,7 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
}),
|
||||
// Collect images for detail items
|
||||
worker.on('item-images-collected', (ev) => {
|
||||
updateDetailCache(ev);
|
||||
if (ev.imageUrls?.length) updateDetailCache(ev);
|
||||
}),
|
||||
// Collect A+ content screenshots and upload
|
||||
worker.on('item-aplus-screenshot-collect', async (ev) => {
|
||||
@ -161,15 +214,25 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
/**
|
||||
* Task wrapper: commit changes by interval during task execution
|
||||
*/
|
||||
const taskWrapper1 = <T extends (...params: any[]) => Promise<void>>(func: T) => {
|
||||
const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value;
|
||||
clearAllCaches();
|
||||
const taskWrapper1 = <T extends (...params: any[]) => Promise<void>>(
|
||||
func: T,
|
||||
objectType: 'search' | 'review',
|
||||
) => {
|
||||
return (...params: Parameters<T>) =>
|
||||
startTask(async () => {
|
||||
const interval = setInterval(() => commitChange(), commitChangeIngerval);
|
||||
await func(...params);
|
||||
clearInterval(interval);
|
||||
await commitChange();
|
||||
settings.value = { ...settings.value, objects: [objectType] };
|
||||
clearAllCaches();
|
||||
const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value;
|
||||
const interval = setInterval(
|
||||
() => void commitChange().catch(console.error),
|
||||
commitChangeIngerval,
|
||||
);
|
||||
try {
|
||||
await func(...params);
|
||||
} finally {
|
||||
clearInterval(interval);
|
||||
await commitChange();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
@ -179,9 +242,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const taskWrapper2 = <T extends (input: any, options?: taskOptionBase) => Promise<void>>(
|
||||
func: T,
|
||||
) => {
|
||||
clearAllCaches();
|
||||
return (...params: Parameters<T>) =>
|
||||
startTask(async () => {
|
||||
settings.value = { ...settings.value, objects: ['detail'] };
|
||||
clearAllCaches();
|
||||
if (!params?.[1]) {
|
||||
params[1] = {};
|
||||
}
|
||||
@ -196,20 +260,24 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
await commitChange();
|
||||
};
|
||||
}
|
||||
await func(params[0], params[1]);
|
||||
await commitChange();
|
||||
try {
|
||||
await func(params[0], params[1]);
|
||||
} finally {
|
||||
await commitChange();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Wrapped task runners for each page type
|
||||
const runSearchPageTask = taskWrapper1(worker.runSearchPageTask.bind(worker));
|
||||
const runSearchPageTask = taskWrapper1(worker.runSearchPageTask.bind(worker), 'search');
|
||||
const runDetailPageTask = taskWrapper2(worker.runDetailPageTask.bind(worker));
|
||||
const runReviewPageTask = taskWrapper1(worker.runReviewPageTask.bind(worker));
|
||||
const runReviewPageTask = taskWrapper1(worker.runReviewPageTask.bind(worker), 'review');
|
||||
|
||||
// Expose API for the composable
|
||||
return {
|
||||
settings,
|
||||
isRunning,
|
||||
lastError,
|
||||
runSearchPageTask,
|
||||
runDetailPageTask,
|
||||
runReviewPageTask,
|
||||
|
||||
@ -9,6 +9,12 @@ import { isForbiddenUrl } from '~/env';
|
||||
import { BaseWorker } from './base';
|
||||
import { AmazonPageWorker, AmazonPageWorkerEvents } from '../interfaces/amazon';
|
||||
import { taskOptionBase } from '../interfaces/common';
|
||||
import {
|
||||
extractAmazonDetailAsin,
|
||||
hasReachedAmazonDetailPage,
|
||||
hasReachedAmazonNavigationTarget,
|
||||
} from '../page-navigation';
|
||||
import { isAmazonReviewRatingFilterIgnored } from '../web-injectors/amazon-review-page';
|
||||
|
||||
/**
|
||||
* AmazonPageWorkerImpl can run on background & sidepanel & popup,
|
||||
@ -43,24 +49,70 @@ class AmazonPageWorkerImpl
|
||||
return tab;
|
||||
}
|
||||
|
||||
private async navigateTabAndWait(
|
||||
tabId: number,
|
||||
url: string,
|
||||
options: { allowDetailRedirect?: boolean } = {},
|
||||
): Promise<string> {
|
||||
const previousTab = await browser.tabs.get(tabId);
|
||||
const previousUrl = previousTab.url;
|
||||
const updatedTab = await browser.tabs.update(tabId, { url, active: true });
|
||||
let navigationObserved =
|
||||
updatedTab.status === 'loading' ||
|
||||
(Boolean(updatedTab.url) && updatedTab.url !== previousUrl);
|
||||
let lastUrl = updatedTab.url;
|
||||
for (let i = 0; i < 150; i++) {
|
||||
const currentTab = await browser.tabs.get(tabId);
|
||||
lastUrl = currentTab.url;
|
||||
navigationObserved ||=
|
||||
currentTab.status === 'loading' ||
|
||||
(Boolean(currentTab.url) && currentTab.url !== previousUrl);
|
||||
const reachedTarget = hasReachedAmazonNavigationTarget(currentTab.url, url);
|
||||
const reachedRedirectedDetail =
|
||||
options.allowDetailRedirect && hasReachedAmazonDetailPage(currentTab.url, url);
|
||||
if (navigationObserved && currentTab.status === 'complete' && (reachedTarget || reachedRedirectedDetail)) {
|
||||
return currentTab.url!;
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
throw new Error(`等待目标页面加载超时: ${url}；当前页面: ${lastUrl || '未知'}`);
|
||||
}
|
||||
|
||||
private async wanderSearchSinglePage(tab: Tabs.Tab) {
|
||||
const injector = new AmazonSearchPageInjector(tab);
|
||||
const runStage = async <T>(stage: string, operation: () => Promise<T>): Promise<T> => {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`搜索页${stage}失败: ${message}`);
|
||||
}
|
||||
};
|
||||
// Wait for the Next button to appear, indicating that the product items have finished loading
|
||||
await injector.waitForPageLoaded();
|
||||
await runStage('加载等待', () => injector.waitForPageLoaded());
|
||||
// Determine the type of product search page https://github.com/primedigitaltech/azon_seeker/issues/1
|
||||
const pagePattern = await injector.getPagePattern();
|
||||
const pagePattern = await runStage('布局识别', () => injector.getPagePattern());
|
||||
// Retrieve key nodes and their information from the critical product search page
|
||||
const data = await injector.getPageData(pagePattern);
|
||||
const data = await runStage('数据解析', () => injector.getPageData(pagePattern));
|
||||
const explicitNoResults =
|
||||
data?.length === 0
|
||||
? await runStage('空结果识别', () => injector.isExplicitNoResultsPage())
|
||||
: false;
|
||||
// get current page
|
||||
const page = await injector.getCurrentPage();
|
||||
// Determine if it is the last page, otherwise navigate to the next page
|
||||
const hasNextPage = await injector.determineHasNextPage();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
if (data === null || typeof hasNextPage !== 'boolean') {
|
||||
const page = await runStage('页码识别', () => injector.getCurrentPage());
|
||||
// Read the next URL without navigating inside the injected script. Clicking here would move
|
||||
// the current Amazon frame into BFCache before executeScript can return its result.
|
||||
const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl());
|
||||
if (data === null) {
|
||||
this.emit('error', { message: '爬取单页信息失败', url: tab.url });
|
||||
throw new Error('爬取单页信息失败');
|
||||
}
|
||||
return { data, hasNextPage, page };
|
||||
if (data.length === 0 && !explicitNoResults) {
|
||||
throw new Error(
|
||||
'Amazon 搜索页未发现可采集商品，且页面没有明确显示“无结果”；请检查是否出现临时错误页、验证码或页面结构变化',
|
||||
);
|
||||
}
|
||||
return { data, nextPageUrl, page };
|
||||
}
|
||||
|
||||
@withErrorHandling
|
||||
@ -69,14 +121,27 @@ class AmazonPageWorkerImpl
|
||||
url.searchParams.append('k', keywords);
|
||||
let tab = await this.getCurrentTab();
|
||||
if (!tab.url?.startsWith('http')) {
|
||||
// A newly created tab may not have a URL in the returned Tab object yet, so never pass its
|
||||
// transient `tab.url` to `new URL()`. Navigate by tab id and wait for the target explicitly.
|
||||
tab = await this.createNewTab('https://www.amazon.com/');
|
||||
tab.url = 'https://www.amazon.com/';
|
||||
await this.navigateTabAndWait(tab.id!, url.toString());
|
||||
return url.toString();
|
||||
}
|
||||
const currentUrl = new URL(tab.url!);
|
||||
if (currentUrl.hostname !== url.hostname || currentUrl.searchParams.get('k') !== keywords) {
|
||||
tab = await browser.tabs.update(tab.id, { url: url.toString(), active: true });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 1000));
|
||||
const sameSearch =
|
||||
currentUrl.hostname === url.hostname && currentUrl.searchParams.get('k') === keywords;
|
||||
if (sameSearch) {
|
||||
try {
|
||||
// Preserve the old behavior: a healthy same-keyword page continues from its current page.
|
||||
await new AmazonSearchPageInjector(tab).waitForPageLoaded();
|
||||
return currentUrl.toString();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes('临时错误页')) throw error;
|
||||
// Only a known Amazon transient error page is recovered by returning to page one.
|
||||
}
|
||||
}
|
||||
await this.navigateTabAndWait(tab.id!, url.toString());
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
@ -84,22 +149,46 @@ class AmazonPageWorkerImpl
|
||||
public async wanderSearchPage(): Promise<void> {
|
||||
const tab = await this.getCurrentTab();
|
||||
let offset = 0;
|
||||
let placementOffset = 0;
|
||||
const adOffsets = new Map<AmazonAdType, number>();
|
||||
while (true) {
|
||||
const { hasNextPage, data, page } = await this.wanderSearchSinglePage(tab);
|
||||
const { nextPageUrl, data, page } = await this.wanderSearchSinglePage(tab);
|
||||
const keywords = new URL(tab.url!).searchParams.get('k')!;
|
||||
const objs = data.map((r, i) => ({
|
||||
...r,
|
||||
keywords,
|
||||
page,
|
||||
rank: offset + 1 + i,
|
||||
createTime: dayjs().format('YYYY/M/D HH:mm:ss'),
|
||||
asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0],
|
||||
}));
|
||||
const pageAdCounts = new Map<AmazonAdType, number>();
|
||||
const objs = data.map((r, i) => {
|
||||
const localAdRank = r.adRank;
|
||||
const localPlacementRank = r.placementRank;
|
||||
const adRank =
|
||||
r.adType && r.adType !== 'organic' && localAdRank
|
||||
? (adOffsets.get(r.adType) || 0) + localAdRank
|
||||
: undefined;
|
||||
if (r.adType && r.adType !== 'organic') {
|
||||
pageAdCounts.set(r.adType, Math.max(pageAdCounts.get(r.adType) || 0, localAdRank || 0));
|
||||
}
|
||||
const placementRank = localPlacementRank ? placementOffset + localPlacementRank : undefined;
|
||||
return {
|
||||
...r,
|
||||
keywords,
|
||||
page,
|
||||
rank: offset + 1 + i,
|
||||
pageRank: r.pageRank || i + 1,
|
||||
adRank,
|
||||
placementRank,
|
||||
placementId: placementRank
|
||||
? `${encodeURIComponent(keywords)}:${page}:${placementRank}`
|
||||
: undefined,
|
||||
createTime: dayjs().format('YYYY/M/D HH:mm:ss'),
|
||||
};
|
||||
});
|
||||
this.emit('item-links-collected', { objs });
|
||||
offset += data.length;
|
||||
if (!hasNextPage) {
|
||||
break;
|
||||
}
|
||||
placementOffset += Math.max(0, ...data.map((item) => item.placementRank || 0));
|
||||
pageAdCounts.forEach((count, adType) => {
|
||||
adOffsets.set(adType, (adOffsets.get(adType) || 0) + count);
|
||||
});
|
||||
if (!nextPageUrl) break;
|
||||
await this.navigateTabAndWait(tab.id!, nextPageUrl);
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
@ -113,8 +202,7 @@ class AmazonPageWorkerImpl
|
||||
const { image = false, aplus = false, extra = false } = options;
|
||||
const params = { asin: '', url: '' };
|
||||
if (entry.match(/^https?:\/\/www\.amazon\.com.*\/dp\/[A-Z0-9]{10}/)) {
|
||||
const [asin] = /\/\/dp\/[A-Z0-9]{10}/.exec(entry)!;
|
||||
params.asin = asin;
|
||||
params.asin = extractAmazonDetailAsin(entry) || '';
|
||||
params.url = entry;
|
||||
} else if (entry.match(/^[A-Z0-9]{10}$/)) {
|
||||
params.asin = entry;
|
||||
@ -124,9 +212,8 @@ class AmazonPageWorkerImpl
|
||||
if (!tab.url || isForbiddenUrl(tab.url)) {
|
||||
tab = await this.createNewTab(params.url);
|
||||
} else {
|
||||
tab = await browser.tabs.update(tab.id, {
|
||||
url: params.url,
|
||||
});
|
||||
await this.navigateTabAndWait(tab.id!, params.url, { allowDetailRedirect: true });
|
||||
tab = await browser.tabs.get(tab.id!);
|
||||
}
|
||||
const injector = new AmazonDetailPageInjector(tab);
|
||||
|
||||
@ -142,6 +229,16 @@ class AmazonPageWorkerImpl
|
||||
//#region Await Production Introduction Element Loaded
|
||||
await injector.waitForPageLoaded();
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000)); // Wait 3 seconds.
|
||||
tab = await browser.tabs.get(tab.id!);
|
||||
const requestedAsin = params.asin;
|
||||
const resolvedAsin = extractAmazonDetailAsin(tab.url);
|
||||
if (!resolvedAsin) {
|
||||
throw new Error(`Amazon 详情页未能识别实际 ASIN：${tab.url || '未知页面'}`);
|
||||
}
|
||||
params.asin = resolvedAsin;
|
||||
if (requestedAsin !== resolvedAsin) {
|
||||
await this.emit('item-detail-redirected', { requestedAsin, resolvedAsin });
|
||||
}
|
||||
//#endregion
|
||||
//#region Fetch Base Info
|
||||
const baseInfo = await injector.getBaseInfo();
|
||||
@ -183,11 +280,10 @@ class AmazonPageWorkerImpl
|
||||
//#region Fetch Goods' Images
|
||||
if (image) {
|
||||
const imageUrls = await injector.getImageUrls();
|
||||
imageUrls.length > 0 &&
|
||||
(await this.emit('item-images-collected', {
|
||||
asin: params.asin,
|
||||
imageUrls: Array.from(new Set(imageUrls)),
|
||||
}));
|
||||
await this.emit('item-images-collected', {
|
||||
asin: params.asin,
|
||||
imageUrls: Array.from(new Set(imageUrls)),
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds.
|
||||
}
|
||||
//#endregion
|
||||
@ -217,24 +313,72 @@ class AmazonPageWorkerImpl
|
||||
@withErrorHandling
|
||||
public async wanderReviewPage(asin: string, options: { recent?: boolean } = {}) {
|
||||
const { recent } = options;
|
||||
const url = new URL(
|
||||
const baseUrl = new URL(
|
||||
`https://www.amazon.com/product-reviews/${asin}/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews`,
|
||||
);
|
||||
recent && url.searchParams.set('sortBy', 'recent');
|
||||
const tab = await this.createNewTab(url.toString());
|
||||
recent && baseUrl.searchParams.set('sortBy', 'recent');
|
||||
const tab = await this.createNewTab(baseUrl.toString());
|
||||
const injector = new AmazonReviewPageInjector(tab);
|
||||
await injector.waitForPageLoad();
|
||||
for (let star = 1; star <= 5; star++) {
|
||||
await injector.showStarsDropDownMenu();
|
||||
await injector.selectStar(star);
|
||||
const runStage = async <T>(stage: string, operation: () => Promise<T>): Promise<T> => {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`评论页${stage}失败: ${message}`);
|
||||
}
|
||||
};
|
||||
await runStage('初始加载等待', () => injector.waitForPageLoad());
|
||||
const starFilters = [
|
||||
{ name: 'one_star', star: 1 },
|
||||
{ name: 'two_star', star: 2 },
|
||||
{ name: 'three_star', star: 3 },
|
||||
{ name: 'four_star', star: 4 },
|
||||
{ name: 'five_star', star: 5 },
|
||||
] as const;
|
||||
const collectedReviewIds = new Set<string>();
|
||||
|
||||
for (const { name: filterByStar, star } of starFilters) {
|
||||
// The original collector gathered every rating bucket. Build the same Amazon filter URL
|
||||
// directly instead of requiring a particular dropdown DOM structure just to discover it.
|
||||
const starUrl = new URL(baseUrl);
|
||||
starUrl.searchParams.set('filterByStar', filterByStar);
|
||||
starUrl.searchParams.delete('pageNumber');
|
||||
await runStage(`${star} 星筛选导航`, () =>
|
||||
this.navigateTabAndWait(tab.id!, starUrl.toString()),
|
||||
);
|
||||
const visitedPageUrls = new Set<string>();
|
||||
let page = 1;
|
||||
let filterIgnored = false;
|
||||
while (true) {
|
||||
await injector.waitForPageLoad();
|
||||
const reviews = await injector.getSinglePageReviews();
|
||||
reviews.length > 0 && this.emit('item-review-collected', { asin, reviews });
|
||||
const hasNextPage = await injector.jumpToNextPageIfExist();
|
||||
if (!hasNextPage) {
|
||||
break;
|
||||
const currentTab = await browser.tabs.get(tab.id!);
|
||||
if (currentTab.url) visitedPageUrls.add(currentTab.url);
|
||||
// Match v0.7.1.12-beta semantics: collect the rating parsed from each review card. The
|
||||
// requested filter is only a navigation aid and must not overwrite or reject real data.
|
||||
await runStage(`${star} 星第 ${page} 页加载等待`, () => injector.waitForPageLoad());
|
||||
const reviews = await runStage('评论解析', () => injector.getSinglePageReviews());
|
||||
if (page === 1 && isAmazonReviewRatingFilterIgnored(reviews, star)) {
|
||||
// Amazon currently experiments with review pages that preserve filterByStar in the URL
|
||||
// but return the same unfiltered review cards. Keep the cards' real ratings, then stop
|
||||
// revisiting the same eight reviews for the remaining rating buckets.
|
||||
filterIgnored = true;
|
||||
}
|
||||
const uniqueReviews = reviews.filter((review) => {
|
||||
if (collectedReviewIds.has(review.id)) return false;
|
||||
collectedReviewIds.add(review.id);
|
||||
return true;
|
||||
});
|
||||
uniqueReviews.length > 0 &&
|
||||
this.emit('item-review-collected', { asin, reviews: uniqueReviews });
|
||||
const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl());
|
||||
if (!nextPageUrl) break;
|
||||
if (visitedPageUrls.has(nextPageUrl)) {
|
||||
throw new Error(`评论页分页出现循环，已停止重复采集: ${nextPageUrl}`);
|
||||
}
|
||||
await runStage('分页导航', () => this.navigateTabAndWait(tab.id!, nextPageUrl));
|
||||
page += 1;
|
||||
}
|
||||
if (filterIgnored) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
setTimeout(() => browser.tabs.remove(tab.id!), 1000);
|
||||
@ -254,7 +398,7 @@ class AmazonPageWorkerImpl
|
||||
const kw = remains.shift()!;
|
||||
await this.doSearch(kw);
|
||||
await this.wanderSearchPage();
|
||||
progress && progress(remains);
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
@ -272,7 +416,10 @@ class AmazonPageWorkerImpl
|
||||
while (remains.length > 0 && !interrupt) {
|
||||
const asin = remains.shift()!;
|
||||
await this.wanderDetailPage(asin, options);
|
||||
progress && progress(remains);
|
||||
// The progress callback persists the completed item locally and enqueues
|
||||
// it for server synchronization. Do not start the next ASIN until that
|
||||
// handoff has completed, otherwise a batch can outrun its result writes.
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
@ -290,7 +437,7 @@ class AmazonPageWorkerImpl
|
||||
while (remains.length > 0 && !interrupt) {
|
||||
const asin = remains.shift()!;
|
||||
await this.wanderReviewPage(asin, options);
|
||||
progress && progress(remains);
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
|
||||
@ -4,6 +4,9 @@ export interface AmazonPageWorkerEvents {
|
||||
/** The event is fired when worker collected links to items on the Amazon search page. */
|
||||
['item-links-collected']: { objs: AmazonSearchItem[] };
|
||||
|
||||
/** Amazon resolved a requested detail ASIN to a canonical or variation ASIN. */
|
||||
['item-detail-redirected']: { requestedAsin: string; resolvedAsin: string };
|
||||
|
||||
/** The event is fired when worker collected goods' base info on the Amazon detail page.*/
|
||||
['item-base-info-collected']: Pick<
|
||||
AmazonDetailItem,
|
||||
@ -11,6 +14,7 @@ export interface AmazonPageWorkerEvents {
|
||||
| 'title'
|
||||
| 'boughtInfo'
|
||||
| 'price'
|
||||
| 'strikethroughPrice'
|
||||
| 'rating'
|
||||
| 'ratingCount'
|
||||
| 'categories'
|
||||
@ -41,7 +45,10 @@ export interface AmazonPageWorkerEvents {
|
||||
};
|
||||
|
||||
/** The event is fired when reviews collected in all review page */
|
||||
['item-review-collected']: { asin: string; reviews: AmazonReview[] };
|
||||
['item-review-collected']: {
|
||||
asin: string;
|
||||
reviews: AmazonReview[];
|
||||
};
|
||||
|
||||
/** Error event that occurs when there is an issue with the Amazon page worker*/
|
||||
['error']: { message: string; url?: string };
|
||||
|
||||
@ -9,4 +9,5 @@ export interface taskOptionBase {
|
||||
export type WorkerComposable<Base, S = {}> = Base & {
|
||||
settings: Ref<S>;
|
||||
isRunning: Ref<boolean>;
|
||||
lastError?: Ref<unknown>;
|
||||
};
|
||||
|
||||
58
src/page-worker/page-navigation.ts
Normal file
58
src/page-worker/page-navigation.ts
Normal file
@ -0,0 +1,58 @@
|
||||
const navigationQueryKeys = ['k', 'page', 'pageNumber', 'filterByStar'] as const;
|
||||
|
||||
function routeIdentity(url: URL) {
|
||||
const reviewMatch = url.pathname.match(/^\/product-reviews\/([A-Z0-9]{10})/i);
|
||||
if (reviewMatch) return `/product-reviews/${reviewMatch[1].toUpperCase()}`;
|
||||
const detailMatch = url.pathname.match(/\/dp\/([A-Z0-9]{10})/i);
|
||||
if (detailMatch) return `/dp/${detailMatch[1].toUpperCase()}`;
|
||||
return url.pathname.replace(/\/ref=.*$/, '') || '/';
|
||||
}
|
||||
|
||||
export function extractAmazonDetailAsin(url: string | undefined) {
|
||||
if (!url) return null;
|
||||
try {
|
||||
return new URL(url).pathname.match(/\/dp\/([A-Z0-9]{10})/i)?.[1]?.toUpperCase() || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Accepts an Amazon canonical/variation redirect only when both URLs are detail pages. */
|
||||
export function hasReachedAmazonDetailPage(
|
||||
currentUrl: string | undefined,
|
||||
targetUrl: string,
|
||||
) {
|
||||
if (!currentUrl) return false;
|
||||
try {
|
||||
const current = new URL(currentUrl);
|
||||
const target = new URL(targetUrl);
|
||||
return (
|
||||
current.origin === target.origin &&
|
||||
Boolean(extractAmazonDetailAsin(currentUrl)) &&
|
||||
Boolean(extractAmazonDetailAsin(targetUrl))
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Amazon may rewrite tracking parameters while navigating. Compare the stable route and the
|
||||
* query parameters which identify the requested search/review page instead of the whole URL.
|
||||
*/
|
||||
export function hasReachedAmazonNavigationTarget(currentUrl: string | undefined, targetUrl: string) {
|
||||
if (!currentUrl) return false;
|
||||
try {
|
||||
const current = new URL(currentUrl);
|
||||
const target = new URL(targetUrl);
|
||||
if (current.origin !== target.origin || routeIdentity(current) !== routeIdentity(target)) {
|
||||
return false;
|
||||
}
|
||||
return navigationQueryKeys.every((key) => {
|
||||
const expected = target.searchParams.get(key);
|
||||
return expected === null || current.searchParams.get(key) === expected;
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
import { createAlova } from 'alova';
|
||||
import adapterFetch from 'alova/fetch';
|
||||
import { remoteHost } from '~/env';
|
||||
import { remoteBaseUrl } from '~/env';
|
||||
import VueHook from 'alova/vue';
|
||||
import { exportKey, generateKeyPair, signData } from '~/logic/crypto';
|
||||
|
||||
@ -8,7 +8,7 @@ const keyPairPromise = generateKeyPair();
|
||||
const publicKeyBase64Promise = keyPairPromise.then(async ({ publicKey }) => exportKey(publicKey));
|
||||
|
||||
const httpClient = createAlova({
|
||||
baseURL: `${__DEV__ ? 'http' : 'https'}://${remoteHost}`,
|
||||
baseURL: remoteBaseUrl,
|
||||
requestAdapter: adapterFetch(),
|
||||
timeout: 10000,
|
||||
statesHook: VueHook,
|
||||
|
||||
103
src/page-worker/web-injectors/amazon-detail-images.ts
Normal file
103
src/page-worker/web-injectors/amazon-detail-images.ts
Normal file
@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Collect product image URLs from an Amazon detail page.
|
||||
*
|
||||
* Keep this function self-contained: browser.scripting.executeScript serializes
|
||||
* it before running it in the target tab, so it must not depend on closures.
|
||||
*/
|
||||
export async function collectAmazonDetailImageUrls(): Promise<string[]> {
|
||||
const delay = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
const isProductImage = (value: string) =>
|
||||
/^https:\/\/(?:m\.media-amazon\.com|images-na\.ssl-images-amazon\.com)\/images\/I\//.test(
|
||||
value,
|
||||
);
|
||||
const unique = (values: string[]) =>
|
||||
Array.from(new Set(values.map((value) => value.replaceAll('\\/', '/')))).filter(
|
||||
isProductImage,
|
||||
);
|
||||
|
||||
const highResolutionUrls: string[] = [];
|
||||
const largeUrls: string[] = [];
|
||||
|
||||
// Amazon has changed the surrounding P.when(...) wrapper several times.
|
||||
// Scan every inline script for the same hiRes/large metadata used by the old parser.
|
||||
for (const scriptNode of Array.from(document.scripts)) {
|
||||
const script = (scriptNode.textContent || '').replaceAll('\\/', '/');
|
||||
for (const match of script.matchAll(
|
||||
/"hiRes"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g,
|
||||
)) {
|
||||
highResolutionUrls.push(match[1]);
|
||||
}
|
||||
for (const match of script.matchAll(
|
||||
/"large"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g,
|
||||
)) {
|
||||
largeUrls.push(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
let urls = unique(highResolutionUrls.length > 0 ? highResolutionUrls : largeUrls);
|
||||
|
||||
// Current detail pages expose the main high-resolution image directly.
|
||||
if (urls.length === 0) {
|
||||
const landingImage = document.querySelector<HTMLImageElement>('#landingImage');
|
||||
const oldHighResolutionUrl = landingImage?.getAttribute('data-old-hires');
|
||||
if (oldHighResolutionUrl) urls.push(oldHighResolutionUrl);
|
||||
|
||||
const dynamicImages = landingImage?.getAttribute('data-a-dynamic-image');
|
||||
if (dynamicImages) {
|
||||
try {
|
||||
const candidates = Object.entries(JSON.parse(dynamicImages) as Record<string, number[]>).sort(
|
||||
([, firstSize], [, secondSize]) =>
|
||||
(secondSize[0] || 0) * (secondSize[1] || 0) -
|
||||
(firstSize[0] || 0) * (firstSize[1] || 0),
|
||||
);
|
||||
if (candidates[0]?.[0]) urls.push(candidates[0][0]);
|
||||
} catch {
|
||||
// Continue to the legacy gallery fallback below.
|
||||
}
|
||||
}
|
||||
urls = unique(urls);
|
||||
}
|
||||
|
||||
// Restore the proven v0.7.2 fallback (commit b062d96): open the image
|
||||
// viewer and read each full-size image. Current selectors are included too.
|
||||
if (urls.length === 0) {
|
||||
const viewerTrigger = document.querySelector<HTMLElement>(
|
||||
'.overlayRestOfImages, #canvasCaption > a',
|
||||
);
|
||||
viewerTrigger?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await delay(1000);
|
||||
|
||||
const thumbnailNodes = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.ivRow div.ivThumbImage, #ivThumbs .ivThumb, #ivThumbs img',
|
||||
),
|
||||
);
|
||||
for (const thumbnailNode of thumbnailNodes) {
|
||||
thumbnailNode.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await delay(500);
|
||||
|
||||
for (let retryCount = 0; retryCount < 5; retryCount++) {
|
||||
const fullImage = document.querySelector<HTMLImageElement>('#ivLargeImage > img');
|
||||
const source = fullImage?.src;
|
||||
if (
|
||||
source &&
|
||||
!source.includes('/ui/loadIndicators/loading-large_labeled') &&
|
||||
isProductImage(source)
|
||||
) {
|
||||
urls.push(source);
|
||||
break;
|
||||
}
|
||||
await delay(500);
|
||||
}
|
||||
}
|
||||
urls = unique(urls);
|
||||
}
|
||||
|
||||
document
|
||||
.querySelector<HTMLElement>(
|
||||
'header > [data-action="a-popover-close"], body > div.a-modal-scroller.a-declarative',
|
||||
)
|
||||
?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
return urls;
|
||||
}
|
||||
32
src/page-worker/web-injectors/amazon-detail-page-ready.ts
Normal file
32
src/page-worker/web-injectors/amazon-detail-page-ready.ts
Normal file
@ -0,0 +1,32 @@
|
||||
/** Wait until the fields required by the detail parser are available. */
|
||||
export async function waitForAmazonDetailPageReady() {
|
||||
const deadline = Date.now() + 20000;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 详情页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 详情页触发了验证码');
|
||||
}
|
||||
|
||||
const unavailable = document.querySelector<HTMLElement>(
|
||||
'#availability .a-color-price, #outOfStock, #exports_desktop_qualifiedBuybox',
|
||||
);
|
||||
const title = document.querySelector<HTMLElement>('#title, #productTitle');
|
||||
const titleReady = Boolean(title?.textContent?.trim());
|
||||
if (document.readyState !== 'loading' && titleReady) {
|
||||
title?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (document.readyState !== 'loading' && unavailable) {
|
||||
throw new Error('Amazon 商品当前不可用，详情标题未加载');
|
||||
}
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
throw new Error('Amazon 详情标题在 20 秒内未加载完成');
|
||||
}
|
||||
73
src/page-worker/web-injectors/amazon-review-page.ts
Normal file
73
src/page-worker/web-injectors/amazon-review-page.ts
Normal file
@ -0,0 +1,73 @@
|
||||
type ReviewReadyOptions = { timeoutMs?: number };
|
||||
|
||||
/** Detects Amazon pages that keep filterByStar in the URL but return other ratings. */
|
||||
export function isAmazonReviewRatingFilterIgnored(
|
||||
reviews: Pick<AmazonReview, 'rating'>[],
|
||||
expectedStar: number,
|
||||
) {
|
||||
return (
|
||||
reviews.length > 0 &&
|
||||
reviews.some((review) => Number.parseInt(review.rating, 10) !== expectedStar)
|
||||
);
|
||||
}
|
||||
|
||||
/** Waits until an Amazon review page, including its requested star filter, is ready. */
|
||||
export async function waitForAmazonReviewPageReady(options: ReviewReadyOptions = {}) {
|
||||
const { timeoutMs = 20000 } = options;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 评论页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 评论页触发了验证码');
|
||||
}
|
||||
|
||||
const reviewList = document.querySelector<HTMLElement>('#cm_cr-review_list');
|
||||
const reviewsContent = document.querySelector<HTMLElement>('.reviews-content');
|
||||
const loadingNode = document.querySelector<HTMLElement>('.reviews-loading');
|
||||
const contentReady = document.readyState !== 'loading' && Boolean(reviewList || reviewsContent);
|
||||
const loadingFinished = !loadingNode || loadingNode.getClientRects().length === 0;
|
||||
if (contentReady && loadingFinished) {
|
||||
(reviewList || reviewsContent)?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
return;
|
||||
}
|
||||
await sleep(300);
|
||||
}
|
||||
throw new Error(`Amazon 评论内容在 ${Math.ceil(timeoutMs / 1000)} 秒内未加载完成`);
|
||||
}
|
||||
|
||||
/** Reads the next review-page URL without navigating from the injected script. */
|
||||
export async function collectAmazonNextReviewPageUrl() {
|
||||
// Amazon may not render the pagination controls until the last review has entered the
|
||||
// viewport. The original collector deliberately scrolled here; keep that behavior while
|
||||
// returning the URL instead of clicking it inside the injected script (which can close the
|
||||
// extension port when Chrome moves the old document into BFCache).
|
||||
const reviewCards = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'#cm_cr-review_list [data-hook="review"], #cm_cr-review_list div[id*="review-card"]',
|
||||
),
|
||||
);
|
||||
reviewCards.at(-1)?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
const pagination =
|
||||
document.querySelector<HTMLElement>('[data-hook="pagination-bar"] .a-pagination') ||
|
||||
document.querySelector<HTMLElement>('[data-hook="pagination-bar"]') ||
|
||||
document.querySelector<HTMLElement>('ul.a-pagination');
|
||||
if (!pagination) return null;
|
||||
|
||||
const nextItem =
|
||||
pagination.querySelector<HTMLElement>('li.a-last') ||
|
||||
pagination.querySelector<HTMLElement>(':scope > *:nth-last-child(1)');
|
||||
if (!nextItem || nextItem.classList.contains('a-disabled')) return null;
|
||||
|
||||
const nextAnchor =
|
||||
nextItem.matches('a[href]')
|
||||
? (nextItem as HTMLAnchorElement)
|
||||
: nextItem.querySelector<HTMLAnchorElement>('a[href]');
|
||||
const href = nextAnchor?.getAttribute('href');
|
||||
return href ? new URL(href, window.location.href).toString() : null;
|
||||
}
|
||||
107
src/page-worker/web-injectors/amazon-search-page-ready.ts
Normal file
107
src/page-worker/web-injectors/amazon-search-page-ready.ts
Normal file
@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Waits for the useful search-page DOM rather than every spinner on the page.
|
||||
* Amazon ad, carousel and recommendation widgets can keep a visible `.a-spinner`
|
||||
* indefinitely even after the search results are ready to collect.
|
||||
*/
|
||||
export async function waitForAmazonSearchPageReady() {
|
||||
const deadline = Date.now() + 20000;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
let nextButton: HTMLElement | null = null;
|
||||
let pageBecameReady = false;
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 搜索页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 搜索页触发了验证码');
|
||||
}
|
||||
const bodyText = document.body?.innerText || document.body?.textContent || '';
|
||||
if (
|
||||
/sorry[,.\s]+something went wrong on our end/i.test(bodyText) ||
|
||||
/please go back and try again/i.test(bodyText)
|
||||
) {
|
||||
throw new Error('Amazon 返回临时错误页，可能触发了限流或风控，请稍后重试');
|
||||
}
|
||||
|
||||
const resultCount = document.querySelectorAll(
|
||||
'[data-component-type="s-search-result"][data-asin], .puis-card-container',
|
||||
).length;
|
||||
nextButton = document.querySelector<HTMLElement>('.s-pagination-next');
|
||||
const noResults = /no results for/i.test(bodyText);
|
||||
const pageReady =
|
||||
document.readyState !== 'loading' && (resultCount > 0 || nextButton !== null || noResults);
|
||||
|
||||
if (pageReady) {
|
||||
pageBecameReady = true;
|
||||
break;
|
||||
}
|
||||
|
||||
window.scrollBy(0, 500 + ~~(Math.random() * 500));
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
if (!pageBecameReady) {
|
||||
throw new Error('Amazon 搜索结果在 20 秒内未加载完成');
|
||||
}
|
||||
|
||||
// Visit the full result stream instead of jumping straight to pagination. Amazon loads
|
||||
// recommendation and sponsored modules when their region approaches the viewport.
|
||||
for (const fraction of [0.15, 0.35, 0.55, 0.75, 0.92]) {
|
||||
const documentHeight = Math.max(
|
||||
document.documentElement.scrollHeight,
|
||||
document.body?.scrollHeight || 0,
|
||||
document.documentElement.offsetHeight,
|
||||
document.body?.offsetHeight || 0,
|
||||
);
|
||||
const top = Math.max(0, (documentHeight - window.innerHeight) * fraction);
|
||||
window.scrollTo({ top, behavior: 'auto' });
|
||||
await sleep(180);
|
||||
}
|
||||
nextButton?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
|
||||
const relevantDomFingerprint = () => {
|
||||
const root = document.querySelector('#search') || document.body;
|
||||
if (!root) return '';
|
||||
const asins = Array.from(root.querySelectorAll<HTMLElement>('[data-asin]'))
|
||||
.map((node) => node.dataset.asin || '')
|
||||
.filter((asin) => /^[A-Z0-9]{10}$/.test(asin));
|
||||
const productLinks = Array.from(
|
||||
root.querySelectorAll<HTMLAnchorElement>(
|
||||
'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
),
|
||||
).map((anchor) => anchor.href);
|
||||
const sponsoredLabels = root.querySelectorAll(
|
||||
'[aria-label*="Sponsored" i], [title*="Sponsored" i]',
|
||||
).length;
|
||||
const cards = root.querySelectorAll(
|
||||
'[data-component-type="s-search-result"], .puis-card-container, .puisg-row, [class*="shop-by-type"]',
|
||||
).length;
|
||||
return [
|
||||
Array.from(new Set(asins)).sort().join(','),
|
||||
Array.from(new Set(productLinks)).sort().join(','),
|
||||
sponsoredLabels,
|
||||
cards,
|
||||
root.querySelectorAll('video').length,
|
||||
root.querySelectorAll('img[src]').length,
|
||||
].join('|');
|
||||
};
|
||||
|
||||
const stabilityStartedAt = Date.now();
|
||||
const minimumStabilityWait = 1500;
|
||||
const stableForRequired = 1000;
|
||||
const maximumStabilityWait = 3500;
|
||||
let fingerprint = relevantDomFingerprint();
|
||||
let stableSince = Date.now();
|
||||
while (Date.now() - stabilityStartedAt < maximumStabilityWait) {
|
||||
await sleep(250);
|
||||
const nextFingerprint = relevantDomFingerprint();
|
||||
if (nextFingerprint !== fingerprint) {
|
||||
fingerprint = nextFingerprint;
|
||||
stableSince = Date.now();
|
||||
}
|
||||
const elapsed = Date.now() - stabilityStartedAt;
|
||||
if (elapsed >= minimumStabilityWait && Date.now() - stableSince >= stableForRequired) return;
|
||||
}
|
||||
}
|
||||
64
src/page-worker/web-injectors/amazon-search-pagination.ts
Normal file
64
src/page-worker/web-injectors/amazon-search-pagination.ts
Normal file
@ -0,0 +1,64 @@
|
||||
/**
|
||||
* Resolves the page represented by the currently rendered search results.
|
||||
* Amazon can update the URL before replacing the visible result DOM, so the
|
||||
* rendered selected-page control is authoritative and the URL is only a fallback.
|
||||
*/
|
||||
export async function collectAmazonCurrentPage(): Promise<number> {
|
||||
const currentUrl = new URL(window.location.href);
|
||||
const parsePage = (value: string | null | undefined) => {
|
||||
const page = Number.parseInt(value || '', 10);
|
||||
return Number.isInteger(page) && page > 0 ? page : null;
|
||||
};
|
||||
|
||||
const isRendered = (node: HTMLElement) => {
|
||||
for (let current: HTMLElement | null = node; current; current = current.parentElement) {
|
||||
if (current.hidden || current.getAttribute('aria-hidden') === 'true') return false;
|
||||
const style = current.ownerDocument.defaultView?.getComputedStyle(current);
|
||||
if (
|
||||
style &&
|
||||
(style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.visibility === 'collapse' ||
|
||||
style.opacity === '0')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (document.documentElement.getClientRects().length === 0) return true;
|
||||
return Array.from(node.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0);
|
||||
};
|
||||
|
||||
const renderedSelectedPages = Array.from(
|
||||
document.querySelectorAll<HTMLElement>('.s-pagination-item.s-pagination-selected'),
|
||||
)
|
||||
.filter(isRendered)
|
||||
.map((node) => parsePage(node.textContent?.trim()))
|
||||
.filter((page): page is number => page !== null);
|
||||
if (renderedSelectedPages.length > 0) return renderedSelectedPages[0];
|
||||
|
||||
const urlPage = parsePage(currentUrl.searchParams.get('page'));
|
||||
if (urlPage) return urlPage;
|
||||
|
||||
const refPage = parsePage(
|
||||
/(?:^|_)pg_(\d+)(?:_|$)/.exec(currentUrl.searchParams.get('ref') || '')?.[1],
|
||||
);
|
||||
if (refPage) return refPage;
|
||||
|
||||
const anySelected = document.querySelector<HTMLElement>(
|
||||
'.s-pagination-item.s-pagination-selected',
|
||||
);
|
||||
return parsePage(anySelected?.textContent?.trim()) || 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the next search-page URL without navigating from the injected script.
|
||||
* Navigation must happen only after browser.scripting.executeScript has returned,
|
||||
* otherwise Amazon can move the frame into BFCache and destroy the script context.
|
||||
*/
|
||||
export async function collectAmazonNextPageUrl() {
|
||||
const nextButton = document.querySelector<HTMLAnchorElement>('.s-pagination-next');
|
||||
if (!nextButton || nextButton.classList.contains('s-pagination-disabled')) return null;
|
||||
|
||||
const href = nextButton.getAttribute('href');
|
||||
return href ? new URL(href, window.location.href).toString() : null;
|
||||
}
|
||||
685
src/page-worker/web-injectors/amazon-search-parser.ts
Normal file
685
src/page-worker/web-injectors/amazon-search-parser.ts
Normal file
@ -0,0 +1,685 @@
|
||||
export type AmazonSearchPagePattern = 'pattern-1' | 'pattern-2';
|
||||
|
||||
type SearchPageCollectorPayload = {
|
||||
pattern: AmazonSearchPagePattern;
|
||||
};
|
||||
|
||||
/**
|
||||
* Collects product exposures from an Amazon search page.
|
||||
*
|
||||
* Keep this function self-contained: browser.scripting.executeScript serializes
|
||||
* the function and cannot provide imported closures in the page context.
|
||||
*/
|
||||
export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) {
|
||||
type CollectedItem = Pick<
|
||||
AmazonSearchItem,
|
||||
| 'asin'
|
||||
| 'link'
|
||||
| 'title'
|
||||
| 'imageSrc'
|
||||
| 'price'
|
||||
| 'strikethroughPrice'
|
||||
| 'adType'
|
||||
| 'adFormat'
|
||||
| 'adRank'
|
||||
| 'pageRank'
|
||||
| 'placement'
|
||||
| 'placementRank'
|
||||
| 'placementId'
|
||||
| 'adProductIndex'
|
||||
>;
|
||||
|
||||
const text = (node: Element | null | undefined) =>
|
||||
(node?.textContent || '').replace(/\s+/g, ' ').trim();
|
||||
|
||||
const isVisible = (node: HTMLElement) => {
|
||||
for (let current: HTMLElement | null = node; current; current = current.parentElement) {
|
||||
if (current.hidden || current.getAttribute('aria-hidden') === 'true') return false;
|
||||
const style = current.ownerDocument.defaultView?.getComputedStyle(current);
|
||||
if (
|
||||
style &&
|
||||
(style.display === 'none' ||
|
||||
style.visibility === 'hidden' ||
|
||||
style.visibility === 'collapse' ||
|
||||
style.opacity === '0' ||
|
||||
style.getPropertyValue('content-visibility') === 'hidden')
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// JSDOM has no layout engine, so computed-style checks above are the authoritative test path.
|
||||
// In Chromium, require the card or one of its key visual descendants to own a real render box.
|
||||
if (node.ownerDocument.documentElement.getClientRects().length === 0) return true;
|
||||
return [node, ...Array.from(node.querySelectorAll<HTMLElement>('img, h2, h3'))].some(
|
||||
(candidate) =>
|
||||
Array.from(candidate.getClientRects()).some((rect) => rect.width > 0 && rect.height > 0),
|
||||
);
|
||||
};
|
||||
|
||||
const isVisuallyHiddenText = (node: HTMLElement) => {
|
||||
if (!isVisible(node)) return true;
|
||||
const style = node.ownerDocument.defaultView?.getComputedStyle(node);
|
||||
if (!style) return false;
|
||||
|
||||
const width = Number.parseFloat(style.width);
|
||||
const height = Number.parseFloat(style.height);
|
||||
const screenReaderOnlyBox =
|
||||
(style.position === 'absolute' || style.position === 'fixed') &&
|
||||
style.overflow === 'hidden' &&
|
||||
Number.isFinite(width) &&
|
||||
Number.isFinite(height) &&
|
||||
width <= 1 &&
|
||||
height <= 1;
|
||||
const zeroClip = /rect\(\s*0(?:px)?[, ]+\s*0(?:px)?[, ]+\s*0(?:px)?[, ]+\s*0(?:px)?\s*\)/i.test(
|
||||
style.clip || '',
|
||||
);
|
||||
const insetClip = /inset\(\s*(?:50%|100%)/i.test(style.clipPath || '');
|
||||
return screenReaderOnlyBox || zeroClip || insetClip;
|
||||
};
|
||||
|
||||
const decodeRepeatedly = (value: string) => {
|
||||
let decoded = value;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
try {
|
||||
const next = decodeURIComponent(decoded);
|
||||
if (next === decoded) break;
|
||||
decoded = next;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
};
|
||||
|
||||
const parseAsinFromText = (value: string) => {
|
||||
const decoded = decodeRepeatedly(value);
|
||||
const patterns = [
|
||||
/\/(?:dp|gp\/product|gp\/aw\/d)\/([A-Z0-9]{10})(?:[/?]|$)/i,
|
||||
/[?&#]asin=([A-Z0-9]{10})(?:[&]|$)/i,
|
||||
/[?&#]lp_asins=([A-Z0-9]{10})/i,
|
||||
/\/dp\/([A-Z0-9]{10})/i,
|
||||
];
|
||||
for (const pattern of patterns) {
|
||||
const match = pattern.exec(decoded);
|
||||
if (match) return match[1].toUpperCase();
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const parseAsin = (node: HTMLElement, href?: string) => {
|
||||
const dataAsin = node.closest<HTMLElement>('[data-asin]')?.dataset.asin || node.dataset.asin;
|
||||
if (dataAsin && /^[A-Z0-9]{10}$/.test(dataAsin)) return dataAsin;
|
||||
|
||||
const candidates = [href || '', decodeRepeatedly(href || '')];
|
||||
for (const candidate of candidates) {
|
||||
const asin = parseAsinFromText(candidate);
|
||||
if (asin) return asin;
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const sponsoredDisclosureCandidates = (node: Element) =>
|
||||
[
|
||||
...(node.matches('[aria-label], [title], span') ? [node] : []),
|
||||
...Array.from(node.querySelectorAll('[aria-label], [title], span')),
|
||||
].filter((candidate): candidate is HTMLElement => candidate instanceof HTMLElement);
|
||||
|
||||
const isSponsoredDisclosure = (candidate: HTMLElement) => {
|
||||
if (isVisuallyHiddenText(candidate)) return false;
|
||||
const accessibleLabels = [
|
||||
candidate.getAttribute('aria-label'),
|
||||
candidate.getAttribute('title'),
|
||||
];
|
||||
if (accessibleLabels.some((value) => /(?:^|\b)Sponsored(?:\b|\s|$)|赞助/i.test(value || ''))) {
|
||||
return true;
|
||||
}
|
||||
return candidate.matches('span') && /^(Sponsored|赞助)(?:\b|\s|$)/i.test(text(candidate));
|
||||
};
|
||||
|
||||
const hasSponsoredDisclosure = (node: Element) =>
|
||||
sponsoredDisclosureCandidates(node).some(isSponsoredDisclosure);
|
||||
|
||||
const closestAsinHost = (node: Element) => {
|
||||
const host = node.closest<HTMLElement>('[data-asin]');
|
||||
return host && /^[A-Z0-9]{10}$/.test(host.dataset.asin || '') ? host : null;
|
||||
};
|
||||
|
||||
const hasModuleSponsoredDisclosure = (node: Element) =>
|
||||
sponsoredDisclosureCandidates(node).some(
|
||||
(candidate) => !closestAsinHost(candidate) && isSponsoredDisclosure(candidate),
|
||||
);
|
||||
|
||||
const metadata = (node: Element) =>
|
||||
[
|
||||
node.getAttribute('data-component-type'),
|
||||
node.getAttribute('data-cel-widget'),
|
||||
node.getAttribute('data-ad-details'),
|
||||
node.id,
|
||||
node.className,
|
||||
]
|
||||
.filter((value) => typeof value === 'string')
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
const isMainResultsBoundary = (node: Element) =>
|
||||
node.id === 'search' ||
|
||||
node.matches('.s-main-slot, [data-component-type="s-search-results"], body');
|
||||
|
||||
const uniqueAsinsIn = (node: Element) => {
|
||||
const asins = new Set<string>();
|
||||
node.querySelectorAll<HTMLElement>('[data-asin]').forEach((candidate) => {
|
||||
const asin = candidate.dataset.asin;
|
||||
if (asin && /^[A-Z0-9]{10}$/.test(asin)) asins.add(asin);
|
||||
});
|
||||
node.querySelectorAll<HTMLAnchorElement>('a[href]').forEach((anchor) => {
|
||||
const asin = parseAsinFromText(anchor.href);
|
||||
if (asin) asins.add(asin);
|
||||
});
|
||||
return asins.size;
|
||||
};
|
||||
|
||||
const hasExplicitAdMetadata = (node: Element) =>
|
||||
/sponsored.?brand|brand.?ad|display.?ad|sponsored.?display|adholder/.test(metadata(node));
|
||||
|
||||
const hasVisibleElement = (node: Element, selector: string) =>
|
||||
Array.from(node.querySelectorAll<HTMLElement>(selector)).some(isVisible);
|
||||
|
||||
const isSearchResultCard = (node: Element) =>
|
||||
node.matches('[data-component-type="s-search-result"]');
|
||||
|
||||
const hasModuleHeadline = (node: Element) =>
|
||||
Array.from(
|
||||
node.querySelectorAll<HTMLElement>(
|
||||
'h1, h2, h3, [class*="headline"], [class*="sb-headline"], [class*="sbv-"]',
|
||||
),
|
||||
).some(
|
||||
(heading) =>
|
||||
!heading.closest('[data-asin]') && isVisible(heading) && text(heading).length > 0,
|
||||
);
|
||||
|
||||
const isOutsideProductCard = (node: Element, root: Element) => {
|
||||
if (node.closest('[data-component-type="s-search-result"]')) return false;
|
||||
const asinHost = node.closest<HTMLElement>('[data-asin]');
|
||||
return !asinHost || asinHost === root;
|
||||
};
|
||||
|
||||
const hasModuleLevelVideo = (node: Element) =>
|
||||
Array.from(node.querySelectorAll('video')).some((video) => isOutsideProductCard(video, node));
|
||||
|
||||
const hasModuleLevelBrandVisual = (node: Element) => {
|
||||
if (hasModuleLevelVideo(node)) return true;
|
||||
const playPause = Array.from(
|
||||
node.querySelectorAll<HTMLElement>(
|
||||
'[aria-label*="Pause" i], [aria-label*="Play" i], [title*="Pause" i], [title*="Play" i]',
|
||||
),
|
||||
).some((candidate) => isOutsideProductCard(candidate, node) && isVisible(candidate));
|
||||
if (playPause) return true;
|
||||
const outsideImages = Array.from(node.querySelectorAll<HTMLElement>('img')).filter(
|
||||
(image) => isOutsideProductCard(image, node) && isVisible(image),
|
||||
);
|
||||
return outsideImages.length >= 2;
|
||||
};
|
||||
|
||||
const hasInfluencerThematicCopy = (node: Element) =>
|
||||
Array.from(node.querySelectorAll<HTMLElement>('h1, h2, h3, span, a, [class*="headline"]')).some(
|
||||
(candidate) => {
|
||||
if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false;
|
||||
return /picks from amazon influencers|influencer storefronts|amazon influencer/i.test(
|
||||
text(candidate),
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
const hasShopByTypeCopy = (node: Element) =>
|
||||
Array.from(node.querySelectorAll<HTMLElement>('h1, h2, h3, span, a, [class*="headline"]')).some(
|
||||
(candidate) => {
|
||||
if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false;
|
||||
return /\bshop\b.+\bby type\b/i.test(text(candidate));
|
||||
},
|
||||
);
|
||||
|
||||
const thematicMetadata = (node: Element) =>
|
||||
[
|
||||
metadata(node),
|
||||
node.getAttribute('cel_widget_id') || '',
|
||||
node.getAttribute('data-multi-ad-feedback-form-trigger') || '',
|
||||
node.getAttribute('data-component-props') || '',
|
||||
]
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
const hasDirectThematicSpSignal = (node: Element) => {
|
||||
const blob = thematicMetadata(node);
|
||||
if (/brand-footer|multi-brand-creative|brands-related|desktop-hsa-3psl/.test(blob)) {
|
||||
return false;
|
||||
}
|
||||
return /featured.?asins.?list|sponsored.?products|sp_search_thematic|loom-desktop|influencer|shop.?by.?type|shop_by_type/.test(
|
||||
blob,
|
||||
);
|
||||
};
|
||||
|
||||
const hasThematicSpSignal = (node: Element) => {
|
||||
if (hasDirectThematicSpSignal(node)) return true;
|
||||
return hasInfluencerThematicCopy(node) || hasShopByTypeCopy(node);
|
||||
};
|
||||
|
||||
const isSponsoredProductsThematic = (node: Element) => {
|
||||
if (hasThematicSpSignal(node)) return true;
|
||||
if (
|
||||
Array.from(
|
||||
node.querySelectorAll(
|
||||
'[cel_widget_id], [data-multi-ad-feedback-form-trigger], [id*="loom"], [class*="loom"]',
|
||||
),
|
||||
).some(hasThematicSpSignal)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
for (
|
||||
let current = node.parentElement;
|
||||
current && !isMainResultsBoundary(current);
|
||||
current = current.parentElement
|
||||
) {
|
||||
if (hasThematicSpSignal(current)) return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const hasVisibleBrandCreative = (node: Element) => {
|
||||
if (isSearchResultCard(node)) return false;
|
||||
if (isSponsoredProductsThematic(node)) return false;
|
||||
|
||||
const visibleBrandDestination = hasVisibleElement(
|
||||
node,
|
||||
'a[href*="/stores/"], a[href*="/brands/"], [class*="brand-logo"], [class*="brandLogo"]',
|
||||
);
|
||||
const explicitBrandModule = /sponsored.?brand|brand.?ad|brands-related|brand-card|\bsbv-/.test(
|
||||
metadata(node),
|
||||
);
|
||||
const brandsRelatedCopy = /brands related to your search/i.test(text(node));
|
||||
const moduleSponsored = hasModuleSponsoredDisclosure(node) || hasSponsoredDisclosure(node);
|
||||
const videoStrip = hasModuleLevelBrandVisual(node) && moduleSponsored;
|
||||
const headlineUnit = hasModuleHeadline(node) && moduleSponsored;
|
||||
return (
|
||||
visibleBrandDestination ||
|
||||
explicitBrandModule ||
|
||||
brandsRelatedCopy ||
|
||||
videoStrip ||
|
||||
headlineUnit
|
||||
);
|
||||
};
|
||||
|
||||
const findAdModule = (card: HTMLElement) => {
|
||||
let current: HTMLElement | null = card;
|
||||
let singleProductModule: HTMLElement | undefined;
|
||||
for (; current; current = current.parentElement) {
|
||||
if (isMainResultsBoundary(current)) break;
|
||||
const explicitModule = hasExplicitAdMetadata(current);
|
||||
const moduleDisclosure = current !== card && hasModuleSponsoredDisclosure(current);
|
||||
const brandedSponsoredModule = moduleDisclosure && hasVisibleBrandCreative(current);
|
||||
const sharedSponsoredProductsModule = moduleDisclosure && uniqueAsinsIn(current) > 1;
|
||||
const singleProductSponsored = moduleDisclosure && uniqueAsinsIn(current) === 1;
|
||||
if (explicitModule || brandedSponsoredModule || sharedSponsoredProductsModule) return current;
|
||||
if (singleProductSponsored && !singleProductModule) singleProductModule = current;
|
||||
}
|
||||
return singleProductModule || card;
|
||||
};
|
||||
|
||||
const classifyAd = (card: HTMLElement, module: HTMLElement) => {
|
||||
const cardMeta = metadata(card);
|
||||
const moduleMeta = metadata(module);
|
||||
const combinedMeta = `${cardMeta} ${moduleMeta}`;
|
||||
const sidebar = Boolean(
|
||||
card.closest(
|
||||
'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"]',
|
||||
),
|
||||
);
|
||||
const sponsored =
|
||||
hasSponsoredDisclosure(card) || (module !== card && hasModuleSponsoredDisclosure(module));
|
||||
|
||||
if (!sponsored) {
|
||||
return {
|
||||
adType: 'organic' as const,
|
||||
placement: undefined,
|
||||
adFormat: undefined,
|
||||
};
|
||||
}
|
||||
if (sidebar || /display.?ad|sponsored.?display/.test(combinedMeta)) {
|
||||
return {
|
||||
adType: 'sponsoredDisplay' as const,
|
||||
placement: 'sidebar' as const,
|
||||
adFormat: 'standard' as const,
|
||||
};
|
||||
}
|
||||
if (isSearchResultCard(card) || isSponsoredProductsThematic(module)) {
|
||||
return {
|
||||
adType: 'sponsoredProducts' as const,
|
||||
placement: 'middleOfSearch' as const,
|
||||
adFormat: hasVisibleElement(card, 'video') ? ('video' as const) : ('standard' as const),
|
||||
};
|
||||
}
|
||||
if (hasVisibleBrandCreative(module)) {
|
||||
return {
|
||||
adType: 'sponsoredBrands' as const,
|
||||
placement: 'middleOfSearch' as const,
|
||||
adFormat: hasModuleLevelVideo(module) ? ('video' as const) : ('standard' as const),
|
||||
};
|
||||
}
|
||||
return {
|
||||
adType: 'sponsoredProducts' as const,
|
||||
placement: 'middleOfSearch' as const,
|
||||
adFormat: hasVisibleElement(module, 'video') ? ('video' as const) : ('standard' as const),
|
||||
};
|
||||
};
|
||||
|
||||
const hasCollectableProductLink = (node: Element) =>
|
||||
Boolean(
|
||||
node.querySelector(
|
||||
'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
),
|
||||
);
|
||||
|
||||
const addCandidate = (node: HTMLElement) => {
|
||||
if (
|
||||
isVisible(node) &&
|
||||
(node.dataset.asin ||
|
||||
hasCollectableProductLink(node) ||
|
||||
parseAsin(node, node.querySelector<HTMLAnchorElement>('a[href]')?.href))
|
||||
) {
|
||||
candidateSet.add(node);
|
||||
}
|
||||
};
|
||||
|
||||
const findProductCardFromAnchor = (anchor: HTMLAnchorElement, module: HTMLElement) => {
|
||||
const asinHost = closestAsinHost(anchor);
|
||||
if (asinHost && module.contains(asinHost) && asinHost !== module) return asinHost;
|
||||
|
||||
let current: HTMLElement | null = anchor.parentElement;
|
||||
while (current && current !== module) {
|
||||
if (current.querySelector('img') && current.querySelector('h2, h3')) return current;
|
||||
current = current.parentElement;
|
||||
}
|
||||
return anchor.parentElement instanceof HTMLElement && anchor.parentElement !== module
|
||||
? anchor.parentElement
|
||||
: null;
|
||||
};
|
||||
|
||||
const candidateSet = new Set<HTMLElement>();
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'[data-component-type="s-search-result"][data-asin], .puis-card-container, .puisg-row',
|
||||
)
|
||||
.forEach((node) => {
|
||||
const stableCard =
|
||||
node.closest<HTMLElement>('[data-component-type="s-search-result"]') || node;
|
||||
addCandidate(stableCard);
|
||||
});
|
||||
|
||||
document.querySelectorAll<HTMLElement>('[data-asin]').forEach((node) => {
|
||||
if (
|
||||
isVisible(node) &&
|
||||
/^[A-Z0-9]{10}$/.test(node.dataset.asin || '') &&
|
||||
node.querySelector('h2, h3, img')
|
||||
) {
|
||||
const stableCard =
|
||||
node.closest<HTMLElement>('[data-component-type="s-search-result"]') || node;
|
||||
candidateSet.add(stableCard);
|
||||
}
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"], [data-component-type*="sponsored-display"]',
|
||||
)
|
||||
.forEach((sidebarAd) => {
|
||||
if (
|
||||
isVisible(sidebarAd) &&
|
||||
hasSponsoredDisclosure(sidebarAd) &&
|
||||
sidebarAd.querySelector('h2, h3') &&
|
||||
sidebarAd.querySelector('img[src], img[data-src]') &&
|
||||
hasCollectableProductLink(sidebarAd)
|
||||
) {
|
||||
candidateSet.add(sidebarAd);
|
||||
}
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'.AdHolder, [class*="brands-related"], [class*="brand-card"], [data-component-type="sponsored-brand"]',
|
||||
)
|
||||
.forEach((module) => {
|
||||
if (!isVisible(module)) return;
|
||||
const nestedCards = module.querySelectorAll<HTMLElement>(
|
||||
'[data-asin], [class*="brand-card"], .sbv-product, [data-slot*="3psl"], [data-slot*="hsa"]',
|
||||
);
|
||||
if (nestedCards.length > 0) {
|
||||
nestedCards.forEach(addCandidate);
|
||||
return;
|
||||
}
|
||||
addCandidate(module);
|
||||
});
|
||||
|
||||
const collectThematicModuleCards = (module: HTMLElement) => {
|
||||
module.querySelectorAll<HTMLElement>('[data-asin]').forEach(addCandidate);
|
||||
module.querySelectorAll<HTMLAnchorElement>('a[href]').forEach((anchor) => {
|
||||
const href = anchor.getAttribute('href') || anchor.href;
|
||||
if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
|
||||
const card = findProductCardFromAnchor(anchor, module);
|
||||
if (card) addCandidate(card);
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
* Amazon frequently changes the wrapper/template names used for ads. Discover a
|
||||
* sponsored unit from its visible disclosure first, then find the closest
|
||||
* ancestor that actually owns one or more product links. This keeps template
|
||||
* names and marketing headings out of the SP/SB/SD classification contract.
|
||||
*/
|
||||
const findSponsoredProductScope = (disclosure: HTMLElement) => {
|
||||
for (
|
||||
let current = disclosure.parentElement;
|
||||
current && !isMainResultsBoundary(current);
|
||||
current = current.parentElement
|
||||
) {
|
||||
if (
|
||||
isVisible(current) &&
|
||||
(hasExplicitAdMetadata(current) || hasDirectThematicSpSignal(current)) &&
|
||||
hasCollectableProductLink(current) &&
|
||||
current.querySelector('img[src], img[data-src], img[srcset]')
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
|
||||
const searchResult = disclosure.closest<HTMLElement>('[data-component-type="s-search-result"]');
|
||||
if (searchResult) return searchResult;
|
||||
|
||||
const asinHost = closestAsinHost(disclosure);
|
||||
if (asinHost) return asinHost;
|
||||
|
||||
for (
|
||||
let current = disclosure.parentElement;
|
||||
current && !isMainResultsBoundary(current);
|
||||
current = current.parentElement
|
||||
) {
|
||||
if (
|
||||
isVisible(current) &&
|
||||
hasCollectableProductLink(current) &&
|
||||
current.querySelector('img[src], img[data-src], img[srcset]')
|
||||
) {
|
||||
return current;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
sponsoredDisclosureCandidates(document.body).forEach((disclosure) => {
|
||||
if (!isSponsoredDisclosure(disclosure)) return;
|
||||
const scope = findSponsoredProductScope(disclosure);
|
||||
if (!scope) return;
|
||||
collectThematicModuleCards(scope);
|
||||
addCandidate(scope);
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"], [cel_widget_id*="SHOP_BY_TYPE" i], [cel_widget_id*="shop-by-type" i]',
|
||||
)
|
||||
.forEach((module) => {
|
||||
if (!isVisible(module) || !isSponsoredProductsThematic(module)) return;
|
||||
collectThematicModuleCards(module);
|
||||
});
|
||||
|
||||
document.querySelectorAll<HTMLElement>('h1, h2, h3, [class*="headline"]').forEach((heading) => {
|
||||
if (closestAsinHost(heading) || !/\bshop\b.+\bby type\b/i.test(text(heading))) return;
|
||||
const module =
|
||||
heading.closest<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], section',
|
||||
) || heading.parentElement;
|
||||
if (module instanceof HTMLElement && isVisible(module)) collectThematicModuleCards(module);
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLAnchorElement>(
|
||||
'a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
)
|
||||
.forEach((anchor) => {
|
||||
if (!isVisible(anchor)) return;
|
||||
const href = anchor.getAttribute('href') || anchor.href;
|
||||
if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
|
||||
const module =
|
||||
anchor.closest<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"]',
|
||||
) || document.body;
|
||||
const card = findProductCardFromAnchor(anchor, module);
|
||||
if (card) addCandidate(card);
|
||||
});
|
||||
|
||||
const cards = Array.from(candidateSet)
|
||||
.filter(
|
||||
(card, index, all) =>
|
||||
!all.some(
|
||||
(other, otherIndex) =>
|
||||
index !== otherIndex &&
|
||||
card.contains(other) &&
|
||||
parseAsin(card, card.querySelector<HTMLAnchorElement>('a[href]')?.href) ===
|
||||
parseAsin(other, other.querySelector<HTMLAnchorElement>('a[href]')?.href),
|
||||
),
|
||||
)
|
||||
.sort((left, right) => {
|
||||
if (left === right) return 0;
|
||||
return left.compareDocumentPosition(right) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1;
|
||||
});
|
||||
|
||||
const organicCandidates = cards.filter((card) => {
|
||||
const module = findAdModule(card);
|
||||
return classifyAd(card, module).adType === 'organic';
|
||||
});
|
||||
const firstOrganicCandidate = organicCandidates[0];
|
||||
const lastOrganicCandidate = organicCandidates[organicCandidates.length - 1];
|
||||
const placementRanks = new Map<HTMLElement, number>();
|
||||
const adRanks = new Map<AmazonAdType, number>();
|
||||
const moduleProductIndexes = new Map<HTMLElement, number>();
|
||||
const moduleAsins = new Map<HTMLElement, Set<string>>();
|
||||
let nextPlacementRank = 1;
|
||||
|
||||
return cards.reduce<CollectedItem[]>((items, card) => {
|
||||
const anchors = Array.from(card.querySelectorAll<HTMLAnchorElement>('a[href]'));
|
||||
const productAnchor = anchors.find((anchor) => parseAsin(card, anchor.href));
|
||||
const asin = parseAsin(card, productAnchor?.href);
|
||||
if (!asin) return items;
|
||||
|
||||
const titleNode =
|
||||
card.querySelector<HTMLElement>('h2[aria-label], h2') ||
|
||||
card.querySelector<HTMLElement>('h3[aria-label], h3');
|
||||
const imageNode = card.querySelector<HTMLImageElement>(
|
||||
'img.s-image, img[src], img[data-src], img[srcset]',
|
||||
);
|
||||
const title =
|
||||
titleNode?.getAttribute('aria-label')?.trim() ||
|
||||
text(titleNode) ||
|
||||
imageNode?.alt?.trim() ||
|
||||
asin;
|
||||
const dataSrc = imageNode?.getAttribute('data-src') || imageNode?.dataset.src;
|
||||
const srcsetUrl = (imageNode?.getAttribute('srcset') || '')
|
||||
.split(',')[0]
|
||||
?.trim()
|
||||
.split(/\s+/)[0];
|
||||
const rawSrc = imageNode?.src && !imageNode.src.startsWith('data:') ? imageNode.src : undefined;
|
||||
const placeholderSrc = /grey-pixel|transparent-pixel|\/1x1(?:\.|$)/i.test(rawSrc || '');
|
||||
const imageSrc = (placeholderSrc ? undefined : rawSrc) || dataSrc || srcsetUrl;
|
||||
if (!imageSrc) return items;
|
||||
|
||||
const currentPriceNode = card.querySelector<HTMLElement>(
|
||||
'.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen',
|
||||
);
|
||||
const strikethroughPriceNode = card.querySelector<HTMLElement>(
|
||||
'[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString',
|
||||
);
|
||||
const price = text(currentPriceNode) || undefined;
|
||||
const rawStrikethroughPrice = text(strikethroughPriceNode) || undefined;
|
||||
const strikethroughPrice =
|
||||
rawStrikethroughPrice && rawStrikethroughPrice !== price ? rawStrikethroughPrice : undefined;
|
||||
|
||||
const module = findAdModule(card);
|
||||
const classification = classifyAd(card, module);
|
||||
if (classification.adType !== 'organic') {
|
||||
const seenAsins = moduleAsins.get(module) || new Set<string>();
|
||||
if (seenAsins.has(asin)) return items;
|
||||
seenAsins.add(asin);
|
||||
moduleAsins.set(module, seenAsins);
|
||||
}
|
||||
let placementRank: number | undefined;
|
||||
let placementId: string | undefined;
|
||||
let adRank: number | undefined;
|
||||
let adProductIndex: number | undefined;
|
||||
let placement: AmazonAdPlacement | undefined = classification.placement;
|
||||
|
||||
if (classification.adType !== 'organic') {
|
||||
if (!placementRanks.has(module)) placementRanks.set(module, nextPlacementRank++);
|
||||
placementRank = placementRanks.get(module)!;
|
||||
placementId = `search-page-placement-${placementRank}`;
|
||||
adRank = (adRanks.get(classification.adType) || 0) + 1;
|
||||
adRanks.set(classification.adType, adRank);
|
||||
adProductIndex = (moduleProductIndexes.get(module) || 0) + 1;
|
||||
moduleProductIndexes.set(module, adProductIndex);
|
||||
|
||||
if (placement !== 'sidebar') {
|
||||
const beforeFirstOrganic =
|
||||
firstOrganicCandidate &&
|
||||
Boolean(
|
||||
module.compareDocumentPosition(firstOrganicCandidate) &
|
||||
Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
);
|
||||
const afterLastOrganic =
|
||||
lastOrganicCandidate &&
|
||||
Boolean(
|
||||
module.compareDocumentPosition(lastOrganicCandidate) & Node.DOCUMENT_POSITION_PRECEDING,
|
||||
);
|
||||
placement = beforeFirstOrganic
|
||||
? 'topOfSearch'
|
||||
: afterLastOrganic
|
||||
? 'bottomOfSearch'
|
||||
: 'middleOfSearch';
|
||||
}
|
||||
}
|
||||
|
||||
items.push({
|
||||
asin,
|
||||
link: `https://www.amazon.com/dp/${asin}`,
|
||||
title,
|
||||
imageSrc,
|
||||
price,
|
||||
strikethroughPrice,
|
||||
adType: classification.adType,
|
||||
adFormat: classification.adFormat,
|
||||
adRank,
|
||||
pageRank: items.length + 1,
|
||||
placement,
|
||||
placementRank,
|
||||
placementId,
|
||||
adProductIndex,
|
||||
});
|
||||
return items;
|
||||
}, []);
|
||||
}
|
||||
@ -1,182 +1,15 @@
|
||||
import { BaseInjector } from './base';
|
||||
import { collectAmazonSearchPageData, type AmazonSearchPagePattern } from './amazon-search-parser';
|
||||
import { collectAmazonDetailImageUrls } from './amazon-detail-images';
|
||||
import { waitForAmazonDetailPageReady } from './amazon-detail-page-ready';
|
||||
import { collectAmazonCurrentPage, collectAmazonNextPageUrl } from './amazon-search-pagination';
|
||||
import { waitForAmazonSearchPageReady } from './amazon-search-page-ready';
|
||||
import { collectAmazonNextReviewPageUrl, waitForAmazonReviewPageReady } from './amazon-review-page';
|
||||
|
||||
export class AmazonSearchPageInjector extends BaseInjector {
|
||||
public waitForPageLoaded() {
|
||||
return this.run(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
while (true) {
|
||||
const targetNode = document.querySelector('.s-pagination-next');
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500));
|
||||
|
||||
const h = Math.max(
|
||||
document.documentElement.scrollHeight,
|
||||
document.body.scrollHeight,
|
||||
document.documentElement.offsetHeight,
|
||||
document.body.offsetHeight,
|
||||
document.documentElement.clientHeight,
|
||||
);
|
||||
|
||||
// await modernScrollTo({ top: h * (0.4 + Math.random() * 0.2), behavior: 'smooth' });
|
||||
|
||||
if (targetNode || document.readyState === 'complete') {
|
||||
// await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
// 第一段：滚到中下部
|
||||
// window.scrollTo({
|
||||
// top: h * (1 - ~~(Math.random() * 50)),
|
||||
// behavior: 'smooth'
|
||||
// });
|
||||
await modernScrollTo({ top: h * (0.9 + Math.random() * 0.08), behavior: 'smooth' });
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
// targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await scrollIntoViewAndWait(targetNode as HTMLElement, {
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// const spins = Array.from(document.querySelectorAll<HTMLElement>('.a-spinner')).filter(
|
||||
// (e) => e.getClientRects().length > 0,
|
||||
// );
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
// const spins = Array.from(document.querySelectorAll('.a-carousel-card-empty'))
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
const pagination = document.querySelectorAll('.rhf-sign-in-button');
|
||||
if (pagination.length > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 平滑滚动并等待滚动结束
|
||||
* @param options 原生 ScrollToOptions 接口 (包含 top, left, behavior)
|
||||
*/
|
||||
async function modernScrollTo(options: ScrollToOptions): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
// 1. 针对已经处于目标位置的情况做兜底处理
|
||||
// 如果目标位置与当前位置一致，部分浏览器可能不会触发 scrollend
|
||||
const targetTop = options.top ?? window.scrollY;
|
||||
const targetLeft = options.left ?? window.scrollX;
|
||||
|
||||
if (targetTop === window.scrollY && targetLeft === window.scrollX) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 监听原生滚动结束事件
|
||||
// 注意：scrollend 是 2023-2024 年起普及的标准，2026 年已是主流
|
||||
document.addEventListener(
|
||||
'scrollend',
|
||||
() => {
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
|
||||
// 3. 执行滚动
|
||||
window.scrollTo(options);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 将元素滚动到视野中心并等待动画结束
|
||||
* @param element 目标 DOM 元素
|
||||
* @param options 滚动配置
|
||||
*/
|
||||
async function scrollIntoViewAndWait(
|
||||
element: HTMLElement,
|
||||
options: ScrollIntoViewOptions = { behavior: 'smooth', block: 'center' },
|
||||
): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
// 监听滚动结束事件
|
||||
// scrollend 会在滚动圆满完成或被中断时触发
|
||||
document.addEventListener(
|
||||
'scrollend',
|
||||
() => {
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
|
||||
// 触发滚动
|
||||
element.scrollIntoView(options);
|
||||
|
||||
// 兼容性兜底：如果元素已经在视野内，浏览器可能不触发滚动
|
||||
// 可以在此处检查位置，若无位移则直接 resolve
|
||||
});
|
||||
}
|
||||
});
|
||||
return this.run(waitForAmazonSearchPageReady);
|
||||
}
|
||||
// public waitForPageLoaded() {
|
||||
// return this.run(async () => {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
|
||||
// // 第一个 while 循环：添加 12 秒超时刷新机制
|
||||
// const timeoutMs = 12000; // 12 秒
|
||||
// let startTime = Date.now();
|
||||
// let refreshCount = 0;
|
||||
// const maxRefresh = 1; // 最多刷新 1 次
|
||||
// console.log('waitForPageLoaded');
|
||||
|
||||
// while (true) {
|
||||
// const targetNode = document.querySelector('.s-pagination-next');
|
||||
// window.scrollBy(0, ~~(Math.random() * 500) + 500);
|
||||
// await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500));
|
||||
|
||||
// if (targetNode || document.readyState === 'complete') {
|
||||
// targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
// break;
|
||||
// }
|
||||
|
||||
// // 检查是否超时
|
||||
// if (Date.now() - startTime > timeoutMs) {
|
||||
// if (refreshCount < maxRefresh) {
|
||||
// // 刷新页面
|
||||
// console.log('检查是否超时，刷新页面');
|
||||
|
||||
// browser.tabs.query({ active: true, currentWindow: true }).then(tabs => {
|
||||
// browser.tabs.reload(tabs[0].id);
|
||||
// });
|
||||
|
||||
// await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始
|
||||
|
||||
// // 重置计时器和刷新计数
|
||||
// refreshCount++;
|
||||
// startTime = Date.now();
|
||||
// continue; // 重新从 while 循环开始
|
||||
// } else {
|
||||
// // 已刷新过一次，仍然超时，抛出错误
|
||||
// throw new Error('等待页面加载超时（已尝试刷新）');
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// // 第二个 while 循环：等待 spinner 消失
|
||||
// while (true) {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// const spins = Array.from(document.querySelectorAll('.a-spinner')).filter(
|
||||
// (e) => e.getClientRects().length > 0
|
||||
// );
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
public async getPagePattern() {
|
||||
return this.run(async () => {
|
||||
@ -190,192 +23,24 @@ export class AmazonSearchPageInjector extends BaseInjector {
|
||||
});
|
||||
}
|
||||
|
||||
public async getPageData(pattern: 'pattern-1' | 'pattern-2') {
|
||||
let data: Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[] | null = null;
|
||||
switch (pattern) {
|
||||
// 处理商品以列表形式展示的情况
|
||||
case 'pattern-1':
|
||||
data = await this.run(async () => {
|
||||
const items = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.puisg-row:has(.a-section.a-spacing-small.a-spacing-top-small:not(.a-text-right))',
|
||||
),
|
||||
).filter((e) => e.getClientRects().length > 0);
|
||||
const linkObjs = items.reduce<
|
||||
Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[]
|
||||
>((objs, el) => {
|
||||
const link = el.querySelector<HTMLAnchorElement>('a')?.href;
|
||||
const title = el
|
||||
.querySelector<HTMLHeadingElement>('h2.a-color-base')!
|
||||
.getAttribute('aria-label')!;
|
||||
const imageSrc = el.querySelector<HTMLImageElement>('img.s-image')!.src!;
|
||||
const price =
|
||||
el.querySelector<HTMLElement>('.a-price:not(.a-text-price) .a-offscreen')
|
||||
?.innerText ||
|
||||
(
|
||||
document.evaluate(
|
||||
`.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`,
|
||||
el,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLSpanElement | null
|
||||
)?.innerText;
|
||||
link && objs.push({ link, title, imageSrc, price });
|
||||
return objs;
|
||||
}, []);
|
||||
return linkObjs;
|
||||
});
|
||||
break;
|
||||
// 处理商品以二维图片格展示的情况
|
||||
case 'pattern-2':
|
||||
data = await this.run(async () => {
|
||||
const items = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.puis-card-container',
|
||||
) as unknown as HTMLElement[],
|
||||
).filter((e) => e.getClientRects().length > 0);
|
||||
const linkObjs = items.reduce<
|
||||
Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[]
|
||||
>((objs, el) => {
|
||||
const link = el.querySelector<HTMLAnchorElement>('a.a-link-normal')?.href;
|
||||
const title = el.querySelector<HTMLHeadingElement>('h2.a-color-base')!.innerText;
|
||||
const imageSrc = el.querySelector<HTMLImageElement>('img.s-image')!.src!;
|
||||
const price =
|
||||
el.querySelector<HTMLElement>('.a-price:not(.a-text-price) .a-offscreen')
|
||||
?.innerText ||
|
||||
(
|
||||
document.evaluate(
|
||||
`.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`,
|
||||
el,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLSpanElement | null
|
||||
)?.innerText;
|
||||
link && objs.push({ link, title, imageSrc, price });
|
||||
return objs;
|
||||
}, []);
|
||||
return linkObjs;
|
||||
});
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
data = data && data.filter((r) => new URL(r.link).pathname.includes('/dp/')); // No advertisement only
|
||||
return data;
|
||||
public async getPageData(pattern: AmazonSearchPagePattern) {
|
||||
return this.run(collectAmazonSearchPageData, { pattern });
|
||||
}
|
||||
|
||||
public async isExplicitNoResultsPage() {
|
||||
return this.run(async () => {
|
||||
const bodyText = document.body?.innerText || document.body?.textContent || '';
|
||||
return /no results for/i.test(bodyText);
|
||||
});
|
||||
}
|
||||
|
||||
public async getCurrentPage() {
|
||||
return this.run(async () => {
|
||||
const node = document.querySelector<HTMLElement>('.s-pagination-item.s-pagination-selected');
|
||||
return node ? Number(node.innerText) : 1;
|
||||
});
|
||||
return this.run(collectAmazonCurrentPage);
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测当前亚马逊搜索页面是否有下一页，并自动点击翻页按钮。
|
||||
*
|
||||
* 该方法在页面上下文中执行，查找亚马逊标准分页按钮（'.s-pagination-next'），
|
||||
* 检查按钮是否未被禁用（'s-pagination-disabled' 类），然后模拟用户点击。
|
||||
* 点击前会随机等待 500-1000 毫秒以避免被识别为机器人。
|
||||
*
|
||||
* @returns `true` 表示有下一页且已点击翻页按钮，页面正在加载下一页内容；
|
||||
* `false` 表示没有下一页（按钮不存在或被禁用）。
|
||||
*/
|
||||
public async determineHasNextPage() {
|
||||
return this.run(async () => {
|
||||
const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
if (nextButton) {
|
||||
if (!nextButton.classList.contains('s-pagination-disabled')) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
nextButton.click();
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
public async getNextPageUrl() {
|
||||
return this.run(collectAmazonNextPageUrl);
|
||||
}
|
||||
// /**
|
||||
// * 检测并执行亚马逊搜索页面翻页，等待页面刷新完成。
|
||||
// * @returns 能否翻页（true=翻页成功，false=没有下一页或翻页失败）
|
||||
// */
|
||||
// public async determineHasNextPage(): Promise<boolean> {
|
||||
// return this.run(async () => {
|
||||
// const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
|
||||
// if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) {
|
||||
// // 记录当前页码
|
||||
// const initialPage = await this.getCurrentPage();
|
||||
|
||||
// // 随机等待后点击
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// nextButton.click();
|
||||
|
||||
// // 等待页面刷新完成
|
||||
// await this.waitForPageLoaded();
|
||||
|
||||
// // 验证翻页是否成功
|
||||
// const newPage = await this.getCurrentPage();
|
||||
|
||||
// // 只有页码真正变化才算翻页成功
|
||||
// return newPage !== null && newPage !== initialPage;
|
||||
// }
|
||||
|
||||
// return false;
|
||||
// });
|
||||
// }
|
||||
|
||||
/**
|
||||
* 检测并执行亚马逊搜索页面翻页，通过 URL 变化确认刷新完成。
|
||||
* @returns 能否翻页（true=已翻页，false=已是最后一页）
|
||||
*/
|
||||
// public async determineHasNextPage(): Promise<boolean> {
|
||||
// return this.run(async () => {
|
||||
// const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
|
||||
// if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) {
|
||||
// // 1. 记录当前 URL
|
||||
// const initialUrl = window.location.href;
|
||||
|
||||
// // 2. 随机等待后点击
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// nextButton.click();
|
||||
|
||||
// // 3. 等待 URL 变化（表示页面已开始导航）
|
||||
// await new Promise<void>((resolve) => {
|
||||
// const checkUrl = () => {
|
||||
// if (window.location.href !== initialUrl) {
|
||||
// resolve();
|
||||
// } else {
|
||||
// setTimeout(checkUrl, 100);
|
||||
// }
|
||||
// };
|
||||
// checkUrl();
|
||||
// });
|
||||
|
||||
// // 4. 等待页面稳定（document.readyState === 'complete'）
|
||||
// await new Promise<void>((resolve) => {
|
||||
// const checkReadyState = () => {
|
||||
// if (document.readyState === 'complete') {
|
||||
// resolve();
|
||||
// } else {
|
||||
// setTimeout(checkReadyState, 100);
|
||||
// }
|
||||
// };
|
||||
// checkReadyState();
|
||||
// });
|
||||
|
||||
// // 5. 额外等待确保内容加载
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// return false;
|
||||
// });
|
||||
// }
|
||||
}
|
||||
|
||||
export class AmazonDetailPageInjector extends BaseInjector {
|
||||
@ -389,35 +54,27 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
/**等待页面加载完成 */
|
||||
public async waitForPageLoaded() {
|
||||
return this.run(async () => {
|
||||
while (true) {
|
||||
window.scrollBy(0, ~~(Math.random() * 500) + 500);
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 100) + 200));
|
||||
const targetNode = document.querySelector(
|
||||
'#prodDetails:has(td), #detailBulletsWrapper_feature_div:has(li), .av-page-desktop, #productDescription_feature_div',
|
||||
);
|
||||
const exceptionalNodeSelectors = ['.music-detail-header', '.avu-retail-page'];
|
||||
for (const selector of exceptionalNodeSelectors) {
|
||||
if (document.querySelector(selector)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (targetNode && document.readyState !== 'loading') {
|
||||
targetNode.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
break;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return this.run(waitForAmazonDetailPageReady);
|
||||
}
|
||||
|
||||
/**获取基本信息 */
|
||||
public async getBaseInfo() {
|
||||
return this.run(async () => {
|
||||
const title = document.querySelector<HTMLElement>('#title')!.innerText;
|
||||
const price = document.querySelector<HTMLElement>(
|
||||
'.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen',
|
||||
const priceRoot =
|
||||
document.querySelector<HTMLElement>(
|
||||
'#corePrice_feature_div, #corePriceDisplay_desktop_feature_div, #apex_desktop',
|
||||
) || document.body;
|
||||
const price = priceRoot.querySelector<HTMLElement>(
|
||||
'.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen',
|
||||
)?.innerText;
|
||||
const rawStrikethroughPrice = priceRoot.querySelector<HTMLElement>(
|
||||
'[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString',
|
||||
)?.innerText;
|
||||
const strikethroughPrice =
|
||||
rawStrikethroughPrice && rawStrikethroughPrice !== price
|
||||
? rawStrikethroughPrice
|
||||
: undefined;
|
||||
const boughtInfo = document.querySelector<HTMLElement>(
|
||||
`#social-proofing-faceout-title-tk_bought`,
|
||||
)?.innerText;
|
||||
@ -436,7 +93,16 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
'#fulfillerInfoFeature_feature_div > *:last-of-type',
|
||||
)?.innerText;
|
||||
const soldBy = document.querySelector<HTMLElement>(`#sellerProfileTriggerId`)?.innerText;
|
||||
return { title, price, boughtInfo, availableDate, categories, shipFrom, soldBy };
|
||||
return {
|
||||
title,
|
||||
price,
|
||||
strikethroughPrice,
|
||||
boughtInfo,
|
||||
availableDate,
|
||||
categories,
|
||||
shipFrom,
|
||||
soldBy,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@ -488,78 +154,7 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
/**获取图像链接 */
|
||||
public async getImageUrls() {
|
||||
return this.run(async () => {
|
||||
const overlay = document.querySelector<HTMLElement>('.overlayRestOfImages');
|
||||
|
||||
if (overlay) {
|
||||
if (document.querySelector<HTMLElement>('#ivThumbs')!.getClientRects().length === 0) {
|
||||
// Click to see full view
|
||||
document.querySelector<HTMLElement>('#canvasCaption > a')?.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
} else {
|
||||
// finally click to see full view if not clicked before
|
||||
document.querySelector<HTMLElement>('#canvasCaption > a')?.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
let script = document.evaluate(
|
||||
`//script[starts-with(text(), "\nP.when(\'A\').register") or contains(text(), "\nP.when('A').register")]`,
|
||||
document,
|
||||
null,
|
||||
XPathResult.STRING_TYPE,
|
||||
).stringValue;
|
||||
const extractUrls = (pattern: RegExp) =>
|
||||
Array.from(script.matchAll(pattern)).map((e) => e[0]);
|
||||
let urls = extractUrls(
|
||||
/(?<="hiRes":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g,
|
||||
);
|
||||
if (urls.length === 0) {
|
||||
urls = extractUrls(
|
||||
/(?<="large":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g,
|
||||
);
|
||||
}
|
||||
|
||||
if (urls.length === 0) {
|
||||
const divs: NodeListOf<HTMLDivElement> =
|
||||
document.querySelectorAll('.ivRow div.ivThumbImage');
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
for (const div of divs) {
|
||||
// click div com
|
||||
div.click();
|
||||
|
||||
// wait for image to load
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
|
||||
// if img src is not loading gif, then get url and break, otherwise continue to wait, with a max retry count to avoid infinite loop
|
||||
let retryCount = 0;
|
||||
const maxRetries = 5;
|
||||
|
||||
while (true) {
|
||||
const loadingGifUrl =
|
||||
'https://m.media-amazon.com/images/G/01/ui/loadIndicators/loading-large_labeled._CB485921664_.gif';
|
||||
// get img dom
|
||||
const imgDom = document.querySelector('#ivLargeImage > img');
|
||||
|
||||
// get url from img dom src attribute
|
||||
if (imgDom && 'src' in imgDom && imgDom.src !== loadingGifUrl) {
|
||||
urls.push(imgDom.src as string);
|
||||
break;
|
||||
} else {
|
||||
// wait for 500ms and try again
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
retryCount++;
|
||||
if (retryCount >= maxRetries) {
|
||||
console.error('在detail页面下载图片，已达最大重试次数:');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
document.querySelector<HTMLElement>('body > div.a-modal-scroller.a-declarative')?.click();
|
||||
return urls;
|
||||
});
|
||||
return this.run(collectAmazonDetailImageUrls);
|
||||
}
|
||||
|
||||
/**获取精选评论 */
|
||||
@ -697,30 +292,7 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
export class AmazonReviewPageInjector extends BaseInjector {
|
||||
public async waitForPageLoad() {
|
||||
return this.run(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
while (true) {
|
||||
const targetNode = document.querySelector(
|
||||
'.reviews-content, #cm_cr-review_list ul[role="list"]:not(.histogram)',
|
||||
);
|
||||
targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
if (
|
||||
targetNode &&
|
||||
targetNode.getClientRects().length > 0 &&
|
||||
document.readyState !== 'loading'
|
||||
) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
while (true) {
|
||||
const loadingNode = document.querySelector('.reviews-loading');
|
||||
if (loadingNode && loadingNode.getClientRects().length === 0) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
});
|
||||
return this.run(waitForAmazonReviewPageReady);
|
||||
}
|
||||
|
||||
public async getSinglePageReviews() {
|
||||
@ -773,54 +345,7 @@ export class AmazonReviewPageInjector extends BaseInjector {
|
||||
});
|
||||
}
|
||||
|
||||
public jumpToNextPageIfExist() {
|
||||
return this.run(async () => {
|
||||
const latestReview = document.evaluate(
|
||||
`//*[@id='cm_cr-review_list']//li[@data-hook='review'][last()]`,
|
||||
document.body,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLElement | null;
|
||||
latestReview?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
const nextPageNode = document.querySelector<HTMLElement>(
|
||||
'[data-hook="pagination-bar"] .a-pagination > *:nth-of-type(2)',
|
||||
);
|
||||
nextPageNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
const ret = nextPageNode && !nextPageNode.classList.contains('a-disabled');
|
||||
ret && nextPageNode?.querySelector('a')?.click();
|
||||
return ret;
|
||||
});
|
||||
}
|
||||
|
||||
public async showStarsDropDownMenu() {
|
||||
return this.run(async () => {
|
||||
while (true) {
|
||||
const dropdown = document.querySelector<HTMLElement>('#star-count-dropdown')!;
|
||||
dropdown.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
dropdown.click();
|
||||
if (dropdown.getAttribute('aria-expanded') === 'true') {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async selectStar(star: number) {
|
||||
return this.run(
|
||||
async ({ star }) => {
|
||||
const starNode = document.evaluate(
|
||||
`//ul[@role='listbox']/li/a[text()="${star} star only"]`,
|
||||
document.body,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLElement;
|
||||
starNode.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
},
|
||||
{ star },
|
||||
);
|
||||
public getNextPageUrl() {
|
||||
return this.run(collectAmazonNextReviewPageUrl);
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,6 +20,14 @@ worker.on('error', ({ message: msg }) => {
|
||||
});
|
||||
message.error(msg);
|
||||
});
|
||||
worker.on('item-detail-redirected', (ev) => {
|
||||
timelines.value.push({
|
||||
type: 'warning',
|
||||
title: '商品发生重定向',
|
||||
time: new Date().toLocaleString(),
|
||||
content: `请求 ASIN：${ev.requestedAsin}\n实际 ASIN：${ev.resolvedAsin}\n数据将按实际 ASIN 保存`,
|
||||
});
|
||||
});
|
||||
worker.on('item-base-info-collected', (ev) => {
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
@ -40,11 +48,15 @@ worker.on('item-category-rank-collected', (ev) => {
|
||||
});
|
||||
});
|
||||
worker.on('item-images-collected', (ev) => {
|
||||
const imageCount = ev.imageUrls?.length || 0;
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
type: imageCount > 0 ? 'success' : 'warning',
|
||||
title: `商品${ev.asin}图像`,
|
||||
time: new Date().toLocaleString(),
|
||||
content: `图片数： ${ev.imageUrls!.length}`,
|
||||
content:
|
||||
imageCount > 0
|
||||
? `图片数： ${imageCount}`
|
||||
: '未找到商品图片；Amazon 页面未提供可识别的图片数据',
|
||||
});
|
||||
});
|
||||
// worker.on('item-top-reviews-collected', (ev) => {
|
||||
@ -98,11 +110,12 @@ const launch = async () => {
|
||||
},
|
||||
...detailWorkerSettings.value,
|
||||
});
|
||||
const failed = Boolean(worker.lastError?.value);
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
type: failed ? 'error' : 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
content: failed ? '数据采集失败，已保留此前成功采集的数据' : '数据采集完成',
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@ -4,7 +4,10 @@ import { usePageWorker } from '~/page-worker';
|
||||
import { reviewAsinInput, reviewWorkerSettings } from '~/storages/amazon';
|
||||
|
||||
const worker = usePageWorker('amazon', { objects: ['review'] });
|
||||
const collectedReviewIds = new Map<string, Set<string>>();
|
||||
const hadFatalError = ref(false);
|
||||
worker.on('error', ({ message: msg }) => {
|
||||
hadFatalError.value = true;
|
||||
timelines.value.push({
|
||||
type: 'error',
|
||||
title: '错误发生',
|
||||
@ -13,14 +16,16 @@ worker.on('error', ({ message: msg }) => {
|
||||
});
|
||||
});
|
||||
worker.on('item-review-collected', (ev) => {
|
||||
const ids = collectedReviewIds.get(ev.asin) || new Set<string>();
|
||||
ev.reviews.forEach((review) => ids.add(review.id));
|
||||
collectedReviewIds.set(ev.asin, ids);
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
title: `商品${ev.asin}评价`,
|
||||
time: new Date().toLocaleString(),
|
||||
content: `获取到 ${ev.reviews.length} 条评价`,
|
||||
content: `已累计获取 ${ids.size} 条评价`,
|
||||
});
|
||||
});
|
||||
|
||||
const asinInputRef = useTemplateRef('asin-input');
|
||||
|
||||
const message = useMessage();
|
||||
@ -29,6 +34,8 @@ const timelines = ref<Timeline[]>([]);
|
||||
|
||||
const launch = async () => {
|
||||
const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0);
|
||||
collectedReviewIds.clear();
|
||||
hadFatalError.value = false;
|
||||
timelines.value = [
|
||||
{
|
||||
type: 'info',
|
||||
@ -43,12 +50,21 @@ const launch = async () => {
|
||||
},
|
||||
recent: reviewWorkerSettings.value.recent,
|
||||
});
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
});
|
||||
if (hadFatalError.value || worker.lastError?.value) {
|
||||
timelines.value.push({
|
||||
type: 'error',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集失败，已保留此前成功采集的数据',
|
||||
});
|
||||
} else {
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleStart = () => {
|
||||
|
||||
8
src/storages/amazon-sync.ts
Normal file
8
src/storages/amazon-sync.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { useWebExtensionStorage } from '~/composables/useWebExtensionStorage';
|
||||
import { AMAZON_SYNC_STATE_KEY, EMPTY_AMAZON_SYNC_STATE } from '~/logic/amazon-sync-shared';
|
||||
|
||||
export const amazonSyncState = useWebExtensionStorage<AmazonSyncState>(
|
||||
AMAZON_SYNC_STATE_KEY,
|
||||
EMPTY_AMAZON_SYNC_STATE,
|
||||
{ flush: 'sync' },
|
||||
);
|
||||
@ -7,8 +7,20 @@ export const detailAsinInput = useWebExtensionStorage<string>('detailAsinInputTe
|
||||
export const reviewAsinInput = useWebExtensionStorage<string>('reviewAsinInputText', '');
|
||||
|
||||
export const itemColumnSettings = useWebExtensionStorage<
|
||||
Set<keyof Pick<AmazonItem, 'keywords' | 'page' | 'rank' | 'createTime' | 'timestamp'>>
|
||||
>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'createTime']));
|
||||
Set<
|
||||
keyof Pick<
|
||||
AmazonItem,
|
||||
| 'keywords'
|
||||
| 'page'
|
||||
| 'rank'
|
||||
| 'pageRank'
|
||||
| 'adRank'
|
||||
| 'placementRank'
|
||||
| 'createTime'
|
||||
| 'timestamp'
|
||||
>
|
||||
>
|
||||
>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'adRank', 'createTime']));
|
||||
|
||||
export const detailWorkerSettings = useWebExtensionStorage('amazon-detail-worker-settings', {
|
||||
image: true,
|
||||
@ -55,8 +67,17 @@ export const allItems = computed<AmazonItem[]>({
|
||||
'title',
|
||||
'imageSrc',
|
||||
'price',
|
||||
'strikethroughPrice',
|
||||
'link',
|
||||
'rank',
|
||||
'pageRank',
|
||||
'adType',
|
||||
'adRank',
|
||||
'placement',
|
||||
'placementRank',
|
||||
'placementId',
|
||||
'adProductIndex',
|
||||
'adFormat',
|
||||
'createTime',
|
||||
];
|
||||
searchItems.value = newValue
|
||||
|
||||
25
src/types/amazon-sync.d.ts
vendored
Normal file
25
src/types/amazon-sync.d.ts
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
import type { ProtocolWithReturn } from 'webext-bridge';
|
||||
|
||||
declare global {
|
||||
type AmazonSyncPayload = {
|
||||
searchItems?: AmazonSearchItem[];
|
||||
detailItems?: AmazonDetailItem[];
|
||||
reviews?: ({ asin: string } & AmazonReview)[];
|
||||
};
|
||||
|
||||
type AmazonSyncStatus = 'idle' | 'pending' | 'syncing' | 'synced' | 'failed';
|
||||
|
||||
type AmazonSyncState = {
|
||||
status: AmazonSyncStatus;
|
||||
pendingCount: number;
|
||||
lastSuccessAt?: string;
|
||||
lastError?: string;
|
||||
};
|
||||
}
|
||||
|
||||
declare module 'webext-bridge' {
|
||||
export interface ProtocolMap {
|
||||
'amazon-sync-enqueue': ProtocolWithReturn<AmazonSyncPayload, AmazonSyncState>;
|
||||
'amazon-sync-flush': ProtocolWithReturn<undefined, AmazonSyncState>;
|
||||
}
|
||||
}
|
||||
43
src/types/amazon.d.ts
vendored
43
src/types/amazon.d.ts
vendored
@ -1,4 +1,25 @@
|
||||
declare type AmazonAdType =
|
||||
| 'organic'
|
||||
| 'sponsoredProducts'
|
||||
| 'sponsoredBrands'
|
||||
| 'sponsoredDisplay'
|
||||
| 'unknownSponsored';
|
||||
|
||||
declare type AmazonAdPlacement =
|
||||
| 'topOfSearch'
|
||||
| 'middleOfSearch'
|
||||
| 'bottomOfSearch'
|
||||
/** 兼容旧版采集数据 */
|
||||
| 'restOfSearch'
|
||||
| 'sidebar'
|
||||
| 'productPage'
|
||||
| 'unknown';
|
||||
|
||||
declare type AmazonAdFormat = 'standard' | 'video' | 'unknown';
|
||||
|
||||
declare type AmazonSearchItem = {
|
||||
/** 客户端生成的幂等同步标识 */
|
||||
syncId?: string;
|
||||
/** 搜索关键词 */
|
||||
keywords: string;
|
||||
/** 商品排名 */
|
||||
@ -13,6 +34,24 @@ declare type AmazonSearchItem = {
|
||||
asin: string;
|
||||
/** 商品价格（可选） */
|
||||
price?: string;
|
||||
/** 页面展示的划线参考价格 */
|
||||
strikethroughPrice?: string;
|
||||
/** 商品在当前页商品流中的排名 */
|
||||
pageRank?: number;
|
||||
/** 广告类型；历史数据可能为空 */
|
||||
adType?: AmazonAdType;
|
||||
/** 同一关键词、同一广告类型中的累计广告排名 */
|
||||
adRank?: number;
|
||||
/** 广告在页面中的位置 */
|
||||
placement?: AmazonAdPlacement;
|
||||
/** 同一关键词采集过程中广告模块的累计顺序 */
|
||||
placementRank?: number;
|
||||
/** 同一广告模块内商品的关联标识 */
|
||||
placementId?: string;
|
||||
/** 商品在广告模块内的顺序 */
|
||||
adProductIndex?: number;
|
||||
/** 广告素材形式 */
|
||||
adFormat?: AmazonAdFormat;
|
||||
/** 商品图片链接 */
|
||||
imageSrc: string;
|
||||
/** 创建时间 */
|
||||
@ -20,6 +59,8 @@ declare type AmazonSearchItem = {
|
||||
};
|
||||
|
||||
declare type AmazonDetailItem = {
|
||||
/** 客户端生成的幂等同步标识 */
|
||||
syncId?: string;
|
||||
/** 商品的 ASIN（亚马逊标准识别号） */
|
||||
asin: string;
|
||||
/** 商品标题 */
|
||||
@ -30,6 +71,8 @@ declare type AmazonDetailItem = {
|
||||
boughtInfo?: string;
|
||||
/** 商品价格 */
|
||||
price?: string;
|
||||
/** 页面展示的划线参考价格 */
|
||||
strikethroughPrice?: string;
|
||||
/** 商品评分 */
|
||||
rating?: number;
|
||||
/** 评分数量 */
|
||||
|
||||
@ -279,6 +279,10 @@ browser.runtime.onConnect.addListener((incomingPort) => {
|
||||
});
|
||||
|
||||
incomingPort.onDisconnect.addListener(() => {
|
||||
// Reading lastError prevents Chrome from reporting an expected BFCache port closure as
|
||||
// an unchecked runtime error in the extension service worker console.
|
||||
void browser.runtime.lastError;
|
||||
|
||||
// sometimes previous content script's onDisconnect is called *after* the fresh content-script's
|
||||
// onConnect. So without this fingerprint equality check, we would remove the new port from map
|
||||
if (connMap.get(connArgs.endpointName)?.fingerprint === connArgs.fingerprint)
|
||||
|
||||
@ -7,6 +7,7 @@ import { encodeConnectionArgs } from './connection-args';
|
||||
import { createDeliveryLogger } from './delivery-logger';
|
||||
import type { StatusMessage } from './port-message';
|
||||
import { PortMessage } from './port-message';
|
||||
import { isBackForwardCachePortError } from './port-disconnect';
|
||||
|
||||
/**
|
||||
* Manfiest V3 extensions can have their service worker terminated at any point
|
||||
@ -18,6 +19,8 @@ import { PortMessage } from './port-message';
|
||||
export const createPersistentPort = (name = '') => {
|
||||
const fingerprint = createFingerprint();
|
||||
let port: Runtime.Port;
|
||||
let connected = false;
|
||||
let inBackForwardCache = false;
|
||||
let undeliveredQueue: ReadonlyArray<QueuedMessage> = [];
|
||||
const pendingResponses = createDeliveryLogger();
|
||||
const onMessageListeners = new Set<(message: InternalMessage, port: Runtime.Port) => void>();
|
||||
@ -77,14 +80,28 @@ export const createPersistentPort = (name = '') => {
|
||||
};
|
||||
|
||||
const connect = () => {
|
||||
if (connected || inBackForwardCache) return;
|
||||
|
||||
port = browser.runtime.connect({
|
||||
name: encodeConnectionArgs({
|
||||
endpointName: name,
|
||||
fingerprint,
|
||||
}),
|
||||
});
|
||||
connected = true;
|
||||
const connectedPort = port;
|
||||
port.onMessage.addListener(handleMessage);
|
||||
port.onDisconnect.addListener(connect);
|
||||
port.onDisconnect.addListener(() => {
|
||||
const disconnectMessage = browser.runtime.lastError?.message;
|
||||
if (port !== connectedPort) return;
|
||||
|
||||
connected = false;
|
||||
if (isBackForwardCachePortError(disconnectMessage)) {
|
||||
inBackForwardCache = true;
|
||||
return;
|
||||
}
|
||||
connect();
|
||||
});
|
||||
|
||||
PortMessage.toBackground(port, {
|
||||
type: 'sync',
|
||||
@ -95,6 +112,17 @@ export const createPersistentPort = (name = '') => {
|
||||
});
|
||||
};
|
||||
|
||||
if (typeof window !== 'undefined') {
|
||||
window.addEventListener('pagehide', (event) => {
|
||||
if (event.persisted) inBackForwardCache = true;
|
||||
});
|
||||
window.addEventListener('pageshow', (event) => {
|
||||
if (!event.persisted) return;
|
||||
inBackForwardCache = false;
|
||||
connect();
|
||||
});
|
||||
}
|
||||
|
||||
connect();
|
||||
|
||||
return {
|
||||
|
||||
3
webext-bridge/src/internal/port-disconnect.ts
Normal file
3
webext-bridge/src/internal/port-disconnect.ts
Normal file
@ -0,0 +1,3 @@
|
||||
export function isBackForwardCachePortError(message?: string): boolean {
|
||||
return message?.includes('back/forward cache') ?? false;
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user