mirror of
https://github.com/primedigitaltech/azon_seeker.git
synced 2026-10-05 07:13:06 +08:00
feat: add Amazon collection metadata and reliable sync
This commit is contained in:
parent
b062d96742
commit
bcbdc321c8
1
.gitignore
vendored
1
.gitignore
vendored
@ -14,6 +14,7 @@ dist-ssr
|
||||
extension/manifest.json
|
||||
extension-firefox/manifest.json
|
||||
node_modules
|
||||
.runtime
|
||||
src/auto-imports.d.ts
|
||||
src/components.d.ts
|
||||
.eslintcache
|
||||
|
||||
16
README.md
16
README.md
@ -11,6 +11,7 @@ Azon Seeker is a browser extension designed to collect and manage product inform
|
||||
- 🛒 **Collect Product Data:** Easily gather product details and reviews from Amazon and other websites.
|
||||
- 📦 **Import/Export:** Supports importing and exporting data in Excel format for further analysis.
|
||||
- ☁️ **Cloud Export:** Export your data to the cloud for backup or sharing.
|
||||
- 🔄 **Reliable Server Sync:** Collected Amazon data is durably queued, retried in the background, and removed from the queue only after the server confirms persistence.
|
||||
- 📊 **Data Table:** View, filter, and manage collected data in a user-friendly table.
|
||||
- 🖼️ **Image Preview:** Hover to preview product images.
|
||||
- 🌐 **Multi-browser Support:** Works with Chromium-based browsers and Firefox.
|
||||
@ -80,4 +81,17 @@ pnpm run build-firefox # For Firefox
|
||||
pnpm run pack:7z
|
||||
```
|
||||
|
||||
After the build is completed, the extension package will be generated in `build/extension.7z`
|
||||
After the build is completed, the Chromium package will be generated in
|
||||
`build/extension-v<version>.7z`.
|
||||
|
||||
## Amazon Server Synchronization
|
||||
|
||||
- Development builds use `127.0.0.1:8001` by default.
|
||||
- Production builds use `amzplugin.rekeymed.com` over HTTPS/WSS by default.
|
||||
- Override either target at build time with `VITE_REMOTE_HOST=host:port`.
|
||||
- The extension persists unsent batches in `browser.storage.local` and retries once per minute.
|
||||
- Search and detail records include a client-generated `syncId`; the server uses it as a unique idempotency key.
|
||||
- The side panel displays pending, syncing, synchronized, and failed states and provides a manual retry action.
|
||||
|
||||
Before publishing a production package, deploy the backend migrations and verify that the
|
||||
production `/openapi.json` contains `syncId`, `adType`, and `strikethroughPrice`.
|
||||
|
||||
11
package.json
11
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "azon-seeker",
|
||||
"displayName": "Azon Seeker v0.7.1.10-beta",
|
||||
"displayName": "Azon Seeker v0.7.1.12-beta",
|
||||
"version": "0.7.2",
|
||||
"private": true,
|
||||
"description": "Starter modify by honestfox101 and PetrichorFun",
|
||||
@ -23,6 +23,15 @@
|
||||
"clear": "rimraf --glob extension/dist extension/manifest.json extension.* ",
|
||||
"clear-firefox": "rimraf --glob extension-firefox/dist extension-firefox/manifest.json extension.*",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test:amazon-parser": "esno scripts/test-amazon-search-parser.ts",
|
||||
"test:amazon-page-ready": "esno scripts/test-amazon-search-page-ready.ts",
|
||||
"test:amazon-pagination": "esno scripts/test-amazon-search-pagination.ts",
|
||||
"test:amazon-detail-images": "esno scripts/test-amazon-detail-images.ts",
|
||||
"test:amazon-detail-page-ready": "esno scripts/test-amazon-detail-page-ready.ts",
|
||||
"test:amazon-review-page": "esno scripts/test-amazon-review-page.ts",
|
||||
"test:amazon-page-navigation": "esno scripts/test-amazon-page-navigation.ts",
|
||||
"test:execute-script": "esno scripts/test-execute-script.ts",
|
||||
"test:extension-runtime": "node scripts/verify-extension-runtime.mjs",
|
||||
"prepare": "husky"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@ -30,7 +30,15 @@ async function getManifest() {
|
||||
48: './assets/icon-512.png',
|
||||
128: './assets/icon-512.png',
|
||||
},
|
||||
permissions: ['tabs', 'storage', 'activeTab', 'scripting', 'unlimitedStorage', 'contextMenus'],
|
||||
permissions: [
|
||||
'tabs',
|
||||
'storage',
|
||||
'activeTab',
|
||||
'scripting',
|
||||
'unlimitedStorage',
|
||||
'contextMenus',
|
||||
'alarms',
|
||||
],
|
||||
content_security_policy: {
|
||||
extension_pages: isDev
|
||||
? `script-src \'self\' http://localhost:${port}; object-src \'self\';`
|
||||
|
||||
@ -1,17 +1,24 @@
|
||||
import _7z from '7zip-min';
|
||||
import { basename } from 'path';
|
||||
import fs from 'fs-extra';
|
||||
import { r, log } from './utils';
|
||||
import packageJson from '../package.json';
|
||||
|
||||
const scriptPath = ['extension', 'extension-firefox'];
|
||||
|
||||
scriptPath.forEach(async (src) => {
|
||||
const srcPath = r(src);
|
||||
const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`);
|
||||
try {
|
||||
const output = await _7z.pack(srcPath, distPath);
|
||||
log('Info', `stdout of the 7za command execution ${output}`);
|
||||
} catch (err) {
|
||||
log('Error', `${err}`);
|
||||
async function main() {
|
||||
for (const src of scriptPath) {
|
||||
const srcPath = r(src);
|
||||
if (!(await fs.pathExists(r(src, 'manifest.json')))) continue;
|
||||
const distPath = r('build', `${basename(src)}-v${packageJson.version}.7z`);
|
||||
try {
|
||||
const output = await _7z.pack(srcPath, distPath);
|
||||
log('Info', `stdout of the 7za command execution ${output}`);
|
||||
} catch (err) {
|
||||
log('Error', `${err}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void main();
|
||||
|
||||
51
scripts/test-amazon-detail-images.ts
Normal file
51
scripts/test-amazon-detail-images.ts
Normal file
@ -0,0 +1,51 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { collectAmazonDetailImageUrls } from '../src/page-worker/web-injectors/amazon-detail-images';
|
||||
|
||||
async function runWithHtml(html: string) {
|
||||
const dom = new JSDOM(html, { url: 'https://www.amazon.com/dp/B0DMNDL5NH' });
|
||||
const previousDocument = globalThis.document;
|
||||
const previousMouseEvent = globalThis.MouseEvent;
|
||||
Object.assign(globalThis, {
|
||||
document: dom.window.document,
|
||||
MouseEvent: dom.window.MouseEvent,
|
||||
});
|
||||
try {
|
||||
return await collectAmazonDetailImageUrls();
|
||||
} finally {
|
||||
Object.assign(globalThis, {
|
||||
document: previousDocument,
|
||||
MouseEvent: previousMouseEvent,
|
||||
});
|
||||
dom.window.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const scriptUrls = await runWithHtml(`
|
||||
<script>
|
||||
window.imageData = {
|
||||
"colorImages": {
|
||||
"initial": [
|
||||
{"hiRes":"https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg","large":"https://m.media-amazon.com/images/I/first-large.jpg"},
|
||||
{"hiRes":"https:\\/\\/m.media-amazon.com\\/images\\/I\\/second._AC_SL1500_.jpg","large":"https://m.media-amazon.com/images/I/second-large.jpg"}
|
||||
]
|
||||
}
|
||||
};
|
||||
</script>
|
||||
`);
|
||||
assert.deepEqual(scriptUrls, [
|
||||
'https://m.media-amazon.com/images/I/first._AC_SL1500_.jpg',
|
||||
'https://m.media-amazon.com/images/I/second._AC_SL1500_.jpg',
|
||||
]);
|
||||
|
||||
const landingUrl = 'https://m.media-amazon.com/images/I/landing._AC_SL1500_.jpg';
|
||||
const landingUrls = await runWithHtml(
|
||||
`<img id="landingImage" data-old-hires="${landingUrl}" data-a-dynamic-image="{}">`,
|
||||
);
|
||||
assert.deepEqual(landingUrls, [landingUrl]);
|
||||
|
||||
console.log('Amazon detail image parser tests passed.');
|
||||
}
|
||||
|
||||
void main();
|
||||
49
scripts/test-amazon-detail-page-ready.ts
Normal file
49
scripts/test-amazon-detail-page-ready.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { waitForAmazonDetailPageReady } from '../src/page-worker/web-injectors/amazon-detail-page-ready';
|
||||
|
||||
const installDom = (html: string, url = 'https://www.amazon.com/dp/B0DMNDL5NH') => {
|
||||
const dom = new JSDOM(html, { url });
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
return dom;
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
const modernDetail = installDom('<span id="productTitle">Ready product</span>');
|
||||
await waitForAmazonDetailPageReady();
|
||||
modernDetail.window.close();
|
||||
|
||||
const legacyDetail = installDom('<span id="title">Legacy product</span>');
|
||||
await waitForAmazonDetailPageReady();
|
||||
legacyDetail.window.close();
|
||||
|
||||
const signIn = installDom('<main>Sign in</main>', 'https://www.amazon.com/ap/signin');
|
||||
await waitForAmazonDetailPageReady().then(
|
||||
() => {
|
||||
throw new Error('Sign-in page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('需要重新登录')) throw error;
|
||||
},
|
||||
);
|
||||
signIn.window.close();
|
||||
|
||||
const captcha = installDom('<input id="captchacharacters">');
|
||||
await waitForAmazonDetailPageReady().then(
|
||||
() => {
|
||||
throw new Error('Captcha page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('验证码')) throw error;
|
||||
},
|
||||
);
|
||||
captcha.window.close();
|
||||
|
||||
console.log('Amazon detail page readiness tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
81
scripts/test-amazon-page-navigation.ts
Normal file
81
scripts/test-amazon-page-navigation.ts
Normal file
@ -0,0 +1,81 @@
|
||||
import {
|
||||
extractAmazonDetailAsin,
|
||||
hasReachedAmazonDetailPage,
|
||||
hasReachedAmazonNavigationTarget,
|
||||
} from '../src/page-worker/page-navigation';
|
||||
|
||||
const assert = (condition: boolean, message: string) => {
|
||||
if (!condition) throw new Error(message);
|
||||
};
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?reviewerType=all_reviews&filterByStar=one_star',
|
||||
),
|
||||
'An unfiltered completed page must not satisfy a star-filter navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_viewopt_sr?filterByStar=one_star&pageNumber=2',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL/ref=cm_cr_arp_d_paging_btm_next_2?filterByStar=one_star&pageNumber=2',
|
||||
),
|
||||
'Tracking ref rewrites must not prevent matching the requested review page.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=1',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?filterByStar=one_star&pageNumber=2',
|
||||
),
|
||||
'A previous review page must not satisfy the next-page navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/s?k=security+camera&page=17&ref=sr_pg_17',
|
||||
'https://www.amazon.com/s?ref=sr_pg_17&page=17&k=security+camera',
|
||||
),
|
||||
'Search navigation should tolerate query ordering differences.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH/ref=sr_1_1',
|
||||
'https://www.amazon.com/dp/B0DMNDL5NH',
|
||||
),
|
||||
'A canonical product slug must still match the requested detail ASIN.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonNavigationTarget(
|
||||
'https://www.amazon.com/Product-Name/dp/B0DMNDL5NH',
|
||||
'https://www.amazon.com/dp/B0BKPNRLC8',
|
||||
),
|
||||
'The previous detail ASIN must not satisfy the next-ASIN navigation.',
|
||||
);
|
||||
|
||||
assert(
|
||||
hasReachedAmazonDetailPage(
|
||||
'https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1',
|
||||
'https://www.amazon.com/dp/B0D2MC2865',
|
||||
),
|
||||
'A canonical Amazon detail redirect must be accepted as a loaded detail page.',
|
||||
);
|
||||
|
||||
assert(
|
||||
!hasReachedAmazonDetailPage(
|
||||
'https://www.amazon.com/',
|
||||
'https://www.amazon.com/dp/B0D2MC2865',
|
||||
),
|
||||
'The Amazon home page must not be accepted as a detail redirect.',
|
||||
);
|
||||
|
||||
assert(
|
||||
extractAmazonDetailAsin('https://www.amazon.com/Product-Name/dp/B0CCXB29LF?th=1') ===
|
||||
'B0CCXB29LF',
|
||||
'The resolved ASIN must be extracted from canonical detail URLs.',
|
||||
);
|
||||
|
||||
console.log('Amazon page navigation tests passed.');
|
||||
82
scripts/test-amazon-review-page.ts
Normal file
82
scripts/test-amazon-review-page.ts
Normal file
@ -0,0 +1,82 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import {
|
||||
collectAmazonNextReviewPageUrl,
|
||||
isAmazonReviewRatingFilterIgnored,
|
||||
waitForAmazonReviewPageReady,
|
||||
} from '../src/page-worker/web-injectors/amazon-review-page';
|
||||
|
||||
const installDom = (html: string, url = 'https://www.amazon.com/product-reviews/B08XM6S6ZL') => {
|
||||
const dom = new JSDOM(html, { url });
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
return dom;
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
if (
|
||||
!isAmazonReviewRatingFilterIgnored(
|
||||
[{ rating: '5.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }],
|
||||
3,
|
||||
)
|
||||
) {
|
||||
throw new Error('Mixed ratings must be detected when Amazon ignores filterByStar');
|
||||
}
|
||||
if (
|
||||
isAmazonReviewRatingFilterIgnored(
|
||||
[{ rating: '3.0 out of 5 stars' }, { rating: '3.0 out of 5 stars' }],
|
||||
3,
|
||||
)
|
||||
) {
|
||||
throw new Error('A correctly filtered review page must not be treated as limited');
|
||||
}
|
||||
|
||||
const readyDom = installDom('<div id="cm_cr-review_list"></div>');
|
||||
await waitForAmazonReviewPageReady({});
|
||||
readyDom.window.close();
|
||||
|
||||
const paginationDom = installDom(
|
||||
'<div data-hook="pagination-bar"><ul class="a-pagination"><li>Previous</li><li class="a-last"><a href="/product-reviews/B08XM6S6ZL?pageNumber=2">Next</a></li></ul></div>',
|
||||
);
|
||||
const nextUrl = await collectAmazonNextReviewPageUrl();
|
||||
if (nextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2') {
|
||||
throw new Error(`Unexpected next review URL: ${nextUrl}`);
|
||||
}
|
||||
paginationDom.window.close();
|
||||
|
||||
const standalonePaginationDom = installDom(
|
||||
'<div id="cm_cr-review_list"><div id="R1-review-card" data-hook="review"></div></div><ul class="a-pagination"><li>Previous</li><li class="a-last"><a href="/product-reviews/B08XM6S6ZL?pageNumber=3">Next</a></li></ul>',
|
||||
'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=2',
|
||||
);
|
||||
const standaloneNextUrl = await collectAmazonNextReviewPageUrl();
|
||||
if (standaloneNextUrl !== 'https://www.amazon.com/product-reviews/B08XM6S6ZL?pageNumber=3') {
|
||||
throw new Error(`Unexpected standalone review pagination URL: ${standaloneNextUrl}`);
|
||||
}
|
||||
standalonePaginationDom.window.close();
|
||||
|
||||
const lastPageDom = installDom(
|
||||
'<div data-hook="pagination-bar"><ul class="a-pagination"><li class="a-last a-disabled">Next</li></ul></div>',
|
||||
);
|
||||
if ((await collectAmazonNextReviewPageUrl()) !== null) {
|
||||
throw new Error('Disabled review pagination must not return a URL');
|
||||
}
|
||||
lastPageDom.window.close();
|
||||
|
||||
const signInDom = installDom('<main>Sign in</main>', 'https://www.amazon.com/ap/signin');
|
||||
await waitForAmazonReviewPageReady({}).then(
|
||||
() => {
|
||||
throw new Error('Sign-in page should fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('需要重新登录')) throw error;
|
||||
},
|
||||
);
|
||||
signInDom.window.close();
|
||||
|
||||
console.log('Amazon review page tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
65
scripts/test-amazon-search-page-ready.ts
Normal file
65
scripts/test-amazon-search-page-ready.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { waitForAmazonSearchPageReady } from '../src/page-worker/web-injectors/amazon-search-page-ready';
|
||||
|
||||
const main = async () => {
|
||||
const dom = new JSDOM(
|
||||
'<main><div class="a-spinner">Persistent widget spinner</div><div data-component-type="s-search-result" data-asin="B000000001"></div><a class="s-pagination-next" href="/s?page=2">Next</a></main>',
|
||||
{ url: 'https://www.amazon.com/s?k=security+camera' },
|
||||
);
|
||||
Object.defineProperty(dom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
Object.assign(dom.window.HTMLElement.prototype, { scrollIntoView: () => undefined });
|
||||
|
||||
const started = Date.now();
|
||||
await waitForAmazonSearchPageReady();
|
||||
const elapsed = Date.now() - started;
|
||||
dom.window.close();
|
||||
|
||||
if (elapsed >= 2000) {
|
||||
throw new Error(`Ready search results were delayed by a persistent spinner (${elapsed}ms)`);
|
||||
}
|
||||
|
||||
const errorDom = new JSDOM(
|
||||
'<main><h1>SORRY</h1><p>Something went wrong on our end.</p><p>Please go back and try again.</p></main>',
|
||||
{ url: 'https://www.amazon.com/s?k=protein+powder' },
|
||||
);
|
||||
Object.defineProperty(errorDom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: errorDom.window.document, window: errorDom.window });
|
||||
await waitForAmazonSearchPageReady().then(
|
||||
() => {
|
||||
throw new Error('Amazon transient error page must not be accepted as an empty result page');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('临时错误页')) throw error;
|
||||
},
|
||||
);
|
||||
errorDom.window.close();
|
||||
|
||||
const captchaDom = new JSDOM('<input id="captchacharacters">', {
|
||||
url: 'https://www.amazon.com/s?k=protein+powder',
|
||||
});
|
||||
Object.defineProperty(captchaDom.window.document, 'readyState', {
|
||||
configurable: true,
|
||||
value: 'complete',
|
||||
});
|
||||
Object.assign(globalThis, { document: captchaDom.window.document, window: captchaDom.window });
|
||||
await waitForAmazonSearchPageReady().then(
|
||||
() => {
|
||||
throw new Error('Amazon captcha page must fail immediately');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('验证码')) throw error;
|
||||
},
|
||||
);
|
||||
captchaDom.window.close();
|
||||
|
||||
console.log('Amazon search page-ready tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
33
scripts/test-amazon-search-pagination.ts
Normal file
33
scripts/test-amazon-search-pagination.ts
Normal file
@ -0,0 +1,33 @@
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { collectAmazonNextPageUrl } from '../src/page-worker/web-injectors/amazon-search-pagination';
|
||||
|
||||
const test = async (name: string, html: string, expected: string | null) => {
|
||||
const dom = new JSDOM(html, { url: 'https://www.amazon.com/s?k=security+camera&page=1' });
|
||||
const previousDocument = globalThis.document;
|
||||
const previousWindow = globalThis.window;
|
||||
Object.assign(globalThis, { document: dom.window.document, window: dom.window });
|
||||
try {
|
||||
const actual = await collectAmazonNextPageUrl();
|
||||
if (actual !== expected) throw new Error(`${name}: expected ${expected}, received ${actual}`);
|
||||
} finally {
|
||||
Object.assign(globalThis, { document: previousDocument, window: previousWindow });
|
||||
dom.window.close();
|
||||
}
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
await test(
|
||||
'active next page',
|
||||
'<a class="s-pagination-next" href="/s?k=security+camera&page=2">Next</a>',
|
||||
'https://www.amazon.com/s?k=security+camera&page=2',
|
||||
);
|
||||
await test(
|
||||
'disabled next page',
|
||||
'<span class="s-pagination-next s-pagination-disabled">Next</span>',
|
||||
null,
|
||||
);
|
||||
await test('missing next page', '<main>No pagination</main>', null);
|
||||
console.log('Amazon search pagination tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
104
scripts/test-amazon-search-parser.ts
Normal file
104
scripts/test-amazon-search-parser.ts
Normal file
@ -0,0 +1,104 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { JSDOM } from 'jsdom';
|
||||
import { collectAmazonSearchPageData } from '../src/page-worker/web-injectors/amazon-search-parser';
|
||||
|
||||
const html = `
|
||||
<main id="search">
|
||||
<section data-component-type="sponsored-brand">
|
||||
<span aria-label="Sponsored">Sponsored</span>
|
||||
<article data-asin="B000000001">
|
||||
<h3>Brand product one</h3>
|
||||
<a href="/dp/B000000001"><img src="https://example.com/brand-1.jpg" /></a>
|
||||
</article>
|
||||
<article data-asin="B000000002">
|
||||
<h3>Brand product two</h3>
|
||||
<a href="/dp/B000000002"><img src="https://example.com/brand-2.jpg" /></a>
|
||||
</article>
|
||||
</section>
|
||||
|
||||
<div class="s-main-slot">
|
||||
<div class="ordinary-result-grid">
|
||||
<article data-component-type="s-search-result" data-asin="B000000003">
|
||||
<span>Amazon's Choice</span>
|
||||
<h2>Organic product</h2>
|
||||
<a href="/dp/B000000003"><img class="s-image" src="https://example.com/organic.jpg" /></a>
|
||||
<span class="a-price"><span class="a-offscreen">$29.99</span></span>
|
||||
</article>
|
||||
|
||||
<article data-component-type="s-search-result" data-asin="B000000004">
|
||||
<a aria-label="View Sponsored information or leave ad feedback"></a>
|
||||
<h2>Sponsored product</h2>
|
||||
<a href="/sspa/click?url=%2Fdp%2FB000000004"><img class="s-image" src="https://example.com/sp.jpg" /></a>
|
||||
<span class="a-price"><span class="a-offscreen">$33.93</span></span>
|
||||
<span class="a-price a-text-price" data-a-strike="true"><span class="a-offscreen">$39.99</span></span>
|
||||
<span class="a-price a-text-price apex-priceperunit-value"><span class="a-offscreen">$0.28</span></span>
|
||||
</article>
|
||||
|
||||
<article data-component-type="s-search-result" data-asin="B000000007">
|
||||
<h2>Second organic product</h2>
|
||||
<a href="/dp/B000000007"><img class="s-image" src="https://example.com/organic-2.jpg" /></a>
|
||||
</article>
|
||||
</div>
|
||||
|
||||
<section class="custom-sponsored-module">
|
||||
<span>Sponsored</span>
|
||||
<article data-asin="B000000005">
|
||||
<h3>Unknown sponsored product</h3>
|
||||
<a href="/dp/B000000005"><img src="https://example.com/unknown.jpg" /></a>
|
||||
</article>
|
||||
</section>
|
||||
</div>
|
||||
|
||||
<aside id="rightResultsATF" data-component-type="sponsored-display">
|
||||
<span>Sponsored</span>
|
||||
<article data-asin="B000000006">
|
||||
<h3>Display product</h3>
|
||||
<a href="/dp/B000000006"><img src="https://example.com/display.jpg" /></a>
|
||||
</article>
|
||||
</aside>
|
||||
</main>
|
||||
`;
|
||||
|
||||
const dom = new JSDOM(html, { url: 'https://www.amazon.com/s?k=mirror' });
|
||||
Object.assign(globalThis, {
|
||||
document: dom.window.document,
|
||||
Node: dom.window.Node,
|
||||
Element: dom.window.Element,
|
||||
HTMLElement: dom.window.HTMLElement,
|
||||
HTMLAnchorElement: dom.window.HTMLAnchorElement,
|
||||
HTMLImageElement: dom.window.HTMLImageElement,
|
||||
});
|
||||
|
||||
async function main() {
|
||||
const items = await collectAmazonSearchPageData({ pattern: 'pattern-2' });
|
||||
assert.equal(items.length, 7);
|
||||
|
||||
const byAsin = new Map(items.map((item) => [item.asin, item]));
|
||||
assert.equal(byAsin.get('B000000003')?.adType, 'organic');
|
||||
assert.equal(byAsin.get('B000000004')?.adType, 'sponsoredProducts');
|
||||
assert.equal(byAsin.get('B000000004')?.price, '$33.93');
|
||||
assert.equal(byAsin.get('B000000004')?.strikethroughPrice, '$39.99');
|
||||
assert.equal(byAsin.get('B000000004')?.link, 'https://www.amazon.com/dp/B000000004');
|
||||
assert.equal(byAsin.get('B000000004')?.placement, 'middleOfSearch');
|
||||
assert.equal(byAsin.get('B000000001')?.adType, 'sponsoredBrands');
|
||||
assert.equal(byAsin.get('B000000002')?.adType, 'sponsoredBrands');
|
||||
assert.equal(byAsin.get('B000000001')?.placement, 'topOfSearch');
|
||||
assert.equal(byAsin.get('B000000001')?.placementId, byAsin.get('B000000002')?.placementId);
|
||||
assert.equal(byAsin.get('B000000001')?.adProductIndex, 1);
|
||||
assert.equal(byAsin.get('B000000002')?.adProductIndex, 2);
|
||||
assert.equal(byAsin.get('B000000005')?.adType, 'unknownSponsored');
|
||||
assert.equal(byAsin.get('B000000005')?.placement, 'bottomOfSearch');
|
||||
assert.equal(byAsin.get('B000000006')?.adType, 'sponsoredDisplay');
|
||||
assert.equal(byAsin.get('B000000006')?.placement, 'sidebar');
|
||||
assert.deepEqual(
|
||||
items.map((item) => item.pageRank),
|
||||
[1, 2, 3, 4, 5, 6, 7],
|
||||
);
|
||||
|
||||
console.log('Amazon search parser fixtures passed.');
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
57
scripts/test-execute-script.ts
Normal file
57
scripts/test-execute-script.ts
Normal file
@ -0,0 +1,57 @@
|
||||
import { exec } from '../src/logic/execute-script';
|
||||
|
||||
const setBrowser = (
|
||||
executeScript: () => Promise<unknown>,
|
||||
getTab: () => Promise<{ status: string }> = async () => ({ status: 'complete' }),
|
||||
) => {
|
||||
Object.assign(globalThis, {
|
||||
browser: {
|
||||
tabs: { get: getTab },
|
||||
scripting: { executeScript },
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
const main = async () => {
|
||||
setBrowser(async () => {
|
||||
throw new Error('Frame was removed');
|
||||
});
|
||||
const rejectionStarted = Date.now();
|
||||
await exec(1, async () => true, undefined, { timeout: 1000 }).then(
|
||||
() => {
|
||||
throw new Error('Expected executeScript to reject');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('Frame was removed')) throw error;
|
||||
if (Date.now() - rejectionStarted >= 1000) {
|
||||
throw new Error('The original executeScript error was delayed until the timeout');
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
setBrowser(async () => [{ result: 42 }]);
|
||||
const result = await exec(1, async () => 42, undefined, { timeout: 1000 });
|
||||
if (result !== 42) throw new Error(`Expected 42, received ${result}`);
|
||||
|
||||
let injectionAttempted = false;
|
||||
setBrowser(
|
||||
async () => {
|
||||
injectionAttempted = true;
|
||||
return [{ result: true }];
|
||||
},
|
||||
async () => ({ status: 'loading' }),
|
||||
);
|
||||
await exec(1, async () => true, undefined, { timeout: 100 }).then(
|
||||
() => {
|
||||
throw new Error('A perpetually loading page should time out before injection');
|
||||
},
|
||||
(error) => {
|
||||
if (!String(error).includes('页面加载超时')) throw error;
|
||||
},
|
||||
);
|
||||
if (injectionAttempted) throw new Error('Script was injected into a still-loading page');
|
||||
|
||||
console.log('Execute-script error propagation tests passed.');
|
||||
};
|
||||
|
||||
void main();
|
||||
71
scripts/verify-extension-runtime.mjs
Normal file
71
scripts/verify-extension-runtime.mjs
Normal file
@ -0,0 +1,71 @@
|
||||
const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9225';
|
||||
const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
const worker = targets.find(
|
||||
(target) => target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'),
|
||||
);
|
||||
if (!worker) throw new Error('Azon Seeker background service worker was not found.');
|
||||
|
||||
const socket = new WebSocket(worker.webSocketDebuggerUrl);
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
|
||||
socket.addEventListener('message', (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (!message.id || !pending.has(message.id)) return;
|
||||
const { resolve, reject } = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
message.error ? reject(new Error(message.error.message)) : resolve(message.result);
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
socket.addEventListener('open', resolve, { once: true });
|
||||
socket.addEventListener('error', reject, { once: true });
|
||||
});
|
||||
|
||||
function request(method, params = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
socket.send(JSON.stringify({ id, method, params }));
|
||||
});
|
||||
}
|
||||
|
||||
const evaluation = await request('Runtime.evaluate', {
|
||||
expression: `(async () => ({
|
||||
manifest: chrome.runtime.getManifest(),
|
||||
alarm: await chrome.alarms.get('amazon-sync-retry'),
|
||||
storage: await chrome.storage.local.get([
|
||||
'amazon-sync-queue-v1',
|
||||
'amazon-sync-state-v1'
|
||||
])
|
||||
}))()`,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
socket.close();
|
||||
|
||||
if (evaluation.exceptionDetails) {
|
||||
throw new Error(evaluation.exceptionDetails.text || 'Extension evaluation failed.');
|
||||
}
|
||||
const runtime = evaluation.result.value;
|
||||
if (runtime.manifest.version !== '0.7.2') throw new Error('Unexpected extension version.');
|
||||
if (!runtime.manifest.permissions.includes('alarms')) throw new Error('Missing alarms permission.');
|
||||
if (runtime.alarm?.name !== 'amazon-sync-retry') throw new Error('Retry alarm was not created.');
|
||||
const state = JSON.parse(runtime.storage['amazon-sync-state-v1']);
|
||||
if (state.pendingCount !== 0 || state.status !== 'synced') {
|
||||
throw new Error(`Unexpected initial sync state: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
extensionId: new URL(worker.url).host,
|
||||
version: runtime.manifest.version,
|
||||
retryAlarm: runtime.alarm.name,
|
||||
syncState: state,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
115
scripts/verify-extension-sync-e2e.mjs
Normal file
115
scripts/verify-extension-sync-e2e.mjs
Normal file
@ -0,0 +1,115 @@
|
||||
const debugPort = process.env.CHROMIUM_DEBUG_PORT || '9226';
|
||||
const syncId = process.env.AZON_E2E_SYNC_ID;
|
||||
const expectFailure = process.env.AZON_EXPECT_SYNC_FAILURE === '1';
|
||||
if (!syncId) throw new Error('AZON_E2E_SYNC_ID is required.');
|
||||
|
||||
let worker;
|
||||
for (let attempt = 0; attempt < 60 && !worker; attempt++) {
|
||||
const targets = await fetch(`http://127.0.0.1:${debugPort}/json/list`).then((response) =>
|
||||
response.json(),
|
||||
);
|
||||
worker = targets.find(
|
||||
(target) =>
|
||||
target.type === 'service_worker' && target.url.endsWith('/dist/background/index.mjs'),
|
||||
);
|
||||
if (!worker) await new Promise((resolve) => setTimeout(resolve, 250));
|
||||
}
|
||||
if (!worker) throw new Error('Azon Seeker background service worker was not found.');
|
||||
|
||||
const socket = new WebSocket(worker.webSocketDebuggerUrl);
|
||||
const pending = new Map();
|
||||
let nextId = 1;
|
||||
socket.addEventListener('message', (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
if (!message.id || !pending.has(message.id)) return;
|
||||
const { resolve, reject } = pending.get(message.id);
|
||||
pending.delete(message.id);
|
||||
message.error ? reject(new Error(message.error.message)) : resolve(message.result);
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
socket.addEventListener('open', resolve, { once: true });
|
||||
socket.addEventListener('error', reject, { once: true });
|
||||
});
|
||||
function request(method, params = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const id = nextId++;
|
||||
pending.set(id, { resolve, reject });
|
||||
socket.send(JSON.stringify({ id, method, params }));
|
||||
});
|
||||
}
|
||||
async function evaluate(expression) {
|
||||
const result = await request('Runtime.evaluate', {
|
||||
expression,
|
||||
awaitPromise: true,
|
||||
returnByValue: true,
|
||||
});
|
||||
if (result.exceptionDetails) throw new Error(result.exceptionDetails.text);
|
||||
return result.result.value;
|
||||
}
|
||||
|
||||
const searchItem = {
|
||||
syncId,
|
||||
asin: 'B000000098',
|
||||
keywords: 'extension queue e2e',
|
||||
rank: 1,
|
||||
page: 1,
|
||||
pageRank: 1,
|
||||
link: 'https://www.amazon.com/dp/B000000098',
|
||||
title: 'Extension queue end-to-end fixture',
|
||||
price: '$33.93',
|
||||
strikethroughPrice: '$39.99',
|
||||
adType: 'sponsoredDisplay',
|
||||
adRank: 1,
|
||||
placement: 'sidebar',
|
||||
placementRank: 1,
|
||||
placementId: 'extension-e2e-placement',
|
||||
adProductIndex: 1,
|
||||
adFormat: 'standard',
|
||||
imageSrc: 'https://example.com/extension-e2e.jpg',
|
||||
createTime: '2026/08/31 11:45:00',
|
||||
};
|
||||
await evaluate(`(async () => {
|
||||
await chrome.storage.local.set({
|
||||
'amazon-sync-queue-v1': ${JSON.stringify({
|
||||
searchItems: [searchItem],
|
||||
detailItems: [],
|
||||
reviews: [],
|
||||
})},
|
||||
'amazon-sync-state-v1': JSON.stringify({status: 'pending', pendingCount: 1})
|
||||
});
|
||||
await chrome.alarms.create('amazon-sync-retry', { when: Date.now() + 200 });
|
||||
return true;
|
||||
})()`);
|
||||
|
||||
let state;
|
||||
for (let attempt = 0; attempt < 30; attempt++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
state = await evaluate(`(async () => {
|
||||
const stored = await chrome.storage.local.get(['amazon-sync-queue-v1', 'amazon-sync-state-v1']);
|
||||
return {
|
||||
queue: stored['amazon-sync-queue-v1'],
|
||||
state: JSON.parse(stored['amazon-sync-state-v1'])
|
||||
};
|
||||
})()`);
|
||||
if (state.state.status === 'failed') {
|
||||
if (expectFailure) break;
|
||||
throw new Error(state.state.lastError);
|
||||
}
|
||||
if (state.state.status === 'synced' && state.state.pendingCount === 0) break;
|
||||
}
|
||||
socket.close();
|
||||
if (expectFailure) {
|
||||
if (
|
||||
state.state.status !== 'failed' ||
|
||||
state.state.pendingCount !== 1 ||
|
||||
state.queue.searchItems.length !== 1
|
||||
) {
|
||||
throw new Error(`Failed synchronization was not retained: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(JSON.stringify({ syncId, retained: true, ...state.state }, null, 2));
|
||||
process.exit(0);
|
||||
}
|
||||
if (state.state.status !== 'synced' || state.queue.searchItems.length !== 0) {
|
||||
throw new Error(`Queue did not drain: ${JSON.stringify(state)}`);
|
||||
}
|
||||
console.log(JSON.stringify({ syncId, ...state.state }, null, 2));
|
||||
214
src/background/amazon-sync.ts
Normal file
214
src/background/amazon-sync.ts
Normal file
@ -0,0 +1,214 @@
|
||||
import { onMessage } from 'webext-bridge/background';
|
||||
import { flattenObject } from '~/logic/convert';
|
||||
import { exportKey, generateKeyPair, signData } from '~/logic/crypto';
|
||||
import { remoteHost } from '~/env';
|
||||
import {
|
||||
AMAZON_SYNC_ALARM,
|
||||
AMAZON_SYNC_QUEUE_KEY,
|
||||
AMAZON_SYNC_STATE_KEY,
|
||||
EMPTY_AMAZON_SYNC_STATE,
|
||||
} from '~/logic/amazon-sync-shared';
|
||||
|
||||
type AmazonSyncQueue = Required<AmazonSyncPayload>;
|
||||
|
||||
const emptyQueue = (): AmazonSyncQueue => ({ searchItems: [], detailItems: [], reviews: [] });
|
||||
const keyPairPromise = generateKeyPair();
|
||||
const publicKeyPromise = keyPairPromise.then(({ publicKey }) => exportKey(publicKey));
|
||||
let queueMutation: Promise<unknown> = Promise.resolve();
|
||||
let flushOperation: Promise<AmazonSyncState> | undefined;
|
||||
let flushAgain = false;
|
||||
|
||||
function mutateQueue<T>(operation: () => Promise<T>): Promise<T> {
|
||||
const result = queueMutation.then(operation, operation);
|
||||
queueMutation = result.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function readQueue(): Promise<AmazonSyncQueue> {
|
||||
const stored = await browser.storage.local.get(AMAZON_SYNC_QUEUE_KEY);
|
||||
return { ...emptyQueue(), ...(stored[AMAZON_SYNC_QUEUE_KEY] as AmazonSyncQueue | undefined) };
|
||||
}
|
||||
|
||||
function pendingCount(queue: AmazonSyncQueue) {
|
||||
return queue.searchItems.length + queue.detailItems.length + queue.reviews.length;
|
||||
}
|
||||
|
||||
async function writeQueue(queue: AmazonSyncQueue) {
|
||||
await browser.storage.local.set({ [AMAZON_SYNC_QUEUE_KEY]: queue });
|
||||
}
|
||||
|
||||
async function writeState(state: AmazonSyncState) {
|
||||
await browser.storage.local.set({ [AMAZON_SYNC_STATE_KEY]: JSON.stringify(state) });
|
||||
return state;
|
||||
}
|
||||
|
||||
function mergeUnique<T>(current: T[], incoming: T[], key: (item: T) => string | undefined) {
|
||||
const values = new Map(current.map((item) => [key(item), item]));
|
||||
for (const item of incoming) values.set(key(item), item);
|
||||
values.delete(undefined);
|
||||
return Array.from(values.values());
|
||||
}
|
||||
|
||||
async function signedPost(path: string, data: unknown[]) {
|
||||
if (data.length === 0) return;
|
||||
const body = JSON.stringify(data);
|
||||
const { privateKey } = await keyPairPromise;
|
||||
const response = await fetch(`https://${remoteHost}${path}`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-signature': await signData(privateKey, body),
|
||||
'x-public-key': await publicKeyPromise,
|
||||
},
|
||||
body,
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!response.ok) throw new Error(`${path} HTTP ${response.status}`);
|
||||
const result = (await response.json()) as { success?: boolean; count?: number };
|
||||
if (result.success !== true || result.count !== data.length) {
|
||||
throw new Error(`${path} 返回的确认数量不匹配`);
|
||||
}
|
||||
}
|
||||
|
||||
function serializeDetailItems(items: AmazonDetailItem[]) {
|
||||
return items.map((item) =>
|
||||
flattenObject(
|
||||
item.imageUrls
|
||||
? {
|
||||
...item,
|
||||
imageUrls: item.imageUrls.join(';'),
|
||||
}
|
||||
: item,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function serializeReviews(items: ({ asin: string } & AmazonReview)[]) {
|
||||
return items.map((item) => ({ ...item, imageSrc: item.imageSrc?.join(';') }));
|
||||
}
|
||||
|
||||
async function removeSynced(
|
||||
field: keyof AmazonSyncQueue,
|
||||
sentKeys: Set<string>,
|
||||
key: (item: any) => string | undefined,
|
||||
) {
|
||||
await mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
(current[field] as any[]) = (current[field] as any[]).filter((item) => {
|
||||
const value = key(item);
|
||||
return !value || !sentKeys.has(value);
|
||||
});
|
||||
await writeQueue(current);
|
||||
});
|
||||
}
|
||||
|
||||
async function flushQueueInternal(): Promise<AmazonSyncState> {
|
||||
let queue = await mutateQueue(readQueue);
|
||||
if (pendingCount(queue) === 0) {
|
||||
return mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
return writeState({
|
||||
...EMPTY_AMAZON_SYNC_STATE,
|
||||
status: pendingCount(current) === 0 ? 'synced' : 'pending',
|
||||
pendingCount: pendingCount(current),
|
||||
});
|
||||
});
|
||||
}
|
||||
await writeState({ status: 'syncing', pendingCount: pendingCount(queue) });
|
||||
try {
|
||||
const searchSnapshot = queue.searchItems;
|
||||
await signedPost('/amazon/search_items', searchSnapshot);
|
||||
await removeSynced(
|
||||
'searchItems',
|
||||
new Set(searchSnapshot.map((item) => item.syncId).filter(Boolean) as string[]),
|
||||
(item: AmazonSearchItem) => item.syncId,
|
||||
);
|
||||
|
||||
queue = await mutateQueue(readQueue);
|
||||
const detailSnapshot = queue.detailItems;
|
||||
await signedPost('/amazon/detail_items', serializeDetailItems(detailSnapshot));
|
||||
await removeSynced(
|
||||
'detailItems',
|
||||
new Set(detailSnapshot.map((item) => item.syncId).filter(Boolean) as string[]),
|
||||
(item: AmazonDetailItem) => item.syncId,
|
||||
);
|
||||
|
||||
queue = await mutateQueue(readQueue);
|
||||
const reviewSnapshot = queue.reviews;
|
||||
await signedPost('/amazon/reviews', serializeReviews(reviewSnapshot));
|
||||
await removeSynced(
|
||||
'reviews',
|
||||
new Set(reviewSnapshot.map((item) => item.id)),
|
||||
(item: AmazonReview) => item.id,
|
||||
);
|
||||
|
||||
return mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
return writeState({
|
||||
status: pendingCount(current) === 0 ? 'synced' : 'pending',
|
||||
pendingCount: pendingCount(current),
|
||||
lastSuccessAt: new Date().toISOString(),
|
||||
});
|
||||
});
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
await mutateQueue(async () => {
|
||||
const current = await readQueue();
|
||||
await writeState({
|
||||
status: 'failed',
|
||||
pendingCount: pendingCount(current),
|
||||
lastError: message,
|
||||
});
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function flushQueue() {
|
||||
if (flushOperation) {
|
||||
flushAgain = true;
|
||||
} else {
|
||||
flushOperation = flushQueueInternal().finally(() => {
|
||||
flushOperation = undefined;
|
||||
if (flushAgain) {
|
||||
flushAgain = false;
|
||||
void flushQueue().catch(() => undefined);
|
||||
}
|
||||
});
|
||||
}
|
||||
return flushOperation;
|
||||
}
|
||||
|
||||
async function enqueue(payload: AmazonSyncPayload) {
|
||||
const state = await mutateQueue(async () => {
|
||||
const queue = await readQueue();
|
||||
queue.searchItems = mergeUnique(
|
||||
queue.searchItems,
|
||||
payload.searchItems || [],
|
||||
(item) => item.syncId,
|
||||
);
|
||||
queue.detailItems = mergeUnique(
|
||||
queue.detailItems,
|
||||
payload.detailItems || [],
|
||||
(item) => item.syncId,
|
||||
);
|
||||
queue.reviews = mergeUnique(queue.reviews, payload.reviews || [], (item) => item.id);
|
||||
await writeQueue(queue);
|
||||
return writeState({ status: 'pending', pendingCount: pendingCount(queue) });
|
||||
});
|
||||
void flushQueue().catch(() => undefined);
|
||||
return state;
|
||||
}
|
||||
|
||||
onMessage('amazon-sync-enqueue', ({ data }) => enqueue(data));
|
||||
onMessage('amazon-sync-flush', () => flushQueue());
|
||||
|
||||
browser.alarms.create(AMAZON_SYNC_ALARM, { periodInMinutes: 1 });
|
||||
browser.alarms.onAlarm.addListener((alarm) => {
|
||||
if (alarm.name === AMAZON_SYNC_ALARM) void flushQueue().catch(() => undefined);
|
||||
});
|
||||
|
||||
void flushQueue().catch(() => undefined);
|
||||
@ -2,6 +2,7 @@ import { isFirefox } from '~/env';
|
||||
|
||||
// https://github.com/serversideup/webext-bridge/issues/67#issuecomment-2676094094
|
||||
import('webext-bridge/background');
|
||||
import('./amazon-sync');
|
||||
|
||||
// only on dev mode
|
||||
if (import.meta.hot) {
|
||||
|
||||
@ -1,17 +1,18 @@
|
||||
export function useLongTask() {
|
||||
const isRunning = ref(false);
|
||||
const lastError = shallowRef<unknown>();
|
||||
|
||||
const startTask = async <T = undefined>(task: () => Promise<T>) => {
|
||||
if (isRunning.value) {
|
||||
throw Error('Task is still running.');
|
||||
}
|
||||
isRunning.value = true;
|
||||
lastError.value = undefined;
|
||||
let result = undefined;
|
||||
try {
|
||||
result = await task();
|
||||
} catch (error) {
|
||||
console.log(error);
|
||||
|
||||
lastError.value = error;
|
||||
console.error('Task failed:', error);
|
||||
}
|
||||
isRunning.value = false;
|
||||
@ -20,6 +21,7 @@ export function useLongTask() {
|
||||
|
||||
return {
|
||||
isRunning,
|
||||
lastError,
|
||||
startTask,
|
||||
};
|
||||
}
|
||||
|
||||
@ -14,9 +14,6 @@ export function isForbiddenUrl(url: string): boolean {
|
||||
|
||||
export const isFirefox = navigator.userAgent.includes('Firefox');
|
||||
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:8000' : '47.251.4.191:8000';
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'vm8nc3zr-18000.usw2.devtunnels.ms';
|
||||
// export const remoteHost = __DEV__ ? 'ogsm-dev.rekeymed.com:8888' : 'amzplugin.rekeymed.com';
|
||||
export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com';
|
||||
// export const remoteHost = __DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com';
|
||||
// export const remoteHost = '47.251.4.191:8000';
|
||||
export const remoteHost =
|
||||
import.meta.env.VITE_REMOTE_HOST ||
|
||||
(__DEV__ ? '127.0.0.1:18000' : 'amzplugin.rekeymed.com');
|
||||
|
||||
9
src/logic/amazon-sync-client.ts
Normal file
9
src/logic/amazon-sync-client.ts
Normal file
@ -0,0 +1,9 @@
|
||||
import { sendMessage } from 'webext-bridge/sidepanel';
|
||||
|
||||
export async function enqueueAmazonSync(payload: AmazonSyncPayload) {
|
||||
return sendMessage('amazon-sync-enqueue', payload, 'background');
|
||||
}
|
||||
|
||||
export async function flushAmazonSyncQueue() {
|
||||
return sendMessage('amazon-sync-flush', undefined, 'background');
|
||||
}
|
||||
12
src/logic/amazon-sync-shared.ts
Normal file
12
src/logic/amazon-sync-shared.ts
Normal file
@ -0,0 +1,12 @@
|
||||
export const AMAZON_SYNC_QUEUE_KEY = 'amazon-sync-queue-v1';
|
||||
export const AMAZON_SYNC_STATE_KEY = 'amazon-sync-state-v1';
|
||||
export const AMAZON_SYNC_ALARM = 'amazon-sync-retry';
|
||||
|
||||
export const EMPTY_AMAZON_SYNC_STATE: AmazonSyncState = {
|
||||
status: 'idle',
|
||||
pendingCount: 0,
|
||||
};
|
||||
|
||||
export function createAmazonSyncId(prefix: 'search' | 'detail') {
|
||||
return `${prefix}-${crypto.randomUUID()}`;
|
||||
}
|
||||
@ -47,45 +47,45 @@ export async function exec<T, P extends Record<string, unknown>>(
|
||||
options: ExecOptions = {},
|
||||
): Promise<T> {
|
||||
const { timeout = 30000 } = options;
|
||||
return new Promise<T>(async (resolve, reject) => {
|
||||
// 基本刷新
|
||||
// await browser.tabs.reload(tabId);
|
||||
// console.log('exec', browser.tabs.get(tabId), tabId);
|
||||
|
||||
// console.log('exec', func, (await browser.tabs.get(tabId)).url);
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise<void>((r) => setTimeout(r, 200));
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status === 'complete') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const deadline = Date.now() + timeout;
|
||||
let pageLoaded = false;
|
||||
while (Date.now() < deadline) {
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status !== 'complete') {
|
||||
await browser.tabs.reload(tabId);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始
|
||||
if (tab.status === 'complete') {
|
||||
pageLoaded = true;
|
||||
break;
|
||||
}
|
||||
const remaining = deadline - Date.now();
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, Math.min(200, remaining)));
|
||||
}
|
||||
if (!pageLoaded) throw new Error(`页面加载超时(${timeout}ms),未执行采集脚本`);
|
||||
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise<void>((r) => setTimeout(r, 200));
|
||||
const tab = await browser.tabs.get(tabId);
|
||||
if (tab.status === 'complete') {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let timeoutId: ReturnType<typeof setTimeout> | undefined;
|
||||
const remainingTimeout = Math.max(1, deadline - Date.now());
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timeoutId = setTimeout(
|
||||
() => reject(new Error(`脚本运行超时(总预算 ${timeout}ms)`)),
|
||||
remainingTimeout,
|
||||
);
|
||||
});
|
||||
|
||||
setTimeout(() => reject(`脚本运行超时, foucs on ${func}`), timeout);
|
||||
try {
|
||||
const injectResults = await browser.scripting.executeScript({
|
||||
try {
|
||||
const injectResults = await Promise.race([
|
||||
browser.scripting.executeScript({
|
||||
target: { tabId },
|
||||
func,
|
||||
args: payload ? [payload] : undefined,
|
||||
});
|
||||
const ret = injectResults.pop();
|
||||
resolve(ret!.result as T);
|
||||
} catch (e) {
|
||||
throw new Error(`脚本运行失败: ${e}`);
|
||||
}
|
||||
});
|
||||
}),
|
||||
timeoutPromise,
|
||||
]);
|
||||
const ret = injectResults.at(-1);
|
||||
if (!ret) throw new Error('脚本未返回执行结果');
|
||||
return ret.result as T;
|
||||
} catch (error) {
|
||||
const cause = error instanceof Error ? error : new Error(String(error));
|
||||
if (cause.message.startsWith('脚本运行超时')) throw cause;
|
||||
throw new Error(`脚本运行失败: ${cause.message}`);
|
||||
} finally {
|
||||
if (timeoutId !== undefined) clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -8,10 +8,41 @@ import { allDetailItems, allItems, itemColumnSettings, reviewItems } from '~/sto
|
||||
const modal = useModal();
|
||||
const excelHelper = useExcelHelper();
|
||||
|
||||
const adTypeLabels: Record<AmazonAdType, string> = {
|
||||
organic: '自然排位',
|
||||
sponsoredProducts: 'Sponsored Products',
|
||||
sponsoredBrands: 'Sponsored Brands',
|
||||
sponsoredDisplay: 'Sponsored Display',
|
||||
unknownSponsored: '未知广告',
|
||||
};
|
||||
|
||||
const placementLabels: Record<AmazonAdPlacement, string> = {
|
||||
topOfSearch: '顶部',
|
||||
middleOfSearch: '中部',
|
||||
bottomOfSearch: '底部',
|
||||
restOfSearch: '其余位置(旧数据)',
|
||||
sidebar: '侧边栏',
|
||||
productPage: '商品详情页',
|
||||
unknown: '未知位置',
|
||||
};
|
||||
|
||||
const renderPlacement = (row: AmazonItem) => {
|
||||
if (!row.placement) return '-';
|
||||
const label = placementLabels[row.placement];
|
||||
if (row.placement === 'productPage') return label;
|
||||
return row.page ? `第 ${row.page} 页${label}` : label;
|
||||
};
|
||||
|
||||
const renderOptionalValue = (value: unknown) => value ?? '-';
|
||||
|
||||
const isDetailOnlyRow = (row: AmazonItem) =>
|
||||
!row.keywords && row.page == null && row.rank == null && row.pageRank == null;
|
||||
|
||||
const filter = ref<{
|
||||
keywords?: string;
|
||||
search?: string;
|
||||
detailOnly?: boolean;
|
||||
adType?: AmazonAdType | 'unclassified';
|
||||
searchDateRange?: [number, number];
|
||||
detailDateRange?: [number, number];
|
||||
}>({});
|
||||
@ -34,18 +65,55 @@ const columns = computed<TableColumn[]>(() => {
|
||||
key: 'keywords',
|
||||
minWidth: 120,
|
||||
hidden: !itemColumnSettings.value.has('keywords'),
|
||||
render: (row) => renderOptionalValue(row.keywords),
|
||||
},
|
||||
{
|
||||
title: '页码',
|
||||
key: 'page',
|
||||
minWidth: 60,
|
||||
hidden: !itemColumnSettings.value.has('page'),
|
||||
render: (row) => renderOptionalValue(row.page),
|
||||
},
|
||||
{
|
||||
title: '排位',
|
||||
key: 'rank',
|
||||
minWidth: 60,
|
||||
hidden: !itemColumnSettings.value.has('rank'),
|
||||
render: (row) => renderOptionalValue(row.rank),
|
||||
},
|
||||
{
|
||||
title: '页内排位',
|
||||
key: 'pageRank',
|
||||
minWidth: 90,
|
||||
hidden: !itemColumnSettings.value.has('pageRank'),
|
||||
render: (row) => renderOptionalValue(row.pageRank),
|
||||
},
|
||||
{
|
||||
title: '广告类型',
|
||||
key: 'adType',
|
||||
minWidth: 150,
|
||||
render: (row) =>
|
||||
isDetailOnlyRow(row) ? '-' : row.adType ? adTypeLabels[row.adType] : '历史数据/未识别',
|
||||
},
|
||||
{
|
||||
title: '广告排位',
|
||||
key: 'adRank',
|
||||
minWidth: 90,
|
||||
hidden: !itemColumnSettings.value.has('adRank'),
|
||||
render: (row) => renderOptionalValue(row.adRank),
|
||||
},
|
||||
{
|
||||
title: '广告位置',
|
||||
key: 'placement',
|
||||
minWidth: 140,
|
||||
render: renderPlacement,
|
||||
},
|
||||
{
|
||||
title: '广告位顺序',
|
||||
key: 'placementRank',
|
||||
minWidth: 110,
|
||||
hidden: !itemColumnSettings.value.has('placementRank'),
|
||||
render: (row) => renderOptionalValue(row.placementRank),
|
||||
},
|
||||
{
|
||||
title: 'ASIN',
|
||||
@ -61,6 +129,11 @@ const columns = computed<TableColumn[]>(() => {
|
||||
key: 'price',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: '划线价格',
|
||||
key: 'strikethroughPrice',
|
||||
minWidth: 100,
|
||||
},
|
||||
{
|
||||
title: '封面图',
|
||||
key: 'imageSrc',
|
||||
@ -181,7 +254,7 @@ const getItemHeaders = () => {
|
||||
};
|
||||
|
||||
const filteredData = computed(() => {
|
||||
const { search, detailOnly, keywords, searchDateRange, detailDateRange } = filter.value;
|
||||
const { search, detailOnly, keywords, adType, searchDateRange, detailDateRange } = filter.value;
|
||||
let data = toRaw(detailOnly ? allDetailItems.value : allItems.value);
|
||||
if (search && search.trim() !== '') {
|
||||
data = data.filter((r) => {
|
||||
@ -193,6 +266,9 @@ const filteredData = computed(() => {
|
||||
if (keywords) {
|
||||
data = data.filter((r) => r.keywords === keywords);
|
||||
}
|
||||
if (adType) {
|
||||
data = data.filter((r) => (adType === 'unclassified' ? !r.adType : r.adType === adType));
|
||||
}
|
||||
if (searchDateRange) {
|
||||
const start = dayjs(searchDateRange[0]);
|
||||
const end = dayjs(searchDateRange[1]);
|
||||
@ -279,6 +355,17 @@ const handleClearData = async () => {
|
||||
}))"
|
||||
/>
|
||||
</n-form-item>
|
||||
<n-form-item label="广告类型">
|
||||
<n-select
|
||||
v-model:value="filter.adType"
|
||||
clearable
|
||||
placeholder="全部"
|
||||
:options="[
|
||||
...Object.entries(adTypeLabels).map(([value, label]) => ({ value, label })),
|
||||
{ value: 'unclassified', label: '历史数据/未识别' },
|
||||
]"
|
||||
/>
|
||||
</n-form-item>
|
||||
<n-form-item label="日期(搜索页)">
|
||||
<n-date-picker
|
||||
type="datetimerange"
|
||||
@ -302,6 +389,9 @@ const handleClearData = async () => {
|
||||
<n-checkbox value="keywords" label="关键词" />
|
||||
<n-checkbox value="page" label="页码" />
|
||||
<n-checkbox value="rank" label="排位" />
|
||||
<n-checkbox value="pageRank" label="页内排位" />
|
||||
<n-checkbox value="adRank" label="广告排位" />
|
||||
<n-checkbox value="placementRank" label="广告位顺序" />
|
||||
<n-checkbox value="createTime" label="获取日期" />
|
||||
<n-checkbox value="timestamp" label="获取日期(详情)" />
|
||||
</n-space>
|
||||
|
||||
@ -44,7 +44,7 @@
|
||||
|
||||
位于搜索页面板,可以根据输入的关键词采集搜索页中呈现的商品信息,主要为**关键词排名**。
|
||||
输入:关键词(最多十个)
|
||||
输出:商品搜索页信息,包含关键词、页码、排位、 ASIN、标题、 价格、 封面图、获取日期。
|
||||
输出:商品搜索页信息,包含关键词、页码、排位、广告类型、广告排位、广告位置、ASIN、标题、当前价格、划线价格、封面图、获取日期。
|
||||
|
||||
### 详情页
|
||||
|
||||
@ -52,7 +52,7 @@
|
||||
|
||||
位于详情页面板,可以根据输入的多个ASIN采集商品详情页的商品信息。
|
||||
输入:ASIN(每行输入一个,可以输入多个)
|
||||
输出:商品详情页信息,包含ASIN、标题、 价格、 封面图、获取日期、评分、评论数、 大类、大类排行、小类、 小类排行、 商品图片链接、A+截图等等。
|
||||
输出:商品详情页信息,包含ASIN、标题、当前价格、划线价格、封面图、获取日期、评分、评论数、大类、大类排行、小类、小类排行、商品图片链接、A+截图等等。
|
||||
|
||||
### 评论页
|
||||
|
||||
@ -86,9 +86,15 @@ Items表记录商品信息,表头信息如下。
|
||||
| 关键词 | 产品的关键词 |
|
||||
| 页码 | 产品在关键词搜索列表中的页码 |
|
||||
| 排位 | 产品的排名 |
|
||||
| 页内排位 | 产品在当前搜索页商品流中的排名 |
|
||||
| 广告类型 | 自然排位、Sponsored Products、Sponsored Brands、Sponsored Display 或未知广告 |
|
||||
| 广告排位 | 同一关键词、同一广告类型中的累计排名 |
|
||||
| 广告位置 | 搜索顶部、搜索其余位置或侧边栏 |
|
||||
| 广告位顺序 | 同一关键词采集过程中广告模块的累计顺序 |
|
||||
| ASIN | 亚马逊标准识别号 |
|
||||
| 标题 | 产品的标题 |
|
||||
| 价格 | 产品的价格 |
|
||||
| 价格 | 产品的当前价格 |
|
||||
| 划线价格 | 页面中以划线形式展示的参考价格 |
|
||||
| 封面图 | 产品的封面图片链接 |
|
||||
| 获取日期 | 数据获取的日期 |
|
||||
| 商品链接 | 产品的链接 |
|
||||
|
||||
@ -3,7 +3,8 @@ import amazon from '../impls/amazon';
|
||||
import { uploadImage } from '~/logic/upload-image';
|
||||
import { detailItems, reviewItems, searchItems } from '~/storages/amazon';
|
||||
import { createGlobalState } from '@vueuse/core';
|
||||
import { useAmazonService } from '~/page-worker/services/amazon';
|
||||
import { enqueueAmazonSync } from '~/logic/amazon-sync-client';
|
||||
import { createAmazonSyncId } from '~/logic/amazon-sync-shared';
|
||||
import { taskOptionBase, WorkerComposable } from '../interfaces/common';
|
||||
import { AmazonPageWorker } from '../interfaces/amazon';
|
||||
|
||||
@ -18,10 +19,7 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
// Reactive settings object
|
||||
const settings = shallowRef<AmazonPageWorkerSettings>({});
|
||||
// Long task management
|
||||
const { isRunning, startTask } = useLongTask();
|
||||
// Amazon service instance
|
||||
const service = useAmazonService();
|
||||
|
||||
const { isRunning, lastError, startTask } = useLongTask();
|
||||
// Get the worker instance from implementation
|
||||
const worker = amazon.getAmazonPageWorker();
|
||||
|
||||
@ -29,10 +27,13 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const searchCache = [] as AmazonSearchItem[];
|
||||
const detailCache = new Map<string, AmazonDetailItem>();
|
||||
const reviewCache = new Map<string, AmazonReview[]>();
|
||||
let commitOperation: Promise<void> = Promise.resolve();
|
||||
|
||||
// Add search items to cache
|
||||
const updateSearchCache = (data: AmazonSearchItem[]) => {
|
||||
searchCache.push(...data);
|
||||
searchCache.push(
|
||||
...data.map((item) => ({ ...item, syncId: item.syncId || createAmazonSyncId('search') })),
|
||||
);
|
||||
};
|
||||
|
||||
// Update or add detail item in cache
|
||||
@ -42,7 +43,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const origin = detailCache.get(data.asin);
|
||||
detailCache.set(asin, { ...origin, ...data } as AmazonDetailItem);
|
||||
} else {
|
||||
detailCache.set(asin, data as AmazonDetailItem);
|
||||
detailCache.set(asin, {
|
||||
...data,
|
||||
syncId: data.syncId || createAmazonSyncId('detail'),
|
||||
} as AmazonDetailItem);
|
||||
}
|
||||
};
|
||||
|
||||
@ -52,7 +56,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const values = reviewCache.get(asin) || [];
|
||||
const ids = new Set(values.map((item) => item.id));
|
||||
for (const review of reviews) {
|
||||
ids.has(review.id) || values.push(review);
|
||||
if (!ids.has(review.id)) {
|
||||
values.push(review);
|
||||
ids.add(review.id);
|
||||
}
|
||||
}
|
||||
reviewCache.set(asin, values);
|
||||
};
|
||||
@ -64,44 +71,90 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
reviewCache.clear();
|
||||
};
|
||||
|
||||
// Commit cached changes to storage and service
|
||||
const commitChange = async () => {
|
||||
// Persist locally first, then atomically enqueue for background synchronization.
|
||||
const commitChangeInternal = async () => {
|
||||
const { objects } = settings.value;
|
||||
// Commit search items
|
||||
if (objects?.includes('search')) {
|
||||
searchItems.value = searchItems.value.concat(searchCache);
|
||||
await service.commitSearchItems(searchCache);
|
||||
searchCache.splice(0, searchCache.length);
|
||||
const searchSnapshot = objects?.includes('search') ? searchCache.slice() : [];
|
||||
const detailSnapshot = objects?.includes('detail') ? Array.from(detailCache.values()) : [];
|
||||
const reviewSnapshot = objects?.includes('review')
|
||||
? Array.from(reviewCache.entries()).flatMap(([asin, reviews]) =>
|
||||
reviews.map((review) => ({ asin, ...review })),
|
||||
)
|
||||
: [];
|
||||
|
||||
if (searchSnapshot.length === 0 && detailSnapshot.length === 0 && reviewSnapshot.length === 0) {
|
||||
return;
|
||||
}
|
||||
// Commit detail items
|
||||
if (objects?.includes('detail')) {
|
||||
for (const [k, v] of detailCache.entries()) {
|
||||
if (detailItems.value.has(k)) {
|
||||
const item = detailItems.value.get(k)!;
|
||||
detailItems.value.set(k, { ...item, ...v });
|
||||
|
||||
if (searchSnapshot.length > 0) {
|
||||
const existingIds = new Set(searchItems.value.map((item) => item.syncId).filter(Boolean));
|
||||
searchItems.value = searchItems.value.concat(
|
||||
searchSnapshot.filter((item) => !item.syncId || !existingIds.has(item.syncId)),
|
||||
);
|
||||
}
|
||||
if (detailSnapshot.length > 0) {
|
||||
const nextDetailItems = new Map(detailItems.value);
|
||||
for (const v of detailSnapshot) {
|
||||
const k = v.asin;
|
||||
if (nextDetailItems.has(k)) {
|
||||
const item = nextDetailItems.get(k)!;
|
||||
nextDetailItems.set(k, { ...item, ...v });
|
||||
} else {
|
||||
detailItems.value.set(k, v);
|
||||
nextDetailItems.set(k, v);
|
||||
}
|
||||
}
|
||||
await service.commitDetailItems(detailCache);
|
||||
detailCache.clear();
|
||||
// Replacing the Map gives every extension page a deterministic reactive
|
||||
// change to persist and observe; in-place Map.set could leave the open
|
||||
// results page showing stale data during long batches.
|
||||
detailItems.value = nextDetailItems;
|
||||
}
|
||||
// Commit reviews
|
||||
if (objects?.includes('review')) {
|
||||
for (const [asin, reviews] of reviewCache.entries()) {
|
||||
if (reviewItems.value.has(asin)) {
|
||||
if (reviewSnapshot.length > 0) {
|
||||
const groupedReviews = reviewSnapshot.reduce<Map<string, AmazonReview[]>>((grouped, item) => {
|
||||
const values = grouped.get(item.asin) || [];
|
||||
const { asin: _, ...review } = item;
|
||||
values.push(review);
|
||||
grouped.set(item.asin, values);
|
||||
return grouped;
|
||||
}, new Map());
|
||||
const nextReviewItems = new Map(reviewItems.value);
|
||||
for (const [asin, reviews] of groupedReviews.entries()) {
|
||||
if (nextReviewItems.has(asin)) {
|
||||
const addedIds = new Set(reviews.map((x) => x.id));
|
||||
const origin = reviewItems.value.get(asin)!;
|
||||
const origin = nextReviewItems.get(asin)!;
|
||||
const newReviews = origin.filter((x) => !addedIds.has(x.id)).concat(reviews);
|
||||
newReviews.sort((a, b) => dayjs(b.dateInfo).diff(dayjs(a.dateInfo)));
|
||||
reviewItems.value.set(asin, newReviews);
|
||||
nextReviewItems.set(asin, newReviews);
|
||||
} else {
|
||||
reviewItems.value.set(asin, reviews);
|
||||
nextReviewItems.set(asin, reviews);
|
||||
}
|
||||
}
|
||||
await service.commitReviews(reviewCache);
|
||||
reviewCache.clear();
|
||||
reviewItems.value = nextReviewItems;
|
||||
}
|
||||
|
||||
// The background acknowledges only after the queue is durably written to extension storage.
|
||||
await enqueueAmazonSync({
|
||||
searchItems: searchSnapshot,
|
||||
detailItems: detailSnapshot,
|
||||
reviews: reviewSnapshot,
|
||||
});
|
||||
|
||||
const sentSearchIds = new Set(searchSnapshot.map((item) => item.syncId));
|
||||
const remainingSearch = searchCache.filter((item) => !sentSearchIds.has(item.syncId));
|
||||
searchCache.splice(0, searchCache.length, ...remainingSearch);
|
||||
for (const item of detailSnapshot) {
|
||||
if (detailCache.get(item.asin)?.syncId === item.syncId) detailCache.delete(item.asin);
|
||||
}
|
||||
const sentReviewIds = new Set(reviewSnapshot.map((item) => item.id));
|
||||
for (const [asin, reviews] of reviewCache.entries()) {
|
||||
const remaining = reviews.filter((item) => !sentReviewIds.has(item.id));
|
||||
remaining.length > 0 ? reviewCache.set(asin, remaining) : reviewCache.delete(asin);
|
||||
}
|
||||
};
|
||||
|
||||
const commitChange = () => {
|
||||
const result = commitOperation.then(commitChangeInternal, commitChangeInternal);
|
||||
commitOperation = result.catch(() => undefined);
|
||||
return result;
|
||||
};
|
||||
|
||||
// Register all relevant worker event handlers
|
||||
@ -125,7 +178,7 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
}),
|
||||
// Collect images for detail items
|
||||
worker.on('item-images-collected', (ev) => {
|
||||
updateDetailCache(ev);
|
||||
if (ev.imageUrls?.length) updateDetailCache(ev);
|
||||
}),
|
||||
// Collect A+ content screenshots and upload
|
||||
worker.on('item-aplus-screenshot-collect', async (ev) => {
|
||||
@ -161,15 +214,25 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
/**
|
||||
* Task wrapper: commit changes by interval during task execution
|
||||
*/
|
||||
const taskWrapper1 = <T extends (...params: any[]) => Promise<void>>(func: T) => {
|
||||
const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value;
|
||||
clearAllCaches();
|
||||
const taskWrapper1 = <T extends (...params: any[]) => Promise<void>>(
|
||||
func: T,
|
||||
objectType: 'search' | 'review',
|
||||
) => {
|
||||
return (...params: Parameters<T>) =>
|
||||
startTask(async () => {
|
||||
const interval = setInterval(() => commitChange(), commitChangeIngerval);
|
||||
await func(...params);
|
||||
clearInterval(interval);
|
||||
await commitChange();
|
||||
settings.value = { ...settings.value, objects: [objectType] };
|
||||
clearAllCaches();
|
||||
const { commitChangeInterval: commitChangeIngerval = 10000 } = settings.value;
|
||||
const interval = setInterval(
|
||||
() => void commitChange().catch(console.error),
|
||||
commitChangeIngerval,
|
||||
);
|
||||
try {
|
||||
await func(...params);
|
||||
} finally {
|
||||
clearInterval(interval);
|
||||
await commitChange();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
@ -179,9 +242,10 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
const taskWrapper2 = <T extends (input: any, options?: taskOptionBase) => Promise<void>>(
|
||||
func: T,
|
||||
) => {
|
||||
clearAllCaches();
|
||||
return (...params: Parameters<T>) =>
|
||||
startTask(async () => {
|
||||
settings.value = { ...settings.value, objects: ['detail'] };
|
||||
clearAllCaches();
|
||||
if (!params?.[1]) {
|
||||
params[1] = {};
|
||||
}
|
||||
@ -196,20 +260,24 @@ function buildAmazonPageWorker(): WorkerComposable<AmazonPageWorker, AmazonPageW
|
||||
await commitChange();
|
||||
};
|
||||
}
|
||||
await func(params[0], params[1]);
|
||||
await commitChange();
|
||||
try {
|
||||
await func(params[0], params[1]);
|
||||
} finally {
|
||||
await commitChange();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
// Wrapped task runners for each page type
|
||||
const runSearchPageTask = taskWrapper1(worker.runSearchPageTask.bind(worker));
|
||||
const runSearchPageTask = taskWrapper1(worker.runSearchPageTask.bind(worker), 'search');
|
||||
const runDetailPageTask = taskWrapper2(worker.runDetailPageTask.bind(worker));
|
||||
const runReviewPageTask = taskWrapper1(worker.runReviewPageTask.bind(worker));
|
||||
const runReviewPageTask = taskWrapper1(worker.runReviewPageTask.bind(worker), 'review');
|
||||
|
||||
// Expose API for the composable
|
||||
return {
|
||||
settings,
|
||||
isRunning,
|
||||
lastError,
|
||||
runSearchPageTask,
|
||||
runDetailPageTask,
|
||||
runReviewPageTask,
|
||||
|
||||
@ -9,6 +9,12 @@ import { isForbiddenUrl } from '~/env';
|
||||
import { BaseWorker } from './base';
|
||||
import { AmazonPageWorker, AmazonPageWorkerEvents } from '../interfaces/amazon';
|
||||
import { taskOptionBase } from '../interfaces/common';
|
||||
import {
|
||||
extractAmazonDetailAsin,
|
||||
hasReachedAmazonDetailPage,
|
||||
hasReachedAmazonNavigationTarget,
|
||||
} from '../page-navigation';
|
||||
import { isAmazonReviewRatingFilterIgnored } from '../web-injectors/amazon-review-page';
|
||||
|
||||
/**
|
||||
* AmazonPageWorkerImpl can run on background & sidepanel & popup,
|
||||
@ -43,24 +49,70 @@ class AmazonPageWorkerImpl
|
||||
return tab;
|
||||
}
|
||||
|
||||
private async navigateTabAndWait(
|
||||
tabId: number,
|
||||
url: string,
|
||||
options: { allowDetailRedirect?: boolean } = {},
|
||||
): Promise<string> {
|
||||
const previousTab = await browser.tabs.get(tabId);
|
||||
const previousUrl = previousTab.url;
|
||||
const updatedTab = await browser.tabs.update(tabId, { url, active: true });
|
||||
let navigationObserved =
|
||||
updatedTab.status === 'loading' ||
|
||||
(Boolean(updatedTab.url) && updatedTab.url !== previousUrl);
|
||||
let lastUrl = updatedTab.url;
|
||||
for (let i = 0; i < 150; i++) {
|
||||
const currentTab = await browser.tabs.get(tabId);
|
||||
lastUrl = currentTab.url;
|
||||
navigationObserved ||=
|
||||
currentTab.status === 'loading' ||
|
||||
(Boolean(currentTab.url) && currentTab.url !== previousUrl);
|
||||
const reachedTarget = hasReachedAmazonNavigationTarget(currentTab.url, url);
|
||||
const reachedRedirectedDetail =
|
||||
options.allowDetailRedirect && hasReachedAmazonDetailPage(currentTab.url, url);
|
||||
if (navigationObserved && currentTab.status === 'complete' && (reachedTarget || reachedRedirectedDetail)) {
|
||||
return currentTab.url!;
|
||||
}
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 200));
|
||||
}
|
||||
throw new Error(`等待目标页面加载超时: ${url};当前页面: ${lastUrl || '未知'}`);
|
||||
}
|
||||
|
||||
private async wanderSearchSinglePage(tab: Tabs.Tab) {
|
||||
const injector = new AmazonSearchPageInjector(tab);
|
||||
const runStage = async <T>(stage: string, operation: () => Promise<T>): Promise<T> => {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`搜索页${stage}失败: ${message}`);
|
||||
}
|
||||
};
|
||||
// Wait for the Next button to appear, indicating that the product items have finished loading
|
||||
await injector.waitForPageLoaded();
|
||||
await runStage('加载等待', () => injector.waitForPageLoaded());
|
||||
// Determine the type of product search page https://github.com/primedigitaltech/azon_seeker/issues/1
|
||||
const pagePattern = await injector.getPagePattern();
|
||||
const pagePattern = await runStage('布局识别', () => injector.getPagePattern());
|
||||
// Retrieve key nodes and their information from the critical product search page
|
||||
const data = await injector.getPageData(pagePattern);
|
||||
const data = await runStage('数据解析', () => injector.getPageData(pagePattern));
|
||||
const explicitNoResults =
|
||||
data?.length === 0
|
||||
? await runStage('空结果识别', () => injector.isExplicitNoResultsPage())
|
||||
: false;
|
||||
// get current page
|
||||
const page = await injector.getCurrentPage();
|
||||
// Determine if it is the last page, otherwise navigate to the next page
|
||||
const hasNextPage = await injector.determineHasNextPage();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
if (data === null || typeof hasNextPage !== 'boolean') {
|
||||
const page = await runStage('页码识别', () => injector.getCurrentPage());
|
||||
// Read the next URL without navigating inside the injected script. Clicking here would move
|
||||
// the current Amazon frame into BFCache before executeScript can return its result.
|
||||
const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl());
|
||||
if (data === null) {
|
||||
this.emit('error', { message: '爬取单页信息失败', url: tab.url });
|
||||
throw new Error('爬取单页信息失败');
|
||||
}
|
||||
return { data, hasNextPage, page };
|
||||
if (data.length === 0 && !explicitNoResults) {
|
||||
throw new Error(
|
||||
'Amazon 搜索页未发现可采集商品,且页面没有明确显示“无结果”;请检查是否出现临时错误页、验证码或页面结构变化',
|
||||
);
|
||||
}
|
||||
return { data, nextPageUrl, page };
|
||||
}
|
||||
|
||||
@withErrorHandling
|
||||
@ -69,14 +121,27 @@ class AmazonPageWorkerImpl
|
||||
url.searchParams.append('k', keywords);
|
||||
let tab = await this.getCurrentTab();
|
||||
if (!tab.url?.startsWith('http')) {
|
||||
// A newly created tab may not have a URL in the returned Tab object yet, so never pass its
|
||||
// transient `tab.url` to `new URL()`. Navigate by tab id and wait for the target explicitly.
|
||||
tab = await this.createNewTab('https://www.amazon.com/');
|
||||
tab.url = 'https://www.amazon.com/';
|
||||
await this.navigateTabAndWait(tab.id!, url.toString());
|
||||
return url.toString();
|
||||
}
|
||||
const currentUrl = new URL(tab.url!);
|
||||
if (currentUrl.hostname !== url.hostname || currentUrl.searchParams.get('k') !== keywords) {
|
||||
tab = await browser.tabs.update(tab.id, { url: url.toString(), active: true });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 1000));
|
||||
const sameSearch =
|
||||
currentUrl.hostname === url.hostname && currentUrl.searchParams.get('k') === keywords;
|
||||
if (sameSearch) {
|
||||
try {
|
||||
// Preserve the old behavior: a healthy same-keyword page continues from its current page.
|
||||
await new AmazonSearchPageInjector(tab).waitForPageLoaded();
|
||||
return currentUrl.toString();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (!message.includes('临时错误页')) throw error;
|
||||
// Only a known Amazon transient error page is recovered by returning to page one.
|
||||
}
|
||||
}
|
||||
await this.navigateTabAndWait(tab.id!, url.toString());
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
@ -84,22 +149,46 @@ class AmazonPageWorkerImpl
|
||||
public async wanderSearchPage(): Promise<void> {
|
||||
const tab = await this.getCurrentTab();
|
||||
let offset = 0;
|
||||
let placementOffset = 0;
|
||||
const adOffsets = new Map<AmazonAdType, number>();
|
||||
while (true) {
|
||||
const { hasNextPage, data, page } = await this.wanderSearchSinglePage(tab);
|
||||
const { nextPageUrl, data, page } = await this.wanderSearchSinglePage(tab);
|
||||
const keywords = new URL(tab.url!).searchParams.get('k')!;
|
||||
const objs = data.map((r, i) => ({
|
||||
...r,
|
||||
keywords,
|
||||
page,
|
||||
rank: offset + 1 + i,
|
||||
createTime: dayjs().format('YYYY/M/D HH:mm:ss'),
|
||||
asin: /(?<=\/dp\/)[A-Z0-9]{10}/.exec(r.link as string)![0],
|
||||
}));
|
||||
const pageAdCounts = new Map<AmazonAdType, number>();
|
||||
const objs = data.map((r, i) => {
|
||||
const localAdRank = r.adRank;
|
||||
const localPlacementRank = r.placementRank;
|
||||
const adRank =
|
||||
r.adType && r.adType !== 'organic' && localAdRank
|
||||
? (adOffsets.get(r.adType) || 0) + localAdRank
|
||||
: undefined;
|
||||
if (r.adType && r.adType !== 'organic') {
|
||||
pageAdCounts.set(r.adType, Math.max(pageAdCounts.get(r.adType) || 0, localAdRank || 0));
|
||||
}
|
||||
const placementRank = localPlacementRank ? placementOffset + localPlacementRank : undefined;
|
||||
return {
|
||||
...r,
|
||||
keywords,
|
||||
page,
|
||||
rank: offset + 1 + i,
|
||||
pageRank: r.pageRank || i + 1,
|
||||
adRank,
|
||||
placementRank,
|
||||
placementId: placementRank
|
||||
? `${encodeURIComponent(keywords)}:${page}:${placementRank}`
|
||||
: undefined,
|
||||
createTime: dayjs().format('YYYY/M/D HH:mm:ss'),
|
||||
};
|
||||
});
|
||||
this.emit('item-links-collected', { objs });
|
||||
offset += data.length;
|
||||
if (!hasNextPage) {
|
||||
break;
|
||||
}
|
||||
placementOffset += Math.max(0, ...data.map((item) => item.placementRank || 0));
|
||||
pageAdCounts.forEach((count, adType) => {
|
||||
adOffsets.set(adType, (adOffsets.get(adType) || 0) + count);
|
||||
});
|
||||
if (!nextPageUrl) break;
|
||||
await this.navigateTabAndWait(tab.id!, nextPageUrl);
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
}
|
||||
return new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
@ -113,8 +202,7 @@ class AmazonPageWorkerImpl
|
||||
const { image = false, aplus = false, extra = false } = options;
|
||||
const params = { asin: '', url: '' };
|
||||
if (entry.match(/^https?:\/\/www\.amazon\.com.*\/dp\/[A-Z0-9]{10}/)) {
|
||||
const [asin] = /\/\/dp\/[A-Z0-9]{10}/.exec(entry)!;
|
||||
params.asin = asin;
|
||||
params.asin = extractAmazonDetailAsin(entry) || '';
|
||||
params.url = entry;
|
||||
} else if (entry.match(/^[A-Z0-9]{10}$/)) {
|
||||
params.asin = entry;
|
||||
@ -124,9 +212,8 @@ class AmazonPageWorkerImpl
|
||||
if (!tab.url || isForbiddenUrl(tab.url)) {
|
||||
tab = await this.createNewTab(params.url);
|
||||
} else {
|
||||
tab = await browser.tabs.update(tab.id, {
|
||||
url: params.url,
|
||||
});
|
||||
await this.navigateTabAndWait(tab.id!, params.url, { allowDetailRedirect: true });
|
||||
tab = await browser.tabs.get(tab.id!);
|
||||
}
|
||||
const injector = new AmazonDetailPageInjector(tab);
|
||||
|
||||
@ -142,6 +229,16 @@ class AmazonPageWorkerImpl
|
||||
//#region Await Production Introduction Element Loaded
|
||||
await injector.waitForPageLoaded();
|
||||
await new Promise((resolve) => setTimeout(resolve, 3000)); // Wait 3 seconds.
|
||||
tab = await browser.tabs.get(tab.id!);
|
||||
const requestedAsin = params.asin;
|
||||
const resolvedAsin = extractAmazonDetailAsin(tab.url);
|
||||
if (!resolvedAsin) {
|
||||
throw new Error(`Amazon 详情页未能识别实际 ASIN:${tab.url || '未知页面'}`);
|
||||
}
|
||||
params.asin = resolvedAsin;
|
||||
if (requestedAsin !== resolvedAsin) {
|
||||
await this.emit('item-detail-redirected', { requestedAsin, resolvedAsin });
|
||||
}
|
||||
//#endregion
|
||||
//#region Fetch Base Info
|
||||
const baseInfo = await injector.getBaseInfo();
|
||||
@ -183,11 +280,10 @@ class AmazonPageWorkerImpl
|
||||
//#region Fetch Goods' Images
|
||||
if (image) {
|
||||
const imageUrls = await injector.getImageUrls();
|
||||
imageUrls.length > 0 &&
|
||||
(await this.emit('item-images-collected', {
|
||||
asin: params.asin,
|
||||
imageUrls: Array.from(new Set(imageUrls)),
|
||||
}));
|
||||
await this.emit('item-images-collected', {
|
||||
asin: params.asin,
|
||||
imageUrls: Array.from(new Set(imageUrls)),
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000)); // Wait 2 seconds.
|
||||
}
|
||||
//#endregion
|
||||
@ -217,24 +313,72 @@ class AmazonPageWorkerImpl
|
||||
@withErrorHandling
|
||||
public async wanderReviewPage(asin: string, options: { recent?: boolean } = {}) {
|
||||
const { recent } = options;
|
||||
const url = new URL(
|
||||
const baseUrl = new URL(
|
||||
`https://www.amazon.com/product-reviews/${asin}/ref=cm_cr_dp_d_show_all_btm?ie=UTF8&reviewerType=all_reviews`,
|
||||
);
|
||||
recent && url.searchParams.set('sortBy', 'recent');
|
||||
const tab = await this.createNewTab(url.toString());
|
||||
recent && baseUrl.searchParams.set('sortBy', 'recent');
|
||||
const tab = await this.createNewTab(baseUrl.toString());
|
||||
const injector = new AmazonReviewPageInjector(tab);
|
||||
await injector.waitForPageLoad();
|
||||
for (let star = 1; star <= 5; star++) {
|
||||
await injector.showStarsDropDownMenu();
|
||||
await injector.selectStar(star);
|
||||
const runStage = async <T>(stage: string, operation: () => Promise<T>): Promise<T> => {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(`评论页${stage}失败: ${message}`);
|
||||
}
|
||||
};
|
||||
await runStage('初始加载等待', () => injector.waitForPageLoad());
|
||||
const starFilters = [
|
||||
{ name: 'one_star', star: 1 },
|
||||
{ name: 'two_star', star: 2 },
|
||||
{ name: 'three_star', star: 3 },
|
||||
{ name: 'four_star', star: 4 },
|
||||
{ name: 'five_star', star: 5 },
|
||||
] as const;
|
||||
const collectedReviewIds = new Set<string>();
|
||||
|
||||
for (const { name: filterByStar, star } of starFilters) {
|
||||
// The original collector gathered every rating bucket. Build the same Amazon filter URL
|
||||
// directly instead of requiring a particular dropdown DOM structure just to discover it.
|
||||
const starUrl = new URL(baseUrl);
|
||||
starUrl.searchParams.set('filterByStar', filterByStar);
|
||||
starUrl.searchParams.delete('pageNumber');
|
||||
await runStage(`${star} 星筛选导航`, () =>
|
||||
this.navigateTabAndWait(tab.id!, starUrl.toString()),
|
||||
);
|
||||
const visitedPageUrls = new Set<string>();
|
||||
let page = 1;
|
||||
let filterIgnored = false;
|
||||
while (true) {
|
||||
await injector.waitForPageLoad();
|
||||
const reviews = await injector.getSinglePageReviews();
|
||||
reviews.length > 0 && this.emit('item-review-collected', { asin, reviews });
|
||||
const hasNextPage = await injector.jumpToNextPageIfExist();
|
||||
if (!hasNextPage) {
|
||||
break;
|
||||
const currentTab = await browser.tabs.get(tab.id!);
|
||||
if (currentTab.url) visitedPageUrls.add(currentTab.url);
|
||||
// Match v0.7.1.12-beta semantics: collect the rating parsed from each review card. The
|
||||
// requested filter is only a navigation aid and must not overwrite or reject real data.
|
||||
await runStage(`${star} 星第 ${page} 页加载等待`, () => injector.waitForPageLoad());
|
||||
const reviews = await runStage('评论解析', () => injector.getSinglePageReviews());
|
||||
if (page === 1 && isAmazonReviewRatingFilterIgnored(reviews, star)) {
|
||||
// Amazon currently experiments with review pages that preserve filterByStar in the URL
|
||||
// but return the same unfiltered review cards. Keep the cards' real ratings, then stop
|
||||
// revisiting the same eight reviews for the remaining rating buckets.
|
||||
filterIgnored = true;
|
||||
}
|
||||
const uniqueReviews = reviews.filter((review) => {
|
||||
if (collectedReviewIds.has(review.id)) return false;
|
||||
collectedReviewIds.add(review.id);
|
||||
return true;
|
||||
});
|
||||
uniqueReviews.length > 0 &&
|
||||
this.emit('item-review-collected', { asin, reviews: uniqueReviews });
|
||||
const nextPageUrl = await runStage('分页识别', () => injector.getNextPageUrl());
|
||||
if (!nextPageUrl) break;
|
||||
if (visitedPageUrls.has(nextPageUrl)) {
|
||||
throw new Error(`评论页分页出现循环,已停止重复采集: ${nextPageUrl}`);
|
||||
}
|
||||
await runStage('分页导航', () => this.navigateTabAndWait(tab.id!, nextPageUrl));
|
||||
page += 1;
|
||||
}
|
||||
if (filterIgnored) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
setTimeout(() => browser.tabs.remove(tab.id!), 1000);
|
||||
@ -254,7 +398,7 @@ class AmazonPageWorkerImpl
|
||||
const kw = remains.shift()!;
|
||||
await this.doSearch(kw);
|
||||
await this.wanderSearchPage();
|
||||
progress && progress(remains);
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
@ -272,7 +416,10 @@ class AmazonPageWorkerImpl
|
||||
while (remains.length > 0 && !interrupt) {
|
||||
const asin = remains.shift()!;
|
||||
await this.wanderDetailPage(asin, options);
|
||||
progress && progress(remains);
|
||||
// The progress callback persists the completed item locally and enqueues
|
||||
// it for server synchronization. Do not start the next ASIN until that
|
||||
// handoff has completed, otherwise a batch can outrun its result writes.
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
@ -290,7 +437,7 @@ class AmazonPageWorkerImpl
|
||||
while (remains.length > 0 && !interrupt) {
|
||||
const asin = remains.shift()!;
|
||||
await this.wanderReviewPage(asin, options);
|
||||
progress && progress(remains);
|
||||
if (progress) await progress(remains);
|
||||
}
|
||||
unsubscribe();
|
||||
}
|
||||
|
||||
@ -4,6 +4,9 @@ export interface AmazonPageWorkerEvents {
|
||||
/** The event is fired when worker collected links to items on the Amazon search page. */
|
||||
['item-links-collected']: { objs: AmazonSearchItem[] };
|
||||
|
||||
/** Amazon resolved a requested detail ASIN to a canonical or variation ASIN. */
|
||||
['item-detail-redirected']: { requestedAsin: string; resolvedAsin: string };
|
||||
|
||||
/** The event is fired when worker collected goods' base info on the Amazon detail page.*/
|
||||
['item-base-info-collected']: Pick<
|
||||
AmazonDetailItem,
|
||||
@ -11,6 +14,7 @@ export interface AmazonPageWorkerEvents {
|
||||
| 'title'
|
||||
| 'boughtInfo'
|
||||
| 'price'
|
||||
| 'strikethroughPrice'
|
||||
| 'rating'
|
||||
| 'ratingCount'
|
||||
| 'categories'
|
||||
@ -41,7 +45,10 @@ export interface AmazonPageWorkerEvents {
|
||||
};
|
||||
|
||||
/** The event is fired when reviews collected in all review page */
|
||||
['item-review-collected']: { asin: string; reviews: AmazonReview[] };
|
||||
['item-review-collected']: {
|
||||
asin: string;
|
||||
reviews: AmazonReview[];
|
||||
};
|
||||
|
||||
/** Error event that occurs when there is an issue with the Amazon page worker*/
|
||||
['error']: { message: string; url?: string };
|
||||
|
||||
@ -9,4 +9,5 @@ export interface taskOptionBase {
|
||||
export type WorkerComposable<Base, S = {}> = Base & {
|
||||
settings: Ref<S>;
|
||||
isRunning: Ref<boolean>;
|
||||
lastError?: Ref<unknown>;
|
||||
};
|
||||
|
||||
58
src/page-worker/page-navigation.ts
Normal file
58
src/page-worker/page-navigation.ts
Normal file
@ -0,0 +1,58 @@
|
||||
const navigationQueryKeys = ['k', 'page', 'pageNumber', 'filterByStar'] as const;
|
||||
|
||||
function routeIdentity(url: URL) {
|
||||
const reviewMatch = url.pathname.match(/^\/product-reviews\/([A-Z0-9]{10})/i);
|
||||
if (reviewMatch) return `/product-reviews/${reviewMatch[1].toUpperCase()}`;
|
||||
const detailMatch = url.pathname.match(/\/dp\/([A-Z0-9]{10})/i);
|
||||
if (detailMatch) return `/dp/${detailMatch[1].toUpperCase()}`;
|
||||
return url.pathname.replace(/\/ref=.*$/, '') || '/';
|
||||
}
|
||||
|
||||
export function extractAmazonDetailAsin(url: string | undefined) {
|
||||
if (!url) return null;
|
||||
try {
|
||||
return new URL(url).pathname.match(/\/dp\/([A-Z0-9]{10})/i)?.[1]?.toUpperCase() || null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Accepts an Amazon canonical/variation redirect only when both URLs are detail pages. */
|
||||
export function hasReachedAmazonDetailPage(
|
||||
currentUrl: string | undefined,
|
||||
targetUrl: string,
|
||||
) {
|
||||
if (!currentUrl) return false;
|
||||
try {
|
||||
const current = new URL(currentUrl);
|
||||
const target = new URL(targetUrl);
|
||||
return (
|
||||
current.origin === target.origin &&
|
||||
Boolean(extractAmazonDetailAsin(currentUrl)) &&
|
||||
Boolean(extractAmazonDetailAsin(targetUrl))
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Amazon may rewrite tracking parameters while navigating. Compare the stable route and the
|
||||
* query parameters which identify the requested search/review page instead of the whole URL.
|
||||
*/
|
||||
export function hasReachedAmazonNavigationTarget(currentUrl: string | undefined, targetUrl: string) {
|
||||
if (!currentUrl) return false;
|
||||
try {
|
||||
const current = new URL(currentUrl);
|
||||
const target = new URL(targetUrl);
|
||||
if (current.origin !== target.origin || routeIdentity(current) !== routeIdentity(target)) {
|
||||
return false;
|
||||
}
|
||||
return navigationQueryKeys.every((key) => {
|
||||
const expected = target.searchParams.get(key);
|
||||
return expected === null || current.searchParams.get(key) === expected;
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
103
src/page-worker/web-injectors/amazon-detail-images.ts
Normal file
103
src/page-worker/web-injectors/amazon-detail-images.ts
Normal file
@ -0,0 +1,103 @@
|
||||
/**
|
||||
* Collect product image URLs from an Amazon detail page.
|
||||
*
|
||||
* Keep this function self-contained: browser.scripting.executeScript serializes
|
||||
* it before running it in the target tab, so it must not depend on closures.
|
||||
*/
|
||||
export async function collectAmazonDetailImageUrls(): Promise<string[]> {
|
||||
const delay = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
const isProductImage = (value: string) =>
|
||||
/^https:\/\/(?:m\.media-amazon\.com|images-na\.ssl-images-amazon\.com)\/images\/I\//.test(
|
||||
value,
|
||||
);
|
||||
const unique = (values: string[]) =>
|
||||
Array.from(new Set(values.map((value) => value.replaceAll('\\/', '/')))).filter(
|
||||
isProductImage,
|
||||
);
|
||||
|
||||
const highResolutionUrls: string[] = [];
|
||||
const largeUrls: string[] = [];
|
||||
|
||||
// Amazon has changed the surrounding P.when(...) wrapper several times.
|
||||
// Scan every inline script for the same hiRes/large metadata used by the old parser.
|
||||
for (const scriptNode of Array.from(document.scripts)) {
|
||||
const script = (scriptNode.textContent || '').replaceAll('\\/', '/');
|
||||
for (const match of script.matchAll(
|
||||
/"hiRes"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g,
|
||||
)) {
|
||||
highResolutionUrls.push(match[1]);
|
||||
}
|
||||
for (const match of script.matchAll(
|
||||
/"large"\s*:\s*"(https:\/\/m\.media-amazon\.com\/images\/I\/[^"?]+(?:\?[^" ]*)?)"/g,
|
||||
)) {
|
||||
largeUrls.push(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
let urls = unique(highResolutionUrls.length > 0 ? highResolutionUrls : largeUrls);
|
||||
|
||||
// Current detail pages expose the main high-resolution image directly.
|
||||
if (urls.length === 0) {
|
||||
const landingImage = document.querySelector<HTMLImageElement>('#landingImage');
|
||||
const oldHighResolutionUrl = landingImage?.getAttribute('data-old-hires');
|
||||
if (oldHighResolutionUrl) urls.push(oldHighResolutionUrl);
|
||||
|
||||
const dynamicImages = landingImage?.getAttribute('data-a-dynamic-image');
|
||||
if (dynamicImages) {
|
||||
try {
|
||||
const candidates = Object.entries(JSON.parse(dynamicImages) as Record<string, number[]>).sort(
|
||||
([, firstSize], [, secondSize]) =>
|
||||
(secondSize[0] || 0) * (secondSize[1] || 0) -
|
||||
(firstSize[0] || 0) * (firstSize[1] || 0),
|
||||
);
|
||||
if (candidates[0]?.[0]) urls.push(candidates[0][0]);
|
||||
} catch {
|
||||
// Continue to the legacy gallery fallback below.
|
||||
}
|
||||
}
|
||||
urls = unique(urls);
|
||||
}
|
||||
|
||||
// Restore the proven v0.7.2 fallback (commit b062d96): open the image
|
||||
// viewer and read each full-size image. Current selectors are included too.
|
||||
if (urls.length === 0) {
|
||||
const viewerTrigger = document.querySelector<HTMLElement>(
|
||||
'.overlayRestOfImages, #canvasCaption > a',
|
||||
);
|
||||
viewerTrigger?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await delay(1000);
|
||||
|
||||
const thumbnailNodes = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.ivRow div.ivThumbImage, #ivThumbs .ivThumb, #ivThumbs img',
|
||||
),
|
||||
);
|
||||
for (const thumbnailNode of thumbnailNodes) {
|
||||
thumbnailNode.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
await delay(500);
|
||||
|
||||
for (let retryCount = 0; retryCount < 5; retryCount++) {
|
||||
const fullImage = document.querySelector<HTMLImageElement>('#ivLargeImage > img');
|
||||
const source = fullImage?.src;
|
||||
if (
|
||||
source &&
|
||||
!source.includes('/ui/loadIndicators/loading-large_labeled') &&
|
||||
isProductImage(source)
|
||||
) {
|
||||
urls.push(source);
|
||||
break;
|
||||
}
|
||||
await delay(500);
|
||||
}
|
||||
}
|
||||
urls = unique(urls);
|
||||
}
|
||||
|
||||
document
|
||||
.querySelector<HTMLElement>(
|
||||
'header > [data-action="a-popover-close"], body > div.a-modal-scroller.a-declarative',
|
||||
)
|
||||
?.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true }));
|
||||
return urls;
|
||||
}
|
||||
32
src/page-worker/web-injectors/amazon-detail-page-ready.ts
Normal file
32
src/page-worker/web-injectors/amazon-detail-page-ready.ts
Normal file
@ -0,0 +1,32 @@
|
||||
/** Wait until the fields required by the detail parser are available. */
|
||||
export async function waitForAmazonDetailPageReady() {
|
||||
const deadline = Date.now() + 20000;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 详情页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 详情页触发了验证码');
|
||||
}
|
||||
|
||||
const unavailable = document.querySelector<HTMLElement>(
|
||||
'#availability .a-color-price, #outOfStock, #exports_desktop_qualifiedBuybox',
|
||||
);
|
||||
const title = document.querySelector<HTMLElement>('#title, #productTitle');
|
||||
const titleReady = Boolean(title?.textContent?.trim());
|
||||
if (document.readyState !== 'loading' && titleReady) {
|
||||
title?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
return true;
|
||||
}
|
||||
|
||||
if (document.readyState !== 'loading' && unavailable) {
|
||||
throw new Error('Amazon 商品当前不可用,详情标题未加载');
|
||||
}
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
throw new Error('Amazon 详情标题在 20 秒内未加载完成');
|
||||
}
|
||||
73
src/page-worker/web-injectors/amazon-review-page.ts
Normal file
73
src/page-worker/web-injectors/amazon-review-page.ts
Normal file
@ -0,0 +1,73 @@
|
||||
type ReviewReadyOptions = { timeoutMs?: number };
|
||||
|
||||
/** Detects Amazon pages that keep filterByStar in the URL but return other ratings. */
|
||||
export function isAmazonReviewRatingFilterIgnored(
|
||||
reviews: Pick<AmazonReview, 'rating'>[],
|
||||
expectedStar: number,
|
||||
) {
|
||||
return (
|
||||
reviews.length > 0 &&
|
||||
reviews.some((review) => Number.parseInt(review.rating, 10) !== expectedStar)
|
||||
);
|
||||
}
|
||||
|
||||
/** Waits until an Amazon review page, including its requested star filter, is ready. */
|
||||
export async function waitForAmazonReviewPageReady(options: ReviewReadyOptions = {}) {
|
||||
const { timeoutMs = 20000 } = options;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 评论页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 评论页触发了验证码');
|
||||
}
|
||||
|
||||
const reviewList = document.querySelector<HTMLElement>('#cm_cr-review_list');
|
||||
const reviewsContent = document.querySelector<HTMLElement>('.reviews-content');
|
||||
const loadingNode = document.querySelector<HTMLElement>('.reviews-loading');
|
||||
const contentReady = document.readyState !== 'loading' && Boolean(reviewList || reviewsContent);
|
||||
const loadingFinished = !loadingNode || loadingNode.getClientRects().length === 0;
|
||||
if (contentReady && loadingFinished) {
|
||||
(reviewList || reviewsContent)?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
return;
|
||||
}
|
||||
await sleep(300);
|
||||
}
|
||||
throw new Error(`Amazon 评论内容在 ${Math.ceil(timeoutMs / 1000)} 秒内未加载完成`);
|
||||
}
|
||||
|
||||
/** Reads the next review-page URL without navigating from the injected script. */
|
||||
export async function collectAmazonNextReviewPageUrl() {
|
||||
// Amazon may not render the pagination controls until the last review has entered the
|
||||
// viewport. The original collector deliberately scrolled here; keep that behavior while
|
||||
// returning the URL instead of clicking it inside the injected script (which can close the
|
||||
// extension port when Chrome moves the old document into BFCache).
|
||||
const reviewCards = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'#cm_cr-review_list [data-hook="review"], #cm_cr-review_list div[id*="review-card"]',
|
||||
),
|
||||
);
|
||||
reviewCards.at(-1)?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, 1000));
|
||||
|
||||
const pagination =
|
||||
document.querySelector<HTMLElement>('[data-hook="pagination-bar"] .a-pagination') ||
|
||||
document.querySelector<HTMLElement>('[data-hook="pagination-bar"]') ||
|
||||
document.querySelector<HTMLElement>('ul.a-pagination');
|
||||
if (!pagination) return null;
|
||||
|
||||
const nextItem =
|
||||
pagination.querySelector<HTMLElement>('li.a-last') ||
|
||||
pagination.querySelector<HTMLElement>(':scope > *:nth-last-child(1)');
|
||||
if (!nextItem || nextItem.classList.contains('a-disabled')) return null;
|
||||
|
||||
const nextAnchor =
|
||||
nextItem.matches('a[href]')
|
||||
? (nextItem as HTMLAnchorElement)
|
||||
: nextItem.querySelector<HTMLAnchorElement>('a[href]');
|
||||
const href = nextAnchor?.getAttribute('href');
|
||||
return href ? new URL(href, window.location.href).toString() : null;
|
||||
}
|
||||
46
src/page-worker/web-injectors/amazon-search-page-ready.ts
Normal file
46
src/page-worker/web-injectors/amazon-search-page-ready.ts
Normal file
@ -0,0 +1,46 @@
|
||||
/**
|
||||
* Waits for the useful search-page DOM rather than every spinner on the page.
|
||||
* Amazon ad, carousel and recommendation widgets can keep a visible `.a-spinner`
|
||||
* indefinitely even after the search results are ready to collect.
|
||||
*/
|
||||
export async function waitForAmazonSearchPageReady() {
|
||||
const deadline = Date.now() + 20000;
|
||||
const sleep = (milliseconds: number) =>
|
||||
new Promise<void>((resolve) => setTimeout(resolve, milliseconds));
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
if (window.location.pathname.startsWith('/ap/signin')) {
|
||||
throw new Error('Amazon 搜索页需要重新登录');
|
||||
}
|
||||
if (document.querySelector('form[action*="validateCaptcha"], #captchacharacters')) {
|
||||
throw new Error('Amazon 搜索页触发了验证码');
|
||||
}
|
||||
const bodyText = document.body?.innerText || document.body?.textContent || '';
|
||||
if (
|
||||
/sorry[,.\s]+something went wrong on our end/i.test(bodyText) ||
|
||||
/please go back and try again/i.test(bodyText)
|
||||
) {
|
||||
throw new Error('Amazon 返回临时错误页,可能触发了限流或风控,请稍后重试');
|
||||
}
|
||||
|
||||
const resultCount = document.querySelectorAll(
|
||||
'[data-component-type="s-search-result"][data-asin], .puis-card-container',
|
||||
).length;
|
||||
const nextButton = document.querySelector<HTMLElement>('.s-pagination-next');
|
||||
const noResults = /no results for/i.test(bodyText);
|
||||
const pageReady =
|
||||
document.readyState !== 'loading' &&
|
||||
(resultCount > 0 || nextButton !== null || noResults);
|
||||
|
||||
if (pageReady) {
|
||||
nextButton?.scrollIntoView({ behavior: 'auto', block: 'center' });
|
||||
await sleep(750);
|
||||
return;
|
||||
}
|
||||
|
||||
window.scrollBy(0, 500 + ~~(Math.random() * 500));
|
||||
await sleep(300);
|
||||
}
|
||||
|
||||
throw new Error('Amazon 搜索结果在 20 秒内未加载完成');
|
||||
}
|
||||
12
src/page-worker/web-injectors/amazon-search-pagination.ts
Normal file
12
src/page-worker/web-injectors/amazon-search-pagination.ts
Normal file
@ -0,0 +1,12 @@
|
||||
/**
|
||||
* Reads the next search-page URL without navigating from the injected script.
|
||||
* Navigation must happen only after browser.scripting.executeScript has returned,
|
||||
* otherwise Amazon can move the frame into BFCache and destroy the script context.
|
||||
*/
|
||||
export async function collectAmazonNextPageUrl() {
|
||||
const nextButton = document.querySelector<HTMLAnchorElement>('.s-pagination-next');
|
||||
if (!nextButton || nextButton.classList.contains('s-pagination-disabled')) return null;
|
||||
|
||||
const href = nextButton.getAttribute('href');
|
||||
return href ? new URL(href, window.location.href).toString() : null;
|
||||
}
|
||||
340
src/page-worker/web-injectors/amazon-search-parser.ts
Normal file
340
src/page-worker/web-injectors/amazon-search-parser.ts
Normal file
@ -0,0 +1,340 @@
|
||||
export type AmazonSearchPagePattern = 'pattern-1' | 'pattern-2';
|
||||
|
||||
type SearchPageCollectorPayload = {
|
||||
pattern: AmazonSearchPagePattern;
|
||||
};
|
||||
|
||||
/**
|
||||
* Collects product exposures from an Amazon search page.
|
||||
*
|
||||
* Keep this function self-contained: browser.scripting.executeScript serializes
|
||||
* the function and cannot provide imported closures in the page context.
|
||||
*/
|
||||
export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) {
|
||||
type CollectedItem = Pick<
|
||||
AmazonSearchItem,
|
||||
| 'asin'
|
||||
| 'link'
|
||||
| 'title'
|
||||
| 'imageSrc'
|
||||
| 'price'
|
||||
| 'strikethroughPrice'
|
||||
| 'adType'
|
||||
| 'adFormat'
|
||||
| 'adRank'
|
||||
| 'pageRank'
|
||||
| 'placement'
|
||||
| 'placementRank'
|
||||
| 'placementId'
|
||||
| 'adProductIndex'
|
||||
>;
|
||||
|
||||
const text = (node: Element | null | undefined) =>
|
||||
(node?.textContent || '').replace(/\s+/g, ' ').trim();
|
||||
|
||||
const isVisible = (node: HTMLElement) =>
|
||||
!node.hidden &&
|
||||
node.getAttribute('aria-hidden') !== 'true' &&
|
||||
node.style.display !== 'none' &&
|
||||
node.style.visibility !== 'hidden';
|
||||
|
||||
const decodeRepeatedly = (value: string) => {
|
||||
let decoded = value;
|
||||
for (let i = 0; i < 3; i++) {
|
||||
try {
|
||||
const next = decodeURIComponent(decoded);
|
||||
if (next === decoded) break;
|
||||
decoded = next;
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return decoded;
|
||||
};
|
||||
|
||||
const parseAsin = (node: HTMLElement, href?: string) => {
|
||||
const dataAsin = node.closest<HTMLElement>('[data-asin]')?.dataset.asin || node.dataset.asin;
|
||||
if (dataAsin && /^[A-Z0-9]{10}$/.test(dataAsin)) return dataAsin;
|
||||
|
||||
const candidates = [href || '', decodeRepeatedly(href || '')];
|
||||
for (const candidate of candidates) {
|
||||
const match = /\/(?:dp|gp\/product)\/([A-Z0-9]{10})(?:[/?]|$)/i.exec(candidate);
|
||||
if (match) return match[1].toUpperCase();
|
||||
}
|
||||
return undefined;
|
||||
};
|
||||
|
||||
const hasSponsoredDisclosure = (node: Element) => {
|
||||
const candidates = [
|
||||
node,
|
||||
...Array.from(node.querySelectorAll('[aria-label], [title], span, a')),
|
||||
];
|
||||
return candidates.some((candidate) => {
|
||||
const accessibleLabels = [
|
||||
candidate.getAttribute('aria-label'),
|
||||
candidate.getAttribute('title'),
|
||||
];
|
||||
if (
|
||||
accessibleLabels.some((value) => /(?:^|\b)Sponsored(?:\b|\s|$)|赞助/i.test(value || ''))
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return /^(Sponsored|赞助)(?:\b|\s|$)/i.test(text(candidate));
|
||||
});
|
||||
};
|
||||
|
||||
const hasSponsoredLink = (node: Element) =>
|
||||
Array.from(node.querySelectorAll<HTMLAnchorElement>('a[href]')).some((anchor) => {
|
||||
const href = decodeRepeatedly(anchor.href).toLowerCase();
|
||||
return (
|
||||
href.includes('/sspa/click') ||
|
||||
href.includes('/gp/slredirect') ||
|
||||
href.includes('adid=') ||
|
||||
href.includes('qualifier=')
|
||||
);
|
||||
});
|
||||
|
||||
const metadata = (node: Element) =>
|
||||
[
|
||||
node.getAttribute('data-component-type'),
|
||||
node.getAttribute('data-cel-widget'),
|
||||
node.getAttribute('data-ad-details'),
|
||||
node.id,
|
||||
node.className,
|
||||
]
|
||||
.filter((value) => typeof value === 'string')
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
|
||||
const isMainResultsBoundary = (node: Element) =>
|
||||
node.id === 'search' ||
|
||||
node.matches('.s-main-slot, [data-component-type="s-search-results"], body');
|
||||
|
||||
const uniqueAsinsIn = (node: Element) => {
|
||||
const asins = new Set<string>();
|
||||
node.querySelectorAll<HTMLElement>('[data-asin]').forEach((candidate) => {
|
||||
const asin = candidate.dataset.asin;
|
||||
if (asin && /^[A-Z0-9]{10}$/.test(asin)) asins.add(asin);
|
||||
});
|
||||
return asins.size;
|
||||
};
|
||||
|
||||
const hasExplicitAdMetadata = (node: Element) =>
|
||||
/sponsored.?brand|brand.?ad|display.?ad|sponsored.?display|adholder/.test(metadata(node));
|
||||
|
||||
const hasBrandCreative = (node: Element) =>
|
||||
/sponsored.?brand|brand.?ad/.test(metadata(node)) ||
|
||||
Boolean(node.querySelector('video')) ||
|
||||
(uniqueAsinsIn(node) > 1 &&
|
||||
Boolean(
|
||||
node.querySelector(
|
||||
'a[href*="/stores/"], a[href*="/brands/"], [class*="brand-logo"], [class*="brandLogo"]',
|
||||
),
|
||||
));
|
||||
|
||||
const findAdModule = (card: HTMLElement) => {
|
||||
let current: HTMLElement | null = card;
|
||||
for (let depth = 0; current && depth < 6; depth++, current = current.parentElement) {
|
||||
if (isMainResultsBoundary(current)) break;
|
||||
const explicitModule = hasExplicitAdMetadata(current);
|
||||
const brandedSponsoredModule =
|
||||
current !== card && hasSponsoredDisclosure(current) && hasBrandCreative(current);
|
||||
const singleProductSponsored =
|
||||
current !== card && hasSponsoredDisclosure(current) && uniqueAsinsIn(current) === 1;
|
||||
if (explicitModule || brandedSponsoredModule || singleProductSponsored)
|
||||
return current;
|
||||
}
|
||||
return card;
|
||||
};
|
||||
|
||||
const classifyAd = (card: HTMLElement, module: HTMLElement) => {
|
||||
const cardMeta = metadata(card);
|
||||
const moduleMeta = metadata(module);
|
||||
const combinedMeta = `${cardMeta} ${moduleMeta}`;
|
||||
const sidebar = Boolean(
|
||||
card.closest(
|
||||
'aside, #rightResultsATF, .s-right-column, [data-cel-widget*="right"], [data-component-type*="sidebar"]',
|
||||
),
|
||||
);
|
||||
const sponsored =
|
||||
hasSponsoredDisclosure(card) ||
|
||||
hasSponsoredDisclosure(module) ||
|
||||
hasSponsoredLink(card) ||
|
||||
hasSponsoredLink(module) ||
|
||||
hasExplicitAdMetadata(card) ||
|
||||
hasExplicitAdMetadata(module);
|
||||
|
||||
if (!sponsored) {
|
||||
return {
|
||||
adType: 'organic' as const,
|
||||
placement: undefined,
|
||||
adFormat: undefined,
|
||||
};
|
||||
}
|
||||
if (sidebar || /display.?ad|sponsored.?display/.test(combinedMeta)) {
|
||||
return {
|
||||
adType: 'sponsoredDisplay' as const,
|
||||
placement: 'sidebar' as const,
|
||||
adFormat: 'standard' as const,
|
||||
};
|
||||
}
|
||||
if (hasBrandCreative(module)) {
|
||||
return {
|
||||
adType: 'sponsoredBrands' as const,
|
||||
placement: 'middleOfSearch' as const,
|
||||
adFormat: module.querySelector('video') ? ('video' as const) : ('standard' as const),
|
||||
};
|
||||
}
|
||||
if (hasSponsoredDisclosure(card) || hasSponsoredLink(card)) {
|
||||
return {
|
||||
adType: 'sponsoredProducts' as const,
|
||||
placement: 'middleOfSearch' as const,
|
||||
adFormat: 'standard' as const,
|
||||
};
|
||||
}
|
||||
return {
|
||||
adType: 'unknownSponsored' as const,
|
||||
placement: 'unknown' as const,
|
||||
adFormat: 'unknown' as const,
|
||||
};
|
||||
};
|
||||
|
||||
const candidateSet = new Set<HTMLElement>();
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'[data-component-type="s-search-result"][data-asin], .puis-card-container, .puisg-row',
|
||||
)
|
||||
.forEach((node) => {
|
||||
const stableCard =
|
||||
node.closest<HTMLElement>('[data-component-type="s-search-result"]') || node;
|
||||
if (
|
||||
isVisible(stableCard) &&
|
||||
(stableCard.dataset.asin ||
|
||||
stableCard.querySelector('a[href*="/dp/"], a[href*="%2Fdp%2F"]'))
|
||||
) {
|
||||
candidateSet.add(stableCard);
|
||||
}
|
||||
});
|
||||
|
||||
document.querySelectorAll<HTMLElement>('[data-asin]').forEach((node) => {
|
||||
if (
|
||||
isVisible(node) &&
|
||||
/^[A-Z0-9]{10}$/.test(node.dataset.asin || '') &&
|
||||
node.querySelector('h2, h3, img')
|
||||
) {
|
||||
const stableCard =
|
||||
node.closest<HTMLElement>('[data-component-type="s-search-result"]') || node;
|
||||
candidateSet.add(stableCard);
|
||||
}
|
||||
});
|
||||
|
||||
const cards = Array.from(candidateSet)
|
||||
.filter(
|
||||
(card, index, all) =>
|
||||
!all.some(
|
||||
(other, otherIndex) =>
|
||||
index !== otherIndex &&
|
||||
card.contains(other) &&
|
||||
card.dataset.asin === other.dataset.asin,
|
||||
),
|
||||
)
|
||||
.sort((left, right) => {
|
||||
if (left === right) return 0;
|
||||
return left.compareDocumentPosition(right) & Node.DOCUMENT_POSITION_FOLLOWING ? -1 : 1;
|
||||
});
|
||||
|
||||
const organicCandidates = cards.filter((card) => {
|
||||
const module = findAdModule(card);
|
||||
return classifyAd(card, module).adType === 'organic';
|
||||
});
|
||||
const firstOrganicCandidate = organicCandidates[0];
|
||||
const lastOrganicCandidate = organicCandidates[organicCandidates.length - 1];
|
||||
const placementRanks = new Map<HTMLElement, number>();
|
||||
const adRanks = new Map<AmazonAdType, number>();
|
||||
const moduleProductIndexes = new Map<HTMLElement, number>();
|
||||
let nextPlacementRank = 1;
|
||||
|
||||
return cards.reduce<CollectedItem[]>((items, card) => {
|
||||
const anchors = Array.from(card.querySelectorAll<HTMLAnchorElement>('a[href]'));
|
||||
const productAnchor = anchors.find((anchor) => parseAsin(card, anchor.href));
|
||||
const asin = parseAsin(card, productAnchor?.href);
|
||||
if (!asin) return items;
|
||||
|
||||
const titleNode = card.querySelector<HTMLElement>('h2[aria-label], h2, h3[aria-label], h3');
|
||||
const imageNode = card.querySelector<HTMLImageElement>('img.s-image, img[src]');
|
||||
const title =
|
||||
titleNode?.getAttribute('aria-label')?.trim() ||
|
||||
text(titleNode) ||
|
||||
imageNode?.alt?.trim() ||
|
||||
asin;
|
||||
const imageSrc = imageNode?.src;
|
||||
if (!imageSrc) return items;
|
||||
|
||||
const currentPriceNode = card.querySelector<HTMLElement>(
|
||||
'.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen',
|
||||
);
|
||||
const strikethroughPriceNode = card.querySelector<HTMLElement>(
|
||||
'[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString',
|
||||
);
|
||||
const price = text(currentPriceNode) || undefined;
|
||||
const rawStrikethroughPrice = text(strikethroughPriceNode) || undefined;
|
||||
const strikethroughPrice =
|
||||
rawStrikethroughPrice && rawStrikethroughPrice !== price ? rawStrikethroughPrice : undefined;
|
||||
|
||||
const module = findAdModule(card);
|
||||
const classification = classifyAd(card, module);
|
||||
let placementRank: number | undefined;
|
||||
let placementId: string | undefined;
|
||||
let adRank: number | undefined;
|
||||
let adProductIndex: number | undefined;
|
||||
let placement: AmazonAdPlacement | undefined = classification.placement;
|
||||
|
||||
if (classification.adType !== 'organic') {
|
||||
if (!placementRanks.has(module)) placementRanks.set(module, nextPlacementRank++);
|
||||
placementRank = placementRanks.get(module)!;
|
||||
placementId = `search-page-placement-${placementRank}`;
|
||||
adRank = (adRanks.get(classification.adType) || 0) + 1;
|
||||
adRanks.set(classification.adType, adRank);
|
||||
adProductIndex = (moduleProductIndexes.get(module) || 0) + 1;
|
||||
moduleProductIndexes.set(module, adProductIndex);
|
||||
|
||||
if (placement !== 'sidebar') {
|
||||
const beforeFirstOrganic =
|
||||
firstOrganicCandidate &&
|
||||
Boolean(
|
||||
module.compareDocumentPosition(firstOrganicCandidate) &
|
||||
Node.DOCUMENT_POSITION_FOLLOWING,
|
||||
);
|
||||
const afterLastOrganic =
|
||||
lastOrganicCandidate &&
|
||||
Boolean(
|
||||
module.compareDocumentPosition(lastOrganicCandidate) &
|
||||
Node.DOCUMENT_POSITION_PRECEDING,
|
||||
);
|
||||
placement = beforeFirstOrganic
|
||||
? 'topOfSearch'
|
||||
: afterLastOrganic
|
||||
? 'bottomOfSearch'
|
||||
: 'middleOfSearch';
|
||||
}
|
||||
}
|
||||
|
||||
items.push({
|
||||
asin,
|
||||
link: `https://www.amazon.com/dp/${asin}`,
|
||||
title,
|
||||
imageSrc,
|
||||
price,
|
||||
strikethroughPrice,
|
||||
adType: classification.adType,
|
||||
adFormat: classification.adFormat,
|
||||
adRank,
|
||||
pageRank: items.length + 1,
|
||||
placement,
|
||||
placementRank,
|
||||
placementId,
|
||||
adProductIndex,
|
||||
});
|
||||
return items;
|
||||
}, []);
|
||||
}
|
||||
@ -1,182 +1,18 @@
|
||||
import { BaseInjector } from './base';
|
||||
import { collectAmazonSearchPageData, type AmazonSearchPagePattern } from './amazon-search-parser';
|
||||
import { collectAmazonDetailImageUrls } from './amazon-detail-images';
|
||||
import { waitForAmazonDetailPageReady } from './amazon-detail-page-ready';
|
||||
import { collectAmazonNextPageUrl } from './amazon-search-pagination';
|
||||
import { waitForAmazonSearchPageReady } from './amazon-search-page-ready';
|
||||
import {
|
||||
collectAmazonNextReviewPageUrl,
|
||||
waitForAmazonReviewPageReady,
|
||||
} from './amazon-review-page';
|
||||
|
||||
export class AmazonSearchPageInjector extends BaseInjector {
|
||||
public waitForPageLoaded() {
|
||||
return this.run(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
while (true) {
|
||||
const targetNode = document.querySelector('.s-pagination-next');
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500));
|
||||
|
||||
const h = Math.max(
|
||||
document.documentElement.scrollHeight,
|
||||
document.body.scrollHeight,
|
||||
document.documentElement.offsetHeight,
|
||||
document.body.offsetHeight,
|
||||
document.documentElement.clientHeight,
|
||||
);
|
||||
|
||||
// await modernScrollTo({ top: h * (0.4 + Math.random() * 0.2), behavior: 'smooth' });
|
||||
|
||||
if (targetNode || document.readyState === 'complete') {
|
||||
// await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
// 第一段:滚到中下部
|
||||
// window.scrollTo({
|
||||
// top: h * (1 - ~~(Math.random() * 50)),
|
||||
// behavior: 'smooth'
|
||||
// });
|
||||
await modernScrollTo({ top: h * (0.9 + Math.random() * 0.08), behavior: 'smooth' });
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
// targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await scrollIntoViewAndWait(targetNode as HTMLElement, {
|
||||
behavior: 'smooth',
|
||||
block: 'center',
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 250));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
while (true) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// const spins = Array.from(document.querySelectorAll<HTMLElement>('.a-spinner')).filter(
|
||||
// (e) => e.getClientRects().length > 0,
|
||||
// );
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
// const spins = Array.from(document.querySelectorAll('.a-carousel-card-empty'))
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
|
||||
const pagination = document.querySelectorAll('.rhf-sign-in-button');
|
||||
if (pagination.length > 0) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 平滑滚动并等待滚动结束
|
||||
* @param options 原生 ScrollToOptions 接口 (包含 top, left, behavior)
|
||||
*/
|
||||
async function modernScrollTo(options: ScrollToOptions): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
// 1. 针对已经处于目标位置的情况做兜底处理
|
||||
// 如果目标位置与当前位置一致,部分浏览器可能不会触发 scrollend
|
||||
const targetTop = options.top ?? window.scrollY;
|
||||
const targetLeft = options.left ?? window.scrollX;
|
||||
|
||||
if (targetTop === window.scrollY && targetLeft === window.scrollX) {
|
||||
resolve();
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 监听原生滚动结束事件
|
||||
// 注意:scrollend 是 2023-2024 年起普及的标准,2026 年已是主流
|
||||
document.addEventListener(
|
||||
'scrollend',
|
||||
() => {
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
|
||||
// 3. 执行滚动
|
||||
window.scrollTo(options);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 将元素滚动到视野中心并等待动画结束
|
||||
* @param element 目标 DOM 元素
|
||||
* @param options 滚动配置
|
||||
*/
|
||||
async function scrollIntoViewAndWait(
|
||||
element: HTMLElement,
|
||||
options: ScrollIntoViewOptions = { behavior: 'smooth', block: 'center' },
|
||||
): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
// 监听滚动结束事件
|
||||
// scrollend 会在滚动圆满完成或被中断时触发
|
||||
document.addEventListener(
|
||||
'scrollend',
|
||||
() => {
|
||||
resolve();
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
|
||||
// 触发滚动
|
||||
element.scrollIntoView(options);
|
||||
|
||||
// 兼容性兜底:如果元素已经在视野内,浏览器可能不触发滚动
|
||||
// 可以在此处检查位置,若无位移则直接 resolve
|
||||
});
|
||||
}
|
||||
});
|
||||
return this.run(waitForAmazonSearchPageReady);
|
||||
}
|
||||
// public waitForPageLoaded() {
|
||||
// return this.run(async () => {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
|
||||
// // 第一个 while 循环:添加 12 秒超时刷新机制
|
||||
// const timeoutMs = 12000; // 12 秒
|
||||
// let startTime = Date.now();
|
||||
// let refreshCount = 0;
|
||||
// const maxRefresh = 1; // 最多刷新 1 次
|
||||
// console.log('waitForPageLoaded');
|
||||
|
||||
// while (true) {
|
||||
// const targetNode = document.querySelector('.s-pagination-next');
|
||||
// window.scrollBy(0, ~~(Math.random() * 500) + 500);
|
||||
// await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 50) + 500));
|
||||
|
||||
// if (targetNode || document.readyState === 'complete') {
|
||||
// targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
// break;
|
||||
// }
|
||||
|
||||
// // 检查是否超时
|
||||
// if (Date.now() - startTime > timeoutMs) {
|
||||
// if (refreshCount < maxRefresh) {
|
||||
// // 刷新页面
|
||||
// console.log('检查是否超时,刷新页面');
|
||||
|
||||
// browser.tabs.query({ active: true, currentWindow: true }).then(tabs => {
|
||||
// browser.tabs.reload(tabs[0].id);
|
||||
// });
|
||||
|
||||
// await new Promise((resolve) => setTimeout(resolve, 2000)); // 等待刷新开始
|
||||
|
||||
// // 重置计时器和刷新计数
|
||||
// refreshCount++;
|
||||
// startTime = Date.now();
|
||||
// continue; // 重新从 while 循环开始
|
||||
// } else {
|
||||
// // 已刷新过一次,仍然超时,抛出错误
|
||||
// throw new Error('等待页面加载超时(已尝试刷新)');
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// // 第二个 while 循环:等待 spinner 消失
|
||||
// while (true) {
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// const spins = Array.from(document.querySelectorAll('.a-spinner')).filter(
|
||||
// (e) => e.getClientRects().length > 0
|
||||
// );
|
||||
// if (spins.length === 0) {
|
||||
// break;
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// }
|
||||
|
||||
public async getPagePattern() {
|
||||
return this.run(async () => {
|
||||
@ -190,78 +26,15 @@ export class AmazonSearchPageInjector extends BaseInjector {
|
||||
});
|
||||
}
|
||||
|
||||
public async getPageData(pattern: 'pattern-1' | 'pattern-2') {
|
||||
let data: Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[] | null = null;
|
||||
switch (pattern) {
|
||||
// 处理商品以列表形式展示的情况
|
||||
case 'pattern-1':
|
||||
data = await this.run(async () => {
|
||||
const items = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.puisg-row:has(.a-section.a-spacing-small.a-spacing-top-small:not(.a-text-right))',
|
||||
),
|
||||
).filter((e) => e.getClientRects().length > 0);
|
||||
const linkObjs = items.reduce<
|
||||
Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[]
|
||||
>((objs, el) => {
|
||||
const link = el.querySelector<HTMLAnchorElement>('a')?.href;
|
||||
const title = el
|
||||
.querySelector<HTMLHeadingElement>('h2.a-color-base')!
|
||||
.getAttribute('aria-label')!;
|
||||
const imageSrc = el.querySelector<HTMLImageElement>('img.s-image')!.src!;
|
||||
const price =
|
||||
el.querySelector<HTMLElement>('.a-price:not(.a-text-price) .a-offscreen')
|
||||
?.innerText ||
|
||||
(
|
||||
document.evaluate(
|
||||
`.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`,
|
||||
el,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLSpanElement | null
|
||||
)?.innerText;
|
||||
link && objs.push({ link, title, imageSrc, price });
|
||||
return objs;
|
||||
}, []);
|
||||
return linkObjs;
|
||||
});
|
||||
break;
|
||||
// 处理商品以二维图片格展示的情况
|
||||
case 'pattern-2':
|
||||
data = await this.run(async () => {
|
||||
const items = Array.from(
|
||||
document.querySelectorAll<HTMLElement>(
|
||||
'.puis-card-container',
|
||||
) as unknown as HTMLElement[],
|
||||
).filter((e) => e.getClientRects().length > 0);
|
||||
const linkObjs = items.reduce<
|
||||
Pick<AmazonSearchItem, 'link' | 'title' | 'imageSrc' | 'price'>[]
|
||||
>((objs, el) => {
|
||||
const link = el.querySelector<HTMLAnchorElement>('a.a-link-normal')?.href;
|
||||
const title = el.querySelector<HTMLHeadingElement>('h2.a-color-base')!.innerText;
|
||||
const imageSrc = el.querySelector<HTMLImageElement>('img.s-image')!.src!;
|
||||
const price =
|
||||
el.querySelector<HTMLElement>('.a-price:not(.a-text-price) .a-offscreen')
|
||||
?.innerText ||
|
||||
(
|
||||
document.evaluate(
|
||||
`.//div[@data-cy="secondary-offer-recipe"]//span[@class='a-color-base' and contains(., '$') and not(*)]`,
|
||||
el,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLSpanElement | null
|
||||
)?.innerText;
|
||||
link && objs.push({ link, title, imageSrc, price });
|
||||
return objs;
|
||||
}, []);
|
||||
return linkObjs;
|
||||
});
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
data = data && data.filter((r) => new URL(r.link).pathname.includes('/dp/')); // No advertisement only
|
||||
return data;
|
||||
public async getPageData(pattern: AmazonSearchPagePattern) {
|
||||
return this.run(collectAmazonSearchPageData, { pattern });
|
||||
}
|
||||
|
||||
public async isExplicitNoResultsPage() {
|
||||
return this.run(async () => {
|
||||
const bodyText = document.body?.innerText || document.body?.textContent || '';
|
||||
return /no results for/i.test(bodyText);
|
||||
});
|
||||
}
|
||||
|
||||
public async getCurrentPage() {
|
||||
@ -271,111 +44,9 @@ export class AmazonSearchPageInjector extends BaseInjector {
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 检测当前亚马逊搜索页面是否有下一页,并自动点击翻页按钮。
|
||||
*
|
||||
* 该方法在页面上下文中执行,查找亚马逊标准分页按钮('.s-pagination-next'),
|
||||
* 检查按钮是否未被禁用('s-pagination-disabled' 类),然后模拟用户点击。
|
||||
* 点击前会随机等待 500-1000 毫秒以避免被识别为机器人。
|
||||
*
|
||||
* @returns `true` 表示有下一页且已点击翻页按钮,页面正在加载下一页内容;
|
||||
* `false` 表示没有下一页(按钮不存在或被禁用)。
|
||||
*/
|
||||
public async determineHasNextPage() {
|
||||
return this.run(async () => {
|
||||
const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
if (nextButton) {
|
||||
if (!nextButton.classList.contains('s-pagination-disabled')) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
nextButton.click();
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
public async getNextPageUrl() {
|
||||
return this.run(collectAmazonNextPageUrl);
|
||||
}
|
||||
// /**
|
||||
// * 检测并执行亚马逊搜索页面翻页,等待页面刷新完成。
|
||||
// * @returns 能否翻页(true=翻页成功,false=没有下一页或翻页失败)
|
||||
// */
|
||||
// public async determineHasNextPage(): Promise<boolean> {
|
||||
// return this.run(async () => {
|
||||
// const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
|
||||
// if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) {
|
||||
// // 记录当前页码
|
||||
// const initialPage = await this.getCurrentPage();
|
||||
|
||||
// // 随机等待后点击
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// nextButton.click();
|
||||
|
||||
// // 等待页面刷新完成
|
||||
// await this.waitForPageLoaded();
|
||||
|
||||
// // 验证翻页是否成功
|
||||
// const newPage = await this.getCurrentPage();
|
||||
|
||||
// // 只有页码真正变化才算翻页成功
|
||||
// return newPage !== null && newPage !== initialPage;
|
||||
// }
|
||||
|
||||
// return false;
|
||||
// });
|
||||
// }
|
||||
|
||||
/**
|
||||
* 检测并执行亚马逊搜索页面翻页,通过 URL 变化确认刷新完成。
|
||||
* @returns 能否翻页(true=已翻页,false=已是最后一页)
|
||||
*/
|
||||
// public async determineHasNextPage(): Promise<boolean> {
|
||||
// return this.run(async () => {
|
||||
// const nextButton = document.querySelector<HTMLLinkElement>('.s-pagination-next');
|
||||
|
||||
// if (nextButton && !nextButton.classList.contains('s-pagination-disabled')) {
|
||||
// // 1. 记录当前 URL
|
||||
// const initialUrl = window.location.href;
|
||||
|
||||
// // 2. 随机等待后点击
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
// nextButton.click();
|
||||
|
||||
// // 3. 等待 URL 变化(表示页面已开始导航)
|
||||
// await new Promise<void>((resolve) => {
|
||||
// const checkUrl = () => {
|
||||
// if (window.location.href !== initialUrl) {
|
||||
// resolve();
|
||||
// } else {
|
||||
// setTimeout(checkUrl, 100);
|
||||
// }
|
||||
// };
|
||||
// checkUrl();
|
||||
// });
|
||||
|
||||
// // 4. 等待页面稳定(document.readyState === 'complete')
|
||||
// await new Promise<void>((resolve) => {
|
||||
// const checkReadyState = () => {
|
||||
// if (document.readyState === 'complete') {
|
||||
// resolve();
|
||||
// } else {
|
||||
// setTimeout(checkReadyState, 100);
|
||||
// }
|
||||
// };
|
||||
// checkReadyState();
|
||||
// });
|
||||
|
||||
// // 5. 额外等待确保内容加载
|
||||
// await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
|
||||
// return true;
|
||||
// }
|
||||
|
||||
// return false;
|
||||
// });
|
||||
// }
|
||||
}
|
||||
|
||||
export class AmazonDetailPageInjector extends BaseInjector {
|
||||
@ -389,35 +60,27 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
/**等待页面加载完成 */
|
||||
public async waitForPageLoaded() {
|
||||
return this.run(async () => {
|
||||
while (true) {
|
||||
window.scrollBy(0, ~~(Math.random() * 500) + 500);
|
||||
await new Promise((resolve) => setTimeout(resolve, ~~(Math.random() * 100) + 200));
|
||||
const targetNode = document.querySelector(
|
||||
'#prodDetails:has(td), #detailBulletsWrapper_feature_div:has(li), .av-page-desktop, #productDescription_feature_div',
|
||||
);
|
||||
const exceptionalNodeSelectors = ['.music-detail-header', '.avu-retail-page'];
|
||||
for (const selector of exceptionalNodeSelectors) {
|
||||
if (document.querySelector(selector)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (targetNode && document.readyState !== 'loading') {
|
||||
targetNode.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
break;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return this.run(waitForAmazonDetailPageReady);
|
||||
}
|
||||
|
||||
/**获取基本信息 */
|
||||
public async getBaseInfo() {
|
||||
return this.run(async () => {
|
||||
const title = document.querySelector<HTMLElement>('#title')!.innerText;
|
||||
const price = document.querySelector<HTMLElement>(
|
||||
'.aok-offscreen, .a-price:not(.a-text-price) .a-offscreen',
|
||||
const priceRoot =
|
||||
document.querySelector<HTMLElement>(
|
||||
'#corePrice_feature_div, #corePriceDisplay_desktop_feature_div, #apex_desktop',
|
||||
) || document.body;
|
||||
const price = priceRoot.querySelector<HTMLElement>(
|
||||
'.a-price:not(.a-text-price) .a-offscreen, [data-a-color="price"] .a-offscreen',
|
||||
)?.innerText;
|
||||
const rawStrikethroughPrice = priceRoot.querySelector<HTMLElement>(
|
||||
'[data-a-strike="true"] .a-offscreen, .basisPrice .a-offscreen, .priceBlockStrikePriceString',
|
||||
)?.innerText;
|
||||
const strikethroughPrice =
|
||||
rawStrikethroughPrice && rawStrikethroughPrice !== price
|
||||
? rawStrikethroughPrice
|
||||
: undefined;
|
||||
const boughtInfo = document.querySelector<HTMLElement>(
|
||||
`#social-proofing-faceout-title-tk_bought`,
|
||||
)?.innerText;
|
||||
@ -436,7 +99,16 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
'#fulfillerInfoFeature_feature_div > *:last-of-type',
|
||||
)?.innerText;
|
||||
const soldBy = document.querySelector<HTMLElement>(`#sellerProfileTriggerId`)?.innerText;
|
||||
return { title, price, boughtInfo, availableDate, categories, shipFrom, soldBy };
|
||||
return {
|
||||
title,
|
||||
price,
|
||||
strikethroughPrice,
|
||||
boughtInfo,
|
||||
availableDate,
|
||||
categories,
|
||||
shipFrom,
|
||||
soldBy,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
@ -488,78 +160,7 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
/**获取图像链接 */
|
||||
public async getImageUrls() {
|
||||
return this.run(async () => {
|
||||
const overlay = document.querySelector<HTMLElement>('.overlayRestOfImages');
|
||||
|
||||
if (overlay) {
|
||||
if (document.querySelector<HTMLElement>('#ivThumbs')!.getClientRects().length === 0) {
|
||||
// Click to see full view
|
||||
document.querySelector<HTMLElement>('#canvasCaption > a')?.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
} else {
|
||||
// finally click to see full view if not clicked before
|
||||
document.querySelector<HTMLElement>('#canvasCaption > a')?.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
let script = document.evaluate(
|
||||
`//script[starts-with(text(), "\nP.when(\'A\').register") or contains(text(), "\nP.when('A').register")]`,
|
||||
document,
|
||||
null,
|
||||
XPathResult.STRING_TYPE,
|
||||
).stringValue;
|
||||
const extractUrls = (pattern: RegExp) =>
|
||||
Array.from(script.matchAll(pattern)).map((e) => e[0]);
|
||||
let urls = extractUrls(
|
||||
/(?<="hiRes":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g,
|
||||
);
|
||||
if (urls.length === 0) {
|
||||
urls = extractUrls(
|
||||
/(?<="large":")https:\/\/m.media-amazon.com\/images\/I\/[\w\d\.\-+]+(?=")/g,
|
||||
);
|
||||
}
|
||||
|
||||
if (urls.length === 0) {
|
||||
const divs: NodeListOf<HTMLDivElement> =
|
||||
document.querySelectorAll('.ivRow div.ivThumbImage');
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
for (const div of divs) {
|
||||
// click div com
|
||||
div.click();
|
||||
|
||||
// wait for image to load
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
|
||||
// if img src is not loading gif, then get url and break, otherwise continue to wait, with a max retry count to avoid infinite loop
|
||||
let retryCount = 0;
|
||||
const maxRetries = 5;
|
||||
|
||||
while (true) {
|
||||
const loadingGifUrl =
|
||||
'https://m.media-amazon.com/images/G/01/ui/loadIndicators/loading-large_labeled._CB485921664_.gif';
|
||||
// get img dom
|
||||
const imgDom = document.querySelector('#ivLargeImage > img');
|
||||
|
||||
// get url from img dom src attribute
|
||||
if (imgDom && 'src' in imgDom && imgDom.src !== loadingGifUrl) {
|
||||
urls.push(imgDom.src as string);
|
||||
break;
|
||||
} else {
|
||||
// wait for 500ms and try again
|
||||
await new Promise((resolve) => setTimeout(resolve, 500 + ~~(500 * Math.random())));
|
||||
retryCount++;
|
||||
if (retryCount >= maxRetries) {
|
||||
console.error('在detail页面下载图片,已达最大重试次数:');
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
document.querySelector<HTMLElement>('body > div.a-modal-scroller.a-declarative')?.click();
|
||||
return urls;
|
||||
});
|
||||
return this.run(collectAmazonDetailImageUrls);
|
||||
}
|
||||
|
||||
/**获取精选评论 */
|
||||
@ -697,30 +298,7 @@ export class AmazonDetailPageInjector extends BaseInjector {
|
||||
|
||||
export class AmazonReviewPageInjector extends BaseInjector {
|
||||
public async waitForPageLoad() {
|
||||
return this.run(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
while (true) {
|
||||
const targetNode = document.querySelector(
|
||||
'.reviews-content, #cm_cr-review_list ul[role="list"]:not(.histogram)',
|
||||
);
|
||||
targetNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
if (
|
||||
targetNode &&
|
||||
targetNode.getClientRects().length > 0 &&
|
||||
document.readyState !== 'loading'
|
||||
) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
while (true) {
|
||||
const loadingNode = document.querySelector('.reviews-loading');
|
||||
if (loadingNode && loadingNode.getClientRects().length === 0) {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
}
|
||||
});
|
||||
return this.run(waitForAmazonReviewPageReady);
|
||||
}
|
||||
|
||||
public async getSinglePageReviews() {
|
||||
@ -773,54 +351,7 @@ export class AmazonReviewPageInjector extends BaseInjector {
|
||||
});
|
||||
}
|
||||
|
||||
public jumpToNextPageIfExist() {
|
||||
return this.run(async () => {
|
||||
const latestReview = document.evaluate(
|
||||
`//*[@id='cm_cr-review_list']//li[@data-hook='review'][last()]`,
|
||||
document.body,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLElement | null;
|
||||
latestReview?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
const nextPageNode = document.querySelector<HTMLElement>(
|
||||
'[data-hook="pagination-bar"] .a-pagination > *:nth-of-type(2)',
|
||||
);
|
||||
nextPageNode?.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
await new Promise((resolve) => setTimeout(resolve, 1000));
|
||||
const ret = nextPageNode && !nextPageNode.classList.contains('a-disabled');
|
||||
ret && nextPageNode?.querySelector('a')?.click();
|
||||
return ret;
|
||||
});
|
||||
}
|
||||
|
||||
public async showStarsDropDownMenu() {
|
||||
return this.run(async () => {
|
||||
while (true) {
|
||||
const dropdown = document.querySelector<HTMLElement>('#star-count-dropdown')!;
|
||||
dropdown.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||
dropdown.click();
|
||||
if (dropdown.getAttribute('aria-expanded') === 'true') {
|
||||
break;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public async selectStar(star: number) {
|
||||
return this.run(
|
||||
async ({ star }) => {
|
||||
const starNode = document.evaluate(
|
||||
`//ul[@role='listbox']/li/a[text()="${star} star only"]`,
|
||||
document.body,
|
||||
null,
|
||||
XPathResult.FIRST_ORDERED_NODE_TYPE,
|
||||
).singleNodeValue as HTMLElement;
|
||||
starNode.click();
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
},
|
||||
{ star },
|
||||
);
|
||||
public getNextPageUrl() {
|
||||
return this.run(collectAmazonNextReviewPageUrl);
|
||||
}
|
||||
}
|
||||
|
||||
@ -20,6 +20,14 @@ worker.on('error', ({ message: msg }) => {
|
||||
});
|
||||
message.error(msg);
|
||||
});
|
||||
worker.on('item-detail-redirected', (ev) => {
|
||||
timelines.value.push({
|
||||
type: 'warning',
|
||||
title: '商品发生重定向',
|
||||
time: new Date().toLocaleString(),
|
||||
content: `请求 ASIN:${ev.requestedAsin}\n实际 ASIN:${ev.resolvedAsin}\n数据将按实际 ASIN 保存`,
|
||||
});
|
||||
});
|
||||
worker.on('item-base-info-collected', (ev) => {
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
@ -40,11 +48,15 @@ worker.on('item-category-rank-collected', (ev) => {
|
||||
});
|
||||
});
|
||||
worker.on('item-images-collected', (ev) => {
|
||||
const imageCount = ev.imageUrls?.length || 0;
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
type: imageCount > 0 ? 'success' : 'warning',
|
||||
title: `商品${ev.asin}图像`,
|
||||
time: new Date().toLocaleString(),
|
||||
content: `图片数: ${ev.imageUrls!.length}`,
|
||||
content:
|
||||
imageCount > 0
|
||||
? `图片数: ${imageCount}`
|
||||
: '未找到商品图片;Amazon 页面未提供可识别的图片数据',
|
||||
});
|
||||
});
|
||||
// worker.on('item-top-reviews-collected', (ev) => {
|
||||
@ -98,11 +110,12 @@ const launch = async () => {
|
||||
},
|
||||
...detailWorkerSettings.value,
|
||||
});
|
||||
const failed = Boolean(worker.lastError?.value);
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
type: failed ? 'error' : 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
content: failed ? '数据采集失败,已保留此前成功采集的数据' : '数据采集完成',
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
@ -4,7 +4,10 @@ import { usePageWorker } from '~/page-worker';
|
||||
import { reviewAsinInput, reviewWorkerSettings } from '~/storages/amazon';
|
||||
|
||||
const worker = usePageWorker('amazon', { objects: ['review'] });
|
||||
const collectedReviewIds = new Map<string, Set<string>>();
|
||||
const hadFatalError = ref(false);
|
||||
worker.on('error', ({ message: msg }) => {
|
||||
hadFatalError.value = true;
|
||||
timelines.value.push({
|
||||
type: 'error',
|
||||
title: '错误发生',
|
||||
@ -13,14 +16,16 @@ worker.on('error', ({ message: msg }) => {
|
||||
});
|
||||
});
|
||||
worker.on('item-review-collected', (ev) => {
|
||||
const ids = collectedReviewIds.get(ev.asin) || new Set<string>();
|
||||
ev.reviews.forEach((review) => ids.add(review.id));
|
||||
collectedReviewIds.set(ev.asin, ids);
|
||||
timelines.value.push({
|
||||
type: 'success',
|
||||
title: `商品${ev.asin}评价`,
|
||||
time: new Date().toLocaleString(),
|
||||
content: `获取到 ${ev.reviews.length} 条评价`,
|
||||
content: `已累计获取 ${ids.size} 条评价`,
|
||||
});
|
||||
});
|
||||
|
||||
const asinInputRef = useTemplateRef('asin-input');
|
||||
|
||||
const message = useMessage();
|
||||
@ -29,6 +34,8 @@ const timelines = ref<Timeline[]>([]);
|
||||
|
||||
const launch = async () => {
|
||||
const asinList = reviewAsinInput.value.split(/\n|\s|,|;/).filter((item) => item.length > 0);
|
||||
collectedReviewIds.clear();
|
||||
hadFatalError.value = false;
|
||||
timelines.value = [
|
||||
{
|
||||
type: 'info',
|
||||
@ -43,12 +50,21 @@ const launch = async () => {
|
||||
},
|
||||
recent: reviewWorkerSettings.value.recent,
|
||||
});
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
});
|
||||
if (hadFatalError.value || worker.lastError?.value) {
|
||||
timelines.value.push({
|
||||
type: 'error',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集失败,已保留此前成功采集的数据',
|
||||
});
|
||||
} else {
|
||||
timelines.value.push({
|
||||
type: 'info',
|
||||
title: '结束',
|
||||
time: new Date().toLocaleString(),
|
||||
content: '数据采集完成',
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleStart = () => {
|
||||
|
||||
8
src/storages/amazon-sync.ts
Normal file
8
src/storages/amazon-sync.ts
Normal file
@ -0,0 +1,8 @@
|
||||
import { useWebExtensionStorage } from '~/composables/useWebExtensionStorage';
|
||||
import { AMAZON_SYNC_STATE_KEY, EMPTY_AMAZON_SYNC_STATE } from '~/logic/amazon-sync-shared';
|
||||
|
||||
export const amazonSyncState = useWebExtensionStorage<AmazonSyncState>(
|
||||
AMAZON_SYNC_STATE_KEY,
|
||||
EMPTY_AMAZON_SYNC_STATE,
|
||||
{ flush: 'sync' },
|
||||
);
|
||||
@ -7,8 +7,20 @@ export const detailAsinInput = useWebExtensionStorage<string>('detailAsinInputTe
|
||||
export const reviewAsinInput = useWebExtensionStorage<string>('reviewAsinInputText', '');
|
||||
|
||||
export const itemColumnSettings = useWebExtensionStorage<
|
||||
Set<keyof Pick<AmazonItem, 'keywords' | 'page' | 'rank' | 'createTime' | 'timestamp'>>
|
||||
>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'createTime']));
|
||||
Set<
|
||||
keyof Pick<
|
||||
AmazonItem,
|
||||
| 'keywords'
|
||||
| 'page'
|
||||
| 'rank'
|
||||
| 'pageRank'
|
||||
| 'adRank'
|
||||
| 'placementRank'
|
||||
| 'createTime'
|
||||
| 'timestamp'
|
||||
>
|
||||
>
|
||||
>('itemColumnSettings', new Set(['keywords', 'page', 'rank', 'adRank', 'createTime']));
|
||||
|
||||
export const detailWorkerSettings = useWebExtensionStorage('amazon-detail-worker-settings', {
|
||||
image: true,
|
||||
@ -55,8 +67,17 @@ export const allItems = computed<AmazonItem[]>({
|
||||
'title',
|
||||
'imageSrc',
|
||||
'price',
|
||||
'strikethroughPrice',
|
||||
'link',
|
||||
'rank',
|
||||
'pageRank',
|
||||
'adType',
|
||||
'adRank',
|
||||
'placement',
|
||||
'placementRank',
|
||||
'placementId',
|
||||
'adProductIndex',
|
||||
'adFormat',
|
||||
'createTime',
|
||||
];
|
||||
searchItems.value = newValue
|
||||
|
||||
25
src/types/amazon-sync.d.ts
vendored
Normal file
25
src/types/amazon-sync.d.ts
vendored
Normal file
@ -0,0 +1,25 @@
|
||||
import type { ProtocolWithReturn } from 'webext-bridge';
|
||||
|
||||
declare global {
|
||||
type AmazonSyncPayload = {
|
||||
searchItems?: AmazonSearchItem[];
|
||||
detailItems?: AmazonDetailItem[];
|
||||
reviews?: ({ asin: string } & AmazonReview)[];
|
||||
};
|
||||
|
||||
type AmazonSyncStatus = 'idle' | 'pending' | 'syncing' | 'synced' | 'failed';
|
||||
|
||||
type AmazonSyncState = {
|
||||
status: AmazonSyncStatus;
|
||||
pendingCount: number;
|
||||
lastSuccessAt?: string;
|
||||
lastError?: string;
|
||||
};
|
||||
}
|
||||
|
||||
declare module 'webext-bridge' {
|
||||
export interface ProtocolMap {
|
||||
'amazon-sync-enqueue': ProtocolWithReturn<AmazonSyncPayload, AmazonSyncState>;
|
||||
'amazon-sync-flush': ProtocolWithReturn<undefined, AmazonSyncState>;
|
||||
}
|
||||
}
|
||||
43
src/types/amazon.d.ts
vendored
43
src/types/amazon.d.ts
vendored
@ -1,4 +1,25 @@
|
||||
declare type AmazonAdType =
|
||||
| 'organic'
|
||||
| 'sponsoredProducts'
|
||||
| 'sponsoredBrands'
|
||||
| 'sponsoredDisplay'
|
||||
| 'unknownSponsored';
|
||||
|
||||
declare type AmazonAdPlacement =
|
||||
| 'topOfSearch'
|
||||
| 'middleOfSearch'
|
||||
| 'bottomOfSearch'
|
||||
/** 兼容旧版采集数据 */
|
||||
| 'restOfSearch'
|
||||
| 'sidebar'
|
||||
| 'productPage'
|
||||
| 'unknown';
|
||||
|
||||
declare type AmazonAdFormat = 'standard' | 'video' | 'unknown';
|
||||
|
||||
declare type AmazonSearchItem = {
|
||||
/** 客户端生成的幂等同步标识 */
|
||||
syncId?: string;
|
||||
/** 搜索关键词 */
|
||||
keywords: string;
|
||||
/** 商品排名 */
|
||||
@ -13,6 +34,24 @@ declare type AmazonSearchItem = {
|
||||
asin: string;
|
||||
/** 商品价格(可选) */
|
||||
price?: string;
|
||||
/** 页面展示的划线参考价格 */
|
||||
strikethroughPrice?: string;
|
||||
/** 商品在当前页商品流中的排名 */
|
||||
pageRank?: number;
|
||||
/** 广告类型;历史数据可能为空 */
|
||||
adType?: AmazonAdType;
|
||||
/** 同一关键词、同一广告类型中的累计广告排名 */
|
||||
adRank?: number;
|
||||
/** 广告在页面中的位置 */
|
||||
placement?: AmazonAdPlacement;
|
||||
/** 同一关键词采集过程中广告模块的累计顺序 */
|
||||
placementRank?: number;
|
||||
/** 同一广告模块内商品的关联标识 */
|
||||
placementId?: string;
|
||||
/** 商品在广告模块内的顺序 */
|
||||
adProductIndex?: number;
|
||||
/** 广告素材形式 */
|
||||
adFormat?: AmazonAdFormat;
|
||||
/** 商品图片链接 */
|
||||
imageSrc: string;
|
||||
/** 创建时间 */
|
||||
@ -20,6 +59,8 @@ declare type AmazonSearchItem = {
|
||||
};
|
||||
|
||||
declare type AmazonDetailItem = {
|
||||
/** 客户端生成的幂等同步标识 */
|
||||
syncId?: string;
|
||||
/** 商品的 ASIN(亚马逊标准识别号) */
|
||||
asin: string;
|
||||
/** 商品标题 */
|
||||
@ -30,6 +71,8 @@ declare type AmazonDetailItem = {
|
||||
boughtInfo?: string;
|
||||
/** 商品价格 */
|
||||
price?: string;
|
||||
/** 页面展示的划线参考价格 */
|
||||
strikethroughPrice?: string;
|
||||
/** 商品评分 */
|
||||
rating?: number;
|
||||
/** 评分数量 */
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user