mirror of
https://github.com/primedigitaltech/azon_seeker.git
synced 2026-10-05 15:22:59 +08:00
fix: collect Shop by type thematic SP ads via redirect chains without data-asin
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
syrup
parent
7d446d8f54
commit
4c26b108a7
@ -931,6 +931,202 @@ async function main() {
|
||||
false,
|
||||
);
|
||||
|
||||
const shopByTypeDom = new JSDOM(
|
||||
`
|
||||
<main id="search">
|
||||
<div class="s-main-slot">
|
||||
<article data-component-type="s-search-result" data-asin="B0ORGCAM01">
|
||||
<h2>Organic camera</h2>
|
||||
<a href="/dp/B0ORGCAM01"><img class="s-image" src="https://example.com/org-cam.jpg" /></a>
|
||||
</article>
|
||||
|
||||
<div class="sbv-container AdHolder">
|
||||
<span>Sponsored</span>
|
||||
<video></video>
|
||||
<a href="/stores/stealthcam"><span class="brand-logo">STEALTH CAM</span></a>
|
||||
<div class="sb-headline">STEALTH CAM</div>
|
||||
<div data-asin="B0STEALTH1" class="sbv-product">
|
||||
<h2>Stealth Cam trail camera</h2>
|
||||
<a href="/sspa/click?url=%2Fdp%2FB0STEALTH1">
|
||||
<img src="https://example.com/stealth.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div
|
||||
class="s-widget-container celwidget slot=MAIN template=SHOP_BY_TYPE widget=shop-by-type-desktop"
|
||||
cel_widget_id="MAIN-SHOP_BY_TYPE-8"
|
||||
>
|
||||
<h2>Shop cameras by type</h2>
|
||||
<span>Sponsored</span>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>security system</h3>
|
||||
<div>
|
||||
<h2>Night Owl 2-Way Audio</h2>
|
||||
<a href="https://redirect.amazon.com/x/c/abc123?url=%2Fdp%2FB0NIGHTOWL">
|
||||
<img
|
||||
src="https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/grey-pixel.gif"
|
||||
data-src="https://example.com/nightowl-type.jpg"
|
||||
/>
|
||||
</a>
|
||||
<span class="a-price"><span class="a-offscreen">$549.99</span></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>webcam</h3>
|
||||
<div>
|
||||
<h2>Prisual PTZ Camera</h2>
|
||||
<a href="https://redirect.amazon.com/x/c/def456?url=%2Fdp%2FB0PRISUAL1">
|
||||
<img
|
||||
src="https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/grey-pixel.gif"
|
||||
data-src="https://example.com/prisual-type.jpg"
|
||||
/>
|
||||
</a>
|
||||
<span class="a-price"><span class="a-offscreen">$494.09</span></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>instant print</h3>
|
||||
<div>
|
||||
<h2>Kids Instant Print Camera</h2>
|
||||
<a href="https://redirect.amazon.com/x/c/ghi789?url=%2Fdp%2FB0KIDINST1">
|
||||
<img data-src="https://example.com/instant-type.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>point and shoot</h3>
|
||||
<div>
|
||||
<h2>Digital Camera 4K</h2>
|
||||
<a href="https://redirect.amazon.com/x/c/jkl012?url=%2Fdp%2FB0POINTSH1">
|
||||
<img data-src="https://example.com/pointshoot-type.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>film camera</h3>
|
||||
<div>
|
||||
<h2>Kodak EKTAR H35N</h2>
|
||||
<a href="https://direct.amazon.com/x/c/QmToken/https://www.amazon.com/dp/B0KODAKH35">
|
||||
<img data-src="https://example.com/kodak-type.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
`,
|
||||
{ url: 'https://www.amazon.com/s?k=camera' },
|
||||
);
|
||||
Object.assign(globalThis, {
|
||||
document: shopByTypeDom.window.document,
|
||||
Node: shopByTypeDom.window.Node,
|
||||
Element: shopByTypeDom.window.Element,
|
||||
HTMLElement: shopByTypeDom.window.HTMLElement,
|
||||
HTMLAnchorElement: shopByTypeDom.window.HTMLAnchorElement,
|
||||
HTMLImageElement: shopByTypeDom.window.HTMLImageElement,
|
||||
});
|
||||
const shopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' });
|
||||
const shopByTypeByAsin = new Map(shopByType.map((item) => [item.asin, item]));
|
||||
const shopByTypeAsins = ['B0NIGHTOWL', 'B0PRISUAL1', 'B0KIDINST1', 'B0POINTSH1', 'B0KODAKH35'];
|
||||
const shopByTypeItems = shopByTypeAsins.map((asin) => shopByTypeByAsin.get(asin));
|
||||
assert.ok(shopByTypeItems.filter(Boolean).length > 0, 'Shop by type cards must be collected');
|
||||
assert.equal(shopByTypeItems.filter(Boolean).length, 5);
|
||||
assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.asin, 'B0NIGHTOWL');
|
||||
assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.title, 'Night Owl 2-Way Audio');
|
||||
assert.equal(
|
||||
shopByTypeByAsin.get('B0NIGHTOWL')?.imageSrc,
|
||||
'https://example.com/nightowl-type.jpg',
|
||||
);
|
||||
assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.price, '$549.99');
|
||||
assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.asin, 'B0PRISUAL1');
|
||||
assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.title, 'Prisual PTZ Camera');
|
||||
assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.asin, 'B0KIDINST1');
|
||||
assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.title, 'Kids Instant Print Camera');
|
||||
assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.asin, 'B0POINTSH1');
|
||||
assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.title, 'Digital Camera 4K');
|
||||
assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.asin, 'B0KODAKH35');
|
||||
assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.title, 'Kodak EKTAR H35N');
|
||||
assert.ok(shopByTypeItems.every((item) => item?.asin && item.title && item.imageSrc));
|
||||
assert.equal(shopByTypeByAsin.get('B0ORGCAM01')?.adType, 'organic');
|
||||
assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adType, 'sponsoredBrands');
|
||||
assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adFormat, 'video');
|
||||
assert.ok(shopByTypeItems.every((item) => item?.adType === 'sponsoredProducts'));
|
||||
assert.equal(
|
||||
shopByTypeByAsin.get('B0NIGHTOWL')?.placementId,
|
||||
shopByTypeByAsin.get('B0PRISUAL1')?.placementId,
|
||||
);
|
||||
assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.adProductIndex, 1);
|
||||
assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.adProductIndex, 2);
|
||||
assert.equal(
|
||||
shopByType.some((item) =>
|
||||
/^(security system|webcam|instant print|point and shoot|film camera)$/i.test(item.title),
|
||||
),
|
||||
false,
|
||||
);
|
||||
|
||||
const organicShopByTypeDom = new JSDOM(
|
||||
`
|
||||
<main id="search">
|
||||
<div class="s-main-slot">
|
||||
<article data-component-type="s-search-result" data-asin="B0ORGTYPE0">
|
||||
<h2>Organic camera</h2>
|
||||
<a href="/dp/B0ORGTYPE0"><img class="s-image" src="https://example.com/org-type0.jpg" /></a>
|
||||
</article>
|
||||
|
||||
<div
|
||||
class="s-widget-container celwidget slot=MAIN template=SHOP_BY_TYPE widget=shop-by-type-desktop"
|
||||
cel_widget_id="MAIN-SHOP_BY_TYPE-9"
|
||||
>
|
||||
<h2>Shop cameras by type</h2>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>webcam</h3>
|
||||
<div data-asin="B0ORGTYPE1">
|
||||
<h2>Organic type webcam</h2>
|
||||
<a href="/dp/B0ORGTYPE1">
|
||||
<img class="s-image" src="https://example.com/org-type1.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
<div class="shop-by-type-column">
|
||||
<h3>film camera</h3>
|
||||
<div>
|
||||
<h2>Organic type film camera</h2>
|
||||
<a href="https://redirect.amazon.com/x/c/org?url=%2Fdp%2FB0ORGTYPE2">
|
||||
<img data-src="https://example.com/org-type2.jpg" />
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
`,
|
||||
{ url: 'https://www.amazon.com/s?k=camera' },
|
||||
);
|
||||
Object.assign(globalThis, {
|
||||
document: organicShopByTypeDom.window.document,
|
||||
Node: organicShopByTypeDom.window.Node,
|
||||
Element: organicShopByTypeDom.window.Element,
|
||||
HTMLElement: organicShopByTypeDom.window.HTMLElement,
|
||||
HTMLAnchorElement: organicShopByTypeDom.window.HTMLAnchorElement,
|
||||
HTMLImageElement: organicShopByTypeDom.window.HTMLImageElement,
|
||||
});
|
||||
const organicShopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' });
|
||||
const organicShopByTypeByAsin = new Map(organicShopByType.map((item) => [item.asin, item]));
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.asin, 'B0ORGTYPE1');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.title, 'Organic type webcam');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.asin, 'B0ORGTYPE2');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.title, 'Organic type film camera');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE0')?.adType, 'organic');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.adType, 'organic');
|
||||
assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.adType, 'organic');
|
||||
assert.equal(
|
||||
organicShopByType.some(
|
||||
(item) => item.adType === 'sponsoredProducts' || item.adType === 'sponsoredBrands',
|
||||
),
|
||||
false,
|
||||
);
|
||||
|
||||
console.log('Amazon search parser fixtures passed.');
|
||||
}
|
||||
|
||||
|
||||
@ -68,13 +68,15 @@ export async function waitForAmazonSearchPageReady() {
|
||||
.map((node) => node.dataset.asin || '')
|
||||
.filter((asin) => /^[A-Z0-9]{10}$/.test(asin));
|
||||
const productLinks = Array.from(
|
||||
root.querySelectorAll<HTMLAnchorElement>('a[href*="/dp/"], a[href*="%2Fdp%2F"]'),
|
||||
root.querySelectorAll<HTMLAnchorElement>(
|
||||
'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
),
|
||||
).map((anchor) => anchor.href);
|
||||
const sponsoredLabels = root.querySelectorAll(
|
||||
'[aria-label*="Sponsored" i], [title*="Sponsored" i]',
|
||||
).length;
|
||||
const cards = root.querySelectorAll(
|
||||
'[data-component-type="s-search-result"], .puis-card-container, .puisg-row',
|
||||
'[data-component-type="s-search-result"], .puis-card-container, .puisg-row, [class*="shop-by-type"]',
|
||||
).length;
|
||||
return [
|
||||
Array.from(new Set(asins)).sort().join(','),
|
||||
|
||||
@ -231,6 +231,14 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
},
|
||||
);
|
||||
|
||||
const hasShopByTypeCopy = (node: Element) =>
|
||||
Array.from(node.querySelectorAll<HTMLElement>('h1, h2, h3, span, a, [class*="headline"]')).some(
|
||||
(candidate) => {
|
||||
if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false;
|
||||
return /\bshop\b.+\bby type\b/i.test(text(candidate));
|
||||
},
|
||||
);
|
||||
|
||||
const hasThematicSpSignal = (node: Element) => {
|
||||
const blob = [
|
||||
metadata(node),
|
||||
@ -244,13 +252,13 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
/featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer/.test(
|
||||
/featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer|shop.?by.?type|shop_by_type/.test(
|
||||
blob,
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return hasInfluencerThematicCopy(node);
|
||||
return hasInfluencerThematicCopy(node) || hasShopByTypeCopy(node);
|
||||
};
|
||||
|
||||
const isSponsoredProductsThematic = (node: Element) => {
|
||||
@ -364,7 +372,7 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
const hasCollectableProductLink = (node: Element) =>
|
||||
Boolean(
|
||||
node.querySelector(
|
||||
'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="]',
|
||||
'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
),
|
||||
);
|
||||
|
||||
@ -379,6 +387,20 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
}
|
||||
};
|
||||
|
||||
const findProductCardFromAnchor = (anchor: HTMLAnchorElement, module: HTMLElement) => {
|
||||
const asinHost = closestAsinHost(anchor);
|
||||
if (asinHost && module.contains(asinHost) && asinHost !== module) return asinHost;
|
||||
|
||||
let current: HTMLElement | null = anchor.parentElement;
|
||||
while (current && current !== module) {
|
||||
if (current.querySelector('img') && current.querySelector('h2, h3')) return current;
|
||||
current = current.parentElement;
|
||||
}
|
||||
return anchor.parentElement instanceof HTMLElement && anchor.parentElement !== module
|
||||
? anchor.parentElement
|
||||
: null;
|
||||
};
|
||||
|
||||
const candidateSet = new Set<HTMLElement>();
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
@ -434,6 +456,50 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
addCandidate(module);
|
||||
});
|
||||
|
||||
const collectThematicModuleCards = (module: HTMLElement) => {
|
||||
module.querySelectorAll<HTMLElement>('[data-asin]').forEach(addCandidate);
|
||||
module.querySelectorAll<HTMLAnchorElement>('a[href]').forEach((anchor) => {
|
||||
const href = anchor.getAttribute('href') || anchor.href;
|
||||
if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
|
||||
const card = findProductCardFromAnchor(anchor, module);
|
||||
if (card) addCandidate(card);
|
||||
});
|
||||
};
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"], [cel_widget_id*="SHOP_BY_TYPE" i], [cel_widget_id*="shop-by-type" i]',
|
||||
)
|
||||
.forEach((module) => {
|
||||
if (!isVisible(module) || !isSponsoredProductsThematic(module)) return;
|
||||
collectThematicModuleCards(module);
|
||||
});
|
||||
|
||||
document.querySelectorAll<HTMLElement>('h1, h2, h3, [class*="headline"]').forEach((heading) => {
|
||||
if (closestAsinHost(heading) || !/\bshop\b.+\bby type\b/i.test(text(heading))) return;
|
||||
const module =
|
||||
heading.closest<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], section',
|
||||
) || heading.parentElement;
|
||||
if (module instanceof HTMLElement && isVisible(module)) collectThematicModuleCards(module);
|
||||
});
|
||||
|
||||
document
|
||||
.querySelectorAll<HTMLAnchorElement>(
|
||||
'a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
|
||||
)
|
||||
.forEach((anchor) => {
|
||||
if (!isVisible(anchor)) return;
|
||||
const href = anchor.getAttribute('href') || anchor.href;
|
||||
if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
|
||||
const module =
|
||||
anchor.closest<HTMLElement>(
|
||||
'.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"]',
|
||||
) || document.body;
|
||||
const card = findProductCardFromAnchor(anchor, module);
|
||||
if (card) addCandidate(card);
|
||||
});
|
||||
|
||||
const cards = Array.from(candidateSet)
|
||||
.filter(
|
||||
(card, index, all) =>
|
||||
@ -468,7 +534,9 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
|
||||
const asin = parseAsin(card, productAnchor?.href);
|
||||
if (!asin) return items;
|
||||
|
||||
const titleNode = card.querySelector<HTMLElement>('h2[aria-label], h2, h3[aria-label], h3');
|
||||
const titleNode =
|
||||
card.querySelector<HTMLElement>('h2[aria-label], h2') ||
|
||||
card.querySelector<HTMLElement>('h3[aria-label], h3');
|
||||
const imageNode = card.querySelector<HTMLImageElement>(
|
||||
'img.s-image, img[src], img[data-src], img[srcset]',
|
||||
);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user