diff --git a/scripts/test-amazon-search-parser.ts b/scripts/test-amazon-search-parser.ts index d170297..33f40b5 100644 --- a/scripts/test-amazon-search-parser.ts +++ b/scripts/test-amazon-search-parser.ts @@ -931,6 +931,202 @@ async function main() { false, ); + const shopByTypeDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+ Sponsored + + +
STEALTH CAM
+
+

Stealth Cam trail camera

+ + + +
+
+ +
+

Shop cameras by type

+ Sponsored +
+

security system

+
+

Night Owl 2-Way Audio

+ + + + $549.99 +
+
+
+

webcam

+
+

Prisual PTZ Camera

+ + + + $494.09 +
+
+
+

instant print

+
+

Kids Instant Print Camera

+ + + +
+
+
+

point and shoot

+
+

Digital Camera 4K

+ + + +
+
+
+

film camera

+
+

Kodak EKTAR H35N

+ + + +
+
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=camera' }, + ); + Object.assign(globalThis, { + document: shopByTypeDom.window.document, + Node: shopByTypeDom.window.Node, + Element: shopByTypeDom.window.Element, + HTMLElement: shopByTypeDom.window.HTMLElement, + HTMLAnchorElement: shopByTypeDom.window.HTMLAnchorElement, + HTMLImageElement: shopByTypeDom.window.HTMLImageElement, + }); + const shopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const shopByTypeByAsin = new Map(shopByType.map((item) => [item.asin, item])); + const shopByTypeAsins = ['B0NIGHTOWL', 'B0PRISUAL1', 'B0KIDINST1', 'B0POINTSH1', 'B0KODAKH35']; + const shopByTypeItems = shopByTypeAsins.map((asin) => shopByTypeByAsin.get(asin)); + assert.ok(shopByTypeItems.filter(Boolean).length > 0, 'Shop by type cards must be collected'); + assert.equal(shopByTypeItems.filter(Boolean).length, 5); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.asin, 'B0NIGHTOWL'); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.title, 'Night Owl 2-Way Audio'); + assert.equal( + shopByTypeByAsin.get('B0NIGHTOWL')?.imageSrc, + 'https://example.com/nightowl-type.jpg', + ); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.price, '$549.99'); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.asin, 'B0PRISUAL1'); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.title, 'Prisual PTZ Camera'); + assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.asin, 'B0KIDINST1'); + assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.title, 'Kids Instant Print Camera'); + assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.asin, 'B0POINTSH1'); + assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.title, 'Digital Camera 4K'); + assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.asin, 'B0KODAKH35'); + assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.title, 'Kodak EKTAR H35N'); + assert.ok(shopByTypeItems.every((item) => item?.asin && item.title && item.imageSrc)); + assert.equal(shopByTypeByAsin.get('B0ORGCAM01')?.adType, 'organic'); + assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adType, 'sponsoredBrands'); + assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adFormat, 'video'); + assert.ok(shopByTypeItems.every((item) => item?.adType === 'sponsoredProducts')); + assert.equal( + shopByTypeByAsin.get('B0NIGHTOWL')?.placementId, + shopByTypeByAsin.get('B0PRISUAL1')?.placementId, + ); + assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.adProductIndex, 1); + assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.adProductIndex, 2); + assert.equal( + shopByType.some((item) => + /^(security system|webcam|instant print|point and shoot|film camera)$/i.test(item.title), + ), + false, + ); + + const organicShopByTypeDom = new JSDOM( + ` +
+
+
+

Organic camera

+ +
+ +
+

Shop cameras by type

+
+

webcam

+
+

Organic type webcam

+ + + +
+
+
+

film camera

+
+

Organic type film camera

+ + + +
+
+
+
+
+ `, + { url: 'https://www.amazon.com/s?k=camera' }, + ); + Object.assign(globalThis, { + document: organicShopByTypeDom.window.document, + Node: organicShopByTypeDom.window.Node, + Element: organicShopByTypeDom.window.Element, + HTMLElement: organicShopByTypeDom.window.HTMLElement, + HTMLAnchorElement: organicShopByTypeDom.window.HTMLAnchorElement, + HTMLImageElement: organicShopByTypeDom.window.HTMLImageElement, + }); + const organicShopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' }); + const organicShopByTypeByAsin = new Map(organicShopByType.map((item) => [item.asin, item])); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.asin, 'B0ORGTYPE1'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.title, 'Organic type webcam'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.asin, 'B0ORGTYPE2'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.title, 'Organic type film camera'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE0')?.adType, 'organic'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.adType, 'organic'); + assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.adType, 'organic'); + assert.equal( + organicShopByType.some( + (item) => item.adType === 'sponsoredProducts' || item.adType === 'sponsoredBrands', + ), + false, + ); + console.log('Amazon search parser fixtures passed.'); } diff --git a/src/page-worker/web-injectors/amazon-search-page-ready.ts b/src/page-worker/web-injectors/amazon-search-page-ready.ts index 58b6a5b..e322c88 100644 --- a/src/page-worker/web-injectors/amazon-search-page-ready.ts +++ b/src/page-worker/web-injectors/amazon-search-page-ready.ts @@ -68,13 +68,15 @@ export async function waitForAmazonSearchPageReady() { .map((node) => node.dataset.asin || '') .filter((asin) => /^[A-Z0-9]{10}$/.test(asin)); const productLinks = Array.from( - root.querySelectorAll('a[href*="/dp/"], a[href*="%2Fdp%2F"]'), + root.querySelectorAll( + 'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', + ), ).map((anchor) => anchor.href); const sponsoredLabels = root.querySelectorAll( '[aria-label*="Sponsored" i], [title*="Sponsored" i]', ).length; const cards = root.querySelectorAll( - '[data-component-type="s-search-result"], .puis-card-container, .puisg-row', + '[data-component-type="s-search-result"], .puis-card-container, .puisg-row, [class*="shop-by-type"]', ).length; return [ Array.from(new Set(asins)).sort().join(','), diff --git a/src/page-worker/web-injectors/amazon-search-parser.ts b/src/page-worker/web-injectors/amazon-search-parser.ts index 4068b81..a708a0d 100644 --- a/src/page-worker/web-injectors/amazon-search-parser.ts +++ b/src/page-worker/web-injectors/amazon-search-parser.ts @@ -231,6 +231,14 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) }, ); + const hasShopByTypeCopy = (node: Element) => + Array.from(node.querySelectorAll('h1, h2, h3, span, a, [class*="headline"]')).some( + (candidate) => { + if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false; + return /\bshop\b.+\bby type\b/i.test(text(candidate)); + }, + ); + const hasThematicSpSignal = (node: Element) => { const blob = [ metadata(node), @@ -244,13 +252,13 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) return false; } if ( - /featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer/.test( + /featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer|shop.?by.?type|shop_by_type/.test( blob, ) ) { return true; } - return hasInfluencerThematicCopy(node); + return hasInfluencerThematicCopy(node) || hasShopByTypeCopy(node); }; const isSponsoredProductsThematic = (node: Element) => { @@ -364,7 +372,7 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) const hasCollectableProductLink = (node: Element) => Boolean( node.querySelector( - 'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="]', + 'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', ), ); @@ -379,6 +387,20 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) } }; + const findProductCardFromAnchor = (anchor: HTMLAnchorElement, module: HTMLElement) => { + const asinHost = closestAsinHost(anchor); + if (asinHost && module.contains(asinHost) && asinHost !== module) return asinHost; + + let current: HTMLElement | null = anchor.parentElement; + while (current && current !== module) { + if (current.querySelector('img') && current.querySelector('h2, h3')) return current; + current = current.parentElement; + } + return anchor.parentElement instanceof HTMLElement && anchor.parentElement !== module + ? anchor.parentElement + : null; + }; + const candidateSet = new Set(); document .querySelectorAll( @@ -434,6 +456,50 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) addCandidate(module); }); + const collectThematicModuleCards = (module: HTMLElement) => { + module.querySelectorAll('[data-asin]').forEach(addCandidate); + module.querySelectorAll('a[href]').forEach((anchor) => { + const href = anchor.getAttribute('href') || anchor.href; + if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return; + const card = findProductCardFromAnchor(anchor, module); + if (card) addCandidate(card); + }); + }; + + document + .querySelectorAll( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"], [cel_widget_id*="SHOP_BY_TYPE" i], [cel_widget_id*="shop-by-type" i]', + ) + .forEach((module) => { + if (!isVisible(module) || !isSponsoredProductsThematic(module)) return; + collectThematicModuleCards(module); + }); + + document.querySelectorAll('h1, h2, h3, [class*="headline"]').forEach((heading) => { + if (closestAsinHost(heading) || !/\bshop\b.+\bby type\b/i.test(text(heading))) return; + const module = + heading.closest( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], section', + ) || heading.parentElement; + if (module instanceof HTMLElement && isVisible(module)) collectThematicModuleCards(module); + }); + + document + .querySelectorAll( + 'a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]', + ) + .forEach((anchor) => { + if (!isVisible(anchor)) return; + const href = anchor.getAttribute('href') || anchor.href; + if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return; + const module = + anchor.closest( + '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"]', + ) || document.body; + const card = findProductCardFromAnchor(anchor, module); + if (card) addCandidate(card); + }); + const cards = Array.from(candidateSet) .filter( (card, index, all) => @@ -468,7 +534,9 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload) const asin = parseAsin(card, productAnchor?.href); if (!asin) return items; - const titleNode = card.querySelector('h2[aria-label], h2, h3[aria-label], h3'); + const titleNode = + card.querySelector('h2[aria-label], h2') || + card.querySelector('h3[aria-label], h3'); const imageNode = card.querySelector( 'img.s-image, img[src], img[data-src], img[srcset]', );