fix: collect Shop by type thematic SP ads via redirect chains without data-asin

Co-authored-by: Cursor <cursoragent@cursor.com>

scripts/test-amazon-search-parser.ts

@@ -931,6 +931,202 @@ async function main() {
     false,
   );
 
+  const shopByTypeDom = new JSDOM(
+    `
+      <main id="search">
+        <div class="s-main-slot">
+          <article data-component-type="s-search-result" data-asin="B0ORGCAM01">
+            <h2>Organic camera</h2>
+            <a href="/dp/B0ORGCAM01"><img class="s-image" src="https://example.com/org-cam.jpg" /></a>
+          </article>
+
+          <div class="sbv-container AdHolder">
+            <span>Sponsored</span>
+            <video></video>
+            <a href="/stores/stealthcam"><span class="brand-logo">STEALTH CAM</span></a>
+            <div class="sb-headline">STEALTH CAM</div>
+            <div data-asin="B0STEALTH1" class="sbv-product">
+              <h2>Stealth Cam trail camera</h2>
+              <a href="/sspa/click?url=%2Fdp%2FB0STEALTH1">
+                <img src="https://example.com/stealth.jpg" />
+              </a>
+            </div>
+          </div>
+
+          <div
+            class="s-widget-container celwidget slot=MAIN template=SHOP_BY_TYPE widget=shop-by-type-desktop"
+            cel_widget_id="MAIN-SHOP_BY_TYPE-8"
+          >
+            <h2>Shop cameras by type</h2>
+            <span>Sponsored</span>
+            <div class="shop-by-type-column">
+              <h3>security system</h3>
+              <div>
+                <h2>Night Owl 2-Way Audio</h2>
+                <a href="https://redirect.amazon.com/x/c/abc123?url=%2Fdp%2FB0NIGHTOWL">
+                  <img
+                    src="https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/grey-pixel.gif"
+                    data-src="https://example.com/nightowl-type.jpg"
+                  />
+                </a>
+                <span class="a-price"><span class="a-offscreen">$549.99</span></span>
+              </div>
+            </div>
+            <div class="shop-by-type-column">
+              <h3>webcam</h3>
+              <div>
+                <h2>Prisual PTZ Camera</h2>
+                <a href="https://redirect.amazon.com/x/c/def456?url=%2Fdp%2FB0PRISUAL1">
+                  <img
+                    src="https://images-na.ssl-images-amazon.com/images/G/01/x-locale/common/grey-pixel.gif"
+                    data-src="https://example.com/prisual-type.jpg"
+                  />
+                </a>
+                <span class="a-price"><span class="a-offscreen">$494.09</span></span>
+              </div>
+            </div>
+            <div class="shop-by-type-column">
+              <h3>instant print</h3>
+              <div>
+                <h2>Kids Instant Print Camera</h2>
+                <a href="https://redirect.amazon.com/x/c/ghi789?url=%2Fdp%2FB0KIDINST1">
+                  <img data-src="https://example.com/instant-type.jpg" />
+                </a>
+              </div>
+            </div>
+            <div class="shop-by-type-column">
+              <h3>point and shoot</h3>
+              <div>
+                <h2>Digital Camera 4K</h2>
+                <a href="https://redirect.amazon.com/x/c/jkl012?url=%2Fdp%2FB0POINTSH1">
+                  <img data-src="https://example.com/pointshoot-type.jpg" />
+                </a>
+              </div>
+            </div>
+            <div class="shop-by-type-column">
+              <h3>film camera</h3>
+              <div>
+                <h2>Kodak EKTAR H35N</h2>
+                <a href="https://direct.amazon.com/x/c/QmToken/https://www.amazon.com/dp/B0KODAKH35">
+                  <img data-src="https://example.com/kodak-type.jpg" />
+                </a>
+              </div>
+            </div>
+          </div>
+        </div>
+      </main>
+    `,
+    { url: 'https://www.amazon.com/s?k=camera' },
+  );
+  Object.assign(globalThis, {
+    document: shopByTypeDom.window.document,
+    Node: shopByTypeDom.window.Node,
+    Element: shopByTypeDom.window.Element,
+    HTMLElement: shopByTypeDom.window.HTMLElement,
+    HTMLAnchorElement: shopByTypeDom.window.HTMLAnchorElement,
+    HTMLImageElement: shopByTypeDom.window.HTMLImageElement,
+  });
+  const shopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' });
+  const shopByTypeByAsin = new Map(shopByType.map((item) => [item.asin, item]));
+  const shopByTypeAsins = ['B0NIGHTOWL', 'B0PRISUAL1', 'B0KIDINST1', 'B0POINTSH1', 'B0KODAKH35'];
+  const shopByTypeItems = shopByTypeAsins.map((asin) => shopByTypeByAsin.get(asin));
+  assert.ok(shopByTypeItems.filter(Boolean).length > 0, 'Shop by type cards must be collected');
+  assert.equal(shopByTypeItems.filter(Boolean).length, 5);
+  assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.asin, 'B0NIGHTOWL');
+  assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.title, 'Night Owl 2-Way Audio');
+  assert.equal(
+    shopByTypeByAsin.get('B0NIGHTOWL')?.imageSrc,
+    'https://example.com/nightowl-type.jpg',
+  );
+  assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.price, '$549.99');
+  assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.asin, 'B0PRISUAL1');
+  assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.title, 'Prisual PTZ Camera');
+  assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.asin, 'B0KIDINST1');
+  assert.equal(shopByTypeByAsin.get('B0KIDINST1')?.title, 'Kids Instant Print Camera');
+  assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.asin, 'B0POINTSH1');
+  assert.equal(shopByTypeByAsin.get('B0POINTSH1')?.title, 'Digital Camera 4K');
+  assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.asin, 'B0KODAKH35');
+  assert.equal(shopByTypeByAsin.get('B0KODAKH35')?.title, 'Kodak EKTAR H35N');
+  assert.ok(shopByTypeItems.every((item) => item?.asin && item.title && item.imageSrc));
+  assert.equal(shopByTypeByAsin.get('B0ORGCAM01')?.adType, 'organic');
+  assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adType, 'sponsoredBrands');
+  assert.equal(shopByTypeByAsin.get('B0STEALTH1')?.adFormat, 'video');
+  assert.ok(shopByTypeItems.every((item) => item?.adType === 'sponsoredProducts'));
+  assert.equal(
+    shopByTypeByAsin.get('B0NIGHTOWL')?.placementId,
+    shopByTypeByAsin.get('B0PRISUAL1')?.placementId,
+  );
+  assert.equal(shopByTypeByAsin.get('B0NIGHTOWL')?.adProductIndex, 1);
+  assert.equal(shopByTypeByAsin.get('B0PRISUAL1')?.adProductIndex, 2);
+  assert.equal(
+    shopByType.some((item) =>
+      /^(security system|webcam|instant print|point and shoot|film camera)$/i.test(item.title),
+    ),
+    false,
+  );
+
+  const organicShopByTypeDom = new JSDOM(
+    `
+      <main id="search">
+        <div class="s-main-slot">
+          <article data-component-type="s-search-result" data-asin="B0ORGTYPE0">
+            <h2>Organic camera</h2>
+            <a href="/dp/B0ORGTYPE0"><img class="s-image" src="https://example.com/org-type0.jpg" /></a>
+          </article>
+
+          <div
+            class="s-widget-container celwidget slot=MAIN template=SHOP_BY_TYPE widget=shop-by-type-desktop"
+            cel_widget_id="MAIN-SHOP_BY_TYPE-9"
+          >
+            <h2>Shop cameras by type</h2>
+            <div class="shop-by-type-column">
+              <h3>webcam</h3>
+              <div data-asin="B0ORGTYPE1">
+                <h2>Organic type webcam</h2>
+                <a href="/dp/B0ORGTYPE1">
+                  <img class="s-image" src="https://example.com/org-type1.jpg" />
+                </a>
+              </div>
+            </div>
+            <div class="shop-by-type-column">
+              <h3>film camera</h3>
+              <div>
+                <h2>Organic type film camera</h2>
+                <a href="https://redirect.amazon.com/x/c/org?url=%2Fdp%2FB0ORGTYPE2">
+                  <img data-src="https://example.com/org-type2.jpg" />
+                </a>
+              </div>
+            </div>
+          </div>
+        </div>
+      </main>
+    `,
+    { url: 'https://www.amazon.com/s?k=camera' },
+  );
+  Object.assign(globalThis, {
+    document: organicShopByTypeDom.window.document,
+    Node: organicShopByTypeDom.window.Node,
+    Element: organicShopByTypeDom.window.Element,
+    HTMLElement: organicShopByTypeDom.window.HTMLElement,
+    HTMLAnchorElement: organicShopByTypeDom.window.HTMLAnchorElement,
+    HTMLImageElement: organicShopByTypeDom.window.HTMLImageElement,
+  });
+  const organicShopByType = await collectAmazonSearchPageData({ pattern: 'pattern-2' });
+  const organicShopByTypeByAsin = new Map(organicShopByType.map((item) => [item.asin, item]));
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.asin, 'B0ORGTYPE1');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.title, 'Organic type webcam');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.asin, 'B0ORGTYPE2');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.title, 'Organic type film camera');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE0')?.adType, 'organic');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE1')?.adType, 'organic');
+  assert.equal(organicShopByTypeByAsin.get('B0ORGTYPE2')?.adType, 'organic');
+  assert.equal(
+    organicShopByType.some(
+      (item) => item.adType === 'sponsoredProducts' || item.adType === 'sponsoredBrands',
+    ),
+    false,
+  );
+
   console.log('Amazon search parser fixtures passed.');
 }
 

src/page-worker/web-injectors/amazon-search-page-ready.ts

@@ -68,13 +68,15 @@ export async function waitForAmazonSearchPageReady() {
       .map((node) => node.dataset.asin || '')
       .filter((asin) => /^[A-Z0-9]{10}$/.test(asin));
     const productLinks = Array.from(
-      root.querySelectorAll<HTMLAnchorElement>('a[href*="/dp/"], a[href*="%2Fdp%2F"]'),
+      root.querySelectorAll<HTMLAnchorElement>(
+        'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
+      ),
     ).map((anchor) => anchor.href);
     const sponsoredLabels = root.querySelectorAll(
       '[aria-label*="Sponsored" i], [title*="Sponsored" i]',
     ).length;
     const cards = root.querySelectorAll(
-      '[data-component-type="s-search-result"], .puis-card-container, .puisg-row',
+      '[data-component-type="s-search-result"], .puis-card-container, .puisg-row, [class*="shop-by-type"]',
     ).length;
     return [
       Array.from(new Set(asins)).sort().join(','),

src/page-worker/web-injectors/amazon-search-parser.ts

@@ -231,6 +231,14 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
       },
     );
 
+  const hasShopByTypeCopy = (node: Element) =>
+    Array.from(node.querySelectorAll<HTMLElement>('h1, h2, h3, span, a, [class*="headline"]')).some(
+      (candidate) => {
+        if (closestAsinHost(candidate) || isVisuallyHiddenText(candidate)) return false;
+        return /\bshop\b.+\bby type\b/i.test(text(candidate));
+      },
+    );
+
   const hasThematicSpSignal = (node: Element) => {
     const blob = [
       metadata(node),
@@ -244,13 +252,13 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
       return false;
     }
     if (
-      /featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer/.test(
+      /featured.?asins.?list|sponsoredproductsloom|sp_search_thematic|loom-desktop|influencer|shop.?by.?type|shop_by_type/.test(
         blob,
       )
     ) {
       return true;
     }
-    return hasInfluencerThematicCopy(node);
+    return hasInfluencerThematicCopy(node) || hasShopByTypeCopy(node);
   };
 
   const isSponsoredProductsThematic = (node: Element) => {
@@ -364,7 +372,7 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
   const hasCollectableProductLink = (node: Element) =>
     Boolean(
       node.querySelector(
-        'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="]',
+        'a[href*="/dp/"], a[href*="%2Fdp%2F"], a[href*="/gp/product"], a[href*="%2Fgp%2Fproduct%2F"], a[href*="sspa/click"], a[href*="lp_asins="], a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
       ),
     );
 
@@ -379,6 +387,20 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
     }
   };
 
+  const findProductCardFromAnchor = (anchor: HTMLAnchorElement, module: HTMLElement) => {
+    const asinHost = closestAsinHost(anchor);
+    if (asinHost && module.contains(asinHost) && asinHost !== module) return asinHost;
+
+    let current: HTMLElement | null = anchor.parentElement;
+    while (current && current !== module) {
+      if (current.querySelector('img') && current.querySelector('h2, h3')) return current;
+      current = current.parentElement;
+    }
+    return anchor.parentElement instanceof HTMLElement && anchor.parentElement !== module
+      ? anchor.parentElement
+      : null;
+  };
+
   const candidateSet = new Set<HTMLElement>();
   document
     .querySelectorAll<HTMLElement>(
@@ -434,6 +456,50 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
       addCandidate(module);
     });
 
+  const collectThematicModuleCards = (module: HTMLElement) => {
+    module.querySelectorAll<HTMLElement>('[data-asin]').forEach(addCandidate);
+    module.querySelectorAll<HTMLAnchorElement>('a[href]').forEach((anchor) => {
+      const href = anchor.getAttribute('href') || anchor.href;
+      if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
+      const card = findProductCardFromAnchor(anchor, module);
+      if (card) addCandidate(card);
+    });
+  };
+
+  document
+    .querySelectorAll<HTMLElement>(
+      '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"], [cel_widget_id*="SHOP_BY_TYPE" i], [cel_widget_id*="shop-by-type" i]',
+    )
+    .forEach((module) => {
+      if (!isVisible(module) || !isSponsoredProductsThematic(module)) return;
+      collectThematicModuleCards(module);
+    });
+
+  document.querySelectorAll<HTMLElement>('h1, h2, h3, [class*="headline"]').forEach((heading) => {
+    if (closestAsinHost(heading) || !/\bshop\b.+\bby type\b/i.test(text(heading))) return;
+    const module =
+      heading.closest<HTMLElement>(
+        '.s-widget-container, [class*="celwidget"], [class*="s-widget"], section',
+      ) || heading.parentElement;
+    if (module instanceof HTMLElement && isVisible(module)) collectThematicModuleCards(module);
+  });
+
+  document
+    .querySelectorAll<HTMLAnchorElement>(
+      'a[href*="redirect.amazon.com"], a[href*="direct.amazon.com"], a[href*="/x/c/"]',
+    )
+    .forEach((anchor) => {
+      if (!isVisible(anchor)) return;
+      const href = anchor.getAttribute('href') || anchor.href;
+      if (!parseAsinFromText(anchor.href) && !parseAsinFromText(href)) return;
+      const module =
+        anchor.closest<HTMLElement>(
+          '.s-widget-container, [class*="celwidget"], [class*="s-widget"], [class*="shop-by-type"]',
+        ) || document.body;
+      const card = findProductCardFromAnchor(anchor, module);
+      if (card) addCandidate(card);
+    });
+
   const cards = Array.from(candidateSet)
     .filter(
       (card, index, all) =>
@@ -468,7 +534,9 @@ export async function collectAmazonSearchPageData(_: SearchPageCollectorPayload)
     const asin = parseAsin(card, productAnchor?.href);
     if (!asin) return items;
 
-    const titleNode = card.querySelector<HTMLElement>('h2[aria-label], h2, h3[aria-label], h3');
+    const titleNode =
+      card.querySelector<HTMLElement>('h2[aria-label], h2') ||
+      card.querySelector<HTMLElement>('h3[aria-label], h3');
     const imageNode = card.querySelector<HTMLImageElement>(
       'img.s-image, img[src], img[data-src], img[srcset]',
     );