From 1fd7827e36edf53eddbe698378d8cf25a8f65a88 Mon Sep 17 00:00:00 2001 From: AuYeung Date: Thu, 18 Jul 2024 20:44:40 +0800 Subject: [PATCH] When the query has no content, terminate the loop early --- media_platform/xhs/core.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index a9103aa..5a9293f 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -110,6 +110,9 @@ class XiaoHongShuCrawler(AbstractCrawler): sort=SearchSortType(config.SORT_TYPE) if config.SORT_TYPE != '' else SearchSortType.GENERAL, ) utils.logger.info(f"[XiaoHongShuCrawler.search] Search notes res:{notes_res}") + if(not notes_res or not notes_res.get('has_more', False)): + utils.logger.info("No more content!") + break semaphore = asyncio.Semaphore(config.MAX_CONCURRENCY_NUM) task_list = [ self.get_note_detail(post_item.get("id"), semaphore)