RYDE-WORK/MediaCrawler
1
0
Fork 0
mirror of https://github.com/RYDE-WORK/MediaCrawler.git synced 2026-01-25 17:52:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

chore: 简化判断方式

Browse Source
This commit is contained in:
leantli 2024-04-04 00:11:22 +08:00
parent 133f978477
commit 68a60faa7f
5 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
media_platform/bilibili/core.py
View File

@ -93,7 +93,9 @@ class BilibiliCrawler(AbstractCrawler):
:return:
"""
utils.logger.info("[BilibiliCrawler.search] Begin search bilibli keywords")
bili_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT)) # bilibili limit page fixed value
bili_limit_count =20 # bilibili limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < bili_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = bili_limit_count
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"[BilibiliCrawler.search] Current search keyword: {keyword}")
page = 1

4
media_platform/douyin/core.py
View File

@ -81,10 +81,12 @@ class DouYinCrawler(AbstractCrawler):
async def search(self) -> None:
utils.logger.info("[DouYinCrawler.search] Begin search douyin keywords")
dy_limit_count = 10 # douyin limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < dy_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = dy_limit_count
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"[DouYinCrawler.search] Current keyword: {keyword}")
aweme_list: List[str] = []
dy_limit_count = min(10, max(1, config.CRAWLER_MAX_NOTES_COUNT))
page = 0
while (page + 1) * dy_limit_count <= config.CRAWLER_MAX_NOTES_COUNT:
try:

4
media_platform/kuaishou/core.py
View File

@ -85,7 +85,9 @@ class KuaishouCrawler(AbstractCrawler):
async def search(self):
utils.logger.info("[KuaishouCrawler.search] Begin search kuaishou keywords")
ks_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT)) # kuaishou limit page fixed value
ks_limit_count = 20 # kuaishou limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < ks_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = ks_limit_count
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"[KuaishouCrawler.search] Current search keyword: {keyword}")
page = 1

4
media_platform/weibo/core.py
View File

@ -104,7 +104,9 @@ class WeiboCrawler(AbstractCrawler):
:return:
"""
utils.logger.info("[WeiboCrawler.search] Begin search weibo keywords")
weibo_limit_count = min(10, max(1, config.CRAWLER_MAX_NOTES_COUNT))
weibo_limit_count = 10 # weibo limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < weibo_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = weibo_limit_count
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"[WeiboCrawler.search] Current search keyword: {keyword}")
page = 1

4
media_platform/xhs/core.py
View File

@ -96,7 +96,9 @@ class XiaoHongShuCrawler(AbstractCrawler):
async def search(self) -> None:
"""Search for notes and retrieve their comment information."""
utils.logger.info("[XiaoHongShuCrawler.search] Begin search xiaohongshu keywords")
xhs_limit_count = min(20, max(1, config.CRAWLER_MAX_NOTES_COUNT)) # xhs limit page fixed value
xhs_limit_count = 20 # xhs limit page fixed value
if config.CRAWLER_MAX_NOTES_COUNT < xhs_limit_count:
config.CRAWLER_MAX_NOTES_COUNT = xhs_limit_count
for keyword in config.KEYWORDS.split(","):
utils.logger.info(f"[XiaoHongShuCrawler.search] Current search keyword: {keyword}")
page = 1