update

commons/scheduler.py

@@ -24,7 +24,7 @@ class CrawlerScheduler:
 
         self._lock = threading.Lock()
 
-        self.heartbeat_url = 'http://pricesys2.kailin.com.cn:8083/api/collect_task/heartbeat'
+        self.heartbeat_url = 'http://pricesys2.kailin.com.cn:8082/api/collect_task/heartbeat'
         self.heartbeat_interval = heartbeat_interval
         self.end = False
         self.heartbeat_thread = None
@@ -80,10 +80,8 @@ class CrawlerScheduler:
 
     def get_task(self):
         try:
-            task_api = "http://pricesys2.kailin.com.cn:8083/api/collect_task/pull"
-
+            task_api = "http://pricesys2.kailin.com.cn:8082/api/collect_task/pull"
             headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
-
             params = {
                 'platform': self.platform,
                 'username': self.username
@@ -101,7 +99,7 @@ class CrawlerScheduler:
 
     def post_report(self, data):
         try:
-            url = "http://pricesys2.kailin.com.cn:8083/api/collect_task/report"
+            url = "http://pricesys2.kailin.com.cn:8082/api/collect_task/report"
             print('传给返回接口的数据', data)
             logger.info(f'report上传数据：{data}')
             headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}

spiders/yaoex/yaoex_crawl.py

@@ -16,7 +16,7 @@ TOKEN = "Sm45MzRmREtiaStVTnJORXEySHhYYzNwUmQ2RUprWXlwelRDem4wV2RZUCtUUU5jMGVCVTR
 
 # 平台常量（API 调度用）
 PLATFORM_ID = 6
-YYC_DEVICE_ID = "181930302811"
+YYC_DEVICE_ID = "18193030281"
 
 HEARTBEAT_INTERVAL_SECONDS = 60
 TASK_API_BASE = "http://pricesys2.kailin.com.cn:8083/api/collect_task"
@@ -107,6 +107,7 @@ class YaoexCrawler:
             "username": YYC_DEVICE_ID,
             "current_page": 0,
             "total_pages": 0,
+            "crawled_count": 0,
             "is_finished": 0,
         }
         logger.info("上报开始 task_id=%s", self.task_id)
@@ -119,6 +120,7 @@ class YaoexCrawler:
             "username": YYC_DEVICE_ID,
             "current_page": current_page,
             "total_pages": total_pages,
+            "crawled_count": self.pipeline.crawl_count,
             "is_finished": 0,
         }
         return self._send_report(payload, f"进度上报 page={current_page}")

spiders/yaoex/yaoex_snapshot_crawl.py

@@ -147,6 +147,7 @@ class YaoexSnapshotCrawl:
             "username": YYC_DEVICE_ID,
             "current_page": 0,
             "total_pages": 0,
+            "crawled_count": 0,
             "is_finished": 0,
         }
         logger.info("上报开始 task_id=%s", self.task_id)
@@ -159,6 +160,7 @@ class YaoexSnapshotCrawl:
             "username": YYC_DEVICE_ID,
             "current_page": current_page,
             "total_pages": total_pages,
+            "crawled_count": self.pipeline.crawl_count,
             "is_finished": 0,
         }
         return self._send_report(payload, f"进度上报 page={current_page}")

spiders/ybm/ybm_crawl.py

@@ -378,7 +378,7 @@ def get_current_time():
 
 def random_delay(min_seconds, max_seconds):
     delay = random.uniform(min_seconds, max_seconds)
-    time.sleep(1)
+    time.sleep(delay)
     return delay
 
 
@@ -832,6 +832,7 @@ class YbmCrawler:
             "username": self.username,
             "current_page": 0,
             "total_pages": 0,
+            "crawled_count": 0,
             "is_finished": 0,
         }
         logger.info("上报开始 task_id=%s keyword=%s", self.task_id, self.keyword)
@@ -844,6 +845,7 @@ class YbmCrawler:
             "username": self.username,
             "current_page": current_page,
             "total_pages": total_pages,
+            "crawled_count": self.pipeline.crawl_count,
             "is_finished": 0,
         }
         return self._send_report(payload, f"进度上报 page={current_page}")
@@ -1459,7 +1461,7 @@ class YbmCrawler:
         with sync_playwright() as p:
             browser = p.chromium.launch(
                 channel="chrome",
-                headless=False,
+                headless=True,
                 slow_mo=random.randint(100, 300),
                 args=[
                     "--disable-blink-features=AutomationControlled",
@@ -1594,39 +1596,47 @@ class YbmCrawler:
 # ==================== SECTION 9: ENTRY POINT (测试/直接运行) ====================
 
 if __name__ == '__main__':
-    # 直接运行：使用手动任务进行测试
     load_city_mapping()
-    if USE_MANUAL_TASK:
-        task = MANUAL_TASK
-    else:
-        # 从内部 API 拉取任务
-        scheduler = CrawlerScheduler(YBM_DEVICE_ID, str(PLATFORM_ID))
-        task = scheduler.get_task() or {}
-        if not task:
-            logger.error("拉取任务失败")
-
-    if not task:
-        logger.error("未获取到任何任务，程序退出")
-    else:
-        crawler = YbmCrawler(task)
-        count, success = crawler.run()
-        logger.info(f"测试完成：crawl_count={count}, is_success={success}")
+    scheduler = CrawlerScheduler(YBM_DEVICE_ID, str(PLATFORM_ID))
+    idle_seconds = random.randint(180, 300)
+    logger.info("药帮忙采集启动（直接运行模式），每轮间隔 %s 秒", idle_seconds)
 
-        # 发送飞书通知
+    while True:
+        task = None
         try:
-            spec_items = [item.strip() for item in re.split(r"[|、，,\n\r]+", task.get("product_specs", "") or "") if item.strip()]
-            display_spec = "、".join(spec_items)
-            notice_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-            if success:
-                feishu_send_text(
-                    f"{notice_time} 通知:\n"
-                    f"平台: 药帮忙, 药品: {task.get('product_name', '')}, 品规: {display_spec}, 爬取数据: {count}条"
-                )
-            else:
-                feishu_send_error_card(
-                    task_name=task.get("product_name", ""),
-                    err_msg=f"task_id={task.get('id')}, company_id={task.get('company_id')}, crawl_count={count}",
-                    mention_all=False,
-                )
+            scheduler.start()
+            task = scheduler.get_task() or {}
+            if not task:
+                logger.info("暂无任务，等待 %s 秒后重试", idle_seconds)
+                time.sleep(idle_seconds)
+                continue
+
+            logger.info("开始执行药帮忙爬虫任务 task_id=%s", task.get("id"))
+            crawler = YbmCrawler(task)
+            count, success = crawler.run()
+            logger.info("药帮忙爬虫任务执行完成 task_id=%s count=%s success=%s",
+                        task.get("id"), count, success)
+
+            # 飞书通知
+            try:
+                spec_items = [item.strip() for item in re.split(r"[|、，,\n\r]+", task.get("product_specs", "") or "") if item.strip()]
+                display_spec = "、".join(spec_items)
+                notice_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                if success:
+                    feishu_send_text(
+                        f"{notice_time} 通知:\n"
+                        f"平台: 药帮忙, 药品: {task.get('product_name', '')}, 品规: {display_spec}, 爬取数据: {count}条"
+                    )
+                else:
+                    feishu_send_error_card(
+                        task_name=task.get("product_name", ""),
+                        err_msg=f"task_id={task.get('id')}, company_id={task.get('company_id')}, crawl_count={count}",
+                        mention_all=False,
+                    )
+            except Exception as e:
+                logger.warning("飞书通知发送失败：%s", e)
+
         except Exception as e:
-            logger.warning(f"飞书通知发送失败：{e}")
+            logger.error("药帮忙爬虫任务执行失败: %s", e, exc_info=True)
+
+        time.sleep(idle_seconds)

start_run_ybm.py

@@ -120,7 +120,6 @@ def main_loop():
                     exception_type=type(e).__name__,
                     remark=str(e)[:500],
                 )
-
         time.sleep(idle_seconds)