更新 'spiders/yaoex/yaoex_crawl.py'

spiders/yaoex/yaoex_crawl.py

@@ -17,11 +17,7 @@ TOKEN = "Sm45MzRmREtiaStVTnJORXEySHhYYzNwUmQ2RUprWXlwelRDem4wV2RZUCtUUU5jMGVCVTR
 # 平台常量（API 调度用）
 PLATFORM_ID = 6
 YYC_DEVICE_ID = "18193030281"
-TASK_API_BASE = "http://192.168.2.246:8080/api/collect_task"
-TASK_API_TOKEN = "zhijiayun_crawler_2026"
-TASK_PULL_URL = TASK_API_BASE + "/pull"
-TASK_HEARTBEAT_URL = TASK_API_BASE + "/heartbeat"
-TASK_REPORT_URL = TASK_API_BASE + "/report"
+
 HEARTBEAT_INTERVAL_SECONDS = 60
 
 headers = {
@@ -102,74 +98,6 @@ class YaoexCrawler:
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round", 1)
 
-    # ---------- API report helpers ----------
-
-    def _send_report(self, params: dict, log_msg: str):
-        try:
-            resp = requests.post(
-                TASK_REPORT_URL, json=params, timeout=10,
-                headers={'X-Crawler-Token': TASK_API_TOKEN},
-            )
-            if resp.status_code == 200:
-                try:
-                    data = resp.json()
-                    if data.get("code") == "success":
-                        logger.info("✅ 回告成功：%s", log_msg)
-                        return "success"
-                    else:
-                        msg = data.get("msg", "")
-                        if "已达平台限额" in msg or "限额" in msg:
-                            logger.warning("⛔ 回告接口返回限额错误：code=%s, msg=%s", data.get('code'), msg)
-                            return "limit_reached"
-                        else:
-                            logger.warning("⚠️ 回告接口返回错误：code=%s, msg=%s", data.get('code'), msg)
-                            return "error"
-                except ValueError:
-                    logger.warning("⚠️ 回告响应非 JSON：%s", resp.text[:200])
-                    return "error"
-            else:
-                logger.warning("⚠️ 回告 HTTP %s", resp.status_code)
-                return "error"
-        except Exception as e:
-            logger.error("❌ 回告失败：%s，错误：%s", log_msg, str(e))
-            return "error"
-
-    def _report_start(self):
-        payload = {
-            "task_id": self.task_id,
-            "platform": str(self.platform),
-            "username": YYC_DEVICE_ID,
-            "current_page": 0,
-            "total_pages": 0,
-            "is_finished": 0,
-        }
-        logger.info("上报开始 task_id=%s", self.task_id)
-        return self._send_report(payload, "开始上报（is_finished=0）")
-
-    def _report_progress(self, current_page: int, total_pages: int = 0):
-        payload = {
-            "task_id": self.task_id,
-            "platform": str(self.platform),
-            "username": YYC_DEVICE_ID,
-            "current_page": current_page,
-            "total_pages": total_pages,
-            "is_finished": 0,
-        }
-        return self._send_report(payload, f"进度上报 page={current_page}")
-
-    def _report_end(self, success: bool, real_count: int, page_no: int = 0, total_pages: int = 0):
-        payload = {
-            "task_id": self.task_id,
-            "platform": str(self.platform),
-            "username": YYC_DEVICE_ID,
-            "current_page": page_no,
-            "total_pages": total_pages if total_pages > 0 else page_no,
-            "crawled_count": real_count,
-            "is_finished": 1 if success else 0,
-        }
-        logger.info("上报结束 task_id=%s payload=%s", self.task_id, payload)
-        self._send_report(payload, "结束上报（is_finished=1）")
-
     @staticmethod
     def _timestamp_ms() -> str:
         return str(int(time.time() * 1000))
@@ -433,7 +361,7 @@ class YaoexCrawler:
                 break
             # 每页结束上报进度，检查限额
             self._page_no = page
-            prog_status = self._report_progress(page, total_pages=api_total_pages)
+            prog_status = self.scheduler.report_progress(self.task_id, page, total_pages=api_total_pages)
             if prog_status == "limit_reached":
                 logger.warning("⛔ 进度上报检测到限额，通知停止")
                 self.scheduler.set_flag(True)
@@ -443,7 +371,7 @@ class YaoexCrawler:
         self._page_no = 0
         try:
             self.scheduler.start()
-            self._report_start()
+            self.scheduler.report_start(self.task_id)
             self.search_data()
         except Exception as e:
             logger.error(e)
@@ -451,7 +379,8 @@ class YaoexCrawler:
         finally:
             self.scheduler.stop()
             try:
-                self._report_end(
+                self.scheduler.report_end(
+                    self.task_id,
                     success=self.is_success,
                     real_count=self.pipeline.crawl_count,
                     page_no=self._page_no,