Merge remote-tracking branch 'origin/2.0' into 2.0

spiders/pdd/main.py

@@ -574,6 +574,7 @@ def run_task_worker(task_payload):
             collect_round=task_payload.get("collect_round"),
             collect_equipment_account_id=task_payload.get("collect_equipment_account_id", 1),
             collect_region_id=task_payload.get("collect_region_id", 1),
+            username=task_payload.get("username"),
         )
         completed_normally = pdd.main(device_id, 1, 0)
         if completed_normally:
@@ -588,8 +589,8 @@ def run_task_worker(task_payload):
             end_page = getattr(pdd, "page", end_page)
             pdd.finish_task_abnormally(end_page, f"任务执行异常: {err_msg}")
         else:
-            report_api(task_id, end_page=end_page, start=4, end_time=int(time.time()), finish_status=0,
-                       collect_round=task_payload.get("collect_round"))
+            report_api(task_id, platform=PLATFORM_PDD, username="unknown",
+                       page=end_page, is_finished=0)
         logging.exception(f"[任务 {task_id}] 执行异常，设备: {device_id}，错误: {err_msg}")
         send_error_card(
             task_name=f"PDD任务(task_id={task_id}, device={device_id}, key={search_key})",
@@ -761,24 +762,21 @@ TASK_HEARTBEAT_URL = "http://120.24.26.108:8082/api/collect_task/heartbeat"
 TASK_API_HEADERS = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
 
 
-def report_api(task_id, page=None, start=None, end_page=None, end_time=None, finish_status=None, collect_round=None):
-    """向调度中心上报任务状态（与 YBM 共用同一 API）。"""
+def report_api(task_id, platform, username, page=0, is_finished=0,
+               crawled_count=0, total_pages=0):
+    """向调度中心上报任务状态（完全对齐 YBM _send_report 格式）。"""
     if USE_MANUAL_TASKS:
-        logging.info(
-            f"手动任务模式：跳过接口上报 task_id={task_id}, start={start}, "
-            f"page={page}, end_page={end_page}, finish_status={finish_status}"
-        )
+        logging.info(f"手动任务模式：跳过接口上报 task_id={task_id}")
         return
 
     params = {
-        "collect_task_allocate_id": task_id,
-        "start_page": page if page is not None else '',
-        "end_page": end_page if end_page is not None else '',
-        "collect_round": collect_round if collect_round is not None else '',
-        "status": start,
-        "finish_status": finish_status if finish_status is not None else 0,
-        "start_time": int(time.time()),
-        "end_time": end_time if end_time is not None else '',
+        "task_id": task_id,
+        "platform": str(platform),
+        "username": str(username) if username else "",
+        "current_page": page,
+        "total_pages": total_pages,
+        "crawled_count": crawled_count,
+        "is_finished": is_finished,
     }
     print(params)
     try:
@@ -862,9 +860,11 @@ class PDD:
             collect_round=None,
             collect_equipment_account_id=None,
             collect_region_id=None,
+            username=None,
 
     ):
         # 阶段 1：初始化与 App、OCR、日志、OSS 相关的基础依赖。
+        self.username = username  # 调度系统账号名
         self.package_name = 'com.xunmeng.pinduoduo'
         self.APP_ID = '116857964'
         self.API_KEY = '1gAzACJOAr7BeILKqkqPOETh'
@@ -894,7 +894,7 @@ class PDD:
         self.save_search_key = save_search_key or search_key
         # 起止页在入口阶段先做边界修正，主循环只消费规范化后的值。
         self.start_page = max(parse_optional_int(start_page, 0), 0)
-        self.end_page = max(parse_optional_int(end_page, 0), 0)
+        self.end_page = parse_optional_int(end_page, None)  # None = 不限页数
         self.max_counts_limit = max_counts_limit
         self.collect_config_info = collect_config_info or ""
         self.direct_shop_lookup = direct_shop_lookup
@@ -1004,32 +1004,30 @@ class PDD:
             time.sleep(self.get_sleep_time())
 
     def finish_task_normally(self, end_page, reason):
-        # 功能：以"正常完成"状态结束任务并保证只上报一次。
-        # 多个退出分支都会走到这里，因此先判断是否已经上报过结束状态。
         if not self.finish_reported:
             report_api(
                 self.task_id,
-                end_page=end_page,
-                start=3,
-                end_time=int(time.time()),
-                finish_status=1,
-                collect_round=getattr(self, 'collect_round', 0),
+                platform=self.platform,
+                username=getattr(self, 'username', ''),
+                page=end_page,
+                is_finished=1,
+                crawled_count=getattr(self, 'max_counts', 0),
+                total_pages=end_page,
             )
             self.finish_reported = True
         print(reason)
         return True
 
     def finish_task_abnormally(self, end_page, reason, finish_status=0):
-        # 功能：以"异常结束"状态结束任务并保证只上报一次。
-        # 异常结束与正常结束共享同一幂等保护，避免重复通知外部调度系统。
         if not self.finish_reported:
             report_api(
                 self.task_id,
-                end_page=end_page,
-                start=4,
-                end_time=int(time.time()),
-                finish_status=finish_status,
-                collect_round=getattr(self, 'collect_round', 0),
+                platform=self.platform,
+                username=getattr(self, 'username', ''),
+                page=end_page,
+                is_finished=0,
+                crawled_count=getattr(self, 'max_counts', 0),
+                total_pages=end_page,
             )
             self.finish_reported = True
         print(reason)
@@ -3339,7 +3337,8 @@ class PDD:
         # 阶段 2：向调度系统上报"执行中"，然后开始按页扫描商品列表。
         #  上报状态
         # 进入主循环前先上报"执行中"，让调度系统能看到设备已经开始跑任务。
-        report_api(self.task_id, self.page, 2,finish_status=0)
+        report_api(self.task_id, platform=self.platform, username=getattr(self, 'username', ''),
+                   page=self.page, is_finished=0)
         # ========== 新增：启动心跳上报线程 ==========
         self._heartbeat_running = True
 

spiders/yaoex/yaoex_crawl.py

@@ -23,7 +23,6 @@ TASK_API_BASE = "http://pricesys2.kailin.com.cn:8083/api/collect_task"
 TASK_API_TOKEN = "zhijiayun_crawler_2026"
 TASK_PULL_URL = TASK_API_BASE + "/pull"
 TASK_HEARTBEAT_URL = TASK_API_BASE + "/heartbeat"
-TASK_REPORT_URL = TASK_API_BASE + "/report"
 
 headers = {
     "Accept": "application/json, text/plain, */*",
@@ -60,6 +59,9 @@ class YaoexCrawler:
         self.account_name = None
         self.pipeline = DrugPipeline("yaoex")
         self.scheduler = CrawlerScheduler(YYC_DEVICE_ID, str(PLATFORM_ID), heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS)
+        # 独立回告（不依赖 commons，commons 经常变）
+        self.TASK_REPORT_URL = "http://pricesys2.kailin.com.cn:8083/api/collect_task/report"
+        self.TASK_API_TOKEN = "zhijiayun_crawler_2026"
         if self.task_dict:
             self.get_product_data()
 
@@ -71,8 +73,8 @@ class YaoexCrawler:
     def _send_report(self, params: dict, log_msg: str):
         try:
             resp = requests.post(
-                TASK_REPORT_URL, json=params, timeout=10,
-                headers={'X-Crawler-Token': TASK_API_TOKEN},
+                self.TASK_REPORT_URL, json=params, timeout=10,
+                headers={'X-Crawler-Token': self.TASK_API_TOKEN},
             )
             if resp.status_code == 200:
                 try:

spiders/yaoex/yaoex_snapshot_crawl.py

@@ -26,7 +26,6 @@ PLATFORM_ID = 6
 HEARTBEAT_INTERVAL_SECONDS = 60
 TASK_API_BASE = "http://pricesys2.kailin.com.cn:8083/api/collect_task"
 TASK_API_TOKEN = "zhijiayun_crawler_2026"
-TASK_REPORT_URL = TASK_API_BASE + "/report"
 
 CAPTCHA_TOKEN = "zPzmt1mG1ouCU6GTzsZN2Lmm8pdZypapPcLJTBRETco"
 CAPTCHA_API_URL = "http://api.jfbym.com/api/YmServer/customApi"
@@ -87,6 +86,9 @@ class YaoexSnapshotCrawl:
         self.user_id = YYC_ACCOUNT["user_id"]
         self.token = YYC_ACCOUNT["token"]
         self.scheduler = CrawlerScheduler(YYC_DEVICE_ID, str(PLATFORM_ID), heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS)
+        # 独立回告（不依赖 commons，commons 经常变）
+        self.TASK_REPORT_URL = "http://pricesys2.kailin.com.cn:8083/api/collect_task/report"
+        self.TASK_API_TOKEN = "zhijiayun_crawler_2026"
 
     def get_product_data(self):
         self.task_id = self.task_dict["id"]
@@ -111,8 +113,8 @@ class YaoexSnapshotCrawl:
     def _send_report(self, params: dict, log_msg: str):
         try:
             resp = requests.post(
-                TASK_REPORT_URL, json=params, timeout=10,
-                headers={'X-Crawler-Token': TASK_API_TOKEN},
+                self.TASK_REPORT_URL, json=params, timeout=10,
+                headers={'X-Crawler-Token': self.TASK_API_TOKEN},
             )
             if resp.status_code == 200:
                 try:
@@ -246,7 +248,7 @@ class YaoexSnapshotCrawl:
         self.driver = ChromiumPage(co)
 
     def _is_logged_in(self):
-        return bool(self.driver.ele("xpath=//a[@id='logout']", timeout=5))
+        return bool(self.driver.ele("xpath=//span[@class='login-out']", timeout=5))
 
     def _call_captcha_api(self, image_bytes):
         try:

spiders/ybm/ybm_crawl.py

@@ -48,7 +48,6 @@ TASK_API_TOKEN = "zhijiayun_crawler_2026"
 
 TASK_PULL_URL = TASK_API_BASE + "/pull"
 TASK_HEARTBEAT_URL = TASK_API_BASE + "/heartbeat"
-TASK_REPORT_URL = TASK_API_BASE + "/report"
 
 # 手动任务（调试用）
 USE_MANUAL_TASK = False
@@ -759,6 +758,10 @@ class YbmCrawler:
         # 由 CrawlerScheduler 管理心跳和停止信号
         self.scheduler = CrawlerScheduler(YBM_DEVICE_ID, str(PLATFORM_ID), heartbeat_interval=HEARTBEAT_INTERVAL_SECONDS)
 
+        # 独立回告（不依赖 commons，commons 经常变）
+        self.TASK_REPORT_URL = "http://pricesys2.kailin.com.cn:8083/api/collect_task/report"
+        self.TASK_API_TOKEN = "zhijiayun_crawler_2026"
+
         if self.task_dict:
             self._extract_task_params()
 
@@ -790,13 +793,13 @@ class YbmCrawler:
         }, ensure_ascii=False, default=str)
 
 
-    # ---------- API report helpers ----------
+# ---------- API report helpers ----------
 
     def _send_report(self, params: dict, log_msg: str):
         try:
             resp = requests.post(
-                TASK_REPORT_URL, json=params, timeout=10,
-                headers={'X-Crawler-Token': TASK_API_TOKEN},
+                self.TASK_REPORT_URL, json=params, timeout=10,
+                headers={'X-Crawler-Token': self.TASK_API_TOKEN},
             )
             if resp.status_code == 200:
                 try:
@@ -1456,7 +1459,7 @@ class YbmCrawler:
         with sync_playwright() as p:
             browser = p.chromium.launch(
                 channel="chrome",
-                headless=True,
+                headless=False,
                 slow_mo=random.randint(100, 300),
                 args=[
                     "--disable-blink-features=AutomationControlled",