update

spiders/yaojiujiu/yyjzt_crawl.py

@@ -366,17 +366,20 @@ class YyjztSpider:
 
     # ==================== 主流程 ====================
     def _match_item(self, product):
-        """检查商品是否匹配品牌+品名，返回 True=匹配"""
         product_brand = product.get("product_brand", "")
         product_name = product.get("product_name", "")
+        product_specs = product.get("spec", "")
 
         brand_ok = True
         name_ok = True
+        specs_ok = True
         if self.product_brand.strip():
             brand_ok = self.product_brand.strip() in str(product_brand)
         if self.product_name.strip():
             name_ok = self.product_name.strip() in str(product_name)
-        return brand_ok and name_ok
+        if self.product_specs.strip():
+            specs_ok = self.product_specs.strip() in str(product_specs)
+        return brand_ok and name_ok and specs_ok
 
     def search_data(self):
         if not self.task_dict:

spiders/yaojiujiu/yyjzt_snapshot.py

@@ -418,6 +418,12 @@ class YyjztSnapshotSpider:
         matched = 0
         unmatched_log = []
         for item, _ in items:
+            if self.product_brand and self.product_brand.strip() not in str(item.get("brandName") or ""):
+                continue
+            if self.product_name and self.product_name.strip() not in str(item.get("itemStoreName") or ""):
+                continue
+            if self.product_specs and self.product_specs.strip() not in str(item.get("specs") or ""):
+                continue
             card, fail_detail = self._match(item, dom_cards, used)
             if not card:
                 unmatched_log.append(fail_detail)
@@ -539,7 +545,7 @@ class YyjztSnapshotSpider:
         # 2. 一个浏览器干到底（headless + Linux 兼容）
         with sync_playwright() as p:
             browser = p.chromium.launch(
-                headless=True,
+                headless=True, channel="chrome",
                 args=[
                     "--disable-blink-features=AutomationControlled",
                     "--no-sandbox",
@@ -619,10 +625,10 @@ class YyjztSnapshotSpider:
                 time.sleep(random.uniform(2, 3))
 
             # 写结果文件
-            out_path = f"./snapshots/result_{int(time.time())}.json"
-            with open(out_path, "w", encoding="utf-8") as f:
-                json.dump(self._all_results, f, ensure_ascii=False, indent=2, default=str)
-            logger.info("结果文件: %s (%s 条)", out_path, len(self._all_results))
+            # out_path = f"./snapshots/result_{int(time.time())}.json"
+            # with open(out_path, "w", encoding="utf-8") as f:
+            #     json.dump(self._all_results, f, ensure_ascii=False, indent=2, default=str)
+            # logger.info("结果文件: %s (%s 条)", out_path, len(self._all_results))
 
             browser.close()
 

spiders/yaoshibang/ysb_snapshot_crawl.py

@@ -262,7 +262,8 @@ class YaoShiBangSnapshot:
             co.set_browser_path(chrome_path)
 
         # ---------- 2. 统一使用 set_argument 设置（避免 set_headless 报错） ----------
-        co.set_argument('--headless=new')  # 启用新版无头模式（Windows/Linux 均支持）
+        co.set_argument('--headless')  # 旧版无头（更兼容）
+        co.set_argument('--window-size=1920,1080')
         co.set_argument('--no-sandbox')  # Linux 必需，Windows 下被忽略，无害
         co.set_argument('--disable-dev-shm-usage')  # Linux 解决 /dev/shm 问题，Windows 无害
         co.set_argument('--disable-gpu')  # 无头模式下可减少 GPU 问题，兼容所有系统
@@ -277,23 +278,22 @@ class YaoShiBangSnapshot:
         profile_dir.mkdir(parents=True, exist_ok=True)
         co.set_user_data_path(str(profile_dir))
 
-        # ---------- 4. 可选：本地调试端口（跨平台） ----------
-        # 如果想手动指定端口，可取消注释并自行实现 _get_free_port()
-        # debug_port = self._get_free_port()
-        # co.set_local_port(debug_port)
-        # co.set_argument(f"--remote-debugging-port={debug_port}")
-        # co.set_argument("--remote-debugging-address=127.0.0.1")
+        co.set_argument("--remote-debugging-port=0")
+        co.set_argument("--remote-debugging-address=127.0.0.1")
 
         self.driver = ChromiumPage(co)
 
-    def _solve_slider_captcha(self):
+    def _solve_slider_captcha(self, yidun=None):
         """检测并处理易盾滑块验证码，成功返回 True。"""
         self.driver.wait.doc_loaded()
         time.sleep(2)
 
-        yidun = self.driver.ele("xpath://div[@class='yidun_modal']", timeout=3)
-        if not yidun:
+        captcha_text = self.driver.ele("xpath://*[contains(text(),'安全验证') or contains(text(),'验证码')]", timeout=1)
+        if not captcha_text:
+            captcha_text = self.driver.ele("xpath://div[@class='yidun_modal']", timeout=2)
+        if not captcha_text:
             return True
+        logger.warning("检测到验证码: %s", captcha_text.inner_text()[:50])
 
         logger.info("检测到滑块验证码，开始处理")
         jpg_bytes = yidun.get_screenshot(as_bytes="jpg")
@@ -439,6 +439,9 @@ class YaoShiBangSnapshot:
 
     def _goto_detail_page(self, item_id, detail_url):
         """get 后 refresh 一次，让 SPA 按当前 URL 重新渲染详情。"""
+        if not self._solve_slider_captcha():
+            logger.error("验证码处理失败，跳过当前商品")
+            return False
         for attempt in range(1, DETAIL_NAV_RETRIES + 1):
             try:
                 self.driver.get(detail_url, timeout=5)
@@ -520,7 +523,7 @@ class YaoShiBangSnapshot:
         time.sleep(1)
         self._dismiss_popup_before_screenshot()
         try:
-            jpg_bytes = self.driver.get_screenshot(as_bytes="jpg")
+            jpg_bytes = self.driver.get_screenshot(as_bytes="jpg", full_page=True)
             if not jpg_bytes:
                 logger.warning("截图为空 upload_key=%s", upload_key)
                 return ""
@@ -594,6 +597,17 @@ class YaoShiBangSnapshot:
 
     def _dismiss_popup_before_screenshot(self):
         """截图前关闭或隐藏营销弹窗，避免遮挡。"""
+        self.driver.run_js("""
+        // 暴力清除所有高 z-index 遮罩和弹窗
+        document.querySelectorAll('*').forEach(el => {
+          const s = getComputedStyle(el);
+          const z = parseInt(s.zIndex) || 0;
+          if (z > 100 && (s.position === 'fixed' || s.position === 'absolute') &&
+              (el.clientWidth > 100 || el.clientHeight > 100)) {
+            el.remove();
+          }
+        });
+        """)
         close_locs = [
             "xpath=//div[contains(@class,'dialog')]//i[contains(@class,'close')]",
             "xpath=//div[contains(@class,'popup')]//i[contains(@class,'close')]",
@@ -601,6 +615,8 @@ class YaoShiBangSnapshot:
             "xpath=//button[contains(@class,'close')]",
             "xpath=//span[text()='×']",
             "xpath=//*[contains(text(),'智能采购')]/ancestor::div[1]//*[contains(@class,'close')]",
+            "xpath=//*[contains(text(),'智能采购')]/ancestor::div[3]//*[contains(@class,'close')]",
+            "xpath=//*[contains(text(),'智能采购')]/ancestor::div[3]//button[last()]",
         ]
         for loc in close_locs:
             try:
@@ -764,6 +780,10 @@ class YaoShiBangSnapshot:
             page_start = self.start_page
         self.page = page_start  # 兜底：避免0页时回告 total_pages=0
         for page in range(page_start, 100):
+            if not self._solve_slider_captcha():
+                logger.error("验证码处理失败，停止任务")
+                self._report_exception(3, "验证码处理失败，停止当前任务")
+                return False
             first_search = page == 1
             logger.info("药师帮爬取第%s页 firstSearch=%s stateValue=%s", page, first_search,
                         self._state_value or "(空)")
@@ -815,6 +835,8 @@ class YaoShiBangSnapshot:
                 continue
 
             state_val = self._extract_state_value(json_data, data_block)
+            logger.info("第%s页 解密后 keys: %s, stateValue=%s, has_wholesales=%s", page,
+                        list(json_data.keys())[:20], state_val, bool(json_data.get("wholesales")))
             if state_val:
                 self._state_value = state_val
 

start_run_yaojiujiu.py

@@ -77,8 +77,7 @@ def _report_progress(scheduler, task_id, is_finished, crawled_count=0,
         "is_finished": is_finished,
         "total_pages": total_pages,
     }
-    if crawled_count:
-        data["crawled_count"] = crawled_count
+    data["crawled_count"] = crawled_count
     if exception_type is not None:
         data["exception_type"] = exception_type
     if remark:
@@ -113,7 +112,15 @@ def run_one_task(scheduler, device_id, account_password, account_name):
         spider_cls = YyjztSpider
         logger.info("任务 %s 使用普通模式", task_id)
 
-    result = spider_cls(task_dict).run()
+    try:
+        result = spider_cls(task_dict).run()
+    except Exception as e:
+        logger.exception("任务 %s 执行异常: %s", task_id, e)
+        _report_progress(scheduler, task_id, is_finished=0,
+                         crawled_count=0,
+                         exception_type=5, remark=str(e)[:200])
+        return
+
     if len(result) == 3:
         crawl_count, is_success, total_pages = result
     else: