Merge remote-tracking branch 'origin/2.0' into 2.0

Yjj/main.py

@@ -1163,6 +1163,8 @@ def collect_data(page, keyword, keyword_dict, scheduler=None):
 
                 if no_match_count >= MAX_NO_MATCH:
                     logger.error(f"❌ 关键词「{keyword}」无匹配商品次数已达{MAX_NO_MATCH}次，直接终止当前关键词采集，进入下一个关键词")
+                    _report_page_progress(scheduler, keyword_dict, page_no, len(collect_result),
+                                          is_finished=1, total_pages=page_no)
                     return []
 
                 # 关键词不在标题中，跳过当前商品

Yjj/yjj_cookies.json

@@ -14,7 +14,7 @@
     "value": "6460a7d1a5012c84f19757b5f6a068b5",
     "domain": "quickaplus-he-api-cn-shanghai.aliyuncs.com",
     "path": "/",
-    "expires": 1816055693.319722,
+    "expires": 1816845539.770418,
     "httpOnly": false,
     "secure": true,
     "sameSite": "None"
@@ -24,7 +24,7 @@
     "value": "6460a7d1a5012c84f19757b5f6a068b5",
     "domain": ".yyjzt.com",
     "path": "/",
-    "expires": 1816055692.27729,
+    "expires": 1816845538.73354,
     "httpOnly": false,
     "secure": true,
     "sameSite": "Lax"
@@ -41,30 +41,30 @@
   },
   {
     "name": "acw_tc",
-    "value": "d3fa0861-0ec0-4fbb-a585-944439135c9be6a43251cc558f35eabbaaf0d49c6538",
+    "value": "0788f343-f74c-448e-94d5-00f75218e963fbbadd9aacafb4214b54816999cf65ef",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1781497457.440472,
+    "expires": 1782287304.731589,
     "httpOnly": true,
     "secure": false,
     "sameSite": "Lax"
   },
   {
     "name": "acw_tc",
-    "value": "0a15e17b17814956580202850e658e4a59f91c2d4187327e9a184a4af3b499",
+    "value": "0a15e15e17822855053244392e66059c0b7d88df893bdd6048cac4cef059c9",
     "domain": "s.yyjzt.com",
     "path": "/",
-    "expires": 1781497458.038508,
+    "expires": 1782287305.308821,
     "httpOnly": true,
     "secure": false,
     "sameSite": "Lax"
   },
   {
     "name": "yjj-token",
-    "value": "ssr_2066368815537303561",
+    "value": "ssr_2069681672177860622",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -74,7 +74,7 @@
     "value": "18971731507",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -84,7 +84,7 @@
     "value": "%E6%A2%85%E6%9E%97%E5%85%AD%E8%B7%AF",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -94,7 +94,7 @@
     "value": "18971731507",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -104,7 +104,7 @@
     "value": "1570949407765906055",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -114,7 +114,7 @@
     "value": "18971731507",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -124,7 +124,7 @@
     "value": "1570719328938967459",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -134,7 +134,7 @@
     "value": "%E6%B9%96%E5%8C%97%E5%92%8C%E5%90%AF%E5%85%B4%E9%A6%99%E9%A2%82%E5%A4%A7%E8%8D%AF%E6%88%BF%E8%BF%9E%E9%94%81%E6%9C%89%E9%99%90%E5%85%AC%E5%8F%B8",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877539,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -144,7 +144,7 @@
     "value": "1570715138825075429",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -154,7 +154,7 @@
     "value": "1",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -164,7 +164,7 @@
     "value": "42010500",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -174,7 +174,7 @@
     "value": "91420104591067935X",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877539,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -184,7 +184,7 @@
     "value": "0",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087691,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -194,7 +194,7 @@
     "value": "firmList",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1781495752,
+    "expires": 1782285598,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -204,7 +204,7 @@
     "value": "cartNum",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1781517292,
+    "expires": 1782307138,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -214,7 +214,7 @@
     "value": "category",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1781517292,
+    "expires": 1782307138,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -224,7 +224,7 @@
     "value": "sloganAndBottom",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1781495752,
+    "expires": 1782285598,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -234,7 +234,7 @@
     "value": "0",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -244,7 +244,7 @@
     "value": "",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877538,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -254,7 +254,7 @@
     "value": "https://s.yyjzt.com/jzt-cms-web/20221013/1665666007570af1.png",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877539,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -264,7 +264,7 @@
     "value": "https://s.yyjzt.com/jzt-cms-web/20231222/17032279632862dc.png",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877539,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"
@@ -274,7 +274,7 @@
     "value": "https://s.yyjzt.com/jzt-cms-web/20260420/1776653614972f3a0.png",
     "domain": "www.yyjzt.com",
     "path": "/",
-    "expires": 1784087692,
+    "expires": 1784877539,
     "httpOnly": false,
     "secure": false,
     "sameSite": "Lax"

Yjj/yjj_v2.py

@@ -17,8 +17,8 @@ from io import BytesIO
 from PIL import Image
 import traceback
 import oss2
-from commons.scheduler import CrawlerScheduler
-import platform
+from scheduler import CrawlerScheduler
+
 # ==================== 调度器配置 ====================
 PLATFORM_ID = "7"  # 药九九平台编码
 
@@ -207,21 +207,19 @@ def load_cookies(context, cookie_path=COOKIE_FILE_PATH):
 def is_login(page):
     """验证是否已登录（核心：检测登录态）"""
     try:
-        # 访问需要登录的页面
         page.goto(LOGIN_VALIDATE_URL, timeout=ELEMENT_TIMEOUT)
-        time.sleep(5)
         page.wait_for_load_state("networkidle")
 
-        # 检测是否跳转到登录页（URL包含login则未登录）
         if "login" in page.url.lower():
             logger.warning(" Cookie失效，需要重新登录")
             return False
 
-        # 可选：检测登录后的专属元素（比如用户名、个人中心等）
-        # if page.locator("用户中心选择器").count() > 0:
-        #     return True
-        logger.info(" Cookie有效，已保持登录状态")
-        return True
+        if page.locator(".user-box").count() > 0:
+            logger.info(" Cookie有效，已保持登录状态")
+            return True
+
+        logger.warning(" 已跳离登录页但未检测到用户元素，判定未登录")
+        return False
     except Exception as e:
         logger.error(f" 验证登录状态失败：{e}")
         return False
@@ -1163,6 +1161,7 @@ def collect_data(page, keyword, keyword_dict, scheduler=None):
 
                 if no_match_count >= MAX_NO_MATCH:
                     logger.error(f"❌ 关键词「{keyword}」无匹配商品次数已达{MAX_NO_MATCH}次，直接终止当前关键词采集，进入下一个关键词")
+                    _report_page_progress(scheduler, keyword_dict, page_no, len(collect_result),is_finished=1, total_pages=page_no)
                     return []
 
                 # 关键词不在标题中，跳过当前商品
@@ -1618,10 +1617,8 @@ def main():
     logger.info("调度器已启动，心跳已就绪")
 
     with sync_playwright() as p:
-        is_headless = (platform.system() == "Linux" and not os.environ.get("DISPLAY"))
-
         browser = p.chromium.launch(
-            headless=is_headless,  # 根据环境自动切换
+            headless=False,
             channel="chrome",
             slow_mo=random.randint(100, 300),
             args=_chromium_launch_args(),

spiders/yaofangwang/yaofangwang_crawl.py

@@ -58,7 +58,7 @@ class YaofangwangCrawl:
         self.account_id = self.task_dict.get("collect_equipment_account_id", "")
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round", 1)
-        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':'1','total_pages':1}
+        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':self.scheduler.username,'total_pages':1}
 
     def _error_report(self,data):
             self.report_data.update(data)
@@ -398,7 +398,9 @@ class YaofangwangCrawl:
 
         drug_list = self._search_result_medicine_ids(self.driver.html)
         id_dict = {}
+        print(drug_list)
         for drug in drug_list:
+            
             spec = drug["spec"]
             href = drug["href"]
             if "x" in self.product_desc:

spiders/yaoyigou/yaoyigou_crawl.py

@@ -27,7 +27,7 @@ headers = {
 
 
 class Yaoyigou:
-    def __init__(self, task_dict=None):
+    def __init__(self, task_dict=None,scheduler = None):
         self.hesytoken = None
         self.headers = headers.copy()
         self.platform = 8
@@ -37,10 +37,17 @@ class Yaoyigou:
         self.brand = ""
         self.is_success = True
         self.task_dict = task_dict
-
+        self.scheduler =scheduler
         if self.task_dict:
             self.get_product_data()
-
+    def _error_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.stop()
+            self.scheduler.post_report(self.report_data)
+
+    def _success_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.post_report(self.report_data)
     def get_product_data(self):
         self.task_id = self.task_dict["id"]
         self.company_id = self.task_dict["company_id"]
@@ -56,6 +63,10 @@ class Yaoyigou:
         self.account_id = self.task_dict.get("collect_equipment_account_id", "")
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round",1)
+        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':self.scheduler.username,'total_pages':1}
+
+
+
 
 
     def _post_json_with_retry(self, url, payload, max_retry=1, timeout=20):
@@ -95,6 +106,7 @@ class Yaoyigou:
                 max_retry=1,
                 timeout=20,
             )
+            print(data_json)
             self.hesytoken = data_json.get('content', "")
         except Exception as e:
             logger.error("登录失败: %s", e)
@@ -208,8 +220,8 @@ class Yaoyigou:
 
             for content in contents:
                 company = content.get("manufacturerName", "")
-                if self.brand not in company:
-                    continue
+                # if self.brand not in company:
+                #     continue
                 try:
                     product = self.parse_product(content)
                     if not product.get("item_id"):
@@ -229,7 +241,9 @@ class Yaoyigou:
         self.headers["hesytoken"] = self.hesytoken
         try:
             self.search_data()
+            self._success_report({'is_finished':1,'need_reassign':0,'current_page':1})
         except Exception as e:
             logger.error(f"爬取搜索失败{str(e)}")
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':5})
             self.is_success = False
         return self.pipeline.crawl_count, self.is_success

start_run_yaoshibang_v2.py

@@ -116,7 +116,7 @@ def run_one_task(scheduler, device_id, account_password, account_name):
     logger.info("拉取到任务: %s", json.dumps(log_dict, ensure_ascii=False))
 
     # 根据 snapshot_collect_status 决定走普通爬虫还是快照爬虫
-    snapshot_status = task_dict.get("snapshot_collect_status", 1)
+    snapshot_status = task_dict.get("snapshot_collect_status", 0)
     if snapshot_status == 0:
         spider_cls = YaoShiBangSnapshot
         logger.info("任务 %s 启用快照模式", task_id)

start_run_yaoyigou.py

@@ -1,8 +1,56 @@
 from spiders.yaoyigou.yaoyigou_crawl import Yaoyigou
 from commons.collect_schedule_runner import run_scheduled_loop
-
+from commons.scheduler import CrawlerScheduler
+from commons.Logger import logger
+import time
 PLATFORM_NAME = "药易购"
 PLATFORM_ID = 8
 
+class YYG():
+        
+    def __init__(self):
+        # self.db_online = MySQLPool39()
+        self.crawl_count = ""
+        self.task_id = ""
+        self.task_dict=''
+
+    def run(self):
+
+        spider_schedule =  CrawlerScheduler(1,PLATFORM_ID)
+        spider_schedule.start()
+        time.sleep(3)
+
+        while 1 :
+            print(f'not spider_schedule.end:{not spider_schedule.end}')
+            if(not spider_schedule.end):
+                print('进入工作')
+
+                self.task_dict = spider_schedule.get_task()
+                
+                #self.task_dict = {'id': 149, 'company_id': 0, 'product_name': '藿香正气合剂', 'product_specs': '', 'product_brand': '999', 'product_keyword': '', 'current_page': 0, 'start_offset': 0}
+                print(self.task_dict)
+
+
+
+                if not self.task_dict:
+                    logger.info(f"{PLATFORM_NAME}暂无任务")
+                    time.sleep(35)
+                    continue
+
+                
+                crawl_count, is_success = Yaoyigou(self.task_dict,spider_schedule).run()
+                spider_schedule.stop()
+            else:
+                spider_schedule.start()
+                print('休息')
+                time.sleep(35)
+
+            
+            time.sleep(30)
+
 if __name__ == "__main__":
-    run_scheduled_loop(PLATFORM_NAME, PLATFORM_ID, Yaoyigou)
+
+    import random
+    import time
+    YYG().run()
+