药久久 药师帮 2.0无头兼容

Yjj/scheduler.py

@@ -1,7 +1,6 @@
-import threading
 import time
 import requests
-from commons.Logger import get_spider_logger
+from Logger import get_spider_logger
 import threading
 
 logger = get_spider_logger('scheduler')
@@ -10,7 +9,7 @@ logger = get_spider_logger('scheduler')
 class CrawlerScheduler:
     """爬虫任务调度器"""
 
-    def __init__(self, DEVICE_ID, platform, heartbeat_interval=60):
+    def __init__(self, DEVICE_ID, platform, heartbeat_interval=30):
         """
         初始化调度器
 
@@ -25,7 +24,7 @@ class CrawlerScheduler:
 
         self._lock = threading.Lock()
 
-        self.heartbeat_url = 'http://192.168.2.246:8080/api/collect_task/heartbeat'
+        self.heartbeat_url = 'http://120.24.26.108:8082/api/collect_task/heartbeat'
         self.heartbeat_interval = heartbeat_interval
         self.end = False
         self.heartbeat_thread = None
@@ -47,14 +46,15 @@ class CrawlerScheduler:
                 )
 
                 print(f"[心跳] 发送成功: {response.status_code}")
-                print(response.json())
+                result = response.json()
 
-                if response.json().get('code') != 'success':
-                    logger.error(f'心跳回传：{response.json()}')
+                if self.end or result.get('code') != 'success':
+                    logger.error(f'心跳回传：{result}')
                     self.set_flag(True)
                     # 写日志
                     break
-                logger.info(f'心跳回传：{response.json()}')
+
+                logger.info(f'心跳回传：{result}')
                 self.set_flag(False)
 
                 time.sleep(self.heartbeat_interval)
@@ -75,13 +75,12 @@ class CrawlerScheduler:
 
     def stop(self):
         """停止调度器"""
-
         self.set_flag(True)
         logger.info('心跳停止')
 
     def get_task(self):
         try:
-            task_api = "http://192.168.2.246:8080/api/collect_task/pull"
+            task_api = "http://120.24.26.108:8082/api/collect_task/pull"
 
             headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
 
@@ -102,7 +101,7 @@ class CrawlerScheduler:
 
     def post_report(self, data):
         try:
-            url = "http://192.168.2.246:8080/api/collect_task/report"
+            url = "http://120.24.26.108:8082/api/collect_task/report"
             print('传给返回接口的数据', data)
             logger.info(f'report上传数据：{data}')
             headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
@@ -115,7 +114,6 @@ class CrawlerScheduler:
             if (result.get('code') != 'success'):
                 logger.error(f'翻页回传结果不成功：{result}')
                 self.stop()
-                # 用守护线程 直接杀死该进程
             print(f'任务进度上传 {result}')
         except Exception as e:
             logger.error(e)

Yjj/yjj_v2.py

@@ -18,7 +18,7 @@ from PIL import Image
 import traceback
 import oss2
 from scheduler import CrawlerScheduler
-
+import platform
 # ==================== 调度器配置 ====================
 PLATFORM_ID = "7"  # 药九九平台编码
 
@@ -145,7 +145,7 @@ def init_browser_with_proxy(playwright):
 
     # 启动浏览器（保留原有反爬配置）
     return playwright.chromium.launch(
-        headless=False,  # 非无头模式
+        headless=True,  # 非无头模式
         channel="chrome",  # 使用Chrome内核
         slow_mo=random.randint(100, 300),  # 随机操作延迟
         proxy=proxy_config,  # 代理配置（None则不使用代理）
@@ -1618,8 +1618,10 @@ def main():
     logger.info("调度器已启动，心跳已就绪")
 
     with sync_playwright() as p:
+        is_headless = (platform.system() == "Linux" and not os.environ.get("DISPLAY"))
+
         browser = p.chromium.launch(
-            headless=True,
+            headless=is_headless,  # 根据环境自动切换
             channel="chrome",
             slow_mo=random.randint(100, 300),
             args=_chromium_launch_args(),

commons/scheduler.py

@@ -3,7 +3,6 @@ import requests
 from commons.Logger import get_spider_logger
 import threading
 
-
 logger = get_spider_logger('scheduler')
 
 
@@ -19,17 +18,17 @@ class CrawlerScheduler:
             heartbeat_url: 心跳上报URL
             heartbeat_interval: 心跳间隔时间（秒）
         """
-        
+
         self.username = DEVICE_ID
         self.platform = platform
 
         self._lock = threading.Lock()
-        
-        self.heartbeat_url = 'http://192.168.2.246:8080/api/collect_task/heartbeat'
+
+        self.heartbeat_url = 'http://120.24.26.108:8082/api/collect_task/heartbeat'
         self.heartbeat_interval = heartbeat_interval
         self.end = False
         self.heartbeat_thread = None
-        
+
     def _heartbeat_reporter(self):
         """守护线程：只负责上报心跳"""
         headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
@@ -45,27 +44,26 @@ class CrawlerScheduler:
                     headers=headers,
                     timeout=3
                 )
-                        
 
                 print(f"[心跳] 发送成功: {response.status_code}")
-                result =response.json()
-                
+                result = response.json()
+
                 if self.end or result.get('code') != 'success':
                     logger.error(f'心跳回传：{result}')
                     self.set_flag(True)
                     # 写日志
                     break
-                
+
                 logger.info(f'心跳回传：{result}')
                 self.set_flag(False)
 
                 time.sleep(self.heartbeat_interval)
-                
+
             except Exception as e:
                 print(e)
                 logger.error(e)
                 time.sleep(5)
-    
+
     def start(self):
         """启动调度器"""
         self.set_flag(False)
@@ -74,54 +72,50 @@ class CrawlerScheduler:
             daemon=True
         )
         self.heartbeat_thread.start()
-    
+
     def stop(self):
         """停止调度器"""
         self.set_flag(True)
         logger.info('心跳停止')
 
-
-
     def get_task(self):
         try:
-            task_api = "http://192.168.2.246:8080/api/collect_task/pull"
+            task_api = "http://120.24.26.108:8082/api/collect_task/pull"
+
+            headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
 
-            headers = {'X-Crawler-Token':'zhijiayun_crawler_2026'}
-            
             params = {
-            'platform': self.platform,
-            'username': self.username
+                'platform': self.platform,
+                'username': self.username
             }
-            response = requests.get(task_api, params=params,headers=headers,timeout=5)
+            response = requests.get(task_api, params=params, headers=headers, timeout=5)
             result = response.json()
             logger.info(f'拉取任务：{result}')
-            if result.get('code') =='success':
-                
+            if result.get('code') == 'success':
                 return result.get('data').get('task')
-            
-            
-            print('拉取任务返回',result)
+
+            print('拉取任务返回', result)
 
         except Exception as e:
-            logger.error('获取任务报错',e)
+            logger.error('获取任务报错', e)
 
-    def post_report(self,data):
+    def post_report(self, data):
         try:
-            url = "http://192.168.2.246:8080/api/collect_task/report"
-            print('传给返回接口的数据',data)
+            url = "http://120.24.26.108:8082/api/collect_task/report"
+            print('传给返回接口的数据', data)
             logger.info(f'report上传数据：{data}')
-            headers = {'X-Crawler-Token':'zhijiayun_crawler_2026'}
-            response = requests.post(url, json=data,headers =headers,timeout=5)
-            #记录日志
+            headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
+            response = requests.post(url, json=data, headers=headers, timeout=5)
+            # 记录日志
             result = response.json()
 
             logger.info(result)
 
-            if(result.get('code')!='success'):
+            if (result.get('code') != 'success'):
                 logger.error(f'翻页回传结果不成功：{result}')
                 self.stop()
             print(f'任务进度上传 {result}')
-        except Exception as e :
+        except Exception as e:
             logger.error(e)
 
     def set_flag(self, value):

spiders/yaoshibang/login_yaoshibang.py

@@ -10,6 +10,8 @@ from oss_upload.oss_upload import AliyunOSSUploader
 from commons.config import YSB_ACCOUNT
 from spiders.yaoshibang.yidun_slider_captcha import solve_slider_captcha
 import json
+import os
+import platform
 
 chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
 
@@ -51,19 +53,46 @@ class YaoShiBangLogin:
             s.bind(("127.0.0.1", 0))
             return s.getsockname()[1]
 
+    import platform
+    import os
+
     def init_browser(self):
-        co = ChromiumOptions().set_browser_path(chrome_path)
+        # ---------- 1. 根据操作系统选择 Chrome 路径 ----------
+        system = platform.system()
+        if system == "Windows":
+            chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
+        elif system == "Linux":
+            chrome_path = "/usr/bin/google-chrome"  # 若用 Chromium 则改为 "/usr/bin/chromium-browser"
+        else:  # macOS 或其他
+            chrome_path = None  # 让 DrissionPage 自动尝试查找
+
+        co = ChromiumOptions()
+        if chrome_path:
+            co.set_browser_path(chrome_path)
+
+        # ---------- 2. 用户数据目录（跨平台通用） ----------
+        user_dir = os.path.abspath(f"./spiders/yaoshibang/{self.account_name}")
+        co.set_user_data_path(user_dir)
+
+        # ---------- 3. 调试端口（可选，Windows/Linux 均支持） ----------
         debug_port = self._get_free_port()
-        co.set_user_data_path(f"./spiders/yaoshibang/{self.account_name}")
-
         co.set_local_port(debug_port)
         co.set_argument(f"--remote-debugging-port={debug_port}")
         co.set_argument("--remote-debugging-address=127.0.0.1")
-        # co.set_argument("--disable-blink-features=AutomationControlled")
-        co.set_argument("--disable-dev-shm-usage")
-        co.set_argument("--start-maximized")
-        co.set_argument("--no-first-run")  # 避免首次运行弹窗
-        co.set_argument("--no-default-browser-check")  # 避免默认浏览器检查
+
+        # ---------- 4. 无头模式 + 稳定性参数（全部通过 set_argument 设置，跨平台安全） ----------
+        co.set_argument("--headless")  # 可用 "--headless=new" 替代新版无头
+        co.set_argument("--no-sandbox")  # Linux 必需，Windows 下被 Chrome 忽略
+        co.set_argument("--disable-dev-shm-usage")  # Linux 防 /dev/shm 爆满
+        co.set_argument("--disable-gpu")  # 无头模式下可防 GPU 问题
+        co.set_argument("--disable-setuid-sandbox")
+        co.set_argument("--disable-software-rasterizer")
+        co.set_argument("--disable-extensions")
+        co.set_argument("--disable-plugins")
+        co.set_argument("--no-first-run")
+        co.set_argument("--no-default-browser-check")
+        co.set_argument("--lang=zh-CN")
+
         self.driver = ChromiumPage(co)
 
     def _is_logged_in(self):

spiders/yaoshibang/ysb_snapshot_crawl.py

@@ -21,6 +21,9 @@ from oss_upload.oss_upload import AliyunOSSUploader
 from pipelines.drug_pipelines import DrugPipeline
 from datetime import datetime, timedelta
 from area_info.city_name_to_id import get_city
+import platform
+
+
 
 CAPTCHA_TOKEN = "zPzmt1mG1ouCU6GTzsZN2Lmm8pdZypapPcLJTBRETco"
 CAPTCHA_API_URL = "http://api.jfbym.com/api/YmServer/customApi"
@@ -243,22 +246,44 @@ class YaoShiBangSnapshot:
         return account_dict
 
     def init_browser(self):
-        co = ChromiumOptions().set_browser_path(chrome_path)
-        co.headless(True)
-        debug_port = self._get_free_port()
-        profile_dir = self._resolve_browser_profile_dir()
+        co = ChromiumOptions()
+
+        # ---------- 1. 根据系统选择 Chrome 路径 ----------
+        system = platform.system()
+        if system == "Windows":
+            chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
+            # 如果装在非默认路径，可再追加备选
+        elif system == "Linux":
+            chrome_path = "/usr/bin/google-chrome"  # 或 "/usr/bin/chromium-browser"
+        else:  # macOS 或其他
+            chrome_path = None  # 让 DrissionPage 自动查找
+
+        if chrome_path:
+            co.set_browser_path(chrome_path)
+
+        # ---------- 2. 统一使用 set_argument 设置（避免 set_headless 报错） ----------
+        co.set_argument('--headless=new')  # 启用新版无头模式（Windows/Linux 均支持）
+        co.set_argument('--no-sandbox')  # Linux 必需，Windows 下被忽略，无害
+        co.set_argument('--disable-dev-shm-usage')  # Linux 解决 /dev/shm 问题，Windows 无害
+        co.set_argument('--disable-gpu')  # 无头模式下可减少 GPU 问题，兼容所有系统
+        co.set_argument('--disable-blink-features=AutomationControlled')
+        co.set_argument('--start-maximized')
+        co.set_argument('--no-first-run')
+        co.set_argument('--no-default-browser-check')
+
+        # ---------- 3. 用户数据目录（跨平台） ----------
+        # 例如放在项目目录下的 profiles/ 中，按账号区分
+        profile_dir = Path("./browser_profiles") / self.account_name
         profile_dir.mkdir(parents=True, exist_ok=True)
         co.set_user_data_path(str(profile_dir))
-        logger.info("浏览器用户目录(绝对路径): %s", profile_dir.resolve())
-
-        co.set_local_port(debug_port)
-        co.set_argument(f"--remote-debugging-port={debug_port}")
-        co.set_argument("--remote-debugging-address=127.0.0.1")
-        # co.set_argument("--disable-blink-features=AutomationControlled")
-        co.set_argument("--disable-dev-shm-usage")
-        co.set_argument("--start-maximized")
-        co.set_argument("--no-first-run")  # 避免首次运行弹窗
-        co.set_argument("--no-default-browser-check")  # 避免默认浏览器检查
+
+        # ---------- 4. 可选：本地调试端口（跨平台） ----------
+        # 如果想手动指定端口，可取消注释并自行实现 _get_free_port()
+        # debug_port = self._get_free_port()
+        # co.set_local_port(debug_port)
+        # co.set_argument(f"--remote-debugging-port={debug_port}")
+        # co.set_argument("--remote-debugging-address=127.0.0.1")
+
         self.driver = ChromiumPage(co)
 
     def _solve_slider_captcha(self):