update

commons/conn_mysql.py

@@ -165,7 +165,7 @@ class MySQLPoolOn2(_MySQLPoolBase):
             port=int(os.environ.get("MYSQL_DRUG_PORT", "3307")),
             user=os.environ.get("MYSQL_DRUG_USER", "root"),
             password=os.environ.get("MYSQL_DRUG_PASSWORD", "zhijiayun123456"),
-            database=os.environ.get("MYSQL_DRUG_DB", "drug_retrieve_test"),
+            database=os.environ.get("MYSQL_DRUG_DB", "drug_retrieve"),
         )
         super().__init__(pool)
 

commons/scheduler.py

@@ -3,33 +3,32 @@ import requests
 from commons.Logger import get_spider_logger
 import threading
 
-
 logger = get_spider_logger('scheduler')
 
 
 class CrawlerScheduler:
     """爬虫任务调度器"""
-    
+
     def __init__(self, DEVICE_ID, platform, heartbeat_interval=30):
         """
         初始化调度器
-        
+
         Args:
             platform: 平台名称
             heartbeat_url: 心跳上报URL
             heartbeat_interval: 心跳间隔时间（秒）
         """
-        
+
         self.username = DEVICE_ID
         self.platform = platform
 
         self._lock = threading.Lock()
-        
-        self.heartbeat_url = 'http://120.24.26.108:8082/api/collect_task/heartbeat'
+
+        self.heartbeat_url = 'http://pricesys2.kailin.com.cn:8083/api/collect_task/heartbeat'
         self.heartbeat_interval = heartbeat_interval
         self.end = False
         self.heartbeat_thread = None
-        
+
     def _heartbeat_reporter(self):
         """守护线程：只负责上报心跳"""
         headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
@@ -45,27 +44,26 @@ class CrawlerScheduler:
                     headers=headers,
                     timeout=3
                 )
-                        
 
                 print(f"[心跳] 发送成功: {response.status_code}")
-                result =response.json()
-                
+                result = response.json()
+
                 if self.end or result.get('code') != 'success':
                     logger.error(f'心跳回传：{result}')
                     self.set_flag(True)
                     # 写日志
                     break
-                
+
                 logger.info(f'心跳回传：{result}')
                 self.set_flag(False)
 
                 time.sleep(self.heartbeat_interval)
-                
+
             except Exception as e:
                 print(e)
                 logger.error(e)
                 time.sleep(5)
-    
+
     def start(self):
         """启动调度器"""
         self.set_flag(False)
@@ -74,54 +72,50 @@ class CrawlerScheduler:
             daemon=True
         )
         self.heartbeat_thread.start()
-    
+
     def stop(self):
         """停止调度器"""
         self.set_flag(True)
         logger.info('心跳停止')
 
-
-
     def get_task(self):
         try:
-            task_api = "http://120.24.26.108:8082/api/collect_task/pull"
+            task_api = "http://pricesys2.kailin.com.cn:8083/api/collect_task/pull"
+
+            headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
 
-            headers = {'X-Crawler-Token':'zhijiayun_crawler_2026'}
-            
             params = {
-            'platform': self.platform,
-            'username': self.username
+                'platform': self.platform,
+                'username': self.username
             }
-            response = requests.get(task_api, params=params,headers=headers,timeout=5)
+            response = requests.get(task_api, params=params, headers=headers, timeout=5)
             result = response.json()
             logger.info(f'拉取任务：{result}')
-            if result.get('code') =='success':
-                
+            if result.get('code') == 'success':
                 return result.get('data').get('task')
-            
-            
-            print('拉取任务返回',result)
+
+            print('拉取任务返回', result)
 
         except Exception as e:
-            logger.error('获取任务报错',e)
+            logger.error('获取任务报错', e)
 
-    def post_report(self,data):
+    def post_report(self, data):
         try:
-            url = "http://120.24.26.108:8082/api/collect_task/report"
-            print('传给返回接口的数据',data)
+            url = "http://pricesys2.kailin.com.cn:8083/api/collect_task/report"
+            print('传给返回接口的数据', data)
             logger.info(f'report上传数据：{data}')
-            headers = {'X-Crawler-Token':'zhijiayun_crawler_2026'}
-            response = requests.post(url, json=data,headers =headers,timeout=5)
-            #记录日志
+            headers = {'X-Crawler-Token': 'zhijiayun_crawler_2026'}
+            response = requests.post(url, json=data, headers=headers, timeout=5)
+            # 记录日志
             result = response.json()
 
             logger.info(result)
 
-            if(result.get('code')!='success'):
+            if (result.get('code') != 'success'):
                 logger.error(f'翻页回传结果不成功：{result}')
                 self.stop()
             print(f'任务进度上传 {result}')
-        except Exception as e :
+        except Exception as e:
             logger.error(e)
 
     def set_flag(self, value):

spiders/yaoshibang/ysb_snapshot_crawl.py

@@ -138,7 +138,6 @@ class YaoShiBangSnapshot:
         )
 
     def _report_exception(self, exception_type, remark=""):
-        """异常回告，上报 exception_type (1-6)"""
         scheduler = self.task_dict.get("scheduler")
         if scheduler:
             data = {
@@ -156,6 +155,7 @@ class YaoShiBangSnapshot:
                 data["crawled_count"] = self.pipeline.crawl_count
             scheduler.post_report(data)
 
+
     def _report_page_progress(self, page, crawled_count, is_finished=0, total_pages=0):
         """逐页回告进度，返回 True=code=error 需停止"""
         scheduler = self.task_dict.get("scheduler")
@@ -283,18 +283,14 @@ class YaoShiBangSnapshot:
 
         self.driver = ChromiumPage(co)
 
-    def _solve_slider_captcha(self, yidun=None):
+    def _solve_slider_captcha(self):
         """检测并处理易盾滑块验证码，成功返回 True。"""
         self.driver.wait.doc_loaded()
         time.sleep(2)
 
-        captcha_text = self.driver.ele("xpath://*[contains(text(),'安全验证') or contains(text(),'验证码')]", timeout=1)
-        if not captcha_text:
-            captcha_text = self.driver.ele("xpath://div[@class='yidun_modal']", timeout=2)
-        if not captcha_text:
+        yidun = self.driver.ele("xpath://div[@class='yidun_modal']", timeout=3)
+        if not yidun:
             return True
-        logger.warning("检测到验证码: %s", captcha_text.inner_text()[:50])
-
         logger.info("检测到滑块验证码，开始处理")
         jpg_bytes = yidun.get_screenshot(as_bytes="jpg")
 
@@ -439,9 +435,6 @@ class YaoShiBangSnapshot:
 
     def _goto_detail_page(self, item_id, detail_url):
         """get 后 refresh 一次，让 SPA 按当前 URL 重新渲染详情。"""
-        if not self._solve_slider_captcha():
-            logger.error("验证码处理失败，跳过当前商品")
-            return False
         for attempt in range(1, DETAIL_NAV_RETRIES + 1):
             try:
                 self.driver.get(detail_url, timeout=5)
@@ -780,10 +773,6 @@ class YaoShiBangSnapshot:
             page_start = self.start_page
         self.page = page_start  # 兜底：避免0页时回告 total_pages=0
         for page in range(page_start, 100):
-            if not self._solve_slider_captcha():
-                logger.error("验证码处理失败，停止任务")
-                self._report_exception(3, "验证码处理失败，停止当前任务")
-                return False
             first_search = page == 1
             logger.info("药师帮爬取第%s页 firstSearch=%s stateValue=%s", page, first_search,
                         self._state_value or "(空)")