更新 'spiders/yaofangwang/yaofangwang_crawl.py'

spiders/yaofangwang/yaofangwang_crawl.py

@@ -21,15 +21,17 @@ chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
 
 
 class YaofangwangCrawl:
-    def __init__(self, drug_dict=None):
+    def __init__(self, drug_dict=None,scheduler=None):
         self.driver = None
         self.ip = ""
         self.base_url = "https://www.yaofangwang.com"
+
         self.ua = (
             "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
             "AppleWebKit/537.36 (KHTML, like Gecko) "
             "Chrome/124.0.0.0 Safari/537.36"
         )
+        self.scheduler =scheduler
 
         self.platform = "11"
         self.task_dict = drug_dict or {}
@@ -56,7 +58,16 @@ class YaofangwangCrawl:
         self.account_id = self.task_dict.get("collect_equipment_account_id", "")
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round", 1)
+        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':self.scheduler.username,'total_pages':1}
+
+    def _error_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.stop()
+            self.scheduler.post_report(self.report_data)
 
+    def _success_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.post_report(self.report_data)
     @staticmethod
     def _x1(node, xp):
         vals = node.xpath(xp)
@@ -171,12 +182,16 @@ class YaofangwangCrawl:
         co.set_argument("--no-default-browser-check")  # 避免默认浏览器检查
         co.set_user_agent(
             'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36')
-        # co.headless()
+        co.headless()
+
+        co.set_argument('--no-sandbox') 
+
         if self.ip:
             proxy = self.ip.strip()
             if not proxy.startswith(("http://", "https://")):
                 proxy = f"http://{proxy}"
             co.set_argument(f"--proxy-server={proxy}")
+            
         self.driver = ChromiumPage(co)
 
     def parse_html(self, html):
@@ -188,6 +203,7 @@ class YaofangwangCrawl:
         font_match = re.search(r"(/fonts/\w+\.ttf)", html)
         camp_dict = {}
         if font_match:
+
             font_url = self.base_url + font_match.group(1)
             camp_dict = self.get_font(font_url)
 
@@ -318,7 +334,6 @@ class YaofangwangCrawl:
                 "collect_region_id": self.collect_region_id,
                 "collect_round": self.collect_round,
                 "is_sold_out": 0
-
             }
 
             try:
@@ -351,17 +366,12 @@ class YaofangwangCrawl:
     def _search_result_medicine_ids(self, html):
         """从搜索结果 HTML 解析 medicine_id，避免遍历 DrissionPage 元素导致 ElementLostError。"""
         tree = etree.HTML(html)
-
         if tree is None:
             return []
         li_list = tree.xpath("//div[@id='wrap']//ul[contains(@class,'goodlist_search')]/li")
         id_list = []
         for li in li_list:
-            href_raw = self._x1(li, ".//a[@class='txt sc_medicine']/@href")
-            titles = li.xpath(".//a[@class='txt sc_medicine']/@title")
-            title = "".join(titles).replace("\n", "").strip()
-            if self.product not in title:
-                break
+            href_raw = self._x1(li, ".//a/@href")
             if not href_raw:
                 continue
             href = "https:" + href_raw if href_raw.startswith("//") else href_raw
@@ -394,6 +404,8 @@ class YaofangwangCrawl:
         drug_list = self._search_result_medicine_ids(self.driver.html)
         id_dict = {}
         for drug in drug_list:
+            if self.scheduler.end == True:
+                return 
             spec = drug["spec"]
             href = drug["href"]
             if "x" in self.product_desc:
@@ -417,10 +429,20 @@ class YaofangwangCrawl:
             return 0
         try:
             self.init_drissionpage()
+        except Exception as e :
+            print(e)
+            self.scheduler
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':1})
+        try:
             self.search_data()
+            self._success_report({'is_finished':1,'need_reassign':0,'current_page':1})
         except Exception as e:
             print(f"运行异常: {e}")
             self.is_success = False
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':5})
+
+
+
         finally:
             if self.driver:
                 self.driver.quit()