2.0 药房网

spiders/yaofangwang/yaofangwang_crawl.py

@@ -21,7 +21,7 @@ chrome_path = r"C:\Program Files\Google\Chrome\Application\chrome.exe"
 
 
 class YaofangwangCrawl:
-    def __init__(self, drug_dict=None):
+    def __init__(self, drug_dict=None,scheduler=None):
         self.driver = None
         self.ip = ""
         self.base_url = "https://www.yaofangwang.com"
@@ -30,6 +30,7 @@ class YaofangwangCrawl:
             "AppleWebKit/537.36 (KHTML, like Gecko) "
             "Chrome/124.0.0.0 Safari/537.36"
         )
+        self.scheduler =scheduler
 
         self.platform = "11"
         self.task_dict = drug_dict or {}
@@ -40,6 +41,7 @@ class YaofangwangCrawl:
         self.ossuploader = AliyunOSSUploader()
         self.is_success = True
         self.account_name = ""
+        self.platform = '11'
 
     def get_product_data(self):
         self.task_id = self.task_dict["id"]
@@ -56,7 +58,16 @@ class YaofangwangCrawl:
         self.account_id = self.task_dict.get("collect_equipment_account_id", "")
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round", 1)
+        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':'1','total_pages':1}
 
+    def _error_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.stop()
+            self.scheduler.post_report(self.report_data)
+
+    def _success_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.post_report(self.report_data)
     @staticmethod
     def _x1(node, xp):
         vals = node.xpath(xp)
@@ -171,7 +182,7 @@ class YaofangwangCrawl:
         co.set_argument("--no-default-browser-check")  # 避免默认浏览器检查
         co.set_user_agent(
             'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36')
-        # co.headless()
+        #co.headless()
         if self.ip:
             proxy = self.ip.strip()
             if not proxy.startswith(("http://", "https://")):
@@ -188,6 +199,7 @@ class YaofangwangCrawl:
         font_match = re.search(r"(/fonts/\w+\.ttf)", html)
         camp_dict = {}
         if font_match:
+
             font_url = self.base_url + font_match.group(1)
             camp_dict = self.get_font(font_url)
 
@@ -310,7 +322,6 @@ class YaofangwangCrawl:
                 "product_brand": self.brand or "",
                 "collect_task_id": self.collect_task_id,
                 "search_name": self.product,
-                "company_name": shop,
                 "collect_config_info": json.dumps(
                 {"sampling_cycle": self.sampling_cycle, "sampling_start_time": self.sampling_start_time,
                  "sampling_end_time": self.sampling_end_time}),
@@ -318,7 +329,6 @@ class YaofangwangCrawl:
                 "collect_region_id": self.collect_region_id,
                 "collect_round": self.collect_round,
                 "is_sold_out": 0
-
             }
 
             try:
@@ -351,17 +361,12 @@ class YaofangwangCrawl:
     def _search_result_medicine_ids(self, html):
         """从搜索结果 HTML 解析 medicine_id，避免遍历 DrissionPage 元素导致 ElementLostError。"""
         tree = etree.HTML(html)
-
         if tree is None:
             return []
         li_list = tree.xpath("//div[@id='wrap']//ul[contains(@class,'goodlist_search')]/li")
         id_list = []
         for li in li_list:
-            href_raw = self._x1(li, ".//a[@class='txt sc_medicine']/@href")
-            titles = li.xpath(".//a[@class='txt sc_medicine']/@title")
-            title = "".join(titles).replace("\n", "").strip()
-            if self.product not in title:
-                break
+            href_raw = self._x1(li, ".//a/@href")
             if not href_raw:
                 continue
             href = "https:" + href_raw if href_raw.startswith("//") else href_raw
@@ -417,10 +422,18 @@ class YaofangwangCrawl:
             return 0
         try:
             self.init_drissionpage()
+        except:
+            self.scheduler
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':1})
+        try:
             self.search_data()
+            self._success_report({'is_finished':1,'need_reassign':0,'current_page':1})
         except Exception as e:
             print(f"运行异常: {e}")
             self.is_success = False
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':5})
+
+
         finally:
             if self.driver:
                 self.driver.quit()

start_run_yaofangwang.py

@@ -1,9 +1,54 @@
 from spiders.yaofangwang.yaofangwang_crawl import YaofangwangCrawl
 from commons.collect_schedule_runner import run_scheduled_loop
+from commons.scheduler import CrawlerScheduler
+from commons.Logger import logger
 
 PLATFORM_NAME = "药房网"
 PLATFORM_ID = 11
 
+class YFW():
+        
+    def __init__(self):
+        # self.db_online = MySQLPool39()
+        self.crawl_count = ""
+        self.task_id = ""
+        self.task_dict=''
+
+    def run(self):
+
+        spider_schedule =  CrawlerScheduler(1,11)
+        spider_schedule.start()
+        time.sleep(3)
+
+        while 1 :
+            print(f'not spider_schedule.end:{not spider_schedule.end}')
+            if(not spider_schedule.end):
+                print('进入工作')
+
+                self.task_dict = spider_schedule.get_task()
+                
+                #self.task_dict = {'id': 149, 'company_id': 0, 'product_name': '藿香正气合剂', 'product_specs': '', 'product_brand': '999', 'product_keyword': '', 'current_page': 0, 'start_offset': 0}
+
+                print(self.task_dict)
+
+
+
+                if not self.task_dict:
+                    logger.info(f"{PLATFORM_NAME}暂无任务")
+                    time.sleep(35)
+                    continue
+
+                
+                crawl_count, is_success = YaofangwangCrawl(self.task_dict,spider_schedule).run()
+                spider_schedule.stop()
+            else:
+                spider_schedule.start()
+                print('休息')
+                time.sleep(35)
+
+            
+            time.sleep(30)
+
 TASK_LIST = [
     {"brand": "天和", "product": "骨通贴膏", "spec": ""},
     {"brand": "天和", "product": "追风膏", "spec": ""},
@@ -23,8 +68,7 @@ if __name__ == "__main__":
 
     import random
     import time
-    for task_dict in TASK_LIST[0:12]:
-        YaofangwangCrawl(task_dict).run()
-        time.sleep(random.uniform(300, 600))
+    YFW().run()
     #
     # run_scheduled_loop(PLATFORM_NAME, PLATFORM_ID, YaofangwangCrawl)
+