更新 'spiders/yaoyigou/yaoyigou_crawl.py'

spiders/yaoyigou/yaoyigou_crawl.py

@@ -27,7 +27,7 @@ headers = {
 
 
 class Yaoyigou:
-    def __init__(self, task_dict=None):
+    def __init__(self, task_dict=None,scheduler = None,number=3):
         self.hesytoken = None
         self.headers = headers.copy()
         self.platform = 8
@@ -37,10 +37,18 @@ class Yaoyigou:
         self.brand = ""
         self.is_success = True
         self.task_dict = task_dict
-
+        self.scheduler =scheduler
+        self.number = number
         if self.task_dict:
             self.get_product_data()
-
+    def _error_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.stop()
+            self.scheduler.post_report(self.report_data)
+
+    def _success_report(self,data):
+            self.report_data.update(data)
+            self.scheduler.post_report(self.report_data)
     def get_product_data(self):
         self.task_id = self.task_dict["id"]
         self.company_id = self.task_dict["company_id"]
@@ -56,6 +64,10 @@ class Yaoyigou:
         self.account_id = self.task_dict.get("collect_equipment_account_id", "")
         self.collect_region_id = self.task_dict.get("collect_region_id", "")
         self.collect_round = self.task_dict.get("collect_round",1)
+        self.report_data = {'task_id':self.task_id,'platform':self.platform,'username':self.number,'total_pages':1}
+
+
+
 
 
     def _post_json_with_retry(self, url, payload, max_retry=1, timeout=20):
@@ -95,6 +107,7 @@ class Yaoyigou:
                 max_retry=1,
                 timeout=20,
             )
+            print(data_json)
             self.hesytoken = data_json.get('content', "")
         except Exception as e:
             logger.error("登录失败: %s", e)
@@ -208,8 +221,8 @@ class Yaoyigou:
 
             for content in contents:
                 company = content.get("manufacturerName", "")
-                if self.brand not in company:
-                    continue
+                # if self.brand not in company:
+                #     continue
                 try:
                     product = self.parse_product(content)
                     if not product.get("item_id"):
@@ -229,7 +242,9 @@ class Yaoyigou:
         self.headers["hesytoken"] = self.hesytoken
         try:
             self.search_data()
+            self._success_report({'is_finished':1,'need_reassign':0,'current_page':1})
         except Exception as e:
             logger.error(f"爬取搜索失败{str(e)}")
+            self._error_report({'is_finished':0,'need_reassign':1,'current_page':1,'exception_type':5})
             self.is_success = False
         return self.pipeline.crawl_count, self.is_success