|
|
@@ -1,8 +1,57 @@
|
|
|
from spiders.yaoyigou.yaoyigou_crawl import Yaoyigou
|
|
|
from commons.collect_schedule_runner import run_scheduled_loop
|
|
|
-
|
|
|
+from commons.scheduler import CrawlerScheduler
|
|
|
+from commons.Logger import logger
|
|
|
+import time
|
|
|
PLATFORM_NAME = "药易购"
|
|
|
PLATFORM_ID = 8
|
|
|
|
|
|
+class YYG():
|
|
|
+
|
|
|
+ def __init__(self):
|
|
|
+ # self.db_online = MySQLPool39()
|
|
|
+ self.crawl_count = ""
|
|
|
+ self.task_id = ""
|
|
|
+ self.task_dict=''
|
|
|
+
|
|
|
+ def run(self):
|
|
|
+
|
|
|
+ spider_schedule = CrawlerScheduler(1,PLATFORM_ID)
|
|
|
+ spider_schedule.start()
|
|
|
+ time.sleep(3)
|
|
|
+
|
|
|
+ while 1 :
|
|
|
+ print(f'not spider_schedule.end:{not spider_schedule.end}')
|
|
|
+ if(not spider_schedule.end):
|
|
|
+ print('进入工作')
|
|
|
+
|
|
|
+ self.task_dict = spider_schedule.get_task()
|
|
|
+
|
|
|
+ #self.task_dict = {'id': 149, 'company_id': 0, 'product_name': '藿香正气合剂', 'product_specs': '', 'product_brand': '999', 'product_keyword': '', 'current_page': 0, 'start_offset': 0}
|
|
|
+ self.task_dict = {'id': 149, 'company_id': 0, 'product_name': '感冒灵', 'product_specs': '', 'product_brand': '999', 'product_keyword': '', 'current_page': 0, 'start_offset': 0}
|
|
|
+ print(self.task_dict)
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ if not self.task_dict:
|
|
|
+ logger.info(f"{PLATFORM_NAME}暂无任务")
|
|
|
+ time.sleep(35)
|
|
|
+ continue
|
|
|
+
|
|
|
+
|
|
|
+ crawl_count, is_success = Yaoyigou(self.task_dict,spider_schedule).run()
|
|
|
+ spider_schedule.stop()
|
|
|
+ else:
|
|
|
+ spider_schedule.start()
|
|
|
+ print('休息')
|
|
|
+ time.sleep(35)
|
|
|
+
|
|
|
+
|
|
|
+ time.sleep(30)
|
|
|
+
|
|
|
if __name__ == "__main__":
|
|
|
- run_scheduled_loop(PLATFORM_NAME, PLATFORM_ID, Yaoyigou)
|
|
|
+
|
|
|
+ import random
|
|
|
+ import time
|
|
|
+ YYG().run()
|
|
|
+
|