from spiders.yaofangwang.yaofangwang_crawl import YaofangwangCrawl from commons.collect_schedule_runner import run_scheduled_loop from commons.scheduler import CrawlerScheduler from commons.Logger import logger PLATFORM_NAME = "药房网" PLATFORM_ID = 11 class YFW(): def __init__(self): # self.db_online = MySQLPool39() self.crawl_count = "" self.task_id = "" self.task_dict='' def run(self): spider_schedule = CrawlerScheduler(1,11) spider_schedule.start() time.sleep(3) while 1 : print(f'not spider_schedule.end:{not spider_schedule.end}') if(not spider_schedule.end): print('进入工作') self.task_dict = spider_schedule.get_task() #self.task_dict = {'id': 149, 'company_id': 0, 'product_name': '藿香正气合剂', 'product_specs': '', 'product_brand': '999', 'product_keyword': '', 'current_page': 0, 'start_offset': 0} print(self.task_dict) if not self.task_dict: logger.info(f"{PLATFORM_NAME}暂无任务") time.sleep(35) continue crawl_count, is_success = YaofangwangCrawl(self.task_dict,spider_schedule).run() spider_schedule.stop() else: spider_schedule.start() print('休息') time.sleep(35) time.sleep(30) TASK_LIST = [ {"brand": "天和", "product": "骨通贴膏", "spec": ""}, {"brand": "天和", "product": "追风膏", "spec": ""}, {"brand": "天和", "product": "麝香壮骨膏", "spec": ""}, {"brand": "达因", "product": "维生素D滴剂", "spec": ""}, {"brand": "999", "product": "益血生胶囊", "spec": ""}, {"brand": "999", "product": "尪痹胶囊", "spec": ""}, {"brand": "999", "product": "气滞胃痛颗粒", "spec": ""}, {"brand": "好娃娃", "product": "小儿氨酚烷胺颗粒", "spec": ""}, {"brand": "澳诺", "product": "澳诺葡萄糖酸钙锌口服溶液", "spec": ""}, {"brand": "易善复", "product": "多烯磷脂酰胆碱胶囊", "spec": ""}, {"brand": "达因", "product": "伊可新维生素AD滴剂", "spec": ""}, {"brand": "999", "product": "消痔软膏", "spec": ""}, ] if __name__ == "__main__": import random import time YFW().run() # # run_scheduled_loop(PLATFORM_NAME, PLATFORM_ID, YaofangwangCrawl)