"""药师帮统一入口 — 根据任务 snapshot_collect_status 字段自动切换普通/快照模式。 与公共类 collect_schedule_runner 不同:本入口直接对接 CrawlerScheduler API (拉取任务/心跳/进度上报),并支持快照模式切换、账号注入等药师帮定制逻辑, 不再依赖 CollectScheduleRunner。 """ import random import time import json from spiders.yaoshibang.ysbang_crawl import YsbSpider from spiders.yaoshibang.ysb_snapshot_crawl import YaoShiBangSnapshot from commons.scheduler import CrawlerScheduler from commons.conn_mysql import MySQLPoolOn2 from commons.Logger import logger from commons.feishu_webhook import send_text PLATFORM_NAME = "药师帮" PLATFORM_ID = 5 def get_device_id_from_db(): """从 accounts_platform 表获取一个可用的药师帮账号(platform=5, status=1, equipment_id=3), 用 phone 作为 DEVICE_ID(回告接口的 username 也用它),同时返回密码供 spider 登录。""" db = MySQLPoolOn2() sql = """SELECT `name`, `phone`, `password` FROM `accounts_platform` WHERE `platform`=5 AND `status`=1 AND `equipment_id`=3 ORDER BY `cookie_timestamp` ASC LIMIT 1""" rows = db.select_data(sql) if not rows: logger.error("无可用药师帮账号(platform=5, status=1, equipment_id=3)") return None account = rows[0] logger.info("获取到药师帮账号: name=%s, phone=%s", account["name"], account["phone"]) return account["phone"], account["password"], account["name"] def _build_task_dict(api_task, device_id, account_password, account_name, scheduler): """将调度 API 返回的原始任务字段组装为 spider 需要的 task_dict。""" current_page = api_task.get("current_page", 0) start_page = current_page if current_page > 0 else 1 task_dict = { # 新 API 字段 "id": api_task.get("id"), "company_id": api_task.get("company_id"), "product_name": api_task.get("product_name", ""), "product_specs": api_task.get("product_specs", ""), "product_brand": api_task.get("product_brand", ""), "product_keyword": api_task.get("product_keyword", ""), "current_page": current_page, "start_offset": api_task.get("start_offset", 0), # 兼容旧字段 "start_page": start_page, "end_page": 0, # spider 逐页回告用 "scheduler": scheduler, "_device_id": device_id, "_platform": str(PLATFORM_ID), "collect_task_id": str(api_task.get("id", "")), "snapshot_collect_status": api_task.get("snapshot_collect_status", 1), "sampling_cycle": "", "sampling_start_time": "", "sampling_end_time": "", "collect_equipment_account_id": "", "collect_region_id": "", "collect_equipment_id": "", "collect_round": 1, # 入口传入的账号信息,spider 可复用避免重复查 DB "account_phone": device_id, "account_password": account_password, "account_name": account_name, } return task_dict def _report_progress(scheduler, task_id, is_finished, crawled_count=0, current_page=0, total_pages=0, exception_type=None, remark=None): """通过调度器 API 上报任务进度。""" data = { "task_id": task_id, "platform": str(PLATFORM_ID), "username": scheduler.username, "current_page": current_page, "is_finished": is_finished, "total_pages": total_pages, } if crawled_count: data["crawled_count"] = crawled_count if exception_type is not None: data["exception_type"] = exception_type if remark: data["remark"] = remark try: scheduler.post_report(data) logger.info("任务状态上报成功: is_finished=%s, crawled_count=%s", is_finished, crawled_count) except Exception as e: logger.warning("任务状态上报失败: %s", e) def run_one_task(scheduler, device_id, account_password, account_name): """拉取一个任务,根据 snapshot_collect_status 选 spider 执行并上报。""" # 拉取任务 api_task = scheduler.get_task() if not api_task: logger.info("%s暂无任务", PLATFORM_NAME) return task_id = api_task.get("id", "") task_dict = _build_task_dict(api_task, device_id, account_password, account_name, scheduler) # 打印任务信息(隐藏不可序列化的 scheduler) log_dict = {k: v for k, v in task_dict.items() if k != "scheduler"} logger.info("拉取到任务: %s", json.dumps(log_dict, ensure_ascii=False)) # 根据 snapshot_collect_status 决定走普通爬虫还是快照爬虫 snapshot_status = task_dict.get("snapshot_collect_status", 1) if snapshot_status == 0: spider_cls = YaoShiBangSnapshot logger.info("任务 %s 启用快照模式", task_id) else: spider_cls = YsbSpider logger.info("任务 %s 使用普通模式", task_id) # 执行爬虫 result = spider_cls(task_dict).run() if len(result) == 3: crawl_count, is_success, total_pages = result else: crawl_count, is_success = result total_pages = 0 # 飞书通知 send_text( f"{time.strftime('%Y-%m-%d %H:%M:%S')} 通知:\n" f"平台: {PLATFORM_NAME}, 药品: {task_dict.get('product_name')}, " f"爬取数据: {crawl_count}条" ) # 上报任务结果 if is_success: _report_progress(scheduler, task_id, is_finished=1, crawled_count=crawl_count, current_page=total_pages, total_pages=total_pages) else: _report_progress(scheduler, task_id, is_finished=0, crawled_count=crawl_count, current_page=total_pages, total_pages=total_pages) if __name__ == "__main__": # 1. 获取账号 result = get_device_id_from_db() if result: device_id, account_password, account_name = result else: device_id, account_password, account_name = "YSB", None, None # 2. 初始化调度器(心跳 + 拉任务 + 上报) scheduler = CrawlerScheduler( DEVICE_ID=device_id, platform=str(PLATFORM_ID), ) scheduler.start() time.sleep(2) # 等待首次心跳完成 logger.info("调度器已启动,device_id=%s,心跳已就绪", device_id) # 3. 循环执行 idle_seconds = random.randint(180, 300) logger.info("循环任务已启动,平台=%s,每轮间隔 %s 秒", PLATFORM_NAME, idle_seconds) while True: try: logger.info("开始执行%s爬虫任务", PLATFORM_NAME) run_one_task(scheduler, device_id, account_password, account_name) logger.info("%s爬虫任务执行完成", PLATFORM_NAME) except Exception as e: logger.error("%s爬虫任务执行失败: %s", PLATFORM_NAME, e, exc_info=True) time.sleep(idle_seconds)