project/spides/snapshot_jd.py

import json
import time
import requests
from commons.Logger import logger
from commons.conn_mysql import MySQLPoolOnline, MySQLPool39
from spiders.jd.jd_auto_crawl_snap2 import JdCrawlerV2
from commons.scheduler import CrawlerScheduler
from commons.feishu_webhook import send_text
import random
from commons.config import JD_DEVICE_ID
platform_name = "京东"


class JdMain:
    def __init__(self):
        # self.db_online = MySQLPool39()
        self.db_online = MySQLPoolOnline()
        self.crawl_count = ""
        self.task_id = ""
        self.task_dict = None
        self.driver = None
        self.cumulative_pages = 0
        self.cumulative_items = 0
        self.cumulative_stored = 0
        self.cumulative_skipped = 0


    def run(self):

        spider_schedule = CrawlerScheduler(JD_DEVICE_ID, 2)
        spider_schedule.start()
        time.sleep(3)

        while 1:
            if not spider_schedule.end:
                self.task_dict = spider_schedule.get_task()

                if not self.task_dict:
                    logger.info(f"{platform_name}暂无任务")
                    time.sleep(35)
                    continue

                self.task_id = self.task_dict.get("id", "")
                self.crawl_count, is_success, self.driver, self.cumulative_pages, self.cumulative_items, self.cumulative_stored, self.cumulative_skipped = JdCrawlerV2(self.task_dict, spider_schedule, self.driver, self.cumulative_pages, self.cumulative_items, self.cumulative_stored, self.cumulative_skipped).run()
                spider_schedule.stop()

            else:
                spider_schedule.start()
                print('休息')
                time.sleep(35)

            time.sleep(10)


if __name__ == '__main__':
    while True:
        JdMain().run()
        interval_time = random.randint(180, 300)
        logger.info(f"程序睡眠{interval_time}秒后继续执行")
        time.sleep(interval_time)