删除 'spiders/ybm/config.py'

spiders/ybm/config.py

@@ -1,417 +0,0 @@
-# config.py - 药帮忙采集配置
-from datetime import datetime
-
-from dotenv import load_dotenv
-import os
-import oss2
-from PIL import Image
-from commons.Logger import logger
-from conn_mysql import MySQLPoolOnline
-
-
-# 步骤：加载 .env 文件（如需）
-# load_dotenv() 默认读取当前目录下的 .env；如在其他路径可手动指定
-# load_dotenv()
-
-# MySQL：与 conn_mysql.MySQLPoolOnline 一致（默认 drug_retrieve 线上库）。
-# 覆盖方式：环境变量 MYSQL_DRUG_HOST / MYSQL_DRUG_PORT / MYSQL_DRUG_USER /
-# MYSQL_DRUG_PASSWORD / MYSQL_DRUG_DB
-mysql_pool = MySQLPoolOnline()
-
-# 模糊匹配 URL
-def fuzzy_match_product_url_in_db_mysql(product_url):
-    # 先做非空校验
-    if not product_url:
-        logger.warning("鈿狅笍 寰呭尮閰嶇殑 product_url 涓虹┖锛岃烦杩囨暟鎹烘煡")
-        return None
-
-    # 如有需要可转义 % / _，避免 LIKE 通配符影响
-    # escaped_product_url = product_url.replace("%", "\%").replace("_", "\_")
-
-    try:
-        sql = "SELECT * FROM ybm_drug_middle WHERE product_link LIKE %s"
-        rows = mysql_pool.select_data(sql, (f"%{product_url}%",))
-        return rows[0] if rows else None
-    except Exception as e:
-        logger.error(f"MySQL 模糊匹配失败：{str(e)}")
-        return None
-
-
-
-
-# ==================== 从数据库提取任务 ====================
-def get_search_keywords_from_db(platform= 9, include_collect_config=False):
-    """读取待执行任务（status=3），返回单条任务字典；无任务返回空 dict。"""
-    sql = " SELECT * FROM retrieve_collect_task_allocate WHERE status = 1 AND platform = %s LIMIT 1 "
-
-    try:
-        rows = mysql_pool.select_data(sql, (platform,))
-        data = rows[0] if rows else {}
-        logger.debug(
-            "读取待执行任务完成 platform=%s task_id=%s",
-            platform,
-            data.get("id") if data else None,
-        )
-    except Exception as e:
-        logger.error(f"读取待执行任务失败，platform={platform}，错误：{str(e)}")
-        data = {}
-    print(data)
-    return data
-
-
-def has_running_task(platform: int = 9) -> bool:
-    """检查指定 platform 在当天是否存在执行中任务（status=2）。"""
-    try:
-        day_start_ts = int(datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
-        next_day_ts = day_start_ts + 24 * 60 * 60
-        sql = (
-            "SELECT 1 AS one FROM retrieve_collect_task_allocate "
-            "WHERE status = 2 AND platform = %s "
-            "AND update_time >= %s AND update_time < %s LIMIT 1"
-        )
-        rows = mysql_pool.select_data(sql, (platform, day_start_ts, next_day_ts))
-        return bool(rows)
-    except Exception as e:
-        logger.error(f"检查执行中任务失败，platform={platform}，错误：{str(e)}")
-        return False
-
-
-if __name__ == '__main__':
-    get_search_keywords_from_db()
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# ==================== 2. 反爬配置 ====================
-# 随机延迟范围（模拟真人操作）
-MIN_CLICK_DELAY = 1.5  # 点击最小延迟（秒）
-MAX_CLICK_DELAY = 3.5  # 点击最大延迟（秒）
-MIN_INPUT_DELAY = 0.1  # 输入最小延迟（秒）
-MAX_INPUT_DELAY = 0.3  # 输入最大延迟（秒）
-MIN_PAGE_DELAY = 2.0   # 页面最小等待（秒）
-MAX_PAGE_DELAY = 4.0   # 页面最大等待（秒）
-
-# 关键词之间的随机延迟（秒）
-MIN_KEYWORD_DELAY = 8.0
-MAX_KEYWORD_DELAY = 15.0
-
-# 滚动配置（目标 400px，含随机偏移）
-SCROLL_TARGET_DISTANCE = 400  # 目标滚动距离
-SCROLL_OFFSET_RANGE = 50       # 随机偏移范围
-SCROLL_STEP = 50               # 每次滚动步长
-SCROLL_INTERVAL = 0.05         # 步长间隔（秒）
-
-# ==================== 3. Cookie & 登录配置 ====================
-COOKIE_FILE_PATH = "ybm_cookies.json"  # Cookie 保存路径
-# Cookie 有效性验证页面
-LOGIN_VALIDATE_URL = "https://www.ybm100.com/new/"
-
-# 登录账号密码
-USERNAME = "18008650300"
-PASSWORD = "12345678"
-
-
-# USERNAME = "yjj112031"
-# PASSWORD = "123456"
-
-# 登录URL
-TARGET_LOGIN_URL = "https://www.ybm100.com/new/login"
-# "https://www.yyjzt.com/login?redirect=%2FgoodDetail%3FladderNum%26itemStoreId%3D124250306%26sourceProdetail%3D%252Fsearch%26is_store%3D0"
-
-# ==================== 4. 元素选择器配置 ====================
-# 基础选择器
-USERNAME_SELECTOR = "input[placeholder*=请输入账号]"
-PASSWORD_SELECTOR = "input[placeholder*=请输入密码]"
-LOGIN_BTN_SELECTOR = "button:has(span:text('登录'))"
-SEARCH_INPUT_SELECTOR = "input[placeholder*='药品名称/厂家名称']"
-SEARCH_INPUT_SELECTOR2 = "div.home-search-container-search-head"
-SEARCH_BTN_SELECTOR = "div.home-search-container-search-head-btn[data-scmd=\"text-搜索\"]"
-
-# 采集元素选择器可根据页面实际情况调整
-# 如页面结构变化，请优先更新以上选择器
-PRODUCT_ITEM_SELECTOR = "div.product-list-item"         # 商品项容器
-
-PRODUCT_TITLE_SELECTOR = "div.product-name"        # 商品标题
-PRODUCT_PRICE_SELECTOR = "div.main-price"       # 商品价格
-PRODUCT_STORE_SELECTOR = 'div.prduct-shop-name div.shop-name'  # 店铺名称
-PRODUCT_COMPANY_SELECTOR = "div.product-manufacturer"            # 公司名称
-PRODUCT_VALIDITY_SELECTOR = "div.product-period"     # 有效期
-# div.shop-info-container-left-info-name span
-# ==================== 5. 等待时间配置（秒） ====================
-ELEMENT_TIMEOUT = 10000
-LOGIN_AFTER_CLICK = 5000
-SEARCH_BTN_TIMEOUT = 5000
-COLLECT_DELAY = 3000
-DETAIL_LOAD_TIMEOUT = 5000  # 点击商品后等待详情加载时间
-
-# ==================== 6. 浏览器配置 ====================
-BROWSER_HEADLESS = False
-BROWSER_CHANNEL = "chrome"
-SLOW_MO_MIN = 50
-SLOW_MO_MAX = 100
-
-# ==================== 7. CSV 配置 ====================
-CSV_HEADERS = [
-    "商品标题", "商品采购价格", "商品折扣价格", "规格", "盒数",
-    "店铺名称", "公司名称",
-    "有效日期", "生产日期", "批准文号", "采集时间"
-]  # CSV 表头
-
-# 存放营业执照截图路径（如需）
-# cropped_screenshot_path =
-
-
-# 百度 OCR 配置
-request_url_config = "https://aip.baidubce.com/rest/2.0/ocr/v1/business_license"
-
-AppKey_config = "tRK2RhyItCSh6BzyT4CNVXQa"
-AppSecret_config = "TDgKiPo94i2mOM1sDqOuDnlcK1bG66jh"
-token_url_config = 'https://aip.baidubce.com/oauth/2.0/token'
-
-
-
-
-
-# ---------------------- OSS 配置 ----------------------
-OSS_ACCESS_KEY_ID = 'LTAI5tDwjfteBvivYN41r8sJ'
-OSS_ACCESS_KEY_SECRET = 'yowuOGi2nYYnrqGpO3qcz94C4brcPp'
-OSS_ENDPOINT = "oss-cn-shenzhen.aliyuncs.com"
-OSS_BUCKET_NAME = "zhijiayun-jiansuo"
-OSS_PREFIX = "scrape_data/"
-
-
-
-# 本地截图配置
-LOCAL_SCREENSHOT_DIR = "local_screenshots"  # 本地截图目录
-LOCAL_SCREENSHOT_NAME = None  # 自动生成文件名
-LOCAL_CROPPED_DIR = "./local_cropped_screenshots"  # 裁剪后目录
-
-
-# 图片压缩配置
-IMAGE_COMPRESS_ENABLE = True  # 是否启用压缩（True/False）
-IMAGE_COMPRESS_QUALITY = 30  # JPG 质量（1-95）
-IMAGE_COMPRESS_PNG_LEVEL = 9  # PNG 压缩级别（0-9）
-
-
-# ---------------------- 宸ュ叿鍑芥暟 ----------------------
-def init_local_screenshot_dir():
-    """初始化本地截图目录（不存在则创建）。"""
-    if not os.path.exists(LOCAL_SCREENSHOT_DIR):
-        os.makedirs(LOCAL_SCREENSHOT_DIR)
-        logger.info(f"本地截图目录已创建: {LOCAL_SCREENSHOT_DIR}")
-    else:
-        logger.debug(f"本地截图目录已存在: {LOCAL_SCREENSHOT_DIR}")
-
-
-
-def init_oss_bucket():
-    """初始化 OSS Bucket 对象。"""
-    try:
-        auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
-        bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET_NAME)
-        bucket.get_bucket_info()
-        logger.info("OSS Bucket 初始化成功")
-        return bucket
-    except Exception as e:
-        logger.error(f"OSS Bucket 初始化失败: {str(e)}")
-        raise
-
-
-
-def upload_local_screenshot_to_oss(bucket, local_file_path, oss_file_path=None):
-    """将本地截图上传到 OSS，返回公网访问链接。"""
-    if not os.path.exists(local_file_path):
-        raise FileNotFoundError(f"本地截图文件不存在: {local_file_path}")
-
-    if not oss_file_path:
-        local_file_name = os.path.basename(local_file_path)
-        oss_file_path = f"screenshots/{local_file_name}"
-
-    try:
-        bucket.put_object_from_file(oss_file_path, local_file_path)
-        oss_file_url = f"https://{OSS_BUCKET_NAME}.{OSS_ENDPOINT}/{oss_file_path}"
-        logger.info(f"截图上传 OSS 成功: {oss_file_url}")
-        return oss_file_url
-    except Exception as e:
-        logger.error(f"截图上传 OSS 失败: {str(e)}")
-        raise
-
-# ---------------------- 图片裁剪与压缩 ----------------------
-def crop_local_screenshot(local_file_path, cropped_file_path=None, crop_region=None):
-    """裁剪本地截图并保存，成功后删除原图，返回裁剪后文件路径。"""
-    if not os.path.exists(local_file_path):
-        raise FileNotFoundError(f"原始截图文件不存在: {local_file_path}")
-
-    os.makedirs(LOCAL_CROPPED_DIR, exist_ok=True)
-
-    if not cropped_file_path:
-        file_name = os.path.basename(local_file_path)
-        file_name_no_ext, file_ext = os.path.splitext(file_name)
-        cropped_file_name = f"{file_name_no_ext}_cropped{file_ext}"
-        cropped_file_path = os.path.join(LOCAL_CROPPED_DIR, cropped_file_name)
-
-    with Image.open(local_file_path) as img:
-        img_width, img_height = img.size
-        logger.debug(f"原图尺寸: width={img_width}, height={img_height}")
-
-        if not crop_region:
-            left = 0
-            upper = 0
-            right = int(img_width)
-            lower = int(img_height * 0.3)
-            crop_region = (left, upper, right, lower)
-            logger.debug(f"未指定裁剪区域，使用默认区域: {crop_region}")
-
-        c_left, c_upper, c_right, c_lower = crop_region
-        if c_right > img_width or c_lower > img_height or c_left < 0 or c_upper < 0:
-            raise ValueError(
-                f"裁剪区域超出图片范围，图片尺寸=({img_width}, {img_height})，裁剪区域={crop_region}"
-            )
-
-        cropped_img = img.crop(crop_region)
-
-        file_ext = os.path.splitext(cropped_file_path)[1].lower()
-        try:
-            if IMAGE_COMPRESS_ENABLE:
-                if file_ext in ['.jpg', '.jpeg']:
-                    cropped_img.save(
-                        cropped_file_path,
-                        format='JPEG',
-                        quality=IMAGE_COMPRESS_QUALITY,
-                        optimize=True,
-                        progressive=True,
-                    )
-                else:
-                    cropped_img.save(cropped_file_path)
-                logger.info(f"裁剪图片已保存（压缩开启）: {cropped_file_path}")
-            else:
-                cropped_img.save(cropped_file_path, format='JPEG')
-                logger.info(f"裁剪图片已保存（压缩关闭）: {cropped_file_path}")
-        except Exception as e:
-            cropped_img.save(cropped_file_path, format='JPEG')
-            logger.warning(f"图片压缩失败，已按普通 JPEG 保存: {str(e)}")
-
-    try:
-        if os.path.exists(cropped_file_path):
-            os.remove(local_file_path)
-            logger.debug(f"已删除原始截图: {local_file_path}")
-        else:
-            logger.warning(f"裁剪文件不存在，跳过删除原图: {cropped_file_path}")
-    except OSError as e:
-        logger.warning(f"删除原始截图失败: {str(e)}")
-
-    return cropped_file_path
-
-
-def screenshot_target_page_to_local_then_oss(target_page, local_file_path=None, oss_file_path=None, full_page=True, crop_region=None):
-    """页面截图到本地后裁剪，再上传 OSS，返回(裁剪路径, OSS链接)。"""
-    os.makedirs(LOCAL_SCREENSHOT_DIR, exist_ok=True)
-
-    if not local_file_path:
-        current_time = datetime.now().strftime("%Y%m%d_%H%M%S")
-        local_file_name = f"{current_time}_target_page.jpg"
-        local_file_path = os.path.join(LOCAL_SCREENSHOT_DIR, local_file_name)
-
-    logger.info(f"开始页面截图: {local_file_path}")
-    target_page.screenshot(
-        path=local_file_path,
-        full_page=full_page,
-        omit_background=False,
-        timeout=10000,
-    )
-    logger.debug("页面截图完成")
-
-    cropped_file_path = crop_local_screenshot(
-        local_file_path=local_file_path,
-        crop_region=crop_region,
-    )
-
-    bucket = init_oss_bucket()
-    oss_file_url = upload_local_screenshot_to_oss(bucket, cropped_file_path, oss_file_path)
-
-    return cropped_file_path, oss_file_url