craw_spider/spiders/ybm/config.py

# config.py - 药帮忙采集配置
from datetime import datetime

from dotenv import load_dotenv
import os
import oss2
from PIL import Image
from commons.Logger import logger
from conn_mysql import MySQLPoolOnline


# 步骤：加载 .env 文件（如需）
# load_dotenv() 默认读取当前目录下的 .env；如在其他路径可手动指定
# load_dotenv()

# MySQL：与 conn_mysql.MySQLPoolOnline 一致（默认 drug_retrieve 线上库）。
# 覆盖方式：环境变量 MYSQL_DRUG_HOST / MYSQL_DRUG_PORT / MYSQL_DRUG_USER /
# MYSQL_DRUG_PASSWORD / MYSQL_DRUG_DB
mysql_pool = MySQLPoolOnline()

# 模糊匹配 URL
def fuzzy_match_product_url_in_db_mysql(product_url):
    # 先做非空校验
    if not product_url:
        logger.warning("鈿狅笍 寰呭尮閰嶇殑 product_url 涓虹┖锛岃烦杩囨暟鎹烘煡")
        return None

    # 如有需要可转义 % / _，避免 LIKE 通配符影响
    # escaped_product_url = product_url.replace("%", "\%").replace("_", "\_")

    try:
        sql = "SELECT * FROM ybm_drug_middle WHERE product_link LIKE %s"
        rows = mysql_pool.select_data(sql, (f"%{product_url}%",))
        return rows[0] if rows else None
    except Exception as e:
        logger.error(f"MySQL 模糊匹配失败：{str(e)}")
        return None




# ==================== 从数据库提取任务 ====================
def get_search_keywords_from_db(platform= 9, include_collect_config=False):
    """读取待执行任务（status=3），返回单条任务字典；无任务返回空 dict。"""
    sql = " SELECT * FROM retrieve_collect_task_allocate WHERE status = 1 AND platform = %s LIMIT 1 "

    try:
        rows = mysql_pool.select_data(sql, (platform,))
        data = rows[0] if rows else {}
        logger.debug(
            "读取待执行任务完成 platform=%s task_id=%s",
            platform,
            data.get("id") if data else None,
        )
    except Exception as e:
        logger.error(f"读取待执行任务失败，platform={platform}，错误：{str(e)}")
        data = {}
    print(data)
    return data


def has_running_task(platform: int = 9) -> bool:
    """检查指定 platform 在当天是否存在执行中任务（status=2）。"""
    try:
        day_start_ts = int(datetime.now().replace(hour=0, minute=0, second=0, microsecond=0).timestamp())
        next_day_ts = day_start_ts + 24 * 60 * 60
        sql = (
            "SELECT 1 AS one FROM retrieve_collect_task_allocate "
            "WHERE status = 2 AND platform = %s "
            "AND update_time >= %s AND update_time < %s LIMIT 1"
        )
        rows = mysql_pool.select_data(sql, (platform, day_start_ts, next_day_ts))
        return bool(rows)
    except Exception as e:
        logger.error(f"检查执行中任务失败，platform={platform}，错误：{str(e)}")
        return False


if __name__ == '__main__':
    get_search_keywords_from_db()

























































































# ==================== 2. 反爬配置 ====================
# 随机延迟范围（模拟真人操作）
MIN_CLICK_DELAY = 1.5  # 点击最小延迟（秒）
MAX_CLICK_DELAY = 3.5  # 点击最大延迟（秒）
MIN_INPUT_DELAY = 0.1  # 输入最小延迟（秒）
MAX_INPUT_DELAY = 0.3  # 输入最大延迟（秒）
MIN_PAGE_DELAY = 2.0   # 页面最小等待（秒）
MAX_PAGE_DELAY = 4.0   # 页面最大等待（秒）

# 关键词之间的随机延迟（秒）
MIN_KEYWORD_DELAY = 8.0
MAX_KEYWORD_DELAY = 15.0

# 滚动配置（目标 400px，含随机偏移）
SCROLL_TARGET_DISTANCE = 400  # 目标滚动距离
SCROLL_OFFSET_RANGE = 50       # 随机偏移范围
SCROLL_STEP = 50               # 每次滚动步长
SCROLL_INTERVAL = 0.05         # 步长间隔（秒）

# ==================== 3. Cookie & 登录配置 ====================
COOKIE_FILE_PATH = "ybm_cookies.json"  # Cookie 保存路径
# Cookie 有效性验证页面
LOGIN_VALIDATE_URL = "https://www.ybm100.com/new/"

# 登录账号密码
USERNAME = "18008650300"
PASSWORD = "12345678"


# USERNAME = "yjj112031"
# PASSWORD = "123456"

# 登录URL
TARGET_LOGIN_URL = "https://www.ybm100.com/new/login"
# "https://www.yyjzt.com/login?redirect=%2FgoodDetail%3FladderNum%26itemStoreId%3D124250306%26sourceProdetail%3D%252Fsearch%26is_store%3D0"

# ==================== 4. 元素选择器配置 ====================
# 基础选择器
USERNAME_SELECTOR = "input[placeholder*=请输入账号]"
PASSWORD_SELECTOR = "input[placeholder*=请输入密码]"
LOGIN_BTN_SELECTOR = "button:has(span:text('登录'))"
SEARCH_INPUT_SELECTOR = "input[placeholder*='药品名称/厂家名称']"
SEARCH_INPUT_SELECTOR2 = "div.home-search-container-search-head"
SEARCH_BTN_SELECTOR = "div.home-search-container-search-head-btn[data-scmd=\"text-搜索\"]"

# 采集元素选择器可根据页面实际情况调整
# 如页面结构变化，请优先更新以上选择器
PRODUCT_ITEM_SELECTOR = "div.product-list-item"         # 商品项容器

PRODUCT_TITLE_SELECTOR = "div.product-name"        # 商品标题
PRODUCT_PRICE_SELECTOR = "div.main-price"       # 商品价格
PRODUCT_STORE_SELECTOR = 'div.prduct-shop-name div.shop-name'  # 店铺名称
PRODUCT_COMPANY_SELECTOR = "div.product-manufacturer"            # 公司名称
PRODUCT_VALIDITY_SELECTOR = "div.product-period"     # 有效期
# div.shop-info-container-left-info-name span
# ==================== 5. 等待时间配置（秒） ====================
ELEMENT_TIMEOUT = 10000
LOGIN_AFTER_CLICK = 5000
SEARCH_BTN_TIMEOUT = 5000
COLLECT_DELAY = 3000
DETAIL_LOAD_TIMEOUT = 5000  # 点击商品后等待详情加载时间

# ==================== 6. 浏览器配置 ====================
BROWSER_HEADLESS = False
BROWSER_CHANNEL = "chrome"
SLOW_MO_MIN = 50
SLOW_MO_MAX = 100

# ==================== 7. CSV 配置 ====================
CSV_HEADERS = [
    "商品标题", "商品采购价格", "商品折扣价格", "规格", "盒数",
    "店铺名称", "公司名称",
    "有效日期", "生产日期", "批准文号", "采集时间"
]  # CSV 表头

# 存放营业执照截图路径（如需）
# cropped_screenshot_path =


# 百度 OCR 配置
request_url_config = "https://aip.baidubce.com/rest/2.0/ocr/v1/business_license"

AppKey_config = "tRK2RhyItCSh6BzyT4CNVXQa"
AppSecret_config = "TDgKiPo94i2mOM1sDqOuDnlcK1bG66jh"
token_url_config = 'https://aip.baidubce.com/oauth/2.0/token'





# ---------------------- OSS 配置 ----------------------
OSS_ACCESS_KEY_ID = 'LTAI5tDwjfteBvivYN41r8sJ'
OSS_ACCESS_KEY_SECRET = 'yowuOGi2nYYnrqGpO3qcz94C4brcPp'
OSS_ENDPOINT = "oss-cn-shenzhen.aliyuncs.com"
OSS_BUCKET_NAME = "zhijiayun-jiansuo"
OSS_PREFIX = "scrape_data/"



# 本地截图配置
LOCAL_SCREENSHOT_DIR = "local_screenshots"  # 本地截图目录
LOCAL_SCREENSHOT_NAME = None  # 自动生成文件名
LOCAL_CROPPED_DIR = "./local_cropped_screenshots"  # 裁剪后目录


# 图片压缩配置
IMAGE_COMPRESS_ENABLE = True  # 是否启用压缩（True/False）
IMAGE_COMPRESS_QUALITY = 30  # JPG 质量（1-95）
IMAGE_COMPRESS_PNG_LEVEL = 9  # PNG 压缩级别（0-9）


# ---------------------- 宸ュ叿鍑芥暟 ----------------------
def init_local_screenshot_dir():
    """初始化本地截图目录（不存在则创建）。"""
    if not os.path.exists(LOCAL_SCREENSHOT_DIR):
        os.makedirs(LOCAL_SCREENSHOT_DIR)
        logger.info(f"本地截图目录已创建: {LOCAL_SCREENSHOT_DIR}")
    else:
        logger.debug(f"本地截图目录已存在: {LOCAL_SCREENSHOT_DIR}")



def init_oss_bucket():
    """初始化 OSS Bucket 对象。"""
    try:
        auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
        bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET_NAME)
        bucket.get_bucket_info()
        logger.info("OSS Bucket 初始化成功")
        return bucket
    except Exception as e:
        logger.error(f"OSS Bucket 初始化失败: {str(e)}")
        raise



def upload_local_screenshot_to_oss(bucket, local_file_path, oss_file_path=None):
    """将本地截图上传到 OSS，返回公网访问链接。"""
    if not os.path.exists(local_file_path):
        raise FileNotFoundError(f"本地截图文件不存在: {local_file_path}")

    if not oss_file_path:
        local_file_name = os.path.basename(local_file_path)
        oss_file_path = f"screenshots/{local_file_name}"

    try:
        bucket.put_object_from_file(oss_file_path, local_file_path)
        oss_file_url = f"https://{OSS_BUCKET_NAME}.{OSS_ENDPOINT}/{oss_file_path}"
        logger.info(f"截图上传 OSS 成功: {oss_file_url}")
        return oss_file_url
    except Exception as e:
        logger.error(f"截图上传 OSS 失败: {str(e)}")
        raise

# ---------------------- 图片裁剪与压缩 ----------------------
def crop_local_screenshot(local_file_path, cropped_file_path=None, crop_region=None):
    """裁剪本地截图并保存，成功后删除原图，返回裁剪后文件路径。"""
    if not os.path.exists(local_file_path):
        raise FileNotFoundError(f"原始截图文件不存在: {local_file_path}")

    os.makedirs(LOCAL_CROPPED_DIR, exist_ok=True)

    if not cropped_file_path:
        file_name = os.path.basename(local_file_path)
        file_name_no_ext, file_ext = os.path.splitext(file_name)
        cropped_file_name = f"{file_name_no_ext}_cropped{file_ext}"
        cropped_file_path = os.path.join(LOCAL_CROPPED_DIR, cropped_file_name)

    with Image.open(local_file_path) as img:
        img_width, img_height = img.size
        logger.debug(f"原图尺寸: width={img_width}, height={img_height}")

        if not crop_region:
            left = 0
            upper = 0
            right = int(img_width)
            lower = int(img_height * 0.3)
            crop_region = (left, upper, right, lower)
            logger.debug(f"未指定裁剪区域，使用默认区域: {crop_region}")

        c_left, c_upper, c_right, c_lower = crop_region
        if c_right > img_width or c_lower > img_height or c_left < 0 or c_upper < 0:
            raise ValueError(
                f"裁剪区域超出图片范围，图片尺寸=({img_width}, {img_height})，裁剪区域={crop_region}"
            )

        cropped_img = img.crop(crop_region)

        file_ext = os.path.splitext(cropped_file_path)[1].lower()
        try:
            if IMAGE_COMPRESS_ENABLE:
                if file_ext in ['.jpg', '.jpeg']:
                    cropped_img.save(
                        cropped_file_path,
                        format='JPEG',
                        quality=IMAGE_COMPRESS_QUALITY,
                        optimize=True,
                        progressive=True,
                    )
                else:
                    cropped_img.save(cropped_file_path)
                logger.info(f"裁剪图片已保存（压缩开启）: {cropped_file_path}")
            else:
                cropped_img.save(cropped_file_path, format='JPEG')
                logger.info(f"裁剪图片已保存（压缩关闭）: {cropped_file_path}")
        except Exception as e:
            cropped_img.save(cropped_file_path, format='JPEG')
            logger.warning(f"图片压缩失败，已按普通 JPEG 保存: {str(e)}")

    try:
        if os.path.exists(cropped_file_path):
            os.remove(local_file_path)
            logger.debug(f"已删除原始截图: {local_file_path}")
        else:
            logger.warning(f"裁剪文件不存在，跳过删除原图: {cropped_file_path}")
    except OSError as e:
        logger.warning(f"删除原始截图失败: {str(e)}")

    return cropped_file_path


def screenshot_target_page_to_local_then_oss(target_page, local_file_path=None, oss_file_path=None, full_page=True, crop_region=None):
    """页面截图到本地后裁剪，再上传 OSS，返回(裁剪路径, OSS链接)。"""
    os.makedirs(LOCAL_SCREENSHOT_DIR, exist_ok=True)

    if not local_file_path:
        current_time = datetime.now().strftime("%Y%m%d_%H%M%S")
        local_file_name = f"{current_time}_target_page.jpg"
        local_file_path = os.path.join(LOCAL_SCREENSHOT_DIR, local_file_name)

    logger.info(f"开始页面截图: {local_file_path}")
    target_page.screenshot(
        path=local_file_path,
        full_page=full_page,
        omit_background=False,
        timeout=10000,
    )
    logger.debug("页面截图完成")

    cropped_file_path = crop_local_screenshot(
        local_file_path=local_file_path,
        crop_region=crop_region,
    )

    bucket = init_oss_bucket()
    oss_file_url = upload_local_screenshot_to_oss(bucket, cropped_file_path, oss_file_path)

    return cropped_file_path, oss_file_url