taobao_detail/detail.py

import json
import time
import copy
from commons.Logger import logger
from commons.conn_mysql import MySQLPoolOnline
from spiders.taobao.taobao_login import (TaobaoAutoCrawl)


db_online = MySQLPoolOnline()

sql = "SELECT *,SUBSTRING_INDEX(platform_item_id, '+', 1) AS pre_id FROM retrieve_scrape_data WHERE scrape_date = '2026-06-13' and enterprise_id=11 and platform_item_id LIKE '%+%' group  by pre_id ;"

data_list =db_online.select_data(sql,)
print(data_list)

taobao_auto = TaobaoAutoCrawl('1', {}, 'n')
driver = taobao_auto.run()
for i in data_list:

    url = i.get('link_url')
    driver.get(url)
    try:
        target = driver.eles('xpath://*[@class="content--DIGuLqdf"]')[-1]

        title_list = target.eles('xpath://span')
        title_list[-1].click()
        time.sleep(1)
        new_data = []
        for j in title_list:
            j.click()
            new_price = (driver.eles('xpath://*[@class="block2--MLcO9YdF"]//*[@class="text--LP7Wf49z"]'))[0].text
            new_url = driver.url
            new_title = j.text

            data = copy.copy(i)
            data.update({'product_name': new_title, 'min_price': new_price, 'link_url': new_url})
            new_data.append(data)
            time.sleep(2)
    except Exception as e:
        logger.exception("页面操作失败, url=%s: %s", url, e)
        continue

    if not new_data:
        continue

    pre_id = data.get('pre_id')

    db_online.execute("START TRANSACTION")
    try:
        if not pre_id:
            logger.warning('pre_id 为空，跳过删除')
        else:
            delete_sql = "DELETE FROM retrieve_scrape_data WHERE platform_item_id LIKE %s AND scrape_date = %s"
            db_online.execute(delete_sql, (f'{pre_id}%', '2026-06-13'))

        table_name = "retrieve_scrape_data"

        for product in new_data:
            del product['pre_id']
            del product['id']
            cols = list(product.keys())
            placeholders = ["%s"] * len(cols)

            sql = f"""
            INSERT INTO `{table_name}` ({','.join([f'`{c}`' for c in cols])})
            VALUES ({','.join(placeholders)})
            """
            db_online.execute(sql, list(product.values()))
            logger.info("%s", json.dumps(product, ensure_ascii=False, default=str))

        db_online.execute("COMMIT")
        logger.info("写入 %d 条, pre_id=%s", len(new_data), pre_id)
    except Exception as e:
        db_online.execute("ROLLBACK")
        logger.exception("事务回滚, pre_id=%s: %s", pre_id, e)
driver.quit()