优化日志查询和ip查询

data-pipeline/migrate_image_urls.py

@@ -0,0 +1,184 @@
+#!/usr/bin/env python3
+"""
+图片 URL 迁移脚本：DB sections/chunks 中的旧图片路径 → 新 hash 文件名路径
+
+同时处理两种可能的 DB 现状（增量安全）：
+  - 旧 SQL 已执行:  /images/甲硝唑_rId9.jpg   →  /images/fa123757_rId9.jpg
+  - 旧 SQL 未执行:  /images/drugs/甲硝唑_rId9.jpg →  /images/fa123757_rId9.jpg
+"""
+
+import re
+import sys
+import os
+import hashlib
+import asyncio
+from pathlib import Path
+
+import asyncpg
+
+
+# ======================================================================
+# 配置
+# ======================================================================
+DB_HOST = os.environ.get("POSTGRES_HOST", "localhost")
+DB_PORT = os.environ.get("POSTGRES_PORT", "5432")
+DB_NAME = os.environ.get("POSTGRES_DB", "pharmacopoeia")
+DB_USER = os.environ.get("POSTGRES_USER", "postgres")
+DB_PASSWORD = os.environ.get("POSTGRES_PASSWORD", "postgres")
+
+# 新旧 URL 前缀
+OLD_PREFIX = "/images/drugs/"
+NEW_PREFIX = "/images/"
+
+# 旧文件名安全的规则（与原 docx_ingest.py 的 _safe_drug_name 一致）
+_RE_SAFE = re.compile(r'[\s（）()　]+')
+
+# 匹配所有旧格式图片 URL（兼容 /images/ 和 /images/drugs/ 两种前缀）
+# group(1) = 旧前缀（中文 safe_name）, group(2) = rId, group(3) = 扩展名
+_OLD_IMG_RE = re.compile(
+    r'/images/(?:drugs/)?'              # /images/ 或 /images/drugs/
+    r'([^"_\s]+)'                        # 旧文件名前缀（中文 safe_name）
+    r'_(rId\d+)'                         # _rIdX 引用
+    r'\.(jpg|jpeg|png|gif|bmp)'          # 扩展名
+)
+
+# 只用于计数查询的 LIKE 模式
+_SEARCH_PATTERN = '/images/%_rId%.'
+
+
+def old_safe_name(drug_name: str) -> str:
+    """与原始 docx_ingest.py 的 _safe_drug_name 一致"""
+    return _RE_SAFE.sub("_", drug_name).strip("_")
+
+
+def new_hash_prefix(drug_name: str) -> str:
+    """新的 hash 前缀（与更新后的 docx_ingest.py 一致）"""
+    return hashlib.md5(drug_name.encode()).hexdigest()[:8]
+
+
+def replace_urls(text: str, mappings: dict[str, str]) -> str:
+    """替换文本中所有旧格式图片 URL → 新 hash 格式。"""
+
+    def _replacer(m: re.Match) -> str:
+        old = m.group(1)          # 旧 safe_name（中文）
+        rId = m.group(2)         # rIdX
+        ext = m.group(3)         # jpg/png/gif
+        new = mappings.get(old)
+        if new is None:
+            # 没匹配到的保留原文件名，但确保前缀是 /images/
+            return f"{NEW_PREFIX}{old}_{rId}.{ext}"
+        return f"{NEW_PREFIX}{new}_{rId}.{ext}"
+
+    return _OLD_IMG_RE.sub(_replacer, text)
+
+
+async def main():
+    # 1. 连接数据库
+    conn = await asyncpg.connect(
+        host=DB_HOST, port=DB_PORT, database=DB_NAME,
+        user=DB_USER, password=DB_PASSWORD,
+    )
+
+    # 2. 查有图片引用的所有药品（同时兼容 /images/drugs/ 和 /images/ 两种状态）
+    drugs = await conn.fetch(f"""
+        SELECT DISTINCT name FROM drugs
+        WHERE sections::text LIKE '%{_SEARCH_PATTERN}%'
+    """)
+    drug_names = [row["name"] for row in drugs]
+    print(f"从 drugs 表找到有图片引用的药品: {len(drug_names)} 个")
+
+    # 3. 构建映射: 旧 safe_name → 新 hash
+    mappings: dict[str, str] = {}
+    for name in drug_names:
+        safe = old_safe_name(name)
+        h = new_hash_prefix(name)
+        mappings[safe] = h
+
+    print(f"映射条目: {len(mappings)}")
+    for i, (old, new) in enumerate(mappings.items()):
+        if i >= 10:
+            print(f"  ... 还有 {len(mappings) - 10} 条")
+            break
+        print(f"  {old} → {new}")
+
+    # 4. 统计待更新量
+    sections_count = await conn.fetchval(f"""
+        SELECT COUNT(*) FROM drugs
+        WHERE sections::text LIKE '%{_SEARCH_PATTERN}%'
+    """)
+    chunks_count = await conn.fetchval(f"""
+        SELECT COUNT(*) FROM drug_chunks
+        WHERE content LIKE '%{_SEARCH_PATTERN}%'
+    """)
+    print(f"\n待更新: drugs={sections_count}  chunks={chunks_count}")
+
+    if sections_count == 0 and chunks_count == 0:
+        print("✅ 无需更新")
+        await conn.close()
+        return
+
+    # 5. 更新 drug_chunks.content（纯文本，直接替换）
+    if chunks_count > 0:
+        print("\n📦 更新 drug_chunks ...")
+        rows = await conn.fetch(f"""
+            SELECT id, content FROM drug_chunks
+            WHERE content LIKE '%{_SEARCH_PATTERN}%'
+        """)
+        updates = []
+        for row in rows:
+            new_content = replace_urls(row["content"], mappings)
+            if new_content != row["content"]:
+                updates.append((new_content, row["id"]))
+        if updates:
+            async with conn.transaction():
+                await conn.executemany(
+                    "UPDATE drug_chunks SET content = $1 WHERE id = $2",
+                    updates,
+                )
+            print(f"  ✅ 更新了 {len(updates)} 条")
+        else:
+            print("  ✅ 无需更新")
+
+    # 6. 更新 drugs.sections（jsonb，需 text ↔ jsonb 转换）
+    if sections_count > 0:
+        print("\n📦 更新 drugs.sections ...")
+        rows = await conn.fetch(f"""
+            SELECT drug_id, sections::text AS txt FROM drugs
+            WHERE sections::text LIKE '%{_SEARCH_PATTERN}%'
+        """)
+        updates = []
+        for row in rows:
+            new_text = replace_urls(row["txt"], mappings)
+            if new_text != row["txt"]:
+                updates.append((new_text, row["drug_id"]))
+        if updates:
+            async with conn.transaction():
+                await conn.executemany(
+                    "UPDATE drugs SET sections = $1::jsonb WHERE drug_id = $2",
+                    updates,
+                )
+            print(f"  ✅ 更新了 {len(updates)} 条")
+        else:
+            print("  ✅ 无需更新")
+
+    # 7. 验证残留：确认无旧格式 URL
+    remaining_d = await conn.fetchval(f"""
+        SELECT COUNT(*) FROM drugs
+        WHERE sections::text LIKE '%{_SEARCH_PATTERN}%'
+    """)
+    remaining_c = await conn.fetchval(f"""
+        SELECT COUNT(*) FROM drug_chunks
+        WHERE content LIKE '%{_SEARCH_PATTERN}%'
+    """)
+
+    print(f"\n残留旧格式: drugs={remaining_d}  chunks={remaining_c}")
+    if remaining_d == 0 and remaining_c == 0:
+        print("✅ 迁移完成")
+    else:
+        print("⚠️  仍有残留，请手动检查")
+
+    await conn.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

database/migrate_image_urls.sql

@@ -1,44 +1,17 @@
 -- ============================================================
--- 图片 URL 迁移：/images/drugs/ → /images/
+-- 图片 URL 迁移（Python 版迁移脚本的配套 SQL）
 -- ============================================================
--- 执行方式:
---   docker exec -i pharmacopoeia-pg psql -U postgres -d pharmacopoeia < database/migrate_image_urls.sql
---
--- 回滚方式:
---   将下面 SQL 中的 /images/" 替换为 /images/drugs/" 再执行一遍
+-- 此 SQL 文件仅供回退参考，实际迁移通过 Python 脚本执行：
+--   cd /opt/pharmacopoeia-ai/data-pipeline && source venv/bin/activate && python migrate_image_urls.py
 -- ============================================================
 
-BEGIN;
+-- 查询当前旧 URL 数据量
+SELECT 'drugs' AS tbl, COUNT(*) FROM drugs WHERE sections::text LIKE '%/images/drugs/%'
+UNION ALL
+SELECT 'drug_chunks', COUNT(*) FROM drug_chunks WHERE content LIKE '%/images/drugs/%';
 
--- 1. 更新 drugs.sections (jsonb) 中的 <img> 标签 URL
---    sections 是 jsonb，需要 text → regexp_replace → jsonb 来回转
-UPDATE drugs
-SET sections = regexp_replace(sections::text, '/images/drugs/', '/images/', 'g')::jsonb
-WHERE sections::text LIKE '%/images/drugs/%';
-
--- 2. 更新 drug_chunks.content (text) 中的 <img> 标签 URL
-UPDATE drug_chunks
-SET content = replace(content, '/images/drugs/', '/images/')
-WHERE content LIKE '%/images/drugs/%';
-
--- 3. 验证：确认无残留旧 URL
-DO $$
-DECLARE
-    drugs_old INT;
-    chunks_old INT;
-BEGIN
-    SELECT COUNT(*) INTO drugs_old FROM drugs
-    WHERE sections::text LIKE '%/images/drugs/%';
-    SELECT COUNT(*) INTO chunks_old FROM drug_chunks
-    WHERE content LIKE '%/images/drugs/%';
-
-    RAISE NOTICE '旧 URL 残留: drugs=%  chunks=%', drugs_old, chunks_old;
-
-    IF drugs_old > 0 OR chunks_old > 0 THEN
-        RAISE WARNING '仍有旧 URL 残留，请检查!';
-    ELSE
-        RAISE NOTICE '✅ 迁移完成，无旧 URL 残留';
-    END IF;
-END $$;
-
-COMMIT;
+-- 手动回滚（极端情况下）：
+-- UPDATE drugs SET sections = regexp_replace(sections::text, '/images/([a-f0-9]{8})_', '/images/drugs/', 'g')::jsonb
+--   WHERE sections::text LIKE '%/images/%';
+-- UPDATE drug_chunks SET content = regexp_replace(content, '/images/([a-f0-9]{8})_', '/images/drugs/', 'g')
+--   WHERE content LIKE '%/images/%' AND content NOT LIKE '%/images/drugs/%';

docs/OPERATIONS_GUIDE.md

@@ -385,7 +385,7 @@ TRUNCATE drugs CASCADE;
 docker exec -i pharmacopoeia-pg psql -U postgres -d pharmacopoeia < database/schema.sql
 
 # 4. 创建图片目录
-mkdir -p static/images/drugs
+mkdir -p /opt/static/images
 
 # 5. 导入 Wiki（~2 分钟）
 cd data-pipeline && source venv/bin/activate
@@ -409,13 +409,51 @@ cd /opt/pharmacopoeia-ai && bash tools/java-deploy.sh restart && nginx -s reload
 
 ---
 
+## 十三、增量更新图片（不改数据库，仅替换图片路径）
+
+> 适用：图片提取逻辑或路径变更，但药品数据无需重建
+
+```bash
+cd /opt/pharmacopoeia-ai
+
+# 1. 拉取最新代码
+git pull origin main
+
+# 2. 清理旧图片目录，提取新图片
+rm -rf /opt/static/images/*
+cd data-pipeline && source venv/bin/activate
+
+# 3. 执行图片提取（生成 hash 文件名的图片 + drug_images.json）
+python extract_images.py
+
+# 4. 迁移 DB 中的旧图片 URL（中文前缀 → hash 前缀，同时去掉 /drugs/ 子目录）
+python migrate_image_urls.py
+
+# 5. 更新 nginx 配置（如未改过）并 reload
+cd /opt/pharmacopoeia-ai
+# 确认 deploy/nginx-pharmacopoeia.conf 中 /images/ alias 指向 /opt/static/images/
+nginx -t && nginx -s reload
+
+# 6. 验证
+curl -s "http://localhost:9000/api/v1/drug/H2025-FA1237" | python3 -c "
+import sys, json
+d = json.load(sys.stdin)
+for k, v in (d.get('sections') or {}).items():
+    if '/images/' in v and 'img' in v:
+        print(f'  [{k}]: {v[:120]}...')
+"
+```
+
+---
+
 ## 快速参考
 
 | 路径/命令 | 用途 |
 |-----------|------|
 | `/opt/pharmacopoeia-ai/` | 项目根目录 |
 | `/opt/2025/` | DOCX 药典数据源 |
-| `/opt/pharmacopoeia-ai/static/images/drugs/` | DOCX 提取的图片 |
+| `/opt/static/images/` | DOCX 提取的图片（新路径） |
+| `/opt/pharmacopoeia-ai/static/images/drugs/` | DOCX 提取的图片（旧路径，已废弃） |
 | `docker-compose ps` | 查看 PG/Redis 状态 |
 | `screen -r reimport` | 重连重导任务 |
 | `http://localhost:9000/api/v1/health` | 健康检查 |