""" 文本清洗与纠错 处理 PDF 提取文本中的格式问题 """ import re import logging from typing import Optional logger = logging.getLogger(__name__) class TextCleaner: def __init__(self): self.replacements = [ # 先处理断行连接(在移除不当换行之前),否则这些规则永远无法匹配 (r"([a-z])\n([a-z])", r"\1\2"), # 英文单词断行连接 (r"(\d)\n(\d)", r"\1\2"), # 数字断行连接 (r"(? str: result = self._normalize_width(text) result = self._normalize_units(result) for pattern, replacement in self.replacements: result = re.sub(pattern, replacement, result) return result.strip() def clean_drug_entry(self, text: str) -> str: cleaned = self.clean(text) cleaned = self._fix_common_ocr_errors(cleaned) return cleaned def _fix_common_ocr_errors(self, text: str) -> str: """扩展的药学 OCR 纠错字典,覆盖常见字形混淆和编码问题""" ocr_fixes = { # ---- 已有 ---- "咤": "啶", "唾": "唑", "茶碱": "茶碱", # ---- 化学药常见 OCR 错误 ---- "甘": "苷", # 苦杏仁甘→苦杏仁苷 "甙": "苷", # 异体字规范 "溜": "馏", # 蒸溜→蒸馏 "腊": "蜡", # 腊→蜡 "酿": "酶", # 酿→酶 (OCR) "容": "密", # 容封→密封 "陚": "酯", # 化学式 "肔": "肽", "唼": "哌", "噻昉": "噻吩", "涣": "溴", "氮卓": "氮䓬", "畄": "留", "貭": "质", "千燥": "干燥", "堿": "碱", } for wrong, correct in ocr_fixes.items(): text = text.replace(wrong, correct) return text def _normalize_width(self, text: str) -> str: """全角/半角字符规范化,确保标点和数字一致性""" # 全角 ASCII 标点 → 半角(保留中文逗号句号) width_map = { "!": "!", "?": "?", """: '"', "'": "'", ":": ":", ";": ";", "(": "(", ")": ")", "【": "[", "】": "]", "~": "~", # 全角数字 → ASCII "0": "0", "1": "1", "2": "2", "3": "3", "4": "4", "5": "5", "6": "6", "7": "7", "8": "8", "9": "9", # 全角大写字母 → ASCII "A": "A", "B": "B", "C": "C", "D": "D", "E": "E", "F": "F", "G": "G", "H": "H", "I": "I", "J": "J", "K": "K", "L": "L", "M": "M", "N": "N", "O": "O", "P": "P", "Q": "Q", "R": "R", "S": "S", "T": "T", "U": "U", "V": "V", "W": "W", "X": "X", "Y": "Y", "Z": "Z", "a": "a", "b": "b", "c": "c", "d": "d", "e": "e", "f": "f", "g": "g", "h": "h", "i": "i", "j": "j", "k": "k", "l": "l", "m": "m", "n": "n", "o": "o", "p": "p", "q": "q", "r": "r", "s": "s", "t": "t", "u": "u", "v": "v", "w": "w", "x": "x", "y": "y", "z": "z", } for full, half in width_map.items(): text = text.replace(full, half) return text def _normalize_units(self, text: str) -> str: """药品计量单位规范化""" unit_fixes = [ (r'\bug\b', 'μg'), # ug → μg (r'\buL\b', 'μL'), # uL → μL (r'\bum\b', 'μm'), # um → μm (r'(?i)\bml\b', 'mL'), # ML/ml/Ml → mL (r'(?i)\bul\b', 'μL'), # ul → μL (r'(\d+)h\b', r'\1小时'), # 24h → 24小时 (r'(\d+)min\b', r'\1分钟'), # 30min → 30分钟 ] for pattern, replacement in unit_fixes: text = re.sub(pattern, replacement, text) return text