package com.pharmacopoeia.controller; import com.pharmacopoeia.config.QwenProperties; import com.pharmacopoeia.dto.ChatRequest; import com.pharmacopoeia.dto.FeedbackRequest; import com.pharmacopoeia.dto.ImageChatRequest; import com.pharmacopoeia.dto.MultimodalChatRequest; import com.pharmacopoeia.service.*; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.http.codec.ServerSentEvent; import org.springframework.web.bind.annotation.*; import reactor.core.publisher.Flux; import reactor.core.publisher.Sinks; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.multipart.MultipartFile; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @RestController @RequestMapping("/api/v1/chat") public class ChatController { // 复用 PromptService.SECTION_DISPLAY 统一权威映射,避免两处重复定义导致不一致 private final RetrieverService retrieverService; private final LLMService llmService; private final PromptService promptService; private final ChatPersistenceService persistenceService; private final RerankerService rerankerService; private final JdbcTemplate jdbc; private final QwenProperties props; public ChatController(RetrieverService rs, LLMService ls, PromptService ps, ChatPersistenceService cps, RerankerService rrs, JdbcTemplate jdbc, QwenProperties props) { this.retrieverService = rs; this.llmService = ls; this.promptService = ps; this.persistenceService = cps; this.rerankerService = rrs; this.jdbc = jdbc; this.props = props; } @PostMapping("/ask") public ResponseEntity> chatAsk(@RequestBody ChatRequest request) { String query = request.getMessage(); String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); String intent = retrieverService.classifyIntent(query); List> docs = retrieverService.search(query, intent, 20); docs = rerankerService.rerank(docs, query, 5); // 统一:LLM 回答 + 原文对照 List> messages = promptService.buildPrompt(query, docs, intent); String llmAnswer = cleanAnswer(llmService.chat(messages)); String sourceText = buildSourceQuote(docs); String answer = llmAnswer + sourceText; List> sources = buildSources(docs); persistenceService.saveMessage(cid, "user", query, intent, null); persistenceService.saveMessage(cid, "assistant", answer, intent, sources); return ResponseEntity.ok(Map.of( "answer", answer, "sources", sources, "intent", intent, "conversation_id", cid )); } @PostMapping(value = "/stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE) public Flux> chatStream(@RequestBody ChatRequest request) { String query = request.getMessage(); final String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); final String intent = retrieverService.classifyIntent(query); final List> docs = rerankerService.rerank(retrieverService.search(query, intent, 20), query, 5); Sinks.Many> sink = Sinks.many().unicast().onBackpressureBuffer(); try { sink.tryEmitNext(ServerSentEvent.builder().event("intent").data(intent).build()); sink.tryEmitNext(ServerSentEvent.builder().event("status").data("Retrieving...").build()); sink.tryEmitNext(ServerSentEvent.builder().event("status").data("Matched " + docs.size() + " records, generating...").build()); final List> messages = promptService.buildPrompt(query, docs, intent); StringBuilder fullAnswer = new StringBuilder(); llmService.chatStream(messages) .doOnNext(token -> { fullAnswer.append(token); sink.tryEmitNext(ServerSentEvent.builder().data(token).build()); }) .doOnComplete(() -> { // 附加原文参考 String sourceQuote = buildSourceQuote(docs); sink.tryEmitNext(ServerSentEvent.builder().data(sourceQuote).build()); final List> sources = buildSources(docs); try { String meta = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(Map.of( "intent", intent, "sources", sources, "conversation_id", cid )); sink.tryEmitNext(ServerSentEvent.builder().event("meta").data(meta).build()); } catch (Exception ignored) {} String finalAnswer = cleanAnswer(fullAnswer.toString()) + sourceQuote; persistenceService.saveMessage(cid, "user", query, intent, null); persistenceService.saveMessage(cid, "assistant", finalAnswer, intent, sources); sink.tryEmitComplete(); }) .doOnError(e -> sink.tryEmitError(e)) .subscribe(); } catch (Exception e) { sink.tryEmitError(e); } return sink.asFlux(); } // ============================================================ // 图片对话 API(Qwen VL 分析 + OCR → RAG 检索 → 联网搜索) // ============================================================ @PostMapping("/ask-image") public ResponseEntity> chatAskImage(@RequestBody ImageChatRequest request) { String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); // Step 1: Qwen VL 分析图片 + OCR 提取文字 String ocrText = llmService.analyzeImage( request.getImageBase64(), request.getMimeType(), "请分析这张图片,提取其中所有文字信息(OCR),特别是药品名称、成分、用法用量等关键药学信息。简要输出即可。"); // Step 2: 拼接查询 → RAG 检索 String query = (!request.getMessage().isBlank()) ? request.getMessage() + "\n\n(图片OCR提取内容:" + ocrText + ")" : ocrText; String intent = retrieverService.classifyIntent(query); List> docs = retrieverService.search(query, intent, 20); docs = rerankerService.rerank(docs, query, 5); // Step 3: 构建 Prompt(含图片分析上下文)+ 联网搜索 List> messages = promptService.buildPrompt(query, docs, intent); String imageContext = "\n\n【图片分析结果】\n" + ocrText + "\n"; messages.get(0).put("content", messages.get(0).get("content") + imageContext); String llmAnswer = cleanAnswer(llmService.chat(messages, true)); String sourceText = buildSourceQuote(docs); String answer = llmAnswer + sourceText; List> sources = buildSources(docs); persistenceService.saveMessage(cid, "user", request.getMessage().isBlank() ? "[图片]" : "[图片] " + request.getMessage(), intent, null); persistenceService.saveMessage(cid, "assistant", answer, intent, sources); return ResponseEntity.ok(Map.of( "answer", answer, "sources", sources, "intent", intent, "conversation_id", cid )); } @PostMapping(value = "/stream-image", produces = MediaType.TEXT_EVENT_STREAM_VALUE) public Flux> chatStreamImage(@RequestBody ImageChatRequest request) { final String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); Sinks.Many> sink = Sinks.many().unicast().onBackpressureBuffer(); try { sink.tryEmitNext(ServerSentEvent.builder().event("status").data("正在分析图片(OCR 文字识别)...").build()); // Step 1: Qwen VL 分析图片 StringBuilder ocrBuilder = new StringBuilder(); llmService.analyzeImageStream(request.getImageBase64(), request.getMimeType(), "请分析这张图片,提取其中所有文字信息(OCR),特别是药品名称、成分、用法用量等。简要输出。") .doOnNext(ocrBuilder::append) .doOnComplete(() -> { String ocrText = ocrBuilder.toString(); sink.tryEmitNext(ServerSentEvent.builder().event("status").data("图片分析完成,正在检索药典知识库...").build()); // Step 2: 拼接查询 → RAG String query = (!request.getMessage().isBlank()) ? request.getMessage() + "\n\n(图片OCR提取内容:" + ocrText + ")" : ocrText; final String intent = retrieverService.classifyIntent(query); sink.tryEmitNext(ServerSentEvent.builder().event("intent").data(intent).build()); final List> docs = rerankerService.rerank(retrieverService.search(query, intent, 20), query, 5); sink.tryEmitNext(ServerSentEvent.builder().event("status") .data("已匹配 " + docs.size() + " 条药典资料,生成回答中(已启用联网搜索)...").build()); // Step 3: 构建 Prompt + 联网搜索流式生成 final List> messages = promptService.buildPrompt(query, docs, intent); String imageContext = "\n\n【图片分析结果】\n" + ocrText + "\n"; messages.get(0).put("content", messages.get(0).get("content") + imageContext); StringBuilder fullAnswer = new StringBuilder(); llmService.chatStream(messages, true) .doOnNext(token -> { fullAnswer.append(token); sink.tryEmitNext(ServerSentEvent.builder().data(token).build()); }) .doOnComplete(() -> { String sourceQuote = buildSourceQuote(docs); sink.tryEmitNext(ServerSentEvent.builder().data(sourceQuote).build()); final List> sources = buildSources(docs); try { String meta = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(Map.of( "intent", intent, "sources", sources, "conversation_id", cid, "ocr_text", ocrText.length() > 200 ? ocrText.substring(0, 200) : ocrText )); sink.tryEmitNext(ServerSentEvent.builder().event("meta").data(meta).build()); } catch (Exception ignored) {} String finalAnswer = cleanAnswer(fullAnswer.toString()) + sourceQuote; persistenceService.saveMessage(cid, "user", request.getMessage().isBlank() ? "[图片]" : "[图片] " + request.getMessage(), intent, null); persistenceService.saveMessage(cid, "assistant", finalAnswer, intent, sources); sink.tryEmitComplete(); }) .doOnError(sink::tryEmitError) .subscribe(); }) .doOnError(sink::tryEmitError) .subscribe(); } catch (Exception e) { sink.tryEmitError(e); } return sink.asFlux(); } // ============================================================ // 统一多模态对话 API(文本 + 图片 + 视频) // ============================================================ @PostMapping("/ask-multimodal") public ResponseEntity> chatAskMultimodal(@RequestBody MultimodalChatRequest request) { String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); // Step 1: 媒体分析(如果有附件) String ocrText = ""; String mediaLabel = ""; if (request.getMediaBase64() != null && !request.getMediaBase64().isBlank() && request.getMediaType() != null && !request.getMediaType().isBlank()) { mediaLabel = "video".equals(request.getMediaType()) ? "视频" : "图片"; ocrText = llmService.analyzeMedia( request.getMediaBase64(), request.getMediaType(), request.getMediaMime(), ""); } // Step 2: 拼接查询 String query = request.getMessage() != null ? request.getMessage().trim() : ""; if (!query.isEmpty() && !ocrText.isEmpty()) { query = query + "\n\n(" + mediaLabel + "OCR提取内容:" + ocrText + ")"; } else if (!ocrText.isEmpty()) { query = ocrText; } else if (query.isEmpty()) { query = "请介绍一下自己"; } // Step 3: RAG 检索 String intent = retrieverService.classifyIntent(query); List> docs = retrieverService.search(query, intent, 20); docs = rerankerService.rerank(docs, query, 5); // Step 4: 构建 Prompt + 联网搜索 List> messages = promptService.buildPrompt(query, docs, intent); if (!ocrText.isEmpty()) { messages.get(0).put("content", messages.get(0).get("content") + "\n\n【" + mediaLabel + "分析结果】\n" + ocrText + "\n"); } boolean enableSearch = !ocrText.isEmpty() || props.isEnableWebSearch(); String llmAnswer = cleanAnswer(llmService.chat(messages, enableSearch)); String sourceText = buildSourceQuote(docs); String answer = llmAnswer + sourceText; List> sources = buildSources(docs); String userMsg = !request.getMessage().isBlank() ? request.getMessage() : !ocrText.isEmpty() ? "[" + mediaLabel + "]" : request.getMessage(); persistenceService.saveMessage(cid, "user", userMsg, intent, null); persistenceService.saveMessage(cid, "assistant", answer, intent, sources); return ResponseEntity.ok(Map.of( "answer", answer, "sources", sources, "intent", intent, "conversation_id", cid )); } @PostMapping(value = "/stream-multimodal", produces = MediaType.TEXT_EVENT_STREAM_VALUE) public Flux> chatStreamMultimodal(@RequestBody MultimodalChatRequest request) { final String cid = request.getConversationId() != null && !request.getConversationId().isBlank() ? request.getConversationId() : UUID.randomUUID().toString(); final boolean hasMedia = request.getMediaBase64() != null && !request.getMediaBase64().isBlank() && request.getMediaType() != null && !request.getMediaType().isBlank(); final String mediaLabel = hasMedia && "video".equals(request.getMediaType()) ? "视频" : "图片"; Sinks.Many> sink = Sinks.many().unicast().onBackpressureBuffer(); try { if (hasMedia) { sink.tryEmitNext(ServerSentEvent.builder().event("status") .data("🔍 正在分析" + mediaLabel + "...").build()); // 先发 OCR section header sink.tryEmitNext(ServerSentEvent.builder() .data("【📷 " + mediaLabel + "分析】\n\n").build()); StringBuilder ocrBuilder = new StringBuilder(); llmService.analyzeMediaStream(request.getMediaBase64(), request.getMediaType(), request.getMediaMime(), "") .doOnNext(token -> { ocrBuilder.append(token); sink.tryEmitNext(ServerSentEvent.builder().data(token).build()); }) .doOnComplete(() -> { sink.tryEmitNext(ServerSentEvent.builder().data("\n\n").build()); doStreamAnswer(sink, cid, request, ocrBuilder.toString(), mediaLabel); }) .doOnError(sink::tryEmitError) .subscribe(); } else { doStreamAnswer(sink, cid, request, "", ""); } } catch (Exception e) { sink.tryEmitError(e); } return sink.asFlux(); } // ============================================================ // 文件上传 API(multipart → base64 → 复用已有对话管线) // ============================================================ @PostMapping("/upload-image") public ResponseEntity> uploadImage( @RequestParam("file") MultipartFile file, @RequestParam(defaultValue = "") String message, @RequestParam(defaultValue = "") String conversationId) { // 校验 MIME 类型 Set allowed = Set.of("image/jpeg", "image/png", "image/webp", "image/bmp"); String contentType = file.getContentType(); if (contentType == null || !allowed.contains(contentType)) { throw new IllegalArgumentException( "不支持的图片格式: " + contentType + ",支持 jpg/png/webp/bmp"); } // 校验大小 ≤ 10MB if (file.getSize() > 10 * 1024 * 1024) { throw new IllegalArgumentException("图片大小不能超过 10MB"); } // 转 base64 → 委托给 ask-image String base64; try { base64 = Base64.getEncoder().encodeToString(file.getBytes()); } catch (Exception e) { throw new RuntimeException("读取上传文件失败", e); } ImageChatRequest req = new ImageChatRequest(); req.setImageBase64(base64); req.setMimeType(contentType); req.setMessage(message); req.setConversationId( conversationId.isBlank() ? UUID.randomUUID().toString() : conversationId); return chatAskImage(req); } @PostMapping("/upload-media") public ResponseEntity> uploadMedia( @RequestParam("file") MultipartFile file, @RequestParam(defaultValue = "") String message, @RequestParam(defaultValue = "") String conversationId) { String contentType = file.getContentType(); if (contentType == null) { throw new IllegalArgumentException("无法识别的媒体类型"); } String mediaType; long maxSize; if (contentType.startsWith("image/")) { mediaType = "image"; maxSize = 10 * 1024 * 1024; // 10MB } else if (contentType.startsWith("video/")) { mediaType = "video"; maxSize = 50 * 1024 * 1024; // 50MB } else { throw new IllegalArgumentException( "不支持的媒体格式: " + contentType + ",支持 jpg/png/webp/bmp/mp4/mov/avi/webm"); } if (file.getSize() > maxSize) { throw new IllegalArgumentException( "文件大小不能超过 " + (maxSize / 1024 / 1024) + "MB"); } String base64; try { base64 = Base64.getEncoder().encodeToString(file.getBytes()); } catch (Exception e) { throw new RuntimeException("读取上传文件失败", e); } MultimodalChatRequest req = new MultimodalChatRequest(); req.setMessage(message); req.setMediaType(mediaType); req.setMediaBase64(base64); req.setMediaMime(contentType); req.setConversationId( conversationId.isBlank() ? UUID.randomUUID().toString() : conversationId); return chatAskMultimodal(req); } /** 流式多模态:OCR 完成后,走 RAG + 生成 */ private void doStreamAnswer(Sinks.Many> sink, String cid, MultimodalChatRequest request, String ocrText, String mediaLabel) { if (!ocrText.isEmpty()) { sink.tryEmitNext(ServerSentEvent.builder().event("status") .data("📚 检索药典知识库...").build()); } String query = request.getMessage() != null ? request.getMessage().trim() : ""; if (!query.isEmpty() && !ocrText.isEmpty()) { query = query + "\n\n(" + mediaLabel + "OCR提取内容:" + ocrText + ")"; } else if (!ocrText.isEmpty()) { query = ocrText; } else if (query.isEmpty()) { query = "请介绍一下自己"; } final String intent = retrieverService.classifyIntent(query); sink.tryEmitNext(ServerSentEvent.builder().event("intent").data(intent).build()); final List> docs = rerankerService.rerank(retrieverService.search(query, intent, 20), query, 5); final boolean enableSearch = !ocrText.isEmpty() || props.isEnableWebSearch(); sink.tryEmitNext(ServerSentEvent.builder().event("status") .data("已匹配 " + docs.size() + " 条药典资料,生成回答中" + (enableSearch ? "(已启用联网搜索)" : "") + "...").build()); final List> messages = promptService.buildPrompt(query, docs, intent); if (!ocrText.isEmpty()) { messages.get(0).put("content", messages.get(0).get("content") + "\n\n【" + mediaLabel + "分析结果】\n" + ocrText + "\n"); } // 发送回答 section header sink.tryEmitNext(ServerSentEvent.builder().data("\n【📚 药典参考回答】\n\n").build()); StringBuilder fullAnswer = new StringBuilder(); llmService.chatStream(messages, enableSearch) .doOnNext(token -> { fullAnswer.append(token); sink.tryEmitNext(ServerSentEvent.builder().data(token).build()); }) .doOnComplete(() -> { String sourceQuote = buildSourceQuote(docs); sink.tryEmitNext(ServerSentEvent.builder().data(sourceQuote).build()); final List> sources = buildSources(docs); try { String meta = new com.fasterxml.jackson.databind.ObjectMapper().writeValueAsString(Map.of( "intent", intent, "sources", sources, "conversation_id", cid, "ocr_text", ocrText.length() > 200 ? ocrText.substring(0, 200) : ocrText)); sink.tryEmitNext(ServerSentEvent.builder().event("meta").data(meta).build()); } catch (Exception ignored) {} String finalAnswer = cleanAnswer(fullAnswer.toString()) + sourceQuote; String userMsg = !request.getMessage().isBlank() ? request.getMessage() : !ocrText.isEmpty() ? "[" + mediaLabel + "]" : ""; persistenceService.saveMessage(cid, "user", userMsg, intent, null); persistenceService.saveMessage(cid, "assistant", finalAnswer, intent, sources); sink.tryEmitComplete(); }) .doOnError(sink::tryEmitError) .subscribe(); } @GetMapping("/history") public ResponseEntity> getHistory( @RequestParam(defaultValue = "1") int page, @RequestParam(defaultValue = "20") int pageSize) { // getHistory 现在直接返回包含 items/page/page_size/total/total_pages 的 Map var result = persistenceService.getHistory(page, pageSize); return ResponseEntity.ok(result); } @GetMapping("/history/{cid}") public ResponseEntity> getConversationDetail(@PathVariable String cid) { var msgs = persistenceService.getConversationDetail(cid); return ResponseEntity.ok(Map.of("conversation_id", cid, "messages", msgs)); } @PostMapping("/feedback") public ResponseEntity> submitFeedback(@RequestBody FeedbackRequest request) { persistenceService.updateFeedback(request.getMessageId(), request.getFeedback()); return ResponseEntity.ok(Map.of("status", "ok")); } @GetMapping("/admin/conversations") public ResponseEntity> adminListConversations( @RequestParam(defaultValue = "1") int page, @RequestParam(defaultValue = "20") int pageSize, @RequestParam(required = false) String keyword) { int offset = (page - 1) * pageSize; StringBuilder sql = new StringBuilder(""" SELECT DISTINCT ON (c.conversation_id) c.conversation_id, c.title, c.created_at, m.content AS last_msg, m.role FROM conversations c JOIN messages m ON m.conversation_id = c.conversation_id """); List params = new ArrayList<>(); if (keyword != null && !keyword.isBlank()) { sql.append("WHERE m.content ILIKE ? "); params.add("%" + keyword + "%"); } sql.append(""" ORDER BY c.conversation_id, m.created_at DESC LIMIT ? OFFSET ? """); params.add(pageSize); params.add(offset); List> items = jdbc.queryForList( sql.toString(), params.toArray()); return ResponseEntity.ok(Map.of( "items", items, "page", page, "page_size", pageSize )); } private List> buildSources(List> docs) { return docs.stream() .limit(3) // 最多 3 条 .map(d -> { String content = (String) d.getOrDefault("content", ""); String drugName = (String) d.getOrDefault("name", ""); String storedSection = (String) d.getOrDefault("section", ""); String sourceVersion = (String) d.getOrDefault("source_version", ""); String sourceVolume = (String) d.getOrDefault("source_volume", ""); String category = (String) d.getOrDefault("category", ""); // 优先用 DB 元数据,回退到内容解析 if (drugName == null || drugName.isEmpty()) { drugName = extractDrugName(content); } String sectionDisplay = PromptService.SECTION_DISPLAY.getOrDefault(storedSection, storedSection); if (sectionDisplay == null || sectionDisplay.isEmpty()) { sectionDisplay = realSection(content, storedSection); } // 构建完整来源引用 StringBuilder sourceBuilder = new StringBuilder(); if (!sourceVersion.isEmpty()) sourceBuilder.append(sourceVersion); if (!sourceVolume.isEmpty()) { if (!sourceBuilder.isEmpty()) sourceBuilder.append(" "); sourceBuilder.append(sourceVolume); } String src = (String) d.getOrDefault("source", ""); if (!src.isEmpty()) { if (!sourceBuilder.isEmpty()) sourceBuilder.append(" "); sourceBuilder.append(src); } String fullSource = sourceBuilder.toString(); content = content.replaceAll("\\s*来源:.*$", ""); content = content.replaceAll("[\\r\\n]+", " ").trim(); String excerpt = content.length() > 500 ? content.substring(0, 500) + "…" : content; return Map.of( "name", drugName, "section", sectionDisplay, "category", category != null ? category : "", "source", fullSource, "excerpt", excerpt ); }) .collect(Collectors.toList()); } /** 从检索结果提取精简原文,附在 LLM 回答后面作为验证 */ private String buildSourceQuote(List> docs) { StringBuilder sb = new StringBuilder(); sb.append("\n\n\n——— 原文参考 ———\n"); int count = 0; for (Map d : docs) { if (count >= 2) break; String content = (String) d.getOrDefault("content", ""); if (content == null || content.isEmpty()) continue; // 精简:去来源行,截断到 200 字 content = content.replaceAll("\\s*来源:.*$", "").trim(); if (content.length() > 200) { int cut = content.lastIndexOf('。', 200); if (cut < 100) cut = 200; content = content.substring(0, cut + 1); } sb.append(content).append("\n"); count++; } return sb.toString(); } private String extractDrugName(String content) { if (content == null) return ""; int start = content.indexOf("【"); int end = content.indexOf(" - "); if (start >= 0 && end > start) { return content.substring(start + 1, end); } return content.length() > 20 ? content.substring(0, 20) : content; } /** 从 content 文本中提取真实 section(兜底"正文") */ private String realSection(String content, String storedSection) { if (!"正文".equals(storedSection) || content == null) return storedSection; int sep = content.indexOf(" - "); if (sep < 0) return storedSection; int end = content.indexOf("】", sep); if (end > sep) { return content.substring(sep + 3, end).trim(); } return storedSection; } private String cleanAnswer(String text) { if (text == null) return ""; // 去除多余空白行(保留单个换行),修复 Qwen 常见格式问题 return text .replace("\r\n", "\n") .replaceAll("\\n{3,}", "\n\n") .trim(); } }