[{"data":1,"prerenderedAt":586},["ShallowReactive",2],{"note:\u002Fprojects\u002Frag-knowledge-base\u002F08-质量门禁与评测闭环":3},{"id":4,"title":5,"body":6,"date":565,"description":566,"draft":567,"extension":568,"featured":567,"kind":569,"lastmod":565,"meta":570,"navigation":414,"path":574,"planned":575,"section":576,"seo":577,"source":578,"stem":579,"tags":580,"toc":414,"weight":584,"__hash__":585},"notes\u002Fprojects\u002Frag-knowledge-base\u002F08-质量门禁与评测闭环.md","质量门禁与评测闭环",{"type":7,"value":8,"toc":555},"minimark",[9,13,21,25,36,42,46,108,115,287,302,306,313,319,326,329,335,342,346,349,355,366,372,376,382,389,393,490,501,504,529,533,539,545,551],[10,11,12],"p",{},"第 1 篇开头留了一个问题：改一行阈值，FAQ 直出准确率从 92% 掉到 71%，没人发现。",[10,14,15,16,20],{},"这一篇就是回答它。RAG 系统的退化是",[17,18,19],"strong",{},"静默","的——不报错、不崩溃、接口 200，\n只是答案慢慢变得不对。所以必须有一套不依赖人的机制，在它变坏的时候亮红灯。",[22,23,24],"h2",{"id":24},"先给结论",[26,27,33],"pre",{"className":28,"code":30,"language":31,"meta":32},[29],"language-text","一道闸管入库   ：脏数据不许进库\n二道闸管效果   ：召回、排序、关键词覆盖、命中路径不能掉\n三道闸管性能   ：首 token 和总耗时不能劣化\n一道闭环管演进 ：线上错例 -> 回归用例 -> 下一轮被门禁拦住\n","text","",[34,35,30],"code",{"__ignoreMap":32},[10,37,38,41],{},[17,39,40],{},"门禁负责拦住退化，闭环负责让系统变好。"," 两者缺一不可。",[22,43,45],{"id":44},"一评测集三种问题三套数据集","一、评测集：三种问题，三套数据集",[47,48,49,65],"table",{},[50,51,52],"thead",{},[53,54,55,59,62],"tr",{},[56,57,58],"th",{},"评测集",[56,60,61],{},"回答什么问题",[56,63,64],{},"文件",[66,67,68,82,95],"tbody",{},[53,69,70,74,77],{},[71,72,73],"td",{},"主链路回归",[71,75,76],{},"本来能答对的问题现在还答得对吗",[71,78,79],{},[34,80,81],{},"eval_sets\u002Fenterprise_knowledge_regression.json",[53,83,84,87,90],{},[71,85,86],{},"多轮追问",[71,88,89],{},"带上下文的追问还接得住吗",[71,91,92],{},[34,93,94],{},"eval_sets\u002Fenterprise_knowledge_followup.json",[53,96,97,100,103],{},[71,98,99],{},"性能基线",[71,101,102],{},"延迟有没有劣化",[71,104,105],{},[34,106,107],{},"eval_sets\u002Fenterprise_knowledge_performance.json",[10,109,110,111,114],{},"回归集的样本是",[17,112,113],{},"带契约的","，不是一堆问题文本：",[26,116,120],{"className":117,"code":118,"language":119,"meta":32,"style":32},"language-json shiki shiki-themes github-light github-dark","{\n  \"case_id\": \"enterprise_vpn_faq\",\n  \"source_filter\": \"it\",\n  \"tenant_id\": \"default\", \"dataset_id\": \"default\",\n  \"visibility\": \"public\", \"user_role\": \"public\",\n  \"query\": \"VPN 连不上应该怎么处理？\",\n  \"expected_hit_type\": \"faq_direct\",\n  \"expected_keywords\": [\"网络\", \"账号\", \"重启\", \"IT 工单\"],\n  \"expected_source_contains\": [\"VPN 连不上应该怎么处理\", \"it_support.md\"]\n}\n","json",[34,121,122,131,148,161,184,206,219,232,262,281],{"__ignoreMap":32},[123,124,127],"span",{"class":125,"line":126},"line",1,[123,128,130],{"class":129},"sVt8B","{\n",[123,132,134,138,141,145],{"class":125,"line":133},2,[123,135,137],{"class":136},"sj4cs","  \"case_id\"",[123,139,140],{"class":129},": ",[123,142,144],{"class":143},"sZZnC","\"enterprise_vpn_faq\"",[123,146,147],{"class":129},",\n",[123,149,151,154,156,159],{"class":125,"line":150},3,[123,152,153],{"class":136},"  \"source_filter\"",[123,155,140],{"class":129},[123,157,158],{"class":143},"\"it\"",[123,160,147],{"class":129},[123,162,164,167,169,172,175,178,180,182],{"class":125,"line":163},4,[123,165,166],{"class":136},"  \"tenant_id\"",[123,168,140],{"class":129},[123,170,171],{"class":143},"\"default\"",[123,173,174],{"class":129},", ",[123,176,177],{"class":136},"\"dataset_id\"",[123,179,140],{"class":129},[123,181,171],{"class":143},[123,183,147],{"class":129},[123,185,187,190,192,195,197,200,202,204],{"class":125,"line":186},5,[123,188,189],{"class":136},"  \"visibility\"",[123,191,140],{"class":129},[123,193,194],{"class":143},"\"public\"",[123,196,174],{"class":129},[123,198,199],{"class":136},"\"user_role\"",[123,201,140],{"class":129},[123,203,194],{"class":143},[123,205,147],{"class":129},[123,207,209,212,214,217],{"class":125,"line":208},6,[123,210,211],{"class":136},"  \"query\"",[123,213,140],{"class":129},[123,215,216],{"class":143},"\"VPN 连不上应该怎么处理？\"",[123,218,147],{"class":129},[123,220,222,225,227,230],{"class":125,"line":221},7,[123,223,224],{"class":136},"  \"expected_hit_type\"",[123,226,140],{"class":129},[123,228,229],{"class":143},"\"faq_direct\"",[123,231,147],{"class":129},[123,233,235,238,241,244,246,249,251,254,256,259],{"class":125,"line":234},8,[123,236,237],{"class":136},"  \"expected_keywords\"",[123,239,240],{"class":129},": [",[123,242,243],{"class":143},"\"网络\"",[123,245,174],{"class":129},[123,247,248],{"class":143},"\"账号\"",[123,250,174],{"class":129},[123,252,253],{"class":143},"\"重启\"",[123,255,174],{"class":129},[123,257,258],{"class":143},"\"IT 工单\"",[123,260,261],{"class":129},"],\n",[123,263,265,268,270,273,275,278],{"class":125,"line":264},9,[123,266,267],{"class":136},"  \"expected_source_contains\"",[123,269,240],{"class":129},[123,271,272],{"class":143},"\"VPN 连不上应该怎么处理\"",[123,274,174],{"class":129},[123,276,277],{"class":143},"\"it_support.md\"",[123,279,280],{"class":129},"]\n",[123,282,284],{"class":125,"line":283},10,[123,285,286],{"class":129},"}\n",[10,288,289,290,293,294,297,298,301],{},"注意 ",[34,291,292],{},"expected_hit_type"," 和 ",[34,295,296],{},"expected_source_contains","：",[17,299,300],{},"它不只断言答案包含什么，\n还断言走了哪条路径、来源是哪个文件。"," 否则「答案碰巧对了，但路径错了」这种退化会被漏掉——\n而路径错误往往意味着下一步就会出错。",[22,303,305],{"id":304},"二四道闸的默认阈值","二、四道闸的默认阈值",[10,307,308,309,312],{},"效果门禁（",[34,310,311],{},"scripts\u002Fquality\u002Fcheck_evaluation_gate.py","）：",[26,314,317],{"className":315,"code":316,"language":31,"meta":32},[29],"Recall@K                 >= 0.8    预期来源有没有被召回\nMRR                      >= 0.6    正确来源排得够不够前\n关键词覆盖率             >= 0.7    答案关键事实是否覆盖\n命中路径准确率           >= 0.7    FAQ \u002F RAG \u002F 边界判断是否正确\nFAQ 直出准确率           >= 0.7\nPrompt Profile 路由准确率 >= 0.7\n场景隔离准确率           == 1.0    越权这类错误零容忍\n错误率                   == 0\n",[34,318,316],{"__ignoreMap":32},[10,320,321,322,325],{},"注意最后两项的特殊待遇：",[17,323,324],{},"场景隔离必须 100%，错误率必须为 0。","\n其余指标可以设阈值容忍波动，但「跨租户串数据」和「样本直接报错」不可以。\n门禁阈值按指标性质分开对待，比一刀切更有意义。",[10,327,328],{},"入库门禁（前一篇讲过）和性能门禁同理：",[26,330,333],{"className":331,"code":332,"language":31,"meta":32},[29],"平均总耗时      \u003C= 15s      首 token 平均 \u003C= 8s\nP95 总耗时      \u003C= 30s      首 token P95  \u003C= 15s\n错误率          == 0\n",[34,334,332],{"__ignoreMap":32},[10,336,337,338,341],{},"首 token 单独设阈值是刻意的：",[17,339,340],{},"流式问答的用户感知发生在首 token，不在总耗时。","\n总耗时 3 秒但首 token 2.5 秒，体感比总耗时 8 秒、首 token 0.8 秒差得多。",[22,343,345],{"id":344},"三闭环让错例变成资产","三、闭环：让错例变成资产",[10,347,348],{},"没有闭环的评测是消耗品——每次发布都要重新发现同一批问题。这里的闭环是三段：",[26,350,353],{"className":351,"code":352,"language":31,"meta":32},[29],"1. 从评测报告里抽取失败样本      scripts\u002Fextract_bad_cases_from_report.py\n2. 人工确认（判断是真错还是标注问题）\n3. 提升为回归用例                scripts\u002Fpromote_bad_cases_to_regression.py\n   -> 下一轮门禁自动拦住它\n",[34,354,352],{"__ignoreMap":32},[10,356,357,358,361,362,365],{},"用户反馈也有对应出口：",[34,359,360],{},"scripts\u002Fexport_feedback_bad_cases.py"," 把差评反馈导出成候选错例。\n关键是第 2 步不能省——",[17,363,364],{},"自动提升会把标注错误一起固化进评测集","，\n之后你会花时间优化一个本来就不该是问题的样本。",[10,367,368,371],{},[34,369,370],{},"scripts\u002Fquality\u002Frun_v1_quality_cycle.py"," 把这一整套串成无人值守循环：\n跑评测 → 出门禁判定 → 归档报告。适合放在 CI 里每日跑一次。",[22,373,375],{"id":374},"四测试体系30-个用例文件覆盖主链路每一环","四、测试体系：30 个用例文件，覆盖主链路每一环",[26,377,380],{"className":378,"code":379,"language":31,"meta":32},[29],"意图与策略    test_intent_and_scenarios \u002F test_intent_decision_gateway \u002F test_intent_model_governance\n检索与生成    test_retrieval_and_prompt \u002F test_answer_confidence \u002F test_stream_query_events\n缓存与性能    test_enterprise_cache \u002F test_retrieval_cache \u002F test_retrieval_warmup_wait\n入库与质量    test_ingestion_incremental_contract \u002F test_quality_gates \u002F test_threshold_calibration\n治理与版本    test_mysql_metadata_stores \u002F test_v1_maintenance_bindings\n",[34,381,379],{"__ignoreMap":32},[10,383,384,385,388],{},"这些用例能跑得快，是因为关键逻辑都是纯函数或带假依赖的（",[34,386,387],{},"tests\u002Fconftest.py","），\n不需要真的起 Milvus 和 LLM。第 6 篇说的「把置信度抽成纯逻辑」，回报就在这里。",[22,390,392],{"id":391},"五交付验收发布前跑一遍","五、交付验收：发布前跑一遍",[26,394,398],{"className":395,"code":396,"language":397,"meta":32,"style":32},"language-powershell shiki shiki-themes github-light github-dark","# 1. 单元与契约测试\npython -m pytest tests -q --tb=short\n\n# 2. 项目护栏：目录结构、约定、禁止事项\npython scripts\u002Fcheck_project_guardrails.py\n\n# 3. 主链路评测 + 效果门禁\npython scripts\u002Fevaluate_core_chain.py --dataset eval_sets\u002Fenterprise_knowledge_regression.json --limit 20 --output reports\u002Fevaluation\u002Fenterprise_knowledge_latest.json\npython scripts\u002Fquality\u002Fcheck_evaluation_gate.py --report reports\u002Fevaluation\u002Fenterprise_knowledge_latest.json\n\n# 4. 追问门禁与性能门禁\npython scripts\u002Fevaluate_followup_chain.py --dataset eval_sets\u002Fenterprise_knowledge_followup.json\npython scripts\u002Fquality\u002Fcollect_performance_baseline.py --dataset eval_sets\u002Fenterprise_knowledge_performance.json --limit 6 --output reports\u002Fperformance\u002Fenterprise_knowledge_latest.json\npython scripts\u002Fquality\u002Fcheck_performance_gate.py --report reports\u002Fperformance\u002Fenterprise_knowledge_latest.json\n\n# 5. 发布校验\npython scripts\u002Fverify_v1_release.py\n","powershell",[34,399,400,405,410,416,421,426,430,435,440,445,449,455,461,467,473,478,484],{"__ignoreMap":32},[123,401,402],{"class":125,"line":126},[123,403,404],{},"# 1. 单元与契约测试\n",[123,406,407],{"class":125,"line":133},[123,408,409],{},"python -m pytest tests -q --tb=short\n",[123,411,412],{"class":125,"line":150},[123,413,415],{"emptyLinePlaceholder":414},true,"\n",[123,417,418],{"class":125,"line":163},[123,419,420],{},"# 2. 项目护栏：目录结构、约定、禁止事项\n",[123,422,423],{"class":125,"line":186},[123,424,425],{},"python scripts\u002Fcheck_project_guardrails.py\n",[123,427,428],{"class":125,"line":208},[123,429,415],{"emptyLinePlaceholder":414},[123,431,432],{"class":125,"line":221},[123,433,434],{},"# 3. 主链路评测 + 效果门禁\n",[123,436,437],{"class":125,"line":234},[123,438,439],{},"python scripts\u002Fevaluate_core_chain.py --dataset eval_sets\u002Fenterprise_knowledge_regression.json --limit 20 --output reports\u002Fevaluation\u002Fenterprise_knowledge_latest.json\n",[123,441,442],{"class":125,"line":264},[123,443,444],{},"python scripts\u002Fquality\u002Fcheck_evaluation_gate.py --report reports\u002Fevaluation\u002Fenterprise_knowledge_latest.json\n",[123,446,447],{"class":125,"line":283},[123,448,415],{"emptyLinePlaceholder":414},[123,450,452],{"class":125,"line":451},11,[123,453,454],{},"# 4. 追问门禁与性能门禁\n",[123,456,458],{"class":125,"line":457},12,[123,459,460],{},"python scripts\u002Fevaluate_followup_chain.py --dataset eval_sets\u002Fenterprise_knowledge_followup.json\n",[123,462,464],{"class":125,"line":463},13,[123,465,466],{},"python scripts\u002Fquality\u002Fcollect_performance_baseline.py --dataset eval_sets\u002Fenterprise_knowledge_performance.json --limit 6 --output reports\u002Fperformance\u002Fenterprise_knowledge_latest.json\n",[123,468,470],{"class":125,"line":469},14,[123,471,472],{},"python scripts\u002Fquality\u002Fcheck_performance_gate.py --report reports\u002Fperformance\u002Fenterprise_knowledge_latest.json\n",[123,474,476],{"class":125,"line":475},15,[123,477,415],{"emptyLinePlaceholder":414},[123,479,481],{"class":125,"line":480},16,[123,482,483],{},"# 5. 发布校验\n",[123,485,487],{"class":125,"line":486},17,[123,488,489],{},"python scripts\u002Fverify_v1_release.py\n",[10,491,492,493,496,497,500],{},"五步全绿，才叫「这次改动没有把系统改坏」。所有报告都落盘到 ",[34,494,495],{},"reports\u002F","，\n管理端还有 ",[34,498,499],{},"\u002Fapi\u002Fadmin\u002Fgate_reports"," 直接查看历史门禁结果。",[22,502,503],{"id":503},"取舍与边界",[505,506,507,517,523],"ul",{},[508,509,510,297,513,516],"li",{},[17,511,512],{},"评测集小则统计意义弱",[34,514,515],{},"--limit 20"," 适合日常快速回归，\n正式发布应该跑全量；样本少于几十条时，1% 的波动可能只是一个样本。",[508,518,519,522],{},[17,520,521],{},"阈值太严会「红灯疲劳」","：天天失败的门禁等于没有门禁。\n阈值应该跟着当前真实水平走，每次优化后上调一点，而不是一开始就设在理想值。",[508,524,525,528],{},[17,526,527],{},"评测集必须跟着业务演进","：业务新增了知识分类，评测集要同步补样本，\n否则你会为一条过期的预期反复调试。",[22,530,532],{"id":531},"收尾这一整套东西到底买到了什么","收尾：这一整套东西到底买到了什么",[10,534,535,536],{},"回头看这 8 篇，SHUN RAG 的每一个设计都在回答同一个问题：\n",[17,537,538],{},"怎么让系统在没人盯着的时候也不出错。",[26,540,543],{"className":541,"code":542,"language":31,"meta":32},[29],"架构分层      让链路可读、可调试、无旁路\n启动校验      让环境问题在启动时暴露，而不是在用户面前\n数据契约      让知识入库有边界\n切分与指纹    让检索准、更新省\n意图与策略    让不同问题走不同的路\n混合检索重排  让正确答案进得了上下文\n置信度与引用  让不该答的就不答\n缓存版本隔离  让多租户、多版本 coexist\n门禁与闭环    让退化被拦住，让错例变资产\n",[34,544,542],{"__ignoreMap":32},[10,546,547,548],{},"如果只带走一句话：",[17,549,550],{},"RAG 的工程量，绝大部分不在模型上，而在这些「不回答」和「不通过」的判断里。",[552,553,554],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":32,"searchDepth":133,"depth":133,"links":556},[557,558,559,560,561,562,563,564],{"id":24,"depth":133,"text":24},{"id":44,"depth":133,"text":45},{"id":304,"depth":133,"text":305},{"id":344,"depth":133,"text":345},{"id":374,"depth":133,"text":375},{"id":391,"depth":133,"text":392},{"id":503,"depth":133,"text":503},{"id":531,"depth":133,"text":532},"2026-07-17","用入库、效果、性能三类门禁和线上错例回流，建立 RAG 系统持续评测与发布验收闭环。",false,"md","project-chapter",{"migration":571},{"generator":572,"sourceSha256":573},"nuxt-site\u002Fscripts\u002Fmigrate-content.mjs","a2c77cf83e3612122c19fb879c397f20c1ecced3b82fb712e4431ecb3dddeb76","\u002Fprojects\u002Frag-knowledge-base\u002F08-质量门禁与评测闭环",null,"projects",{"title":5,"description":566},"content\u002Fprojects\u002Frag-knowledge-base\u002F08-质量门禁与评测闭环.md","projects\u002Frag-knowledge-base\u002F08-质量门禁与评测闭环",[581,582,583],"RAG","评估","MySQL",38,"jhFVVcdjUj3lD5zWCUmuSC-AtgrGudW_qCuBn2rYoVE",1791279145450]