[{"data":1,"prerenderedAt":445},["ShallowReactive",2],{"note:\u002Fprojects\u002Frag-knowledge-base\u002F03-知识入库与增量指纹":3},{"id":4,"title":5,"body":6,"date":424,"description":425,"draft":426,"extension":427,"featured":426,"kind":428,"lastmod":424,"meta":429,"navigation":86,"path":433,"planned":434,"section":435,"seo":436,"source":437,"stem":438,"tags":439,"toc":86,"weight":443,"__hash__":444},"notes\u002Fprojects\u002Frag-knowledge-base\u002F03-知识入库与增量指纹.md","知识入库与增量指纹",{"type":7,"value":8,"toc":414},"minimark",[9,13,16,20,23,34,38,48,112,130,133,150,161,165,168,247,261,265,271,274,280,295,299,305,308,312,319,325,328,331,375,382,385,407,410],[10,11,12],"p",{},"检索效果差的锅，八成不在检索，而在入库。",[10,14,15],{},"企业文档的真实状态是：有的 PDF 是扫描件，有的 Word 里塞了表格，有的目录分类全靠文件名，\n还有人今天改了一版制度、明天又撤回。如果入库阶段没有约定，后面所有环节都是在流沙上盖楼。",[17,18,19],"h2",{"id":19},"先给结论",[10,21,22],{},"入库要解决四件事，缺一件都会在后期反噬：",[24,25,31],"pre",{"className":26,"code":28,"language":29,"meta":30},[27],"language-text","1. 契约   —— 每份资料属于哪个 source，由 scenario.toml 说了算\n2. 切分   —— Parent-Child 两级切分，检索用小的、生成用大的\n3. 变更   —— SHA256 内容指纹决定「谁需要重建」，而不是全量重跑\n4. 门禁   —— 入库就有质量闸，脏数据不许进库\n","text","",[32,33,28],"code",{"__ignoreMap":30},[17,35,37],{"id":36},"一数据契约source-不是场景","一、数据契约：source 不是场景",[10,39,40,43,44,47],{},[32,41,42],{},"scenario.toml"," 定义了这个场景的全部契约，其中最关键的是 ",[32,45,46],{},"valid_sources","：",[24,49,53],{"className":50,"code":51,"language":52,"meta":30,"style":30},"language-toml shiki shiki-themes github-light github-dark","scenario_id = \"enterprise_knowledge\"\nvalid_sources = [\"hr\", \"it\", \"finance\"]\nfaq_collection = \"enterprise_faq_hybrid_v1\"\ndoc_collection = \"enterprise_doc_hybrid_v1\"\n\n[source_patterns]\nhr = \"入职|离职|请假|考勤|绩效|社保|公积金|转正|试用期|岗位变更\"\nit = \"VPN|电脑|账号|邮箱|权限|网络|打印机|软件|数据安全|API Token\"\nfinance = \"报销|发票|预算|付款|借款|费用|单据|付款凭证\"\n","toml",[32,54,55,63,69,75,81,88,94,100,106],{"__ignoreMap":30},[56,57,60],"span",{"class":58,"line":59},"line",1,[56,61,62],{},"scenario_id = \"enterprise_knowledge\"\n",[56,64,66],{"class":58,"line":65},2,[56,67,68],{},"valid_sources = [\"hr\", \"it\", \"finance\"]\n",[56,70,72],{"class":58,"line":71},3,[56,73,74],{},"faq_collection = \"enterprise_faq_hybrid_v1\"\n",[56,76,78],{"class":58,"line":77},4,[56,79,80],{},"doc_collection = \"enterprise_doc_hybrid_v1\"\n",[56,82,84],{"class":58,"line":83},5,[56,85,87],{"emptyLinePlaceholder":86},true,"\n",[56,89,91],{"class":58,"line":90},6,[56,92,93],{},"[source_patterns]\n",[56,95,97],{"class":58,"line":96},7,[56,98,99],{},"hr = \"入职|离职|请假|考勤|绩效|社保|公积金|转正|试用期|岗位变更\"\n",[56,101,103],{"class":58,"line":102},8,[56,104,105],{},"it = \"VPN|电脑|账号|邮箱|权限|网络|打印机|软件|数据安全|API Token\"\n",[56,107,109],{"class":58,"line":108},9,[56,110,111],{},"finance = \"报销|发票|预算|付款|借款|费用|单据|付款凭证\"\n",[10,113,114,115,129],{},"这里有一个容易被误读的点：",[116,117,118,121,122,121,125,128],"strong",{},[32,119,120],{},"hr"," \u002F ",[32,123,124],{},"it",[32,126,127],{},"finance"," 是场景内部的知识分类，不是三个业务场景。","\n对外只有一个助手，只在检索和过滤时用到这些分类。把它们当成三个场景来演进，\n规则、阈值和评测集就永远调不准——因为每次调参都在让另一个分类变差。",[10,131,132],{},"FAQ 走的是纯字段契约，没有解析环节：",[24,134,138],{"className":135,"code":136,"language":137,"meta":30,"style":30},"language-csv shiki shiki-themes github-light github-dark","source,question,answer\nhr,年假和调休应该怎么申请？,员工应先提交请假申请，经直属负责人审批后生效。\n","csv",[32,139,140,145],{"__ignoreMap":30},[56,141,142],{"class":58,"line":59},[56,143,144],{},"source,question,answer\n",[56,146,147],{"class":58,"line":65},[56,148,149],{},"hr,年假和调休应该怎么申请？,员工应先提交请假申请，经直属负责人审批后生效。\n",[10,151,152,153,156,157,160],{},"集合也分成两个：",[32,154,155],{},"enterprise_faq_hybrid_v1"," 和 ",[32,158,159],{},"enterprise_doc_hybrid_v1","。\nFAQ 用来做高频问题的直出快路径，文档集合负责知识查询和追问。\n分开存的好处是两条链路可以各自调参，不会互相污染召回结果。",[17,162,164],{"id":163},"二解析格式不同处理方式也不同","二、解析：格式不同，处理方式也不同",[10,166,167],{},"入站格式支持 Markdown、TXT、PDF、Word、PPT、CSV 和 Excel，但路径不完全一样：",[169,170,171,187],"table",{},[172,173,174],"thead",{},[175,176,177,181,184],"tr",{},[178,179,180],"th",{},"内容形态",[178,182,183],{},"处理方式",[178,185,186],{},"代码位置",[188,189,190,204,217,234],"tbody",{},[175,191,192,196,199],{},[193,194,195],"td",{},"普通文本 \u002F Markdown",[193,197,198],{},"直读 + 归一化（空白、编码、标点统一）",[193,200,201],{},[32,202,203],{},"qa_core\u002Findexing\u002Fdocument_loaders.py",[175,205,206,209,212],{},[193,207,208],{},"表格内容",[193,210,211],{},"转成结构化的行描述再入库",[193,213,214],{},[32,215,216],{},"qa_core\u002Findexing\u002Ftable_documents.py",[175,218,219,222,225],{},[193,220,221],{},"扫描件 \u002F 图片型 PDF",[193,223,224],{},"离线 OCR，产出候选后再人工提升",[193,226,227,230,231],{},[32,228,229],{},"qa_core\u002Findexing\u002Focr_documents.py","、",[32,232,233],{},"ocr_review.py",[175,235,236,239,242],{},[193,237,238],{},"含图片风险的内容",[193,240,241],{},"标注风险等级，阻断项不许入库",[193,243,244],{},[32,245,246],{},"qa_core\u002Findexing\u002Fimage_risk.py",[10,248,249,250,47,253,256,257,260],{},"OCR 刻意设计成",[116,251,252],{},"离线跑 + 人工提升",[32,254,255],{},"scripts\u002Focr\u002Frun_offline_ocr.py"," 产出候选文本，\n",[32,258,259],{},"scripts\u002Focr\u002Fpromote_ocr_candidates.py"," 才把它推进知识库。\n理由是 OCR 一定会有错字，错字进库之后，检索到的就是错的证据——比召回不到更危险。",[17,262,264],{"id":263},"三切分为什么是-1000-和-350","三、切分：为什么是 1000 和 350",[24,266,269],{"className":267,"code":268,"language":29,"meta":30},[27],"Parent  chunk_size=1000, overlap=100\nChild   chunk_size=350,  overlap=50\n",[32,270,268],{"__ignoreMap":30},[10,272,273],{},"两级切分的分工很明确：",[24,275,278],{"className":276,"code":277,"language":29,"meta":30},[27],"入库时：Parent 切成 Child，Child 进向量库（粒度小，匹配准）\n生成时：Child 命中 -> 回填它所属的 Parent -> 把 Parent 交给 LLM（上下文完整）\n",[32,279,277],{"__ignoreMap":30},[10,281,282,283,286,287,290,291,294],{},"只用一种粒度必然二选一：chunk 大了匹配不准，chunk 小了答案被截断。\nParent-Child 让「匹配用的小」和「生成用的大」同时成立。\n切分器用中文标点优先的递归分隔符（",[32,284,285],{},"qa_core\u002Findexing\u002Fchunking.py","），\n避免把一句话从中间劈开；切分规则版本记在 ",[32,288,289],{},"chunk_schema_version"," 里（当前 ",[32,292,293],{},"parent_child_validity_v2","），\n规则一变，历史 chunk 就能被识别出来重建。",[17,296,298],{"id":297},"四增量靠指纹不靠时间戳","四、增量：靠指纹，不靠时间戳",[10,300,301,302],{},"每份文档入库前算一次 SHA256 内容指纹，写进 manifest。\n下一轮入库时先比对指纹：",[116,303,304],{},"没变就跳过，变了的才重新解析、切分、写向量。",[10,306,307],{},"不要用文件修改时间判断变更——同步工具会把 mtime 全刷新一遍，你会白跑一次全量重建。\n内容指纹只认内容本身，这是可复现的前提。",[17,309,311],{"id":310},"五门禁脏数据不许进库","五、门禁：脏数据不许进库",[10,313,314,315,318],{},"入库质量门禁（",[32,316,317],{},"qa_core\u002Fquality\u002Fingestion.py","）是一组「必须为 0」的计数：",[24,320,323],{"className":321,"code":322,"language":29,"meta":30},[27],"失败文件 \u002F 不支持格式 \u002F 空文件 \u002F 低质量切分 \u002F 重复 chunk\nFAQ 空问题 \u002F FAQ 空答案 \u002F FAQ 重复问题 \u002F FAQ 非法 source\nFAQ 与文档内容冲突 \u002F 图片风险阻断文件\n",[32,324,322],{"__ignoreMap":30},[10,326,327],{},"默认阈值全是 0，一票否决。听起来很严，但它拦住的正是那种「跑完没报错、\n但线上开始答错」的隐性故障。",[17,329,330],{"id":330},"动手验证",[24,332,336],{"className":333,"code":334,"language":335,"meta":30,"style":30},"language-powershell shiki shiki-themes github-light github-dark","# 全量重建（会重置 collection）\ndocker compose --env-file .env.single run --rm api python scripts\u002Frebuild_scenarios.py --scenarios enterprise_knowledge --reset-collections\n\n# 再跑一次：观察指纹命中，未变更文档被跳过\ndocker compose --env-file .env.single run --rm api python scripts\u002Frebuild_scenarios.py --scenarios enterprise_knowledge --reset-collections\n\n# 日常更新走增量版本：新建版本 -> 从 active 增量 -> 过门禁 -> 激活\ndocker compose --env-file .env.single run --rm api python scripts\u002Frebuild_kb_version.py --scenario enterprise_knowledge --new-version --incremental-from active --quality-gate --activate --description \"enterprise knowledge update\"\n","powershell",[32,337,338,343,348,352,357,361,365,370],{"__ignoreMap":30},[56,339,340],{"class":58,"line":59},[56,341,342],{},"# 全量重建（会重置 collection）\n",[56,344,345],{"class":58,"line":65},[56,346,347],{},"docker compose --env-file .env.single run --rm api python scripts\u002Frebuild_scenarios.py --scenarios enterprise_knowledge --reset-collections\n",[56,349,350],{"class":58,"line":71},[56,351,87],{"emptyLinePlaceholder":86},[56,353,354],{"class":58,"line":77},[56,355,356],{},"# 再跑一次：观察指纹命中，未变更文档被跳过\n",[56,358,359],{"class":58,"line":83},[56,360,347],{},[56,362,363],{"class":58,"line":90},[56,364,87],{"emptyLinePlaceholder":86},[56,366,367],{"class":58,"line":96},[56,368,369],{},"# 日常更新走增量版本：新建版本 -> 从 active 增量 -> 过门禁 -> 激活\n",[56,371,372],{"class":58,"line":102},[56,373,374],{},"docker compose --env-file .env.single run --rm api python scripts\u002Frebuild_kb_version.py --scenario enterprise_knowledge --new-version --incremental-from active --quality-gate --activate --description \"enterprise knowledge update\"\n",[10,376,377,378,381],{},"入库报告落在 ",[32,379,380],{},"reports\u002Fingestion\u002Fenterprise_knowledge\u002F","，里面能看到每个文件的解析结果、\nchunk 数量和指纹命中情况。第二次跑的时候，跳过数应该接近全量——如果不是，说明指纹没生效。",[17,383,384],{"id":384},"取舍与边界",[386,387,388,395,401],"ul",{},[389,390,391,394],"li",{},[116,392,393],{},"切分参数和文档语言强相关","：1000 \u002F 350 是中文企业制度类文档的经验值；\n换成代码仓库或英文长文档，这两个值都要重新标定。",[389,396,397,400],{},[116,398,399],{},"离线 OCR 换来的是准确率，付出的是流程成本","：需要有人看一眼候选文本。",[389,402,403,406],{},[116,404,405],{},"manifest 是有状态的","：删掉它，增量能力就退化成全量重建。",[10,408,409],{},"下一篇进入查询侧：用户问一句话，系统凭什么决定它该走 FAQ 直出、文档检索，还是追问。",[411,412,413],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":30,"searchDepth":65,"depth":65,"links":415},[416,417,418,419,420,421,422,423],{"id":19,"depth":65,"text":19},{"id":36,"depth":65,"text":37},{"id":163,"depth":65,"text":164},{"id":263,"depth":65,"text":264},{"id":297,"depth":65,"text":298},{"id":310,"depth":65,"text":311},{"id":330,"depth":65,"text":330},{"id":384,"depth":65,"text":384},"2026-06-12","通过数据契约、两级切分、内容指纹与质量门禁，构建可增量更新的企业知识入库链路。",false,"md","project-chapter",{"migration":430},{"generator":431,"sourceSha256":432},"nuxt-site\u002Fscripts\u002Fmigrate-content.mjs","780d1716e85097455a0c310fda15d46785e34dd10a088c4a99540be670167921","\u002Fprojects\u002Frag-knowledge-base\u002F03-知识入库与增量指纹",null,"projects",{"title":5,"description":425},"content\u002Fprojects\u002Frag-knowledge-base\u002F03-知识入库与增量指纹.md","projects\u002Frag-knowledge-base\u002F03-知识入库与增量指纹",[440,441,442],"RAG","Milvus","Embedding",33,"lVk2t3j--OqZzd81vbl9Ql9PWnxzSn-d37DbQ9c8HyA",1791279145025]