[{"data":1,"prerenderedAt":1011},["ShallowReactive",2],{"note:\u002Frag\u002F06_rag_离线链路":3},{"id":4,"title":5,"body":6,"date":991,"description":992,"draft":993,"extension":994,"featured":993,"kind":995,"lastmod":991,"meta":996,"navigation":41,"path":1000,"planned":1001,"section":1002,"seo":1003,"source":1004,"stem":1005,"tags":1006,"toc":41,"weight":158,"__hash__":1010},"notes\u002Frag\u002F06_RAG_离线链路.md","RAG 离线链路",{"type":7,"value":8,"toc":983},"minimark",[9,14,18,162,165,173,177,180,185,190,202,207,251,255,261,264,355,359,363,366,368,383,387,390,665,669,676,942,944,950,953,979],[10,11,13],"h2",{"id":12},"一文档加载","一、文档加载",[15,16,17],"p",{},"加载阶段的目标是把不同格式的文档统一读成纯文本。PDF 按页抽取文字，Word 按段落抽取文字，Markdown 等纯文本文件直接读取。",[19,20,25],"pre",{"className":21,"code":22,"language":23,"meta":24,"style":24},"language-python shiki shiki-themes github-light github-dark","from pathlib import Path\n\nfrom docx import Document\nfrom pypdf import PdfReader\n\n\ndef load_document(path: str) -> str:\n    suffix = Path(path).suffix.lower()\n\n    if suffix == \".pdf\":\n        reader = PdfReader(path)\n        pages = [page.extract_text() or \"\" for page in reader.pages]\n        return \"\\n\".join(pages)\n\n    if suffix == \".docx\":\n        doc = Document(path)\n        return \"\\n\".join(p.text for p in doc.paragraphs)\n\n    return Path(path).read_text(encoding=\"utf-8\")\n\n\ntext = load_document(\"员工手册.pdf\")\nprint(f\"加载完成：{len(text)} 个字符\")\n","python","",[26,27,28,36,43,49,55,60,65,71,77,82,88,94,100,106,111,117,123,129,134,140,145,150,156],"code",{"__ignoreMap":24},[29,30,33],"span",{"class":31,"line":32},"line",1,[29,34,35],{},"from pathlib import Path\n",[29,37,39],{"class":31,"line":38},2,[29,40,42],{"emptyLinePlaceholder":41},true,"\n",[29,44,46],{"class":31,"line":45},3,[29,47,48],{},"from docx import Document\n",[29,50,52],{"class":31,"line":51},4,[29,53,54],{},"from pypdf import PdfReader\n",[29,56,58],{"class":31,"line":57},5,[29,59,42],{"emptyLinePlaceholder":41},[29,61,63],{"class":31,"line":62},6,[29,64,42],{"emptyLinePlaceholder":41},[29,66,68],{"class":31,"line":67},7,[29,69,70],{},"def load_document(path: str) -> str:\n",[29,72,74],{"class":31,"line":73},8,[29,75,76],{},"    suffix = Path(path).suffix.lower()\n",[29,78,80],{"class":31,"line":79},9,[29,81,42],{"emptyLinePlaceholder":41},[29,83,85],{"class":31,"line":84},10,[29,86,87],{},"    if suffix == \".pdf\":\n",[29,89,91],{"class":31,"line":90},11,[29,92,93],{},"        reader = PdfReader(path)\n",[29,95,97],{"class":31,"line":96},12,[29,98,99],{},"        pages = [page.extract_text() or \"\" for page in reader.pages]\n",[29,101,103],{"class":31,"line":102},13,[29,104,105],{},"        return \"\\n\".join(pages)\n",[29,107,109],{"class":31,"line":108},14,[29,110,42],{"emptyLinePlaceholder":41},[29,112,114],{"class":31,"line":113},15,[29,115,116],{},"    if suffix == \".docx\":\n",[29,118,120],{"class":31,"line":119},16,[29,121,122],{},"        doc = Document(path)\n",[29,124,126],{"class":31,"line":125},17,[29,127,128],{},"        return \"\\n\".join(p.text for p in doc.paragraphs)\n",[29,130,132],{"class":31,"line":131},18,[29,133,42],{"emptyLinePlaceholder":41},[29,135,137],{"class":31,"line":136},19,[29,138,139],{},"    return Path(path).read_text(encoding=\"utf-8\")\n",[29,141,143],{"class":31,"line":142},20,[29,144,42],{"emptyLinePlaceholder":41},[29,146,148],{"class":31,"line":147},21,[29,149,42],{"emptyLinePlaceholder":41},[29,151,153],{"class":31,"line":152},22,[29,154,155],{},"text = load_document(\"员工手册.pdf\")\n",[29,157,159],{"class":31,"line":158},23,[29,160,161],{},"print(f\"加载完成：{len(text)} 个字符\")\n",[15,163,164],{},"示例输出：",[19,166,171],{"className":167,"code":169,"language":170,"meta":24},[168],"language-text","加载完成：4820 个字符\n","text",[26,172,169],{"__ignoreMap":24},[10,174,176],{"id":175},"二数据切分","二、数据切分",[178,179],"hr",{},[181,182,184],"h3",{"id":183},"_1-递归层级分块recursivehierarchical-chunking","1. 递归\u002F层级分块（Recursive\u002FHierarchical Chunking）",[186,187,189],"h4",{"id":188},"_11-核心理论","1.1 核心理论",[15,191,192,193,197,198,201],{},"递归分块的核心思想是",[194,195,196],"strong",{},"维护语义单元的完整性","。它不会机械地从第一个字符切割，而是按照一个预定义的",[194,199,200],{},"分隔符优先级列表","，从最粗的粒度开始，层层递进地尝试切分。",[15,203,204],{},[194,205,206],{},"工作流程：",[208,209,210,221,231,245],"ol",{},[211,212,213,216,217,220],"li",{},[194,214,215],{},"首选粗粒度分隔符","：例如，优先使用段落分隔符（",[26,218,219],{},"\\n\\n","）进行切分。",[211,222,223,226,227,230],{},[194,224,225],{},"检查块大小","：如果切出的某个块大小仍然超过设定的 ",[26,228,229],{},"chunk_size"," 上限。",[211,232,233,236,237,240,241,244],{},[194,234,235],{},"降级到次级分隔符","：在这个超长块内部，改用下一级分隔符（如句号 ",[26,238,239],{},"。","、换行符 ",[26,242,243],{},"\\n","）再次尝试。",[211,246,247,250],{},[194,248,249],{},"递归进行，直到达标","：此过程递归执行，可能直到句子、词语，甚至字符级别，确保所有块都符合大小要求。",[186,252,254],{"id":253},"_12-代码实现基于-langchain","1.2 代码实现（基于 LangChain）",[15,256,257,260],{},[26,258,259],{},"RecursiveCharacterTextSplitter"," 是这一策略的经典实现，也是 LangChain 官方推荐的通用起点。",[186,262,263],{"id":263},"安装依赖",[19,265,267],{"className":21,"code":266,"language":23,"meta":24,"style":24},"from langchain_text_splitters import RecursiveCharacterTextSplitter\n\n# 1. 初始化分块器\ntext_splitter = RecursiveCharacterTextSplitter(\n    chunk_size=500,        # 每个块的最大字符数\n    chunk_overlap=50,      # 相邻块的重叠字符数，用于缓解边界上下文丢失\n    separators=[\"\\n\\n\", \"\\n\", \"。\", \"，\", \" \", \"\"]  # 分隔符优先级列表\n)\n\n# 2. 待切分的长文本\nlong_text = \"这是第一段。\\n\\n这是第二段，它包含了一些内容。并且它比较长。\"\n\n# 3. 执行切分\nchunks = text_splitter.split_text(long_text)\n\n# 4. 查看结果\nfor i, chunk in enumerate(chunks):\n    print(f\"块 {i+1}:\\n{chunk}\\n{'-'*20}\")\n",[26,268,269,274,278,283,288,293,298,303,308,312,317,322,326,331,336,340,345,350],{"__ignoreMap":24},[29,270,271],{"class":31,"line":32},[29,272,273],{},"from langchain_text_splitters import RecursiveCharacterTextSplitter\n",[29,275,276],{"class":31,"line":38},[29,277,42],{"emptyLinePlaceholder":41},[29,279,280],{"class":31,"line":45},[29,281,282],{},"# 1. 初始化分块器\n",[29,284,285],{"class":31,"line":51},[29,286,287],{},"text_splitter = RecursiveCharacterTextSplitter(\n",[29,289,290],{"class":31,"line":57},[29,291,292],{},"    chunk_size=500,        # 每个块的最大字符数\n",[29,294,295],{"class":31,"line":62},[29,296,297],{},"    chunk_overlap=50,      # 相邻块的重叠字符数，用于缓解边界上下文丢失\n",[29,299,300],{"class":31,"line":67},[29,301,302],{},"    separators=[\"\\n\\n\", \"\\n\", \"。\", \"，\", \" \", \"\"]  # 分隔符优先级列表\n",[29,304,305],{"class":31,"line":73},[29,306,307],{},")\n",[29,309,310],{"class":31,"line":79},[29,311,42],{"emptyLinePlaceholder":41},[29,313,314],{"class":31,"line":84},[29,315,316],{},"# 2. 待切分的长文本\n",[29,318,319],{"class":31,"line":90},[29,320,321],{},"long_text = \"这是第一段。\\n\\n这是第二段，它包含了一些内容。并且它比较长。\"\n",[29,323,324],{"class":31,"line":96},[29,325,42],{"emptyLinePlaceholder":41},[29,327,328],{"class":31,"line":102},[29,329,330],{},"# 3. 执行切分\n",[29,332,333],{"class":31,"line":108},[29,334,335],{},"chunks = text_splitter.split_text(long_text)\n",[29,337,338],{"class":31,"line":113},[29,339,42],{"emptyLinePlaceholder":41},[29,341,342],{"class":31,"line":119},[29,343,344],{},"# 4. 查看结果\n",[29,346,347],{"class":31,"line":125},[29,348,349],{},"for i, chunk in enumerate(chunks):\n",[29,351,352],{"class":31,"line":131},[29,353,354],{},"    print(f\"块 {i+1}:\\n{chunk}\\n{'-'*20}\")\n",[181,356,358],{"id":357},"_2-语义分块semantic-chunking","2. 语义分块（Semantic Chunking）",[186,360,362],{"id":361},"_21-核心理论","2.1 核心理论",[15,364,365],{},"语义分块不依赖固定的分隔符，而是借助机器学习模型理解文本语义。其核心是：将语义上紧密相关的句子聚合在一起，当语义发生明显跳跃时，则在此处切分。",[15,367,206],{},[369,370,371,374,377,380],"ul",{},[211,372,373],{},"句子切分：使用 NLP 工具（如 NLTK）将文本切分成独立的句子。",[211,375,376],{},"向量化：使用嵌入模型（如 all-MiniLM-L6-v2）为每个句子生成向量（Embedding），代表其语义。",[211,378,379],{},"计算相似度：遍历句子，计算相邻句子向量的余弦相似度。",[211,381,382],{},"动态切分：设定一个相似度阈值。如果两个相邻句子的相似度低于阈值，则认为语义发生转变，在此处分块。",[186,384,386],{"id":385},"_22-代码实现基于-sentence-transformers","2.2 代码实现（基于 Sentence-Transformers）",[15,388,389],{},"下面是一个基于 sentence-transformers 库的精炼实现，清晰展示了核心逻辑。",[19,391,393],{"className":21,"code":392,"language":23,"meta":24,"style":24},"from sentence_transformers import SentenceTransformer, util\n\n# 1. 加载轻量级嵌入模型\nmodel = SentenceTransformer(\"all-MiniLM-L6-v2\")\n\ndef semantic_chunk(text, sim_threshold=0.7):\n    \"\"\"\n    根据句子间的语义相似度进行分块。\n\n    Args:\n        text: 输入的整段文本。\n        sim_threshold: 相似度阈值，低于此值则切分。建议从 0.7 开始尝试调整。\n\n    Returns:\n        一个由字符串组成的列表，每个字符串是一个语义块。\n    \"\"\"\n    # 2. 简单分句（实际场景可换用更强大的分句工具）\n    sentences = [s.strip() for s in text.split('。') if s.strip()]\n    if not sentences:\n        return []\n\n    # 3. 计算所有句子的向量\n    embeddings = model.encode(sentences)\n\n    chunks = []\n    current_chunk = [sentences[0]]  # 从第一句开始\n\n    # 4. 遍历并比较相邻句子的相似度\n    for i in range(1, len(sentences)):\n        # 计算当前句子与前一句的余弦相似度\n        sim = util.cos_sim(embeddings[i-1], embeddings[i]).item()\n\n        # 5. 如果相似度低于阈值，则在此切分\n        if sim \u003C sim_threshold:\n            chunks.append(\"。\".join(current_chunk) + \"。\")\n            current_chunk = [sentences[i]]  # 开启新块\n        else:\n            current_chunk.append(sentences[i])\n\n    # 添加最后一个块\n    if current_chunk:\n        chunks.append(\"。\".join(current_chunk) + \"。\")\n\n    return chunks\n\n# --- 使用示例 ---\ndoc = \"人工智能技术发展迅速。机器学习是AI的核心技术。深度学习推动了AI革命。自然语言处理技术不断进步。聊天机器人越来越智能。计算机视觉应用广泛。图像识别准确率不断提高。\"\n\nresult = semantic_chunk(doc, sim_threshold=0.6)\nfor i, chunk in enumerate(result):\n    print(f\"语义块 {i+1}: {chunk}\")\n",[26,394,395,400,404,409,414,418,423,428,433,437,442,447,452,456,461,466,470,475,480,485,490,494,499,504,509,515,521,526,532,538,544,550,555,561,567,573,579,585,591,596,602,608,614,619,625,630,636,642,647,653,659],{"__ignoreMap":24},[29,396,397],{"class":31,"line":32},[29,398,399],{},"from sentence_transformers import SentenceTransformer, util\n",[29,401,402],{"class":31,"line":38},[29,403,42],{"emptyLinePlaceholder":41},[29,405,406],{"class":31,"line":45},[29,407,408],{},"# 1. 加载轻量级嵌入模型\n",[29,410,411],{"class":31,"line":51},[29,412,413],{},"model = SentenceTransformer(\"all-MiniLM-L6-v2\")\n",[29,415,416],{"class":31,"line":57},[29,417,42],{"emptyLinePlaceholder":41},[29,419,420],{"class":31,"line":62},[29,421,422],{},"def semantic_chunk(text, sim_threshold=0.7):\n",[29,424,425],{"class":31,"line":67},[29,426,427],{},"    \"\"\"\n",[29,429,430],{"class":31,"line":73},[29,431,432],{},"    根据句子间的语义相似度进行分块。\n",[29,434,435],{"class":31,"line":79},[29,436,42],{"emptyLinePlaceholder":41},[29,438,439],{"class":31,"line":84},[29,440,441],{},"    Args:\n",[29,443,444],{"class":31,"line":90},[29,445,446],{},"        text: 输入的整段文本。\n",[29,448,449],{"class":31,"line":96},[29,450,451],{},"        sim_threshold: 相似度阈值，低于此值则切分。建议从 0.7 开始尝试调整。\n",[29,453,454],{"class":31,"line":102},[29,455,42],{"emptyLinePlaceholder":41},[29,457,458],{"class":31,"line":108},[29,459,460],{},"    Returns:\n",[29,462,463],{"class":31,"line":113},[29,464,465],{},"        一个由字符串组成的列表，每个字符串是一个语义块。\n",[29,467,468],{"class":31,"line":119},[29,469,427],{},[29,471,472],{"class":31,"line":125},[29,473,474],{},"    # 2. 简单分句（实际场景可换用更强大的分句工具）\n",[29,476,477],{"class":31,"line":131},[29,478,479],{},"    sentences = [s.strip() for s in text.split('。') if s.strip()]\n",[29,481,482],{"class":31,"line":136},[29,483,484],{},"    if not sentences:\n",[29,486,487],{"class":31,"line":142},[29,488,489],{},"        return []\n",[29,491,492],{"class":31,"line":147},[29,493,42],{"emptyLinePlaceholder":41},[29,495,496],{"class":31,"line":152},[29,497,498],{},"    # 3. 计算所有句子的向量\n",[29,500,501],{"class":31,"line":158},[29,502,503],{},"    embeddings = model.encode(sentences)\n",[29,505,507],{"class":31,"line":506},24,[29,508,42],{"emptyLinePlaceholder":41},[29,510,512],{"class":31,"line":511},25,[29,513,514],{},"    chunks = []\n",[29,516,518],{"class":31,"line":517},26,[29,519,520],{},"    current_chunk = [sentences[0]]  # 从第一句开始\n",[29,522,524],{"class":31,"line":523},27,[29,525,42],{"emptyLinePlaceholder":41},[29,527,529],{"class":31,"line":528},28,[29,530,531],{},"    # 4. 遍历并比较相邻句子的相似度\n",[29,533,535],{"class":31,"line":534},29,[29,536,537],{},"    for i in range(1, len(sentences)):\n",[29,539,541],{"class":31,"line":540},30,[29,542,543],{},"        # 计算当前句子与前一句的余弦相似度\n",[29,545,547],{"class":31,"line":546},31,[29,548,549],{},"        sim = util.cos_sim(embeddings[i-1], embeddings[i]).item()\n",[29,551,553],{"class":31,"line":552},32,[29,554,42],{"emptyLinePlaceholder":41},[29,556,558],{"class":31,"line":557},33,[29,559,560],{},"        # 5. 如果相似度低于阈值，则在此切分\n",[29,562,564],{"class":31,"line":563},34,[29,565,566],{},"        if sim \u003C sim_threshold:\n",[29,568,570],{"class":31,"line":569},35,[29,571,572],{},"            chunks.append(\"。\".join(current_chunk) + \"。\")\n",[29,574,576],{"class":31,"line":575},36,[29,577,578],{},"            current_chunk = [sentences[i]]  # 开启新块\n",[29,580,582],{"class":31,"line":581},37,[29,583,584],{},"        else:\n",[29,586,588],{"class":31,"line":587},38,[29,589,590],{},"            current_chunk.append(sentences[i])\n",[29,592,594],{"class":31,"line":593},39,[29,595,42],{"emptyLinePlaceholder":41},[29,597,599],{"class":31,"line":598},40,[29,600,601],{},"    # 添加最后一个块\n",[29,603,605],{"class":31,"line":604},41,[29,606,607],{},"    if current_chunk:\n",[29,609,611],{"class":31,"line":610},42,[29,612,613],{},"        chunks.append(\"。\".join(current_chunk) + \"。\")\n",[29,615,617],{"class":31,"line":616},43,[29,618,42],{"emptyLinePlaceholder":41},[29,620,622],{"class":31,"line":621},44,[29,623,624],{},"    return chunks\n",[29,626,628],{"class":31,"line":627},45,[29,629,42],{"emptyLinePlaceholder":41},[29,631,633],{"class":31,"line":632},46,[29,634,635],{},"# --- 使用示例 ---\n",[29,637,639],{"class":31,"line":638},47,[29,640,641],{},"doc = \"人工智能技术发展迅速。机器学习是AI的核心技术。深度学习推动了AI革命。自然语言处理技术不断进步。聊天机器人越来越智能。计算机视觉应用广泛。图像识别准确率不断提高。\"\n",[29,643,645],{"class":31,"line":644},48,[29,646,42],{"emptyLinePlaceholder":41},[29,648,650],{"class":31,"line":649},49,[29,651,652],{},"result = semantic_chunk(doc, sim_threshold=0.6)\n",[29,654,656],{"class":31,"line":655},50,[29,657,658],{},"for i, chunk in enumerate(result):\n",[29,660,662],{"class":31,"line":661},51,[29,663,664],{},"    print(f\"语义块 {i+1}: {chunk}\")\n",[10,666,668],{"id":667},"三bge-m3-向量化并写入-milvus","三、BGE-M3 向量化并写入 Milvus",[15,670,671,672,675],{},"本步把每个子块同时编码成 Dense 向量和 Sparse 向量，并连同原文、来源和 ",[26,673,674],{},"parent_id"," 一起写入 Milvus Collection。",[19,677,679],{"className":21,"code":678,"language":23,"meta":24,"style":24},"from pymilvus import (\n    Collection,\n    CollectionSchema,\n    DataType,\n    FieldSchema,\n    connections,\n    utility,\n)\nfrom pymilvus.model.hybrid import BGEM3EmbeddingFunction\n\n# 本地文件 uri 会自动启用 Milvus Lite；生产环境可换成 Milvus 服务地址\nconnections.connect(uri=\".\u002Frag_demo.db\")\n\nef = BGEM3EmbeddingFunction(use_fp16=False, device=\"cpu\")\ndense_dim = ef.dim[\"dense\"]\n\ncollection_name = \"rag_docs\"\nif utility.has_collection(collection_name):\n    Collection(collection_name).drop()\n\nfields = [\n    FieldSchema(name=\"pk\", dtype=DataType.VARCHAR, is_primary=True, auto_id=True, max_length=100),\n    FieldSchema(name=\"parent_id\", dtype=DataType.VARCHAR, max_length=64),\n    FieldSchema(name=\"text\", dtype=DataType.VARCHAR, max_length=2000),\n    FieldSchema(name=\"source\", dtype=DataType.VARCHAR, max_length=512),\n    FieldSchema(name=\"sparse_vector\", dtype=DataType.SPARSE_FLOAT_VECTOR),\n    FieldSchema(name=\"dense_vector\", dtype=DataType.FLOAT_VECTOR, dim=dense_dim),\n]\ncollection = Collection(collection_name, schema=CollectionSchema(fields))\n\ntexts = [item[\"text\"] for item in child_chunks]\nembeddings = ef(texts)\n\ncollection.insert([\n    [item[\"parent_id\"] for item in child_chunks],\n    texts,\n    [item[\"source\"] for item in child_chunks],\n    embeddings[\"sparse\"],\n    embeddings[\"dense\"],\n])\n\ncollection.create_index(\n    \"sparse_vector\",\n    {\"index_type\": \"SPARSE_INVERTED_INDEX\", \"metric_type\": \"IP\"},\n)\ncollection.create_index(\n    \"dense_vector\",\n    {\"index_type\": \"AUTOINDEX\", \"metric_type\": \"IP\"},\n)\ncollection.load()\n\nprint(f\"Collection {collection_name} 已创建\")\nprint(f\"入库数量：{collection.num_entities}\")\nprint(f\"Dense 向量维度：{dense_dim}\")\n",[26,680,681,686,691,696,701,706,711,716,720,725,729,734,739,743,748,753,757,762,767,772,776,781,786,791,796,801,806,811,816,821,825,830,835,839,844,849,854,859,864,869,874,878,883,888,893,897,901,906,911,915,920,924,930,936],{"__ignoreMap":24},[29,682,683],{"class":31,"line":32},[29,684,685],{},"from pymilvus import (\n",[29,687,688],{"class":31,"line":38},[29,689,690],{},"    Collection,\n",[29,692,693],{"class":31,"line":45},[29,694,695],{},"    CollectionSchema,\n",[29,697,698],{"class":31,"line":51},[29,699,700],{},"    DataType,\n",[29,702,703],{"class":31,"line":57},[29,704,705],{},"    FieldSchema,\n",[29,707,708],{"class":31,"line":62},[29,709,710],{},"    connections,\n",[29,712,713],{"class":31,"line":67},[29,714,715],{},"    utility,\n",[29,717,718],{"class":31,"line":73},[29,719,307],{},[29,721,722],{"class":31,"line":79},[29,723,724],{},"from pymilvus.model.hybrid import BGEM3EmbeddingFunction\n",[29,726,727],{"class":31,"line":84},[29,728,42],{"emptyLinePlaceholder":41},[29,730,731],{"class":31,"line":90},[29,732,733],{},"# 本地文件 uri 会自动启用 Milvus Lite；生产环境可换成 Milvus 服务地址\n",[29,735,736],{"class":31,"line":96},[29,737,738],{},"connections.connect(uri=\".\u002Frag_demo.db\")\n",[29,740,741],{"class":31,"line":102},[29,742,42],{"emptyLinePlaceholder":41},[29,744,745],{"class":31,"line":108},[29,746,747],{},"ef = BGEM3EmbeddingFunction(use_fp16=False, device=\"cpu\")\n",[29,749,750],{"class":31,"line":113},[29,751,752],{},"dense_dim = ef.dim[\"dense\"]\n",[29,754,755],{"class":31,"line":119},[29,756,42],{"emptyLinePlaceholder":41},[29,758,759],{"class":31,"line":125},[29,760,761],{},"collection_name = \"rag_docs\"\n",[29,763,764],{"class":31,"line":131},[29,765,766],{},"if utility.has_collection(collection_name):\n",[29,768,769],{"class":31,"line":136},[29,770,771],{},"    Collection(collection_name).drop()\n",[29,773,774],{"class":31,"line":142},[29,775,42],{"emptyLinePlaceholder":41},[29,777,778],{"class":31,"line":147},[29,779,780],{},"fields = [\n",[29,782,783],{"class":31,"line":152},[29,784,785],{},"    FieldSchema(name=\"pk\", dtype=DataType.VARCHAR, is_primary=True, auto_id=True, max_length=100),\n",[29,787,788],{"class":31,"line":158},[29,789,790],{},"    FieldSchema(name=\"parent_id\", dtype=DataType.VARCHAR, max_length=64),\n",[29,792,793],{"class":31,"line":506},[29,794,795],{},"    FieldSchema(name=\"text\", dtype=DataType.VARCHAR, max_length=2000),\n",[29,797,798],{"class":31,"line":511},[29,799,800],{},"    FieldSchema(name=\"source\", dtype=DataType.VARCHAR, max_length=512),\n",[29,802,803],{"class":31,"line":517},[29,804,805],{},"    FieldSchema(name=\"sparse_vector\", dtype=DataType.SPARSE_FLOAT_VECTOR),\n",[29,807,808],{"class":31,"line":523},[29,809,810],{},"    FieldSchema(name=\"dense_vector\", dtype=DataType.FLOAT_VECTOR, dim=dense_dim),\n",[29,812,813],{"class":31,"line":528},[29,814,815],{},"]\n",[29,817,818],{"class":31,"line":534},[29,819,820],{},"collection = Collection(collection_name, schema=CollectionSchema(fields))\n",[29,822,823],{"class":31,"line":540},[29,824,42],{"emptyLinePlaceholder":41},[29,826,827],{"class":31,"line":546},[29,828,829],{},"texts = [item[\"text\"] for item in child_chunks]\n",[29,831,832],{"class":31,"line":552},[29,833,834],{},"embeddings = ef(texts)\n",[29,836,837],{"class":31,"line":557},[29,838,42],{"emptyLinePlaceholder":41},[29,840,841],{"class":31,"line":563},[29,842,843],{},"collection.insert([\n",[29,845,846],{"class":31,"line":569},[29,847,848],{},"    [item[\"parent_id\"] for item in child_chunks],\n",[29,850,851],{"class":31,"line":575},[29,852,853],{},"    texts,\n",[29,855,856],{"class":31,"line":581},[29,857,858],{},"    [item[\"source\"] for item in child_chunks],\n",[29,860,861],{"class":31,"line":587},[29,862,863],{},"    embeddings[\"sparse\"],\n",[29,865,866],{"class":31,"line":593},[29,867,868],{},"    embeddings[\"dense\"],\n",[29,870,871],{"class":31,"line":598},[29,872,873],{},"])\n",[29,875,876],{"class":31,"line":604},[29,877,42],{"emptyLinePlaceholder":41},[29,879,880],{"class":31,"line":610},[29,881,882],{},"collection.create_index(\n",[29,884,885],{"class":31,"line":616},[29,886,887],{},"    \"sparse_vector\",\n",[29,889,890],{"class":31,"line":621},[29,891,892],{},"    {\"index_type\": \"SPARSE_INVERTED_INDEX\", \"metric_type\": \"IP\"},\n",[29,894,895],{"class":31,"line":627},[29,896,307],{},[29,898,899],{"class":31,"line":632},[29,900,882],{},[29,902,903],{"class":31,"line":638},[29,904,905],{},"    \"dense_vector\",\n",[29,907,908],{"class":31,"line":644},[29,909,910],{},"    {\"index_type\": \"AUTOINDEX\", \"metric_type\": \"IP\"},\n",[29,912,913],{"class":31,"line":649},[29,914,307],{},[29,916,917],{"class":31,"line":655},[29,918,919],{},"collection.load()\n",[29,921,922],{"class":31,"line":661},[29,923,42],{"emptyLinePlaceholder":41},[29,925,927],{"class":31,"line":926},52,[29,928,929],{},"print(f\"Collection {collection_name} 已创建\")\n",[29,931,933],{"class":31,"line":932},53,[29,934,935],{},"print(f\"入库数量：{collection.num_entities}\")\n",[29,937,939],{"class":31,"line":938},54,[29,940,941],{},"print(f\"Dense 向量维度：{dense_dim}\")\n",[15,943,164],{},[19,945,948],{"className":946,"code":947,"language":170,"meta":24},[168],"Collection rag_docs 已创建\n入库数量：28\nDense 向量维度：1024\n",[26,949,947],{"__ignoreMap":24},[15,951,952],{},"说明：",[369,954,955,966,976],{},[211,956,957,958,961,962,965],{},"Milvus Collection 同时保存 ",[26,959,960],{},"sparse_vector"," 和 ",[26,963,964],{},"dense_vector","，这是后面 Hybrid Search 的数据基础",[211,967,968,971,972,975],{},[26,969,970],{},"AUTOINDEX"," 与 ",[26,973,974],{},"SPARSE_INVERTED_INDEX"," 分别是 Dense 和 Sparse 向量在 Milvus Lite 上的常用索引",[211,977,978],{},"更新知识时不能只替换原文件，必须重新切分、重新向量化并更新 Collection",[980,981,982],"style",{},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":24,"searchDepth":38,"depth":38,"links":984},[985,986,990],{"id":12,"depth":38,"text":13},{"id":175,"depth":38,"text":176,"children":987},[988,989],{"id":183,"depth":45,"text":184},{"id":357,"depth":45,"text":358},{"id":667,"depth":38,"text":668},"2026-05-12","从文档加载、分块到向量入库，拆解 RAG 离线链路的每一步工程实现与常见踩坑。",false,"md","article",{"migration":997},{"generator":998,"sourceSha256":999},"nuxt-site\u002Fscripts\u002Fmigrate-content.mjs","2c83e745a88e5657e2e8986222f61a7f4938b80539a616dc35251768bfbc0bfd","\u002Frag\u002F06_rag_离线链路",null,"rag",{"title":5,"description":992},"content\u002Frag\u002F06_RAG_离线链路.md","rag\u002F06_RAG_离线链路",[1007,1008,1009],"RAG","Milvus","Embedding","gPf9rr_7UCvqmnegODTmjmW5869uzNj2atSN2SXyZPI",1791279145545]