\u4e8c\u3001\u4ec0\u4e48\u662f\u5206\u5757\uff08Chunking\uff09<\/h2>\n
\u5206\u5757\u662f\u5c06\u5927\u5757\u6587\u672c\u5206\u89e3\u6210\u8f83\u5c0f\u6bb5\u843d\u7684\u8fc7\u7a0b\uff0c\u8fd9\u4f7f\u5f97\u6587\u672c\u6570\u636e\u66f4\u6613\u4e8e\u7ba1\u7406\u548c\u5904\u7406\u3002\u901a\u8fc7\u5206\u5757\uff0c\u6211\u4eec\u80fd\u591f\u66f4\u9ad8\u6548\u5730\u8fdb\u884c\u5185\u5bb9\u5d4c\u5165\uff08embedding\uff09\uff0c\u5e76\u663e\u8457\u63d0\u5347\u4ece\u5411\u91cf\u6570\u636e\u5e93\u4e2d\u53ec\u56de\u5185\u5bb9\u7684\u76f8\u5173\u6027\u548c\u51c6\u786e\u6027\u3002<\/p>\n
\u5728\u5b9e\u9645\u64cd\u4f5c\u4e2d\uff0c\u5206\u5757\u7684\u597d\u5904\u662f\u591a\u65b9\u9762\u7684\u3002\u9996\u5148\uff0c\u5b83\u80fd\u591f\u63d0\u9ad8\u6a21\u578b\u5904\u7406\u7684\u6548\u7387\uff0c\u56e0\u4e3a\u8f83\u5c0f\u7684\u6587\u672c\u6bb5\u843d\u66f4\u5bb9\u6613\u8fdb\u884c\u5d4c\u5165\u548c\u68c0\u7d22\u3002<\/p>\n
\u5176\u6b21\uff0c\u5206\u5757\u540e\u7684\u6587\u672c\u80fd\u591f\u66f4\u7cbe\u786e\u5730\u5339\u914d\u7528\u6237\u67e5\u8be2\uff0c\u4ece\u800c\u63d0\u4f9b\u66f4\u76f8\u5173\u7684\u641c\u7d22\u7ed3\u679c\u3002\u8fd9\u5bf9\u4e8e\u9700\u8981\u9ad8\u7cbe\u5ea6\u4fe1\u606f\u68c0\u7d22\u548c\u5185\u5bb9\u751f\u6210\u7684\u5e94\u7528\u7a0b\u5e8f\u5c24\u4e3a\u91cd\u8981\u3002<\/p>\n
\u901a\u8fc7\u4f18\u5316\u5185\u5bb9\u7684\u5206\u5757\u548c\u5d4c\u5165\u7b56\u7565\uff0c\u6211\u4eec\u53ef\u4ee5\u6700\u5927\u5316LLM\u5728\u5404\u79cd\u5e94\u7528\u573a\u666f\u4e2d\u7684\u6027\u80fd\u3002\u5206\u5757\u6280\u672f\u4e0d\u4ec5\u63d0\u9ad8\u4e86\u5185\u5bb9\u53ec\u56de\u7684\u51c6\u786e\u6027\uff0c\u8fd8\u63d0\u5347\u4e86\u6574\u4f53\u7cfb\u7edf\u7684\u54cd\u5e94\u901f\u5ea6\u548c\u7528\u6237\u4f53\u9a8c\u3002<\/p>\n
\u56e0\u6b64\uff0c\u5728\u6784\u5efa\u548c\u4f18\u5316\u57fa\u4e8eLLM\u7684\u5e94\u7528\u7a0b\u5e8f\u65f6\uff0c\u7406\u89e3\u548c\u5e94\u7528\u5206\u5757\u6280\u672f\u662f\u4e0d\u53ef\u6216\u7f3a\u7684\u6b65\u9aa4\u3002<\/p>\n
\u5206\u5757\u8fc7\u7a0b\u4e2d\u4e3b\u8981\u7684\u4e24\u4e2a\u6982\u5ff5\uff1achunk\/_size\u5757\u7684\u5927\u5c0f\uff0cchunk\/_overlap\u91cd\u53e0\u7a97\u53e3\u3002<\/p>\n
<\/p>\n
\u4e09\u3001\u4e3a\u4f55\u8981\u5bf9\u5185\u5bb9\u505a\u5206\u5757\u5904\u7406<\/h2>\n\n- \u6a21\u578b\u4e0a\u4e0b\u6587\u7a97\u53e3\u9650\u5236<\/strong>\uff1aLLM\u65e0\u6cd5\u4e00\u6b21\u5904\u7406\u8d85\u957f\u6587\u672c\u3002\u5206\u5757\u7684\u76ee\u7684\u5728\u4e8e\u5c06\u957f\u6587\u6863\u5207\u6210\u6a21\u578b\u53ef\u7a33\u5b9a\u5904\u7406\u7684\u4e2d\u7b49\u7c92\u5ea6\u7247\u6bb5\uff0c\u5e76\u5c3d\u91cf\u5bf9\u9f50\u81ea\u7136\u8bed\u4e49\u8fb9\u754c\uff08\u5982\u6807\u9898\u3001\u6bb5\u843d\u3001\u53e5\u5b50\u3001\u4ee3\u7801\u5757\uff09\uff0c\u907f\u514d\u786c\u5207\u5bfc\u81f4\u5173\u952e\u4fe1\u606f\u88ab\u622a\u65ad\u6216\u8bed\u4e49\u6f02\u79fb\u3002\u5373\u4fbf\u4f7f\u7528\u957f\u4e0a\u4e0b\u6587\u6a21\u578b\uff0c\u8fc7\u957f\u8f93\u5165\u4e5f\u4f1a\u63a8\u9ad8\u6210\u672c\u5e76\u7a00\u91ca\u4fe1\u606f\u5bc6\u5ea6\uff0c\u5408\u7406\u5206\u5757\u4ecd\u662f\u5fc5\u9700\u7684\u524d\u7f6e\u7ea6\u675f\u3002
*<\/li>\n- \u68c0\u7d22\u7684\u4fe1\u566a\u6bd4<\/strong>\uff1a\u5757\u8fc7\u5927\u65f6\u65e0\u5173\u5185\u5bb9\u4f1a\u7a00\u91ca\u4fe1\u53f7\u3001\u964d\u4f4e\u76f8\u4f3c\u5ea6\u5224\u522b\u529b\uff1b\u5757\u8fc7\u5c0f\u65f6\u8bed\u5883\u4e0d\u8db3\u3001\u5bb9\u6613\u201c\u53ea\u547d\u4e2d\u8bcd\u4e0d\u547d\u4e2d\u4e49\u201d\u3002\u5408\u9002\u7684\u5757\u7c92\u5ea6\u53ef\u5728\u53ec\u56de\u4e0e\u7cbe\u5ea6\u95f4\u53d6\u5f97\u66f4\u597d\u5e73\u8861\uff0c\u65e2\u8986\u76d6\u7528\u6237\u610f\u56fe\uff0c\u53c8\u4e0d\u5f15\u5165\u591a\u4f59\u566a\u58f0\u3002\u5728\u4e00\u5b9a\u7a0b\u5ea6\u4e0a\u63d0\u5347\u68c0\u7d22\u76f8\u5173\u6027\u7684\u540c\u65f6\u53c8\u80fd\u4fdd\u8bc1\u7ed3\u679c\u7a33\u5b9a\u6027\u3002
*<\/li>\n- \u8bed\u4e49\u8fde\u7eed\u6027<\/strong>\uff1a\u8de8\u6bb5\u843d\u6216\u8de8\u7ae0\u8282\u7684\u8bed\u4e49\u5173\u7cfb\u5e38\u5728\u8fb9\u754c\u5904\u88ab\u5207\u65ad\u3002\u901a\u8fc7\u8bbe\u7f6e\u9002\u5ea6\u7684 chunk\/_overlap\uff0c\u53ef\u4fdd\u7559\u8de8\u5757\u7ebf\u7d22\u3001\u51cf\u5c11\u5173\u952e\u5b9a\u4e49\/\u6761\u4ef6\u88ab\u201c\u5207\u5f00\u201d\u7684\u98ce\u9669\u3002\u5bf9\u4e8e\u5f3a\u7ed3\u6784\u6587\u6863\uff0c\u4f18\u5148\u8ba9\u8fb9\u754c\u8d34\u5408\u6807\u9898\u5c42\u7ea7\u4e0e\u53e5\u5b50\u65ad\u70b9\uff1b\u5fc5\u8981\u65f6\u5728\u68c0\u7d22\u9636\u6bb5\u505a\u8f7b\u91cf\u90bb\u8fd1\u6269\u5c55\uff0c\u4ee5\u63d0\u5347\u7b54\u6848\u7684\u8fde\u8d2f\u6027\u4e0e\u53ef\u8ffd\u6eaf\u6027\uff0c\u540c\u65f6\u907f\u514d\u91cd\u590d\u5185\u5bb9\u6324\u5360\u4e0a\u4e0b\u6587\u9884\u7b97\u3002<\/li>\n<\/ul>\n
\u603b\u4e4b\u7406\u60f3\u7684\u5206\u5757\u662f\u5728\u201c\u4e0a\u4e0b\u6587\u5b8c\u6574\u6027\u201d\u548c\u201c\u4fe1\u606f\u5bc6\u5ea6\u201d\u4e4b\u95f4\u53d6\u5f97\u52a8\u6001\u5e73\u8861\uff1achunk\/_size\u51b3\u5b9a\u4fe1\u606f\u627f\u8f7d\u91cf\uff0cchunk\/_overlap \u7528\u4e8e\u5f25\u8865\u8fb9\u754c\u65ad\u88c2\u5e76\u7ef4\u6301\u8bed\u4e49\u8fde\u7eed\u3002\u53ea\u8981\u8fb9\u754c\u5bf9\u9f50\u8bed\u4e49\u3001\u7c92\u5ea6\u8d34\u5408\u5185\u5bb9\uff0c\u68c0\u7d22\u4e0e\u751f\u6210\u7684\u8d28\u91cf\u5c31\u80fd\u63d0\u5347\u3002<\/p>\n
![\"image.png\"](\"https:\/\/segmentfault.com\/img\/bVdmSiR\" "\\"image.png\\"")
<\/p>\n
\u56db\u3001\u5206\u5757\u7b56\u7565\u8be6\u89e3<\/h2>\n4.1 \u57fa\u7840\u5206\u5757<\/h2>\n
\u57fa\u4e8e\u56fa\u5b9a\u957f\u5ea6\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong>\uff1a\u6309\u9884\u8bbe\u5b57\u7b26\u6570 chunk\/_size \u76f4\u63a5\u5207\u5206\uff0c\u4e0d\u8003\u8651\u6587\u672c\u7ed3\u6784\u3002<\/li>\n
- \u4f18\u70b9<\/strong>\uff1a\u5b9e\u73b0\u6700\u7b80\u5355\u3001\u901f\u5ea6\u5feb\u3001\u5bf9\u4efb\u610f\u6587\u672c\u901a\u7528\u3002<\/li>\n
- \u7f3a\u70b9<\/strong>\uff1a\u5bb9\u6613\u7834\u574f\u8bed\u4e49\u8fb9\u754c\uff1b\u5757\u8fc7\u5927\u5bb9\u6613\u5f15\u5165\u8f83\u591a\u566a\u58f0\uff0c\u8fc7\u5c0f\u5219\u4f1a\u5bfc\u81f4\u4e0a\u4e0b\u6587\u4e0d\u8db3\u3002<\/li>\n
- \u9002\u7528\u573a\u666f<\/strong>\uff1a\u7ed3\u6784\u6027\u5f31\u7684\u7eaf\u6587\u672c\uff0c\u6216\u6570\u636e\u9884\u5904\u7406\u521d\u671f\u7684\u57fa\u7ebf\u65b9\u6848\u3002<\/li>\n<\/ul>\n
<!—-><\/p>\n
from langchain_text_splitters import CharacterTextSplitter\n\n\nsplitter = CharacterTextSplitter(\n separator="", # \u7eaf\u6309\u957f\u5ea6\u5207\n chunk_size=600, # \u4f9d\u636e\u5b9e\u9a8c\u4e0e\u6a21\u578b\u4e0a\u9650\u8c03\u6574\n chunk_overlap=90, # 15% \u91cd\u53e0\n)\nchunks = splitter.split_text(text)\n<\/code><\/pre>\n\n- \u53c2\u6570\u5efa\u8bae\uff08\u4ec5\u9650\u4e2d\u6587\u8bed\u6599\u5efa\u8bae\uff09<\/strong> \uff1a<\/li>\n
- \n
\n- chunk\/_size\uff1a300\u2013800 \u5b57\u4f18\u5148\u5c1d\u8bd5\uff1b\u82e5\u5d4c\u5165\u6a21\u578b\u6700\u4f73\u8f93\u5165\u4e3a 512\/1024 tokens\uff0c\u53ef\u6298\u7b97\u4e3a\u7ea6 350\/700 \u4e2d\u6587\u5b57\u7b26\u8d77\u6b65\u3002<\/li>\n
- chunk\/_overlap\uff1a10%\u201320% \u8d77\u6b65\uff1b\u8d85\u8fc7 30% \u901a\u5e38\u5bfc\u81f4\u7d22\u5f15\u4f53\u79ef\u4e0e\u68c0\u7d22\u5f00\u9500\u663e\u8457\u4e0a\u5347\uff0c\u5bf9\u5b9e\u9645\u6027\u80fd\u8d77\u8d1f\u4f5c\u7528\uff0c\u6700\u540e\u7684\u6548\u679c\u5e76\u4e0d\u4f1a\u5f97\u5230\u660e\u663e\u63d0\u5347\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
\u57fa\u4e8e\u53e5\u5b50\u7684\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong>\uff1a\u5148\u6309\u53e5\u5b50\u5207\u5206\uff0c\u518d\u5c06\u82e5\u5e72\u53e5\u5b50\u805a\u5408\u6210\u6ee1\u8db3chunk\/_size\u7684\u5757\uff1b\u4fdd\u8bc1\u6700\u57fa\u672c\u7684\u8bed\u4e49\u5b8c\u6574\u6027\u3002<\/li>\n
- \u4f18\u70b9<\/strong>\uff1a\u53e5\u5b50\u7ea7\u5b8c\u6574\u6027\u6700\u597d\u3002\u5bf9\u95ee\u53e5\/\u7b54\u53e5\u6620\u5c04\u53cb\u597d\u3002\u4fbf\u4e8e\u9ad8\u8d28\u91cf\u5f15\u7528\u3002<\/li>\n
- \u7f3a\u70b9<\/strong>\uff1a\u4e2d\u6587\u5206\u53e5\u9700\u7279\u522b\u5904\u7406\u3002\u4ec5\u53e5\u5b50\u7ea7\u5207\u5206\u53ef\u80fd\u5bfc\u81f4\u5757\u8fc7\u77ed\uff0c\u9700\u540e\u7eed\u805a\u5408\u3002<\/li>\n
- \u9002\u7528\u573a\u666f<\/strong>\uff1a\u6cd5\u5f8b\u6cd5\u89c4\u3001\u65b0\u95fb\u3001\u516c\u544a\u3001FAQ \u7b49\u4ee5\u53e5\u5b50\u4e3a\u4e3b\u7684\u6587\u672c\u3002<\/li>\n
- \u4e2d\u6587\u5206\u53e5\u6ce8\u610f\u4e8b\u9879<\/strong>\uff1a<\/li>\n
- \n
\n- \u4e0d\u8981\u76f4\u63a5\u7528 NLTK \u82f1\u6587 Punkt\uff1a\u65e0\u6cd5\u8bc6\u522b\u4e2d\u6587\u6807\u70b9\uff0c\u5206\u53e5\u4f1a\u5931\u8d25\u6216\u5f02\u5e38\u3002<\/li>\n
- \u53ef\u4ee5\u76f4\u63a5\u4f7f\u7528\u4ee5\u4e0b\u5185\u5bb9\u8fdb\u884c\u5206\u53e5\uff1a<\/li>\n
- \n
\n- \u57fa\u4e8e\u4e2d\u6587\u6807\u70b9\u7684\u6b63\u5219\uff1a\u6309\u201c\u3002\uff01\uff1f\uff1b\u201d\u7b49\u5207\u5206\uff0c\u4fdd\u7559\u5f15\u53f7\u4e0e\u7701\u7565\u53f7\u7b49\u8fb9\u754c\u3002<\/li>\n
- \u4f7f\u7528\u652f\u6301\u4e2d\u6587\u7684 NLP \u5e93\u8fdb\u884c\u66f4\u7cbe\u7ec6\u7684\u5206\u53e5\uff1a<\/li>\n
- HanLP\uff08\u63a8\u8350\uff0c\u5de5\u4e1a\u7ea7\uff0c\u652f\u6301\u7e41\u591a\u8bed\u8a00\u5b66\u7279\u6027\uff09Stanza\uff08\u6e05\u534e\/\u65af\u5766\u798f\u5408\u4f5c\uff0c\u4e2d\u6587\u652f\u6301\u8f83\u597d\uff09spaCy + pkuseg \u63d2\u4ef6\uff08\u6216 zh-core-web-sm\/med\/lg \u751f\u6001\uff09<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n
- \u793a\u4f8b<\/strong>\uff08\u9002\u914d\u5e38\u89c1\u4e2d\u6587\u6807\u70b9\uff0c\u57fa\u4e8e\u6b63\u5219\u7684\u5206\u53e5\uff09\uff1a<\/li>\n<\/ul>\n
<!—-><\/p>\n
import re\n\n\ndef split_sentences_zh(text: str):\n # \u5728\u53e5\u672b\u6807\u70b9\uff08\u3002\uff01\uff1f\uff1b\uff09\u540e\u9762\u5e26\u53ef\u9009\u5f15\u53f7\u7684\u573a\u666f\u65ad\u53e5\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n sentences = [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n return sentences\n\n\ndef sentence_chunk(text: str, chunk_size=600, overlap=80):\n sents = split_sentences_zh(text)\n chunks, buf = [], ""\n for s in sents:\n if len(buf) + len(s) <= chunk_size:\n buf += s\n else:\n if buf:\n chunks.append(buf)\n # \u7b80\u5355\u91cd\u53e0\uff1a\u4ece\u5f53\u524d\u5757\u5c3e\u90e8\u622a\u53d6 overlap \u5b57\u7b26\u4e0e\u4e0b\u4e00\u53e5\u62fc\u63a5\n buf = (buf[-overlap:] if overlap > 0 and len(buf) > overlap else "") + s\n if buf:\n chunks.append(buf)\n return chunks\n\n\nchunks = sentence_chunk(text, chunk_size=600, overlap=90)\n<\/code><\/pre>\nHanLP \u5206\u53e5\u793a\u4f8b\uff1a<\/p>\n
from hanlp_common.constant import ROOT\nimport hanlp\n\n\ntokenizer = hanlp.load('PKU_NAME_MERGED_SIX_MONTHS_CONVSEG') # \u6216\u53e5\u6cd5\/\u53e5\u5b50\u7ea7\u7ba1\u7ebf\n# HanLP \u9ad8\u5c42 API \u901a\u5e38\u901a\u8fc7\u53e5\u6cd5\/\u8bed\u6599\u7ba1\u7ebf\u83b7\u5f97\u53e5\u5b50\u8fb9\u754c\uff0c\u5177\u4f53\u4ee5\u6240\u7528\u7248\u672c API \u4e3a\u51c6\n# \u5c06\u53e5\u5b50\u5217\u8868\u518d\u505a\u805a\u5408\u4e3a chunk_size\n<\/code><\/pre>\n\u57fa\u4e8e\u9012\u5f52\u5b57\u7b26\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong>\uff1a\u7ed9\u5b9a\u4e00\u7ec4\u7531\u201c\u7c97\u5230\u7ec6\u201d\u7684\u5206\u9694\u7b26\uff08\u5982\u6bb5\u843d\u2192\u6362\u884c\u2192\u7a7a\u683c\u2192\u5b57\u7b26\uff09\uff0c\u81ea\u4e0a\u800c\u4e0b\u9012\u5f52\u5207\u5206\uff0c\u5728\u4e0d\u8d85\u51fa chunk\/_size \u7684\u524d\u63d0\u4e0b\u5c3d\u91cf\u4fdd\u7559\u81ea\u7136\u8bed\u4e49\u8fb9\u754c\u3002<\/li>\n
- \u4f18\u70b9<\/strong>\uff1a\u5728\u201c\u4fdd\u6301\u8bed\u4e49\u8fb9\u754c\u201d\u548c\u201c\u63a7\u5236\u5757\u5927\u5c0f\u201d\u4e4b\u95f4\u53d6\u5f97\u7a33\u5065\u5e73\u8861\uff0c\u5bf9\u5927\u591a\u6570\u6587\u672c\u5373\u63d2\u5373\u7528\u3002<\/li>\n
- \u7f3a\u70b9<\/strong>\uff1a\u5206\u9694\u7b26\u914d\u7f6e\u4e0d\u5f53\u4f1a\u5bfc\u81f4\u5757\u7c92\u5ea6\u5931\u8861\uff0c\u6781\u5ea6\u683c\u5f0f\u5316\u6587\u672c\uff08\u8868\u683c\/\u4ee3\u7801\uff09\u6548\u679c\u4e00\u822c\u3002<\/li>\n
- \u9002\u7528\u573a\u666f<\/strong>\uff1a\u7efc\u5408\u6027\u8bed\u6599\u3001\u8bf4\u660e\u6587\u6863\u3001\u62a5\u544a\u3001\u77e5\u8bc6\u5e93\u6761\u76ee\u3002<\/li>\n<\/ul>\n
<!—-><\/p>\n
import re\nfrom langchain_text_splitters import RecursiveCharacterTextSplitter\n\n\nseparators = [\n r"\/n#{1,6}\/s", # \u6807\u9898\n r"\/n\/d+(?:.\/d+)*\/s", # \u6570\u5b57\u7f16\u53f7\u6807\u9898 1. \/ 2.3. \u7b49\n "\/n\/n", # \u6bb5\u843d\n "\/n", # \u884c\n " ", # \u7a7a\u683c\n "", # \u515c\u5e95\u5b57\u7b26\u7ea7\n]\nsplitter = RecursiveCharacterTextSplitter(\n separators=separators,\n chunk_size=700,\n chunk_overlap=100,\n is_separator_regex=True, # \u544a\u8bc9\u5206\u5272\u5668\u4e0a\u9762\u5305\u542b\u6b63\u5219\n)\nchunks = splitter.split_text(text)\n<\/code><\/pre>\n\n- \u53c2\u6570\u4e0e\u5206\u9694\u7b26\u5efa\u8bae<\/strong>\uff08\u4ec5\u4e2d\u6587\u6587\u6863\u5efa\u8bae\uff09\uff1a<\/li>\n
- \n
\n- chunk\/_size\uff1a400\u2013800 \u5b57\u7b26\uff1b\u5982\u679c\u5185\u5bb9\u66f4\u6280\u672f\u5316\u3001\u957f\u53e5\u591a\u65f6\u53ef\u9002\u5f53\u4e0a\u8c03\u8be5\u6570\u503c\u3002<\/li>\n
- chunk\/_overlap\uff1a10%\u201320%\u3002<\/li>\n
- separators\uff08\u7531\u7c97\u5230\u7ec6\uff0c\u6309\u9700\u88c1\u526a\uff09\uff1a<\/li>\n
- \n
\n- \u7ae0\u8282\/\u6807\u9898\uff1a\u6b63\u5219 r”^#{1,6}\/s”\uff08Markdown \u6807\u9898\uff09\u3001r”^\/d+(.\/d+)*\/s”\uff08\u7f16\u53f7\u6807\u9898\uff09<\/li>\n
- \u6bb5\u843d\uff1a”\/n\/n”<\/li>\n
- \u6362\u884c\uff1a”\/n”<\/li>\n
- \u7a7a\u683c\uff1a” “<\/li>\n
- \u515c\u5e95\uff1a””<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
\u603b\u7ed3<\/strong><\/p>\n\n- \u8c03\u4f18\u6d41\u7a0b<\/strong>\uff1a<\/li>\n
- \n
\n- \u56fa\u5b9a\u68c0\u7d22\u4e0e\u91cd\u6392\uff0c\u53ea\u52a8\u5206\u5757\u53c2\u6570\u3002<\/li>\n
- \u7528\u9a8c\u8bc1\u96c6\u8ba1\u7b97 Recall\/@k\u3001nDCG\u3001MRR\u3001\u6765\u6e90\u547d\u4e2d\u6587\u6863\u8986\u76d6\u7387\u3001\u7b54\u6848\u4e8b\u5b9e\u6027\uff08faithfulness\uff09\u3002<\/li>\n
- \u89c2\u5bdf\u5757\u957f\u5206\u5e03\uff1a\u82e5\u957f\u5c3e\u592a\u957f\uff0c\u9002\u5f53\u6536\u7d27chunk\/_size \u6216\u589e\u52a0\u7c97\u7c92\u5ea6\u5206\u9694\u7b26\uff1b\u82e5\u8fc7\u77ed\uff0c\u653e\u5bbdchunk\/_size \u6216\u964d\u4f4e\u5206\u9694\u7b26\u4f18\u5148\u7ea7\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u91cd\u53e0\u7684\u6210\u672c\u4e0e\u6536\u76ca<\/strong>\uff1a<\/li>\n
- \n
\n- \u6536\u76ca\uff1a\u7f13\u89e3\u8fb9\u754c\u65ad\u88c2\uff0c\u63d0\u5347\u7b54\u6848\u8fde\u8d2f\u6027\u4e0e\u53ef\u8ffd\u6eaf\u6027\u3002<\/li>\n
- \u6210\u672c\uff1a\u7d22\u5f15\u5c3a\u5bf8\u589e\u957f\u3001\u53ec\u56de\u91cd\u590d\u5757\u589e\u591a\u3001rerank \u8d1f\u8f7d\u63d0\u5347\u3002\u901a\u5e38\u4e0d\u5efa\u8bae\u8d85\u8fc7 20%\u201325%\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u7ec4\u5408\u6280\u5de7<\/strong>\uff1a<\/li>\n
- \n
\n- \u5148\u9012\u5f52\u5206\u5757\uff0c\u518d\u5bf9\u201c\u5f02\u5e38\u957f\u53e5\u201d\u6216\u201c\u8de8\u6bb5\u5f15\u7528\u201d\u573a\u666f\u52a0\u4e00\u70b9\u70b9\u989d\u5916 overlap\u3002<\/li>\n
- \u5bf9\u6807\u9898\u5757\u6ce8\u5165\u7236\u7ea7\u6807\u9898\u4e0a\u4e0b\u6587\uff0c\u63d0\u9ad8\u5b9a\u4f4d\u80fd\u529b\u4e0e\u53ef\u89e3\u91ca\u6027\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4f55\u65f6\u5207\u6362\u7b56\u7565<\/strong>\uff1a<\/li>\n
- \n
\n- \u82e5\u95ee\u7b54\u9891\u7e41\u4e22\u4e0a\u4e0b\u6587\u6216\u5f15\u7528\u65ad\u88c2\uff1a\u589e\u5927overlap\u6216\u6539\u7528\u53e5\u5b50\/\u7ed3\u6784\u611f\u77e5\u7b56\u7565\u3002<\/li>\n
- \u82e5\u53ec\u56de\u542b\u566a\u8fc7\u591a\uff1a\u51cf\u5c0f chunk\/_size \u6216\u5f15\u5165\u66f4\u5f3a\u7684\u7ed3\u6784\u5206\u9694\u7b26\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
4.2 \u7ed3\u6784\u611f\u77e5\u5206\u5757<\/h2>\n
\u5229\u7528\u6587\u6863\u56fa\u6709\u7ed3\u6784\uff08\u6807\u9898\u5c42\u7ea7\u3001\u5217\u8868\u3001\u4ee3\u7801\u5757\u3001\u8868\u683c\u3001\u5bf9\u8bdd\u8f6e\u6b21\uff09\u4f5c\u4e3a\u5206\u5757\u8fb9\u754c\uff0c\u903b\u8f91\u6e05\u6670\u3001\u53ef\u8ffd\u6eaf\u6027\u5f3a\uff0c\u80fd\u5728\u4fdd\u8bc1\u4e0a\u4e0b\u6587\u5b8c\u6574\u6027\u7684\u540c\u65f6\u63d0\u5347\u68c0\u7d22\u4fe1\u566a\u6bd4\u3002<\/p>\n
\u7ed3\u6784\u5316\u6587\u672c\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong><\/li>\n
- \u4ee5\u6807\u9898\u5c42\u7ea7\uff08H1\u2013H6\u3001\u7f16\u53f7\u6807\u9898\uff09\u6216\u8bed\u4e49\u5757\uff08\u6bb5\u843d\u3001\u5217\u8868\u3001\u8868\u683c\u3001\u4ee3\u7801\u5757\uff09\u4e3a\u6b64\u7c7b\u578b\u6587\u6863\u7684\u5929\u7136\u8fb9\u754c\uff0c\u5bf9\u8fc7\u957f\u7684\u7ed3\u6784\u5757\u518d\u505a\u4e8c\u6b21\u7ec6\u5206\uff0c\u5bf9\u8fc7\u77ed\u7684\u8fdb\u884c\u76f8\u90bb\u5408\u5e76\u3002<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u5b9e\u65bd\u6b65\u9aa4<\/strong><\/li>\n
- \n
\n- \u89e3\u6790\u7ed3\u6784\uff1aMarkdown \u7528\u89e3\u6790\u5668remark\/markdown-it-py\u6216\u6b63\u5219\u8bc6\u522b\u6807\u9898\u4e0e\u8bed\u5757\uff1bHTML\u7528 DOMBeautifulSoup\/Cheerio\u904d\u5386 Hx\u3001p\u3001li\u3001pre\u3001table \u7b49\u3002<\/li>\n
- \u751f\u6210\u7ae0\u8282\uff1a\u4ee5\u6807\u9898\u4e3a\u7236\u8282\u70b9\uff0c\u5c06\u5176\u540e\u7684\u8fde\u7eed\u5144\u5f1f\u8282\u70b9\u7eb3\u5165\u8be5\u7ae0\u8282\uff0c\u76f4\u81f3\u9047\u5230\u540c\u7ea7\u6216\u66f4\u9ad8\u5c42\u7ea7\u6807\u9898\u3002<\/li>\n
- \u4e8c\u6b21\u5207\u5206\uff1a\u7ae0\u8282\u8d85\u51fa chunk\/_size\u65f6\uff0c\u4f18\u5148\u6309\u5b50\u6807\u9898\/\u6bb5\u843d\u5207\uff0c\u518d\u4e0d\u8db3\u65f6\u6309\u53e5\u5b50\u6216\u9012\u5f52\u5b57\u7b26\u5207\u5206\u3002<\/li>\n
- \u5408\u5e76\u77ed\u5757\uff1a\u4f4e\u4e8e min\/_chunk\/_chars \u7684\u5757\u4e0e\u76f8\u90bb\u5757\u5408\u5e76\uff0c\u4f18\u5148\u4e0e\u540c\u4e00\u7236\u6807\u9898\u4e0b\u7684\u524d\u540e\u5757\u3002<\/li>\n
- \u4e0a\u4e0b\u6587\u91cd\u53e0\uff1a\u4f18\u5148\u7528\u201c\u7ed3\u6784\u91cd\u53e0\u201d\uff08\u7236\u7ea7\u6807\u9898\u8def\u5f84\u3001\u524d\u4e00\u5c0f\u8282\u6807\u9898+\u6458\u8981\uff09\uff0c\u518d\u8f85\u4ee5\u5c0f\u6bd4\u4f8b\u5b57\u7b26overlap\uff0810%\u201315%\uff09\u3002<\/li>\n
- \u5199\u5165 metadata\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u793a\u4f8b\u4ee3\u7801<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
import re\nfrom typing import List, Dict\n\n\nheading_pat = re.compile(r'^(#{1,6})\/s+(.*)$') # \u6807\u9898\nfence_pat = re.compile(r'^```') # fenced code fence\n\n\ndef split_markdown_structure(text: str, chunk_size=900, min_chunk=250, overlap_ratio=0.1) -> List[Dict]:\n lines = text.splitlines()\n sections = []\n in_code = False\n current = {"level": 0, "title": "", "content": [], "path": []}\n \n path_stack = [] # [(level, title)]\n \n for ln in lines:\n if fence_pat.match(ln):\n in_code = not in_code\n m = heading_pat.match(ln) if not in_code else None\n if m:\n if current["content"]:\n sections.append(current)\n level = len(m.group(1))\n title = m.group(2).strip()\n\n\n while path_stack and path_stack[-1][0] >= level:\n path_stack.pop()\n path_stack.append((level, title))\n breadcrumbs = [t for _, t in path_stack]\n current = {"level": level, "title": title, "content": [], "path": breadcrumbs}\n else:\n current["content"].append(ln)\n \n if current["content"]:\n sections.append(current)\n \n # \u901a\u8fc7\u4e8c\u6b21\u62c6\u5206\/\u5408\u5e76\u5c06\u90e8\u5206\u5e73\u94fa\u6210\u5757\n chunks = []\n def emit_chunk(text_block: str, path: List[str], level: int):\n chunks.append({\n "text": text_block.strip(),\n "meta": {\n "section_title": path[-1] if path else "",\n "breadcrumbs": path,\n "section_level": level,\n }\n })\n \n for sec in sections:\n raw = "\/n".join(sec["content"]).strip()\n if not raw:\n continue\n if len(raw) <= chunk_size:\n emit_chunk(raw, sec["path"], sec["level"])\n else:\n paras = [p.strip() for p in raw.split("\/n\/n") if p.strip()]\n buf = ""\n for p in paras:\n if len(buf) + len(p) + 2 <= chunk_size:\n buf += (("\/n\/n" + p) if buf else p)\n else:\n if buf:\n emit_chunk(buf, sec["path"], sec["level"])\n buf = p\n if buf:\n emit_chunk(buf, sec["path"], sec["level"])\n \n merged = []\n for ch in chunks:\n if not merged:\n merged.append(ch)\n continue\n if len(ch["text"]) < min_chunk and merged[-1]["meta"]["breadcrumbs"] == ch["meta"]["breadcrumbs"]:\n merged[-1]["text"] += "\/n\/n" + ch["text"]\n else:\n merged.append(ch)\n \n overlap = int(chunk_size * overlap_ratio)\n for ch in merged:\n bc = " > ".join(ch["meta"]["breadcrumbs"][-3:])\n prefix = f"[{bc}]\/n" if bc else ""\n if prefix and not ch["text"].startswith(prefix):\n ch["text"] = prefix + ch["text"]\n # optional character overlap can\u5728\u68c0\u7d22\u9636\u6bb5\u7528\u90bb\u63a5\u805a\u5408\u66ff\u4ee3\uff0c\u8fd9\u91cc\u7565\n \n return merged\n<\/code><\/pre>\n\n- \u53c2\u6570\u5efa\u8bae\uff08\u4e2d\u6587\u6587\u6863\uff09<\/strong><\/li>\n
- \n
\n- chunk\/_size\uff1a600\u20131000 \u5b57\uff1b\u6280\u672f\u6587\/\u957f\u6bb5\u843d\u53ef\u53d6\u4e0a\u9650\uff0c\u7ee7\u7eed\u9002\u5f53\u589e\u52a0\u3002<\/li>\n
- min\/_chunk\/_chars\uff1a200\u2013300 \u5b57\uff08\u5c0f\u4e8e\u5219\u5408\u5e76\uff09\u3002<\/li>\n
- chunk\/_overlap\uff1a10%\u201315%\uff1b\u82e5\u4f7f\u7528\u201c\u7236\u7ea7\u6807\u9898\u8def\u5f84 + \u6458\u8981\u201d\u4f5c\u4e3a\u7ed3\u6784\u91cd\u53e0\uff0c\u53ef\u964d\u81f3 5%\u201310%\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
\u5bf9\u8bdd\u5f0f\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong><\/li>\n
- \u4ee5\u201c\u8f6e\u6b21\/\u8bf4\u8bdd\u4eba\u201d\u4e3a\u8fb9\u754c\uff0c\u4f18\u5148\u6309\u5bf9\u8bdd\u90bb\u63a5\u5bf9\u548c\u5c0f\u6bb5\u8bdd\u9898\u7a97\u53e3\u805a\u5408\u3002\u91cd\u53e0\u91c7\u7528\u201c\u8f6e\u6b21\u91cd\u53e0\u201d\u800c\u975e\u5355\u7eaf\u5b57\u7b26\u91cd\u53e0\uff0c\u4fdd\u8bc1\u4e0a\u4e0b\u6587\u6d41\u7545\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u9002\u7528\u573a\u666f<\/strong><\/li>\n
- \u5ba2\u670d\u5bf9\u8bdd\u3001\u8bbf\u8c08\u3001\u4f1a\u8bae\u7eaa\u8981\u3001\u6280\u672f\u652f\u6301\u5de5\u5355\u7b49\u591a\u8f6e\u4ea4\u6d41\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u68c0\u7d22\u671f\u90bb\u63a5\u805a\u5408<\/strong><\/li>\n
- \u5728\u68c0\u7d22\u9636\u6bb5\u5bf9\u5bf9\u8bdd\u5757\u505a\u201c\u90bb\u63a5\u6269\u5c55\u201d\uff1a\u53d6\u88ab\u53ec\u56de\u5757\u524d\u540e\u5404 1\u20132 \u8f6e\u4e0a\u4e0b\u6587\uff08\u6216\u76f8\u90bb\u5757\u62fc\u63a5\uff09\u4f5c\u4e3a\u6700\u7ec8\u9001\u5ba1\u4e0a\u4e0b\u6587\uff0c\u4ee5\u63d0\u9ad8\u56de\u7b54\u8fde\u8d2f\u6027\u4e0e\u53ef\u8ffd\u6eaf\u6027\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4e0e\u91cd\u6392\u534f\u540c<\/strong><\/li>\n
- \u53ef\u63d0\u5347\u5bf9\u201c\u8c01\u8bf4\u7684\u3001\u5728\u54ea\u6bb5\u8bf4\u7684\u201d\u7684\u5224\u65ad\u529b\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u793a\u4f8b\u4ee3\u7801<\/strong>\uff1a\uff08\u6309\u8f6e\u6b21\u6ed1\u52a8\u7a97\u53e3\u5206\u5757\uff09<\/li>\n<\/ul>\n
<!—-><\/p>\n
from typing import List, Dict\n\n\ndef chunk_dialogue(turns: List[Dict], max_turns=10, max_chars=900, overlap_turns=2):\n """\n turns: [{"speaker":"User","text":"..." , "ts_start":123, "ts_end":130}, ...]\n """\n chunks = []\n i = 0\n while i < len(turns):\n j = i\n char_count = 0\n speakers = set()\n while j < len(turns):\n t = turns[j]\n uttr_len = len(t["text"])\n # \u82e5\u5355\u6761\u8d85\u957f\uff0c\u5141\u8bb8\u5728\u53e5\u7ea7\u4e8c\u6b21\u5207\u5206\uff08\u6b64\u5904\u7565\uff09\uff0c\u4f46\u4e0d\u8de8 speaker\n if (j - i + 1) > max_turns or (char_count + uttr_len) > max_chars:\n break\n char_count += uttr_len\n speakers.add(t["speaker"])\n j += 1\n \n if j > i:\n window = turns[i:j]\n elif i < len(turns):\n window = [turns[i]]\n else:\n break\n text = "\/n".join([f'{t["speaker"]}: {t["text"]}' for t in window])\n meta = {\n "speakers": list(speakers),\n "turns_range": (i, j - 1),\n "ts_start": window[0].get("ts_start"),\n "ts_end": window[-1].get("ts_end"),\n }\n chunks.append({"text": text, "meta": meta})\n \n # \u6309\u8f6e\u6b21\u91cd\u53e0\u56de\u9000\n if j >= len(turns):\n break\n next_start = i + len(window) - overlap_turns\n i = max(next_start, i + 1) # \u786e\u4fdd\u81f3\u5c11\u524d\u8fdb1\u6b65\n return chunks\n<\/code><\/pre>\n\n- \u53c2\u6570\u5efa\u8bae<\/strong><\/li>\n
- \n
\n- max\/_turns\/_per\/_chunk\uff1a6\u201312 \u8f6e\u8d77\u6b65\uff1b\u8bed\u901f\u5feb\u4fe1\u606f\u5bc6\u5ea6\u9ad8\u53ef\u53d6 8\u201310\u3002<\/li>\n
- max\/_chars\/_per\/_chunk\uff1a600\u20131000 \u5b57\uff1b\u82e5\u5b58\u5728\u957f\u6bb5\u72ec\u767d\uff0c\u4f18\u5148\u53e5\u7ea7\u518d\u5207\uff0c\u4e0d\u8de8\u8bf4\u8bdd\u4eba\u3002<\/li>\n
- overlap\/_turns\uff1a1\u20132 \u8f6e\uff1b\u4fdd\u8bc1\u4e0a\u4e00\u95ee\u4e0b\u4e00\u7b54\u7684\u8fde\u7eed\u6027\u3002<\/li>\n
- keep\/_pairing\uff1a\u4e0d\u8981\u62c6\u5f00\u660e\u663e\u7684\u95ee\u7b54\u5bf9\uff1b\u82e5 chunk \u4e34\u754c\uff0c\u5b81\u53ef\u6269\u4e00\u8f6e\u6216\u540e\u79fb\u5207\u5206\u70b9\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
\u603b\u7ed3<\/strong><\/p>\n\n- \u9996\u9009\u7528\u7ed3\u6784\u8fb9\u754c\u505a\u7b2c\u4e00\u6b21\u5207\u5206\uff0c\u518d\u7528\u53e5\u7ea7\/\u9012\u5f52\u7b56\u7565\u505a\u4e8c\u6b21\u7ec6\u5206\u3002<\/li>\n
- \u4f18\u5148\u4f7f\u7528\u201c\u7ed3\u6784\u91cd\u53e0\u201d\uff08\u7236\u6807\u9898\u8def\u5f84\u3001\u4e0a\u6bb5\u6807\u9898+\u6458\u8981\u3001\u76f8\u90bb\u53d1\u8a00\uff09\u66ff\u4ee3\u5927\u6bd4\u4f8b\u5b57\u7b26\u91cd\u53e0\u3002<\/li>\n
- \u4e3a\u6bcf\u4e2a\u5757\u5199\u597d metadata\uff0c\u53ef\u663e\u8457\u63d0\u5347\u68c0\u7d22\u8d28\u91cf\u4e0e\u53ef\u89e3\u91ca\u6027\u3002<\/li>\n
- \u5bf9 PDF\/HTML \u5148\u53bb\u566a\uff08\u9875\u7709\u9875\u811a\u3001\u5bfc\u822a\u3001\u5e7f\u544a\u7b49\uff09\uff0c\u907f\u514d\u628a\u566a\u58f0\u7d22\u5f15\u8fdb\u5e93\u3002<\/li>\n<\/ul>\n
4.3 \u8bed\u4e49\u4e0e\u4e3b\u9898\u5206\u5757<\/h2>\n
\u8be5\u65b9\u6cd5\u4e0d\u4f9d\u8d56\u6587\u6863\u7684\u7269\u7406\u7ed3\u6784\uff0c\u800c\u662f\u4f9d\u636e\u8bed\u4e49\u8fde\u7eed\u6027\u4e0e\u8bdd\u9898\u8f6c\u79fb\u6765\u51b3\u5b9a\u5207\u5206\u70b9\uff0c\u5c24\u5176\u9002\u5408\u5e0c\u671b\u201c\u5757\u5185\u9ad8\u5ea6\u5185\u805a\u3001\u5757\u95f4\u6e05\u6670\u5206\u754c\u201d\u7684\u77e5\u8bc6\u5e93\u4e0e\u7814\u7a76\u7c7b\u6587\u672c\u3002<\/p>\n
\u8bed\u4e49\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong><\/li>\n
- \n
\n- \u5bf9\u6587\u672c\u5148\u505a\u53e5\u7ea7\u5207\u5206\uff0c\u8ba1\u7b97\u53e5\u5b50\u6216\u77ed\u6bb5\u7684\u5411\u91cf\u8868\u793a\uff1b<\/li>\n
- \u5f53\u76f8\u90bb\u8bed\u4e49\u7684\u76f8\u4f3c\u5ea6\u663e\u8457\u4e0b\u964d\uff08\u53d1\u751f\u201c\u8bed\u4e49\u7a81\u53d8\u201d\uff09\u65f6\u8bbe\u4e3a\u5207\u5206\u70b9\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u9002\u7528\u573a\u666f<\/strong><\/li>\n
- \n
\n- \u4e13\u9898\u5316\u3001\u8bba\u8bc1\u7ed3\u6784\u660e\u663e\u7684\u6587\u6863\uff1a<\/li>\n
- \u767d\u76ae\u4e66\u3001\u8bba\u6587\u3001\u6280\u672f\u624b\u518c\u3001FAQ \u805a\u5408\u9875\uff1b<\/li>\n
- \u9700\u8981\u9ad8\u5185\u805a\u68c0\u7d22\u4e0e\u9ad8\u53ef\u8ffd\u6eaf\u6027\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4f7f\u7528\u6d41\u7a0b<\/strong><\/li>\n
- \n
\n- \u53e5\u7ea7\u5207\u5206\uff1a\u5148\u7528\u4e2d\u6587\u5206\u53e5\uff08\u6807\u70b9\/\u4e2d\u6587\u5206\u53e5\u6a21\u578b\uff09\u5f97\u5230\u53e5\u5b50\u5e8f\u5217\u3002<\/li>\n
- \u5411\u91cf\u5316\uff1a\u5bf9\u6bcf\u4e2a\u53e5\u5b50\u7f16\u7801\uff0c\u5f00\u542f\u5f52\u4e00\u5316\uff08normalize\uff09\u4ee5\u4fbf\u7528\u4f59\u5f26\u76f8\u4f3c\u5ea6\u3002<\/li>\n
- \u7a81\u53d8\u68c0\u6d4b\uff1a<\/li>\n
- \n
\n- \u7b80\u5355\u7c97\u66b4\u7684\u65b9\u6cd5\uff1asim(i, i-1) \u4f4e\u4e8e\u9608\u503c\u5219\u5207\u5206\u3002<\/li>\n
- \u7a33\u5065\u7684\u65b9\u6cd5\uff1a\u4e0e\u201c\u524d\u540e\u7a97\u53e3\u7684\u5747\u503c\u5411\u91cf\u201d\u6bd4\u8f83\uff0c\u8ba1\u7b97\u65b0\u9896\u5ea6 novelty = 1 – cos(emb\/_i, mean\/_emb\/_window)\uff0c\u65b0\u9896\u5ea6\u9ad8\u4e8e\u9608\u503c\u5219\u5207\u5206\u3002<\/li>\n
- \u5e73\u6ed1\u7684\u65b9\u6cd5\uff1a\u5bf9\u76f8\u4f3c\u5ea6\/\u65b0\u9896\u5ea6\u505a\u79fb\u52a8\u5e73\u5747\uff0c\u964d\u4f4e\u6296\u52a8\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u7ea6\u675f\u4e0e\u4fee\u6b63\uff1a\u8bbe\u7f6e\u6700\u5c0f\/\u6700\u5927\u5757\u957f\uff0c\u907f\u514d\u8fc7\u788e\u6216\u8fc7\u957f\uff0c\u5fc5\u8981\u65f6\u8fdb\u884c\u76f8\u90bb\u5757\u5408\u5e76\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4e0e\u68c0\u7d22\/\u91cd\u6392\u7684\u534f\u540c<\/strong><\/li>\n
- \u53ec\u56de\u65f6\u53ef\u505a\u201c\u90bb\u63a5\u6269\u5c55\u201d\uff08\u628a\u88ab\u547d\u4e2d\u7684\u5757\u524d\u540e\u5404\u8ffd\u52a0\u4e00\u4e24\u53e5\uff09\uff0c\u518d\u505a\u91cd\u6392\u5e8f\u3002\u8bed\u4e49\u5206\u5757\u7684\u9ad8\u5185\u805a\u53ef\u8ba9 \u91cd\u6392\u5e8f\u66f4\u7cbe\u51c6\u5730\u533a\u5206\u76f8\u8fd1\u5019\u9009\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4ee3\u7801\u793a\u4f8b<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
from typing import List, Dict, Tuple\nimport numpy as np\nfrom sentence_transformers import SentenceTransformer\nimport re\n\n\ndef split_sentences_zh(text: str) -> List[str]:\n # \u7b80\u6613\u4e2d\u6587\u5206\u53e5\uff0c\u53ef\u66ff\u6362\u4e3a HanLP\/Stanza \u66f4\u7a33\u5065\u7684\u5b9e\u73b0\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n return [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n\n\ndef rolling_mean(vecs: np.ndarray, i: int, w: int) -> np.ndarray:\n s = max(0, i - w)\n e = min(len(vecs), i + w + 1)\n return vecs[s:e].mean(axis=0)\n\n\ndef semantic_chunk(\n text: str,\n model_name: str = "BAAI\/bge-m3",\n window_size: int = 2,\n min_chars: int = 350,\n max_chars: int = 1100,\n lambda_std: float = 0.8,\n overlap_chars: int = 80,\n) -> List[Dict]:\n sents = split_sentences_zh(text)\n if not sents:\n return []\n \n model = SentenceTransformer(model_name)\n emb = model.encode(sents, normalize_embeddings=True, batch_size=64, show_progress_bar=False)\n emb = np.asarray(emb)\n \n # \u57fa\u4e8e\u7a97\u53e3\u5747\u503c\u7684\u201c\u65b0\u9896\u5ea6\u201d\u5206\u6570\n novelties = []\n for i in range(len(sents)):\n ref = rolling_mean(emb, i-1, window_size) if i > 0 else emb[0]\n ref = ref \/ (np.linalg.norm(ref) + 1e-8)\n novelty = 1.0 - float(np.dot(emb[i], ref))\n novelties.append(novelty)\n novelties = np.array(novelties)\n \n # \u76f8\u5bf9\u9608\u503c\uff1a\u03bc + \u03bb\u03c3\n mu, sigma = float(novelties.mean()), float(novelties.std() + 1e-8)\n threshold = mu + lambda_std * sigma\n \n chunks, buf, start_idx = [], "", 0\n def flush(end_idx: int):\n nonlocal buf, start_idx\n if buf.strip():\n chunks.append({\n "text": buf.strip(),\n "meta": {"start_sent": start_idx, "end_sent": end_idx-1}\n })\n buf, start_idx = "", end_idx\n \n for i, s in enumerate(sents):\n # \u82e5\u8d85\u957f\u5219\u5148\u51b2\u6d17\n if len(buf) + len(s) > max_chars and len(buf) >= min_chars:\n flush(i)\n # \u7ed3\u6784\u5316\u91cd\u53e0\uff1a\u9644\u52a0\u4e0a\u4e00\u4e2a\u5757\u7684\u5c3e\u90e8\n if overlap_chars > 0 and len(s) < overlap_chars:\n buf = s\n continue\n \n buf += s\n \n # \u8fbe\u5230\u6700\u5c0f\u957f\u5ea6\u540e\u9047\u5230\u7a81\u53d8\u5219\u5207\u5206\n if len(buf) >= min_chars and novelties[i] > threshold:\n flush(i + 1)\n \n if buf:\n flush(len(sents))\n \n return chunks\n<\/code><\/pre>\n\n- \u53c2\u6570\u8c03\u4f18\u8bf4\u660e<\/strong>\uff08\u4ec5\u4f5c\u53c2\u8003\uff09<\/li>\n
- \n
\n- \u9608\u503c\u7684\u542b\u4e49\uff1a\u8bed\u4e49\u53d8\u5316\u654f\u611f\u5ea6\u63a7\u5236\u5668\uff0c\u8d8a\u4f4e\u8d8a\u5bb9\u6613\u5207\u3001\u8d8a\u9ad8\u8d8a\u4fdd\u5b88\u3002<\/li>\n
- \u8bbe\u5b9a\u65b9\u5f0f\uff1a<\/li>\n
- \n
\n- \u7edd\u5bf9\u9608\u503c\uff1a\u4f8b\u5982\u4f7f\u7528\u4f59\u5f26\u76f8\u4f3c\u5ea6\uff0c\u82e5 sim < 0.75 \u5219\u5207\u5206\uff08\u9700\u6309\u8bed\u6599\u6821\u51c6\uff09\u3002<\/li>\n
- \u76f8\u5bf9\u9608\u503c\uff1a\u5bf9\u5168\u7bc7\u7684\u76f8\u4f3c\u5ea6\/\u65b0\u9896\u5ea6\u5206\u5e03\u4f30\u8ba1\u5747\u503c\u03bc\u4e0e\u6807\u51c6\u5dee\u03c3\uff0c\u4f7f\u7528 \u03bc \u00b1 \u03bb\u03c3 \u4f5c\u4e3a\u9608\u503c\uff0c\u66f4\u7a33\u5065\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u521d\u59cb\u7684\u914d\u7f6e\u5efa\u8bae\uff08\u4ec5\u9650\u4e8e\u4e2d\u6587\u6280\u672f\/\u8bf4\u660e\u6587\u6863\uff09\uff1a<\/li>\n
- \n
\n- \u7a97\u53e3\u5927\u5c0f window\/_size\uff1a2\u20134 \u53e5<\/li>\n
- \u6700\u5c0f\/\u6700\u5927\u5757\u957f\uff1amin\/_chunk\/_chars=300\u2013400\uff0cmax\/_chunk\/_chars=1000\u20131200<\/li>\n
- \u9608\u503c\u7b56\u7565\uff1anovelty > \u03bc + 0.8\u03c3 \u6216\u76f8\u4f3c\u5ea6 < \u03bc – 0.8\u03c3\uff08\u5148\u7c97\u8c03\u540e\u5fae\u8c03\uff09<\/li>\n
- overlap\uff1a10% \u5de6\u53f3\u6216\u6309\u201c\u9644\u52a0\u4e0a\u4e00\u53e5\u201d\u505a\u8f7b\u91cf\u8f6e\u6b21\u91cd\u53e0<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
\u4e3b\u9898\u7684\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong><\/li>\n
- \u5229\u7528\u4e3b\u9898\u6a21\u578b\u6216\u805a\u7c7b\u7b97\u6cd5\u5728\u201c\u5b8f\u89c2\u8bdd\u9898\u201d\u53d1\u751f\u5207\u6362\u65f6\u8fdb\u884c\u5207\u5206\uff0c\u66f4\u591a\u7684\u5173\u6ce8\u7ae0\u8282\u7ea7\u3001\u6bb5\u843d\u7ea7\u7684\u4e3b\u9898\u8fb9\u754c\u3002\u8be5\u7c7b\u5206\u5757\u7b56\u7565\u4e3b\u8981\u9002\u5408\u957f\u7bc7\u3001\u591a\u4e3b\u9898\u6750\u6599\u3002<\/strong><\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u9002\u7528\u573a\u666f<\/strong><\/li>\n
- \n
\n- \u62a5\u544a\u3001\u4e66\u7c4d\u3001\u957f\u8c03\u7814\u6587\u6863\u3001\u7efc\u5408\u8bc4\u5ba1\uff1b<\/li>\n
- \u5f53\u6587\u6863\u5185\u90e8\u786e\u6709\u8f83\u7a33\u5b9a\u7684\u201c\u8bdd\u9898\u5757\u201d\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4f7f\u7528\u6d41\u7a0b<\/strong>\uff08\u6700\u597d\u7528\u201c\u53e5\u5411\u91cf + \u805a\u7c7b + \u5e8f\u5217\u5e73\u6ed1\u201d\u800c\u975e\u7eaf LDA\uff09<\/li>\n
- \n
\n- \u53e5\u7ea7\u5207\u5206\u5e76\u7f16\u7801\uff1a\u9996\u5148\u901a\u8fc7\u5411\u91cf\u6a21\u578b\u5f97\u5230\u53e5\u5411\u91cf\uff0cnormalize\u3002<\/li>\n
- \u6587\u6863\u5185\u6216\u8bed\u6599\u7ea7\u805a\u7c7b\uff1a<\/li>\n
- \n
\n- \u6587\u6863\u5185\u5c0f\u89c4\u6a21\uff1aMiniBatchKMeans\uff08k=3\u20138 \u5148\u9a8c\uff09\u6216 SpectralClustering\u3002<\/li>\n
- \u8bed\u6599\u7ea7\u7edf\u4e00\u4e3b\u9898\uff1a\u5728\u5927\u91cf\u6587\u6863\u4e0a\u805a\u7c7b\uff08\u6216\u7528 HDBSCAN+UMAP\uff09\uff0c\u518d\u5c06\u6bcf\u7bc7\u6587\u6863\u7684\u53e5\u5b50\u6620\u5c04\u5230\u6700\u8fd1\u4e3b\u9898\u4e2d\u5fc3\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u5e8f\u5217\u5e73\u6ed1\u4e0e\u89e3\u7801\uff1a<\/li>\n
- \n
\n- \u5bf9\u53e5\u5b50\u7684\u4e3b\u9898\u6807\u7b7e\u505a\u6ed1\u7a97\u591a\u6570\u6295\u7968\u6216\u4e00\u9636\u9a6c\u5c14\u53ef\u592b\u5e73\u6ed1\uff0c\u907f\u514d\u9891\u7e41\u6296\u52a8\u3002<\/li>\n
- \u5f53\u4e3b\u9898\u6807\u7b7e\u7a33\u5b9a\u53d8\u5316\u5e76\u6ee1\u8db3\u6700\u5c0f\u5757\u957f\u65f6\uff0c\u8bbe\u4e3a\u5207\u5206\u70b9\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u4e3b\u9898\u547d\u540d\uff1a<\/li>\n
- \u7528 KeyBERT\/TF-IDF \u5728\u6bcf\u4e2a\u5757\u5185\u62bd\u5173\u952e\u8bcd\uff0c\u6216\u7528\u5c0f\u6a21\u578b\u751f\u6210\u4e00\u53e5\u8bdd\u4e3b\u9898\u6458\u8981\uff0c\u5199\u5165 metadata\u3002<\/li>\n
- \u7ea6\u675f\uff1amin\/max\/_chars\uff0c\u4fdd\u7559\u4ee3\u7801\/\u8868\u683c\u7b49\u539f\u5b50\u5757\uff0c\u5fc5\u8981\u65f6\u4e0e\u7ed3\u6784\u8fb9\u754c\u7ed3\u5408\u4f7f\u7528\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4ee3\u7801\u793a\u4f8b<\/strong>\uff08KMeans \u6587\u6863\u5185\u805a\u7c7b + \u5e8f\u5217\u5e73\u6ed1\uff09<\/li>\n<\/ul>\n
<!—-><\/p>\n
from typing import List, Dict\nimport numpy as np\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.cluster import KMeans\nimport re\n\n\ndef split_sentences_zh(text: str) -> List[str]:\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n return [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n\n\ndef topic_chunk(\n text: str,\n k_topics: int = 5,\n min_chars: int = 500,\n max_chars: int = 1400,\n smooth_window: int = 2,\n model_name: str = "BAAI\/bge-m3"\n) -> List[Dict]:\n sents = split_sentences_zh(text)\n if not sents:\n return []\n \n model = SentenceTransformer(model_name)\n emb = model.encode(sents, normalize_embeddings=True, batch_size=64, show_progress_bar=False)\n emb = np.asarray(emb)\n \n km = KMeans(n_clusters=k_topics, n_init="auto", random_state=42)\n labels = km.fit_predict(emb)\n \n # \u7b80\u5355\u5e8f\u5217\u5e73\u6ed1\uff1a\u6ed1\u7a97\u591a\u6570\u6295\u7968\n smoothed = labels.copy()\n for i in range(len(labels)):\n s = max(0, i - smooth_window)\n e = min(len(labels), i + smooth_window + 1)\n window = labels[s:e]\n vals, counts = np.unique(window, return_counts=True)\n smoothed[i] = int(vals[np.argmax(counts)])\n \n chunks, buf, start_idx, cur_label = [], "", 0, smoothed[0]\n def flush(end_idx: int):\n nonlocal buf, start_idx\n if buf.strip():\n chunks.append({\n "text": buf.strip(),\n "meta": {"start_sent": start_idx, "end_sent": end_idx-1, "topic": int(cur_label)}\n })\n buf, start_idx = "", end_idx\n \n for i, s in enumerate(sents):\n switched = smoothed[i] != cur_label\n over_max = len(buf) + len(s) > max_chars\n under_min = len(buf) < min_chars\n \n # \u5c1d\u8bd5\u5ef6\u540e\u5207\u5206\uff0c\u4fdd\u8bc1\u6700\u5c0f\u5757\u957f\n if switched and not under_min:\n flush(i)\n cur_label = smoothed[i]\n \n if over_max and not under_min:\n flush(i)\n \n buf += s\n \n if buf:\n flush(len(sents))\n \n return chunks\n<\/code><\/pre>\n\n- \u4e00\u4e9b\u53c2\u6570\u5bf9\u7ed3\u679c\u7684\u5f71\u54cd<\/strong><\/li>\n
- \n
\n- k\uff08\u4e3b\u9898\u6570\uff09\uff1a\u96be\u4ee5\u7cbe\u51c6\u9884\u8bbe\uff0c\u53ef\u901a\u8fc7\u8f6e\u5ed3\u7cfb\u6570\uff08silhouette\uff09\/\u8098\u90e8\u6cd5\u521d\u7b5b\uff0c\u518d\u7ed3\u5408\u9886\u57df\u5148\u9a8c\u4e0e\u4eba\u5de5\u6821\u6b63\u3002<\/li>\n
- HDBSCAN\uff1amin\/_cluster\/_size \u5f71\u54cd\u8f83\u5927\uff0c\u8fc7\u5c0f\u4f1a\u788e\u7247\u5316\uff0c\u8fc7\u5927\u5219\u5408\u5e76\u4e0d\u540c\u8bdd\u9898\u3002<\/li>\n
- min\/_topic\/_span\/_sents\uff1a\u5982 5\u20138 \u53e5\uff0c\u9632\u6b62\u6807\u7b7e\u6296\u52a8\u5bfc\u81f4\u8fc7\u5bc6\u5207\u5206\u3002<\/li>\n
- \u5c0f\u6587\u6863\u4e0d\u5b9c\u7528\uff1a\u6837\u672c\u592a\u5c11\u65f6\u4e3b\u9898\u4e0d\u53ef\u5206\uff0c\u4f18\u5148\u7528\u8bed\u4e49\u5206\u5757\u6216\u7ed3\u6784\u5206\u5757\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
4.4 \u9ad8\u7ea7\u5206\u5757<\/h2>\n
\u5c0f-\u5927\u5206\u5757<\/strong><\/p>\n\n- \u5206\u5757\u7b56\u7565<\/strong><\/li>\n
- \u7528\u201c\u5c0f\u7c92\u5ea6\u5757\u201d\uff08\u5982\u53e5\u5b50\/\u77ed\u53e5\uff09\u505a\u9ad8\u7cbe\u5ea6\u53ec\u56de\uff0c\u5b9a\u4f4d\u5230\u6700\u76f8\u5173\u7684\u5fae\u7247\u6bb5\uff1b\u518d\u5c06\u5176\u201c\u6240\u5728\u7684\u5927\u7c92\u5ea6\u5757\u201d\uff08\u5982\u6bb5\u843d\/\u5c0f\u8282\uff09\u4f5c\u4e3a\u4e0a\u4e0b\u6587\u9001\u5165 LLM\uff0c\u4ee5\u517c\u987e\u7cbe\u786e\u6027\u4e0e\u4e0a\u4e0b\u6587\u5b8c\u6574\u6027\u3002<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4f7f\u7528\u6d41\u7a0b<\/strong><\/li>\n
- \n
\n- \u6784\u5efa\u7d22\u5f15\uff08\u79bb\u7ebf\uff09\uff1a<\/li>\n
- \n
\n- Sentence\/\u77ed\u53e5\u7d22\u5f15\uff08\u7d22\u5f15A\uff09\uff1a\u5355\u4f4d\u4e3a\u53e5\u5b50\u6216\u5b50\u53e5\u3002<\/li>\n
- \u6bb5\u843d\/\u5c0f\u8282\u5b58\u50a8\uff08\u5b58\u50a8B\uff09\uff1a\u4fdd\u7559\u539f\u59cb\u5927\u5757\u6587\u672c\u4e0e\u7ed3\u6784\u4fe1\u606f\u3002<\/li>\n<\/ul>\n<\/li>\n
- \u68c0\u7d22\uff08\u5728\u7ebf\uff09\uff1a<\/li>\n
- \n
\n- \u7528\u7d22\u5f15A\u53ec\u56de top\/_k\/_small \u4e2a\u5c0f\u5757\uff08\u5411\u91cf\u68c0\u7d22\uff09\u3002<\/li>\n
- \u5c06\u5c0f\u5757\u6309 parent\/_id \u5206\u7ec4\uff0c\u8ba1\u7b97\u7ec4\u5185\u5206\u6570\uff08max\/mean\/\u52a0\u6743\uff09\uff0c\u9009\u51fa top\/_m\/_big \u4e2a\u7236\u5757\u5019\u9009\u3002<\/li>\n
- \u5bf9\u201c\u67e5\u8be2-\u7236\u5757\u6587\u672c\u201d\u505a\u4ea4\u53c9\u7f16\u7801\u91cd\u6392\uff0c\u63d0\u5347\u76f8\u5173\u6027\u6392\u5e8f\u7684\u7a33\u5b9a\u6027\u3002<\/li>\n
- \u4e0a\u4e0b\u6587\u7ec4\u88c5\uff1a\u5728\u6bcf\u4e2a\u7236\u5757\u4e2d\u9ad8\u4eae\u6216\u4f18\u5148\u4fdd\u7559\u547d\u4e2d\u5c0f\u53e5\u9644\u8fd1\u7684\u4e0a\u4e0b\u6587\uff08\u90bb\u8fd1N\u53e5\u6216\u7a97\u53e3\u5b57\u7b26 w\uff09\uff0c\u5728\u6574\u4f53 token \u9884\u7b97\u5185\u62fc\u63a5\u591a\u5757\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n
<!—-><\/p>\n
\n- \u4ee3\u7801\u793a\u4f8b\uff08\u4f2a\u4ee3\u7801\uff09<\/strong><\/li>\n<\/ul>\n
*<\/li>\n
*<\/li>\n
\u603b\u4e4b\u7406\u60f3\u7684\u5206\u5757\u662f\u5728\u201c\u4e0a\u4e0b\u6587\u5b8c\u6574\u6027\u201d\u548c\u201c\u4fe1\u606f\u5bc6\u5ea6\u201d\u4e4b\u95f4\u53d6\u5f97\u52a8\u6001\u5e73\u8861\uff1achunk\/_size\u51b3\u5b9a\u4fe1\u606f\u627f\u8f7d\u91cf\uff0cchunk\/_overlap \u7528\u4e8e\u5f25\u8865\u8fb9\u754c\u65ad\u88c2\u5e76\u7ef4\u6301\u8bed\u4e49\u8fde\u7eed\u3002\u53ea\u8981\u8fb9\u754c\u5bf9\u9f50\u8bed\u4e49\u3001\u7c92\u5ea6\u8d34\u5408\u5185\u5bb9\uff0c\u68c0\u7d22\u4e0e\u751f\u6210\u7684\u8d28\u91cf\u5c31\u80fd\u63d0\u5347\u3002<\/p>\n
<\/p>\n
\u56db\u3001\u5206\u5757\u7b56\u7565\u8be6\u89e3<\/h2>\n4.1 \u57fa\u7840\u5206\u5757<\/h2>\n
\u57fa\u4e8e\u56fa\u5b9a\u957f\u5ea6\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n \u57fa\u4e8e\u53e5\u5b50\u7684\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n HanLP \u5206\u53e5\u793a\u4f8b\uff1a<\/p>\n \u57fa\u4e8e\u9012\u5f52\u5b57\u7b26\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n \u603b\u7ed3<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n \u5229\u7528\u6587\u6863\u56fa\u6709\u7ed3\u6784\uff08\u6807\u9898\u5c42\u7ea7\u3001\u5217\u8868\u3001\u4ee3\u7801\u5757\u3001\u8868\u683c\u3001\u5bf9\u8bdd\u8f6e\u6b21\uff09\u4f5c\u4e3a\u5206\u5757\u8fb9\u754c\uff0c\u903b\u8f91\u6e05\u6670\u3001\u53ef\u8ffd\u6eaf\u6027\u5f3a\uff0c\u80fd\u5728\u4fdd\u8bc1\u4e0a\u4e0b\u6587\u5b8c\u6574\u6027\u7684\u540c\u65f6\u63d0\u5347\u68c0\u7d22\u4fe1\u566a\u6bd4\u3002<\/p>\n \u7ed3\u6784\u5316\u6587\u672c\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n \u5bf9\u8bdd\u5f0f\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n \u603b\u7ed3<\/strong><\/p>\n \u8be5\u65b9\u6cd5\u4e0d\u4f9d\u8d56\u6587\u6863\u7684\u7269\u7406\u7ed3\u6784\uff0c\u800c\u662f\u4f9d\u636e\u8bed\u4e49\u8fde\u7eed\u6027\u4e0e\u8bdd\u9898\u8f6c\u79fb\u6765\u51b3\u5b9a\u5207\u5206\u70b9\uff0c\u5c24\u5176\u9002\u5408\u5e0c\u671b\u201c\u5757\u5185\u9ad8\u5ea6\u5185\u805a\u3001\u5757\u95f4\u6e05\u6670\u5206\u754c\u201d\u7684\u77e5\u8bc6\u5e93\u4e0e\u7814\u7a76\u7c7b\u6587\u672c\u3002<\/p>\n \u8bed\u4e49\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n \u4e3b\u9898\u7684\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n <!—-><\/p>\n \u5c0f-\u5927\u5206\u5757<\/strong><\/p>\n <!—-><\/p>\n <!—-><\/p>\n\n
from langchain_text_splitters import CharacterTextSplitter\n\n\nsplitter = CharacterTextSplitter(\n separator="", # \u7eaf\u6309\u957f\u5ea6\u5207\n chunk_size=600, # \u4f9d\u636e\u5b9e\u9a8c\u4e0e\u6a21\u578b\u4e0a\u9650\u8c03\u6574\n chunk_overlap=90, # 15% \u91cd\u53e0\n)\nchunks = splitter.split_text(text)\n<\/code><\/pre>\n\n
\n
\n
\n
\n
import re\n\n\ndef split_sentences_zh(text: str):\n # \u5728\u53e5\u672b\u6807\u70b9\uff08\u3002\uff01\uff1f\uff1b\uff09\u540e\u9762\u5e26\u53ef\u9009\u5f15\u53f7\u7684\u573a\u666f\u65ad\u53e5\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n sentences = [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n return sentences\n\n\ndef sentence_chunk(text: str, chunk_size=600, overlap=80):\n sents = split_sentences_zh(text)\n chunks, buf = [], ""\n for s in sents:\n if len(buf) + len(s) <= chunk_size:\n buf += s\n else:\n if buf:\n chunks.append(buf)\n # \u7b80\u5355\u91cd\u53e0\uff1a\u4ece\u5f53\u524d\u5757\u5c3e\u90e8\u622a\u53d6 overlap \u5b57\u7b26\u4e0e\u4e0b\u4e00\u53e5\u62fc\u63a5\n buf = (buf[-overlap:] if overlap > 0 and len(buf) > overlap else "") + s\n if buf:\n chunks.append(buf)\n return chunks\n\n\nchunks = sentence_chunk(text, chunk_size=600, overlap=90)\n<\/code><\/pre>\nfrom hanlp_common.constant import ROOT\nimport hanlp\n\n\ntokenizer = hanlp.load('PKU_NAME_MERGED_SIX_MONTHS_CONVSEG') # \u6216\u53e5\u6cd5\/\u53e5\u5b50\u7ea7\u7ba1\u7ebf\n# HanLP \u9ad8\u5c42 API \u901a\u5e38\u901a\u8fc7\u53e5\u6cd5\/\u8bed\u6599\u7ba1\u7ebf\u83b7\u5f97\u53e5\u5b50\u8fb9\u754c\uff0c\u5177\u4f53\u4ee5\u6240\u7528\u7248\u672c API \u4e3a\u51c6\n# \u5c06\u53e5\u5b50\u5217\u8868\u518d\u505a\u805a\u5408\u4e3a chunk_size\n<\/code><\/pre>\n\n
import re\nfrom langchain_text_splitters import RecursiveCharacterTextSplitter\n\n\nseparators = [\n r"\/n#{1,6}\/s", # \u6807\u9898\n r"\/n\/d+(?:.\/d+)*\/s", # \u6570\u5b57\u7f16\u53f7\u6807\u9898 1. \/ 2.3. \u7b49\n "\/n\/n", # \u6bb5\u843d\n "\/n", # \u884c\n " ", # \u7a7a\u683c\n "", # \u515c\u5e95\u5b57\u7b26\u7ea7\n]\nsplitter = RecursiveCharacterTextSplitter(\n separators=separators,\n chunk_size=700,\n chunk_overlap=100,\n is_separator_regex=True, # \u544a\u8bc9\u5206\u5272\u5668\u4e0a\u9762\u5305\u542b\u6b63\u5219\n)\nchunks = splitter.split_text(text)\n<\/code><\/pre>\n\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
4.2 \u7ed3\u6784\u611f\u77e5\u5206\u5757<\/h2>\n
\n
\n
\n
\n
import re\nfrom typing import List, Dict\n\n\nheading_pat = re.compile(r'^(#{1,6})\/s+(.*)$') # \u6807\u9898\nfence_pat = re.compile(r'^```') # fenced code fence\n\n\ndef split_markdown_structure(text: str, chunk_size=900, min_chunk=250, overlap_ratio=0.1) -> List[Dict]:\n lines = text.splitlines()\n sections = []\n in_code = False\n current = {"level": 0, "title": "", "content": [], "path": []}\n \n path_stack = [] # [(level, title)]\n \n for ln in lines:\n if fence_pat.match(ln):\n in_code = not in_code\n m = heading_pat.match(ln) if not in_code else None\n if m:\n if current["content"]:\n sections.append(current)\n level = len(m.group(1))\n title = m.group(2).strip()\n\n\n while path_stack and path_stack[-1][0] >= level:\n path_stack.pop()\n path_stack.append((level, title))\n breadcrumbs = [t for _, t in path_stack]\n current = {"level": level, "title": title, "content": [], "path": breadcrumbs}\n else:\n current["content"].append(ln)\n \n if current["content"]:\n sections.append(current)\n \n # \u901a\u8fc7\u4e8c\u6b21\u62c6\u5206\/\u5408\u5e76\u5c06\u90e8\u5206\u5e73\u94fa\u6210\u5757\n chunks = []\n def emit_chunk(text_block: str, path: List[str], level: int):\n chunks.append({\n "text": text_block.strip(),\n "meta": {\n "section_title": path[-1] if path else "",\n "breadcrumbs": path,\n "section_level": level,\n }\n })\n \n for sec in sections:\n raw = "\/n".join(sec["content"]).strip()\n if not raw:\n continue\n if len(raw) <= chunk_size:\n emit_chunk(raw, sec["path"], sec["level"])\n else:\n paras = [p.strip() for p in raw.split("\/n\/n") if p.strip()]\n buf = ""\n for p in paras:\n if len(buf) + len(p) + 2 <= chunk_size:\n buf += (("\/n\/n" + p) if buf else p)\n else:\n if buf:\n emit_chunk(buf, sec["path"], sec["level"])\n buf = p\n if buf:\n emit_chunk(buf, sec["path"], sec["level"])\n \n merged = []\n for ch in chunks:\n if not merged:\n merged.append(ch)\n continue\n if len(ch["text"]) < min_chunk and merged[-1]["meta"]["breadcrumbs"] == ch["meta"]["breadcrumbs"]:\n merged[-1]["text"] += "\/n\/n" + ch["text"]\n else:\n merged.append(ch)\n \n overlap = int(chunk_size * overlap_ratio)\n for ch in merged:\n bc = " > ".join(ch["meta"]["breadcrumbs"][-3:])\n prefix = f"[{bc}]\/n" if bc else ""\n if prefix and not ch["text"].startswith(prefix):\n ch["text"] = prefix + ch["text"]\n # optional character overlap can\u5728\u68c0\u7d22\u9636\u6bb5\u7528\u90bb\u63a5\u805a\u5408\u66ff\u4ee3\uff0c\u8fd9\u91cc\u7565\n \n return merged\n<\/code><\/pre>\n\n
\n
\n
\n
\n
\n
\n
from typing import List, Dict\n\n\ndef chunk_dialogue(turns: List[Dict], max_turns=10, max_chars=900, overlap_turns=2):\n """\n turns: [{"speaker":"User","text":"..." , "ts_start":123, "ts_end":130}, ...]\n """\n chunks = []\n i = 0\n while i < len(turns):\n j = i\n char_count = 0\n speakers = set()\n while j < len(turns):\n t = turns[j]\n uttr_len = len(t["text"])\n # \u82e5\u5355\u6761\u8d85\u957f\uff0c\u5141\u8bb8\u5728\u53e5\u7ea7\u4e8c\u6b21\u5207\u5206\uff08\u6b64\u5904\u7565\uff09\uff0c\u4f46\u4e0d\u8de8 speaker\n if (j - i + 1) > max_turns or (char_count + uttr_len) > max_chars:\n break\n char_count += uttr_len\n speakers.add(t["speaker"])\n j += 1\n \n if j > i:\n window = turns[i:j]\n elif i < len(turns):\n window = [turns[i]]\n else:\n break\n text = "\/n".join([f'{t["speaker"]}: {t["text"]}' for t in window])\n meta = {\n "speakers": list(speakers),\n "turns_range": (i, j - 1),\n "ts_start": window[0].get("ts_start"),\n "ts_end": window[-1].get("ts_end"),\n }\n chunks.append({"text": text, "meta": meta})\n \n # \u6309\u8f6e\u6b21\u91cd\u53e0\u56de\u9000\n if j >= len(turns):\n break\n next_start = i + len(window) - overlap_turns\n i = max(next_start, i + 1) # \u786e\u4fdd\u81f3\u5c11\u524d\u8fdb1\u6b65\n return chunks\n<\/code><\/pre>\n\n
\n
\n
4.3 \u8bed\u4e49\u4e0e\u4e3b\u9898\u5206\u5757<\/h2>\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
from typing import List, Dict, Tuple\nimport numpy as np\nfrom sentence_transformers import SentenceTransformer\nimport re\n\n\ndef split_sentences_zh(text: str) -> List[str]:\n # \u7b80\u6613\u4e2d\u6587\u5206\u53e5\uff0c\u53ef\u66ff\u6362\u4e3a HanLP\/Stanza \u66f4\u7a33\u5065\u7684\u5b9e\u73b0\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n return [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n\n\ndef rolling_mean(vecs: np.ndarray, i: int, w: int) -> np.ndarray:\n s = max(0, i - w)\n e = min(len(vecs), i + w + 1)\n return vecs[s:e].mean(axis=0)\n\n\ndef semantic_chunk(\n text: str,\n model_name: str = "BAAI\/bge-m3",\n window_size: int = 2,\n min_chars: int = 350,\n max_chars: int = 1100,\n lambda_std: float = 0.8,\n overlap_chars: int = 80,\n) -> List[Dict]:\n sents = split_sentences_zh(text)\n if not sents:\n return []\n \n model = SentenceTransformer(model_name)\n emb = model.encode(sents, normalize_embeddings=True, batch_size=64, show_progress_bar=False)\n emb = np.asarray(emb)\n \n # \u57fa\u4e8e\u7a97\u53e3\u5747\u503c\u7684\u201c\u65b0\u9896\u5ea6\u201d\u5206\u6570\n novelties = []\n for i in range(len(sents)):\n ref = rolling_mean(emb, i-1, window_size) if i > 0 else emb[0]\n ref = ref \/ (np.linalg.norm(ref) + 1e-8)\n novelty = 1.0 - float(np.dot(emb[i], ref))\n novelties.append(novelty)\n novelties = np.array(novelties)\n \n # \u76f8\u5bf9\u9608\u503c\uff1a\u03bc + \u03bb\u03c3\n mu, sigma = float(novelties.mean()), float(novelties.std() + 1e-8)\n threshold = mu + lambda_std * sigma\n \n chunks, buf, start_idx = [], "", 0\n def flush(end_idx: int):\n nonlocal buf, start_idx\n if buf.strip():\n chunks.append({\n "text": buf.strip(),\n "meta": {"start_sent": start_idx, "end_sent": end_idx-1}\n })\n buf, start_idx = "", end_idx\n \n for i, s in enumerate(sents):\n # \u82e5\u8d85\u957f\u5219\u5148\u51b2\u6d17\n if len(buf) + len(s) > max_chars and len(buf) >= min_chars:\n flush(i)\n # \u7ed3\u6784\u5316\u91cd\u53e0\uff1a\u9644\u52a0\u4e0a\u4e00\u4e2a\u5757\u7684\u5c3e\u90e8\n if overlap_chars > 0 and len(s) < overlap_chars:\n buf = s\n continue\n \n buf += s\n \n # \u8fbe\u5230\u6700\u5c0f\u957f\u5ea6\u540e\u9047\u5230\u7a81\u53d8\u5219\u5207\u5206\n if len(buf) >= min_chars and novelties[i] > threshold:\n flush(i + 1)\n \n if buf:\n flush(len(sents))\n \n return chunks\n<\/code><\/pre>\n\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
from typing import List, Dict\nimport numpy as np\nfrom sentence_transformers import SentenceTransformer\nfrom sklearn.cluster import KMeans\nimport re\n\n\ndef split_sentences_zh(text: str) -> List[str]:\n pattern = re.compile(r'([^\u3002\uff01\uff1f\uff1b]*[\u3002\uff01\uff1f\uff1b]+|[^\u3002\uff01\uff1f\uff1b]+$)')\n return [m.group(0).strip() for m in pattern.finditer(text) if m.group(0).strip()]\n\n\ndef topic_chunk(\n text: str,\n k_topics: int = 5,\n min_chars: int = 500,\n max_chars: int = 1400,\n smooth_window: int = 2,\n model_name: str = "BAAI\/bge-m3"\n) -> List[Dict]:\n sents = split_sentences_zh(text)\n if not sents:\n return []\n \n model = SentenceTransformer(model_name)\n emb = model.encode(sents, normalize_embeddings=True, batch_size=64, show_progress_bar=False)\n emb = np.asarray(emb)\n \n km = KMeans(n_clusters=k_topics, n_init="auto", random_state=42)\n labels = km.fit_predict(emb)\n \n # \u7b80\u5355\u5e8f\u5217\u5e73\u6ed1\uff1a\u6ed1\u7a97\u591a\u6570\u6295\u7968\n smoothed = labels.copy()\n for i in range(len(labels)):\n s = max(0, i - smooth_window)\n e = min(len(labels), i + smooth_window + 1)\n window = labels[s:e]\n vals, counts = np.unique(window, return_counts=True)\n smoothed[i] = int(vals[np.argmax(counts)])\n \n chunks, buf, start_idx, cur_label = [], "", 0, smoothed[0]\n def flush(end_idx: int):\n nonlocal buf, start_idx\n if buf.strip():\n chunks.append({\n "text": buf.strip(),\n "meta": {"start_sent": start_idx, "end_sent": end_idx-1, "topic": int(cur_label)}\n })\n buf, start_idx = "", end_idx\n \n for i, s in enumerate(sents):\n switched = smoothed[i] != cur_label\n over_max = len(buf) + len(s) > max_chars\n under_min = len(buf) < min_chars\n \n # \u5c1d\u8bd5\u5ef6\u540e\u5207\u5206\uff0c\u4fdd\u8bc1\u6700\u5c0f\u5757\u957f\n if switched and not under_min:\n flush(i)\n cur_label = smoothed[i]\n \n if over_max and not under_min:\n flush(i)\n \n buf += s\n \n if buf:\n flush(len(sents))\n \n return chunks\n<\/code><\/pre>\n\n
\n
4.4 \u9ad8\u7ea7\u5206\u5757<\/h2>\n
\n
\n
\n
\n
\n
\n