\u800c Documate<\/a> \u7684\u51fa\u73b0\u5219\u662f\u901a\u8fc7\u5de5\u5177\u94fe\u7684\u96c6\u6210\uff0c\u4ec5\u9700\u4f7f\u7528 CLI \u63d0\u4f9b\u7684\u547d\u4ee4\u548c\u90e8\u7f72\u670d\u52a1\u7aef\u7684\u4ee3\u7801\uff0c\u5c31\u53ef\u4ee5\u5f88\u8f7b\u677e\u5730\u5b9e\u73b0\u4e0a\u8ff0\u7684\u6570\u636e\u6295\u5582\u6a21\u578b + \u63d0\u95ee ChatGPT \u8fc7\u7a0b\u7684\u81ea\u52a8\u5316\uff0c\u8ba9\u4f60\u7684\u6587\u6863\uff08VitePress<\/a>\u3001Docusaurus<\/a>\u3001Docsify<\/a>\uff09\u7ad9\u70b9\u5177\u5907 AI \u5bf9\u8bdd\u529f\u80fd\u3002<\/p>\n Documate \u7684\u5b98\u7f51\u6587\u6863\u5bf9\u5982\u4f55\u4f7f\u7528\u5b83\u8fdb\u884c\u73b0\u6709\u6587\u6863\u7ad9\u70b9\u7684\u63a5\u5165\u4ecb\u7ecd\u7684\u5f88\u4e3a\u8be6\u5c3d\uff0c\u5e76\u4e14\u5176\u4f5c\u8005\uff08\u6708\u5f71<\/a>\uff09\u4e5f\u4e13\u95e8\u5199\u4e86\u3010\u9ed1\u79d1\u6280\u3011\u8ba9\u4f60\u7684 VitePress \u6587\u6863\u7ad9\u652f\u6301 AI \u5bf9\u8bdd\u80fd\u529b<\/a>\u6587\u7ae0\u4ecb\u7ecd\uff0c\u5bf9\u63a5\u5165 Documate \u611f\u5174\u8da3\u7684\u540c\u5b66\u53ef\u4ee5\u81ea\u884c\u9605\u8bfb\u6587\u6863\u6216\u6587\u7ae0\u3002<\/p>\n \u76f8\u4fe1\u5f88\u591a\u540c\u5b66\u548c\u6211\u4e00\u6837\uff0c\u5bf9\u5982\u4f55\u57fa\u4e8e\u6587\u6863\u7684\u5185\u5bb9\u5b9e\u73b0 AI Chat<\/strong> \u7559\u6709\u7591\u95ee\uff0c\u90a3\u4e48\u63a5\u4e0b\u6765\uff0c\u672c\u6587\u5c06\u56f4\u7ed5 Documate \u7684\u5b9e\u73b0\u539f\u7406\u5206\u522b\u5c55\u5f00\u4ecb\u7ecd\uff1a<\/p>\n Documate \u4e3b\u8981\u7531 2 \u90e8\u5206\u6784\u6210\uff0cDocumate CLI<\/a> \u548c\u670d\u52a1\u7aef\uff08Backend\uff09<\/a>\u63a5\u53e3\u5b9e\u73b0\uff0c\u5176\u4e2d Documate CLI \u4e3b\u8981\u804c\u8d23\u662f\u83b7\u53d6\u672c\u5730\u6587\u6863\u8fc7\u7a0b\u7684\u6587\u6863\u548c\u6784\u9020\u6307\u5b9a\u7ed3\u6784\u7684\u6587\u6863\u6570\u636e\uff0c\u6700\u7ec8\u4e0a\u4f20\u6570\u636e\u5230 \u6574\u4f53\u5b9e\u73b0\u673a\u5236\u5982\u4e0b\u56fe\u6240\u793a\uff1a<\/p>\n \u5176\u4e2d\uff0c\u5173\u4e8e Documate CLI \u4e3b\u8981\u652f\u6301\u4e86 \u76f8\u6bd4\u8f83 CLI\uff0c\u5728 Documate \u7684\u670d\u52a1\u7aef\u5b9e\u73b0\u7684\u4e00\u7cfb\u5217\u80fd\u529b\u662f\u652f\u6301\u6587\u6863\u5185\u5bb9\u5bf9\u8bdd\u7684\u5173\u952e\u6280\u672f\u70b9<\/strong>\uff0c\u90a3\u8fd9\u4e9b\u80fd\u529b\u53c8\u662f\u5982\u4f55\u901a\u8fc7\u4ee3\u7801\u5b9e\u73b0\u7684\uff1f\u4e0b\u9762\uff0c\u6211\u4eec\u6765\u5206\u522b\u4ece\u4ee3\u7801\u5c42\u9762\u6df1\u5165\u8ba4\u8bc6\u4e0b Documate \u670d\u52a1\u7aef\u7684\u5404\u4e2a\u80fd\u529b\u7684\u5b9e\u73b0\u3002<\/p>\n Documate \u670d\u52a1\u7aef\u4e3b\u8981\u8d1f\u8d23\u63a5\u6536\u5e76\u5b58\u50a8 \u5176\u4e2d\uff0c\u540e\u8005\u9700\u8981\u4f7f\u7528 OpenAI \u63d0\u4f9b\u7684 Text Embeddings \u6765\u5b9e\u73b0 AI \u5bf9\u8bdd\u7684\u529f\u80fd\uff0c\u6240\u4ee5\uff0c\u6211\u4eec\u5148\u6765\u5bf9 OpenAI Text Embeddings \u5efa\u7acb\u4e00\u4e2a\u57fa\u7840\u7684\u8ba4\u77e5\u3002<\/p>\n \u5728 OpenAI \u7684\u5f00\u53d1\u8005\u5e73\u53f0<\/a> \u63d0\u4f9b\u4e86\u5f88\u591a\u529f\u80fd\u7684 API \u7ed9\u5f00\u53d1\u8005\u8c03\u7528\uff1a<\/p>\n \u57fa\u4e8e\u6587\u6863\u5185\u5bb9\u7684 AI \u5bf9\u8bdd\u7684\u5b9e\u73b0\u672c\u8d28\u662f\u6839\u636e\u5173\u952e\u8bcd\u641c\u7d22\u5f97\u5230\u7b54\u6848\uff0c\u6240\u4ee5\u9700\u8981\u4f7f\u7528\u5230 Embeddings\uff0cEmbedding \u4e3b\u8981\u7528\u4e8e\u8861\u91cf\u6587\u672c\u5b57\u7b26\u4e32\u4e4b\u95f4\u7684\u5173\u8054\u6027\uff0c\u4e00\u4e2a Embedding \u662f\u7531\u6d6e\u70b9\u6570\u5b57\u6784\u6210\u7684\u77e2\u91cf\u6570\u7ec4\uff0c\u4f8b\u5982 \u6587\u6863\u5185\u5bb9\u7684\u5b58\u50a8\u4e3b\u8981\u5206\u4e3a 2 \u4e2a\u6b65\u9aa4\uff1a<\/p>\n 1\u3001\u6839\u636e\u6a21\u578b\u6bcf\u6b21\u80fd\u63a5\u53d7\u7684 Token \u6700\u5927\u957f\u5ea6\u53bb\u5bf9\u5185\u5bb9\u8fdb\u884c\u5206\u7247 chunks<\/strong><\/p>\n OpenAI \u7684\u6a21\u578b\u8c03\u7528\u6240\u80fd\u63a5\u6536\u7684 Token \u7684\u957f\u5ea6\u662f\u6709\u9650\u7684\uff0c\u5bf9\u5e94\u7684 \u9996\u5148\uff0c\u4f1a\u5148\u4f7f\u7528 gpt-3-encoder<\/a> \u6765\u5bf9\u6587\u6863\u5185\u5bb9\u8fdb\u884c Byte Pair Encoding<\/a>\uff0c\u5c06\u6587\u6863\u4ece\u6587\u672c\u5f62\u5f0f\u8f6c\u6210\u4e00\u7cfb\u5217\u6570\u5b57\uff0c\u4ece\u800c\u7528\u4e8e\u540e\u7eed\u6295\u5582\uff08Feed\uff09\u7ed9\u6a21\u578b\u3002\u5176\u4e2d\uff0c BPE \u7b97\u6cd5<\/a> \u7684\u5b9e\u73b0\uff1a<\/p>\n \u7531\u4e8e BPE \u7f16\u7801\u540e\u7684\u7ed3\u679c \u5bf9\u6587\u6863\u5185\u5bb9\u8fdb\u884c\u5206\u7247\u7684\u76ee\u7684\u662f\u7528\u4e8e\u540e\u7eed\u5c06\u6587\u6863\u5185\u5bb9\u6295\u5582\uff08Feed\uff09\u7ed9\u6a21\u578b\u7684\u65f6\u5019\u662f\u6709\u6548\uff08\u4e0d\u4f1a\u8d85\u51fa Token \u6700\u5927\u957f\u5ea6\uff09\u548c\u8fde\u7eed\u7684\u3002<\/p><\/blockquote>\n 2\u3001\u6784\u9020\u6307\u5b9a\u7684\u6570\u636e\u7ed3\u6784 \u56e0\u4e3a\uff0c\u5c06\u6587\u6863\u7684\u6240\u6709\u5185\u5bb9\u5168\u90e8\u6295\u5582\u7ed9\u6a21\u578b\u662f\u6709\u6210\u672c\uff08Token \u8ba1\u8d39\uff09\u5e76\u4e14\u6536\u76ca\u4f4e\uff08\u95ee\u7b54\u5185\u5bb9\u5173\u8054\u6027\u4f4e\uff09<\/strong>\uff0c\u6240\u4ee5\uff0c\u9700\u8981\u5728\u63d0\u95ee\u7684\u73af\u8282\u901a\u8fc7\u77e2\u91cf\u6570\u636e\u5e93\u67e5\u8be2\u7684\u65b9\u5f0f\uff0c\u67e5\u51fa\u5173\u8054\u7684\u6587\u6863\u5185\u5bb9\uff0c\u7136\u540e\u518d\u5c06\u5bf9\u5e94\u7684\u6587\u6863\u5185\u5bb9\u6295\u5582\u7ed9\u6a21\u578b\uff0c\u6a21\u578b\u6839\u636e\u5bf9\u5173\u8054<\/strong>\u6587\u6863\u4e0a\u4e0b\u6587\u548c\u95ee\u9898\u7ed9\u51fa\u5408\u7406\u7684\u56de\u7b54\u3002<\/p>\n \u90a3\u4e48\uff0c\u5728\u524d\u9762\u6839\u636e BPE \u751f\u6210\u7684 Token \u548c\u5206\u7247\u751f\u6210\u7684\u57fa\u7840\u4e0a\uff0c\u9700\u8981\u5c06\u8be5\u7ed3\u679c\u6309\u6307\u5b9a\u7684\u6570\u636e\u7ed3\u6784\uff08\u8def\u5f84\u3001\u6807\u9898\u7b49\uff09\u5b58\u5165\u6570\u636e\u5e93\u4e2d\uff0c\u7528\u4e8e\u540e\u7eed\u63d0\u95ee\u7684\u65f6\u5019\u67e5\u8be2\u77e2\u91cf\u6570\u636e\u5e93\uff1a<\/p>\n \u8fd9\u91cc\u4f1a\u4f7f\u7528\u5230 AirCode \u63d0\u4f9b\u7684\u8868\u64cd\u4f5c\u7684 OpenAI \u8981\u6c42\u8f93\u5165\u7684\u5185\u5bb9\u662f\u9700\u8981\u7b26\u5408\u5b83\u4eec\u89c4\u5b9a\u7684\u5185\u5bb9\u653f\u7b56<\/a>\u7684\uff0c\u6240\u4ee5\u9700\u8981\u5148\u5bf9\u8f93\u5165\u7684\u95ee\u9898\u8fdb\u884c\u5185\u5bb9\u68c0\u67e5\uff0cOpenAI \u4e5f\u63d0\u4f9b\u76f8\u5e94\u7684 API \u7528\u4e8e\u68c0\u67e5\u5185\u5bb9\u5b89\u5168\uff0c\u800c OpenAI \u7684 API \u8c03\u7528\u53ef\u4ee5\u901a\u8fc7 OpenAI Node<\/a> \u6765\u5b9e\u73b0\uff1a<\/p>\n \u5982\u679c\uff0c\u8fd4\u56de\u7684\u7ed3\u679c \u90a3\u4e48\uff0c\u6709\u4e86\u77e2\u91cf\u5750\u6807\u540e\uff0c\u6211\u4eec\u9700\u8981\u7528\u5148\u524d\u5b58\u50a8\u5230\u6570\u636e\u5e93\u7684\u5206\u7247\u6587\u672c\u521b\u5efa\u77e2\u91cf\u6570\u636e\u5e93\uff0c\u8fd9\u53ef\u4ee5\u4f7f\u7528 Orama<\/a> \u5b8c\u6210\uff0c\u5b83\u63d0\u4f9b\u4e86\u5168\u6587\u548c\u77e2\u91cf\u641c\u7d22\u7684\u80fd\u529b\u3002<\/p>\n \u9996\u5148\uff0c\u9700\u8981\u5148\u4ece\u6570\u636e\u5e93\u4e2d\u67e5\u8be2\u51fa\u6240\u6709\u7684\u6587\u6863\u6570\u636e\uff1a<\/p>\n \u7136\u540e\uff0c\u901a\u8fc7 Orama \u63d0\u4f9b\u7684 \u6709\u4e86\u6587\u6863\u5185\u5bb9\u5bf9\u5e94\u7684\u77e2\u91cf\u6570\u636e\u5e93\u540e\uff0c\u6211\u4eec\u5c31\u53ef\u4ee5\u7528\u524d\u9762 Emdeedings \u6839\u636e\u63d0\u95ee\u5185\u5bb9\u751f\u6210\u7684\u77e2\u91cf\u5750\u6807\u8fdb\u884c\u641c\u7d22\uff0c\u4f7f\u7528 Orama \u63d0\u4f9b\u7684 \u90a3\u4e48\uff0c\u4e3a\u4ec0\u4e48\u4f7f\u7528\u7684\u662f\u77e2\u91cf\u641c\u7d22\u800c\u4e0d\u662f\u6587\u672c\u641c\u7d22\uff1f<\/strong> \u56e0\u4e3a\uff0c\u77e2\u91cf\u641c\u7d22\u7684\u4f5c\u7528\u662f\u4e3a\u4e86\u641c\u7d22\u5230\u548c\u6587\u672c\u5bf9\u5e94\u7684\u77e2\u91cf\u4f4d\u7f6e\u76f8\u8fd1\u7684\u5185\u5bb9\uff0c\u7528\u4e8e\u751f\u6210\u4e0a\u4e0b\u6587\u672c GPT \u6574\u7406\u6700\u7ec8\u7684\u56de\u7b54\u3002<\/p>\n \u5176\u4e2d \u7531\u4e8e\uff0c\u5148\u524d\u5c06\u6587\u6863\u5185\u5bb9\u6839\u636e Embeddings \u7684 Token \u6700\u5927\u957f\u5ea6\u5206\u7247\u8fdb\u884c\u5b58\u50a8\uff0c\u6240\u4ee5\uff0c\u8fd9\u91cc\u9700\u8981\u5c06 \u5230\u8fd9\u91cc\uff0c\u6211\u4eec\u5df2\u7ecf\u6709\u4e86\u95ee\u9898\u548c\u95ee\u9898\u5173\u8054\u7684\u5185\u5bb9<\/strong>\uff0c\u53ef\u4ee5\u7528\u5b83\u4eec\u6784\u9020\u4e00\u4e2a Prompt \u7528\u4e8e\u540e\u7eed AI \u5bf9\u8bdd\u4f7f\u7528\uff1a<\/p>\n \u4e0b\u9762\uff0c\u6211\u4eec\u5c31\u53ef\u4ee5\u8c03\u7528 OpenAI \u7684 API \u8fdb\u884c AI \u5bf9\u8bdd\uff1a<\/p>\n \u5176\u4e2d\uff0c \u901a\u8fc7\u5b66\u4e60 Documate \u5185\u90e8\u7684\u5b9e\u73b0\u539f\u7406\uff0c\u6211\u4eec\u53ef\u4ee5\u77e5\u9053\u4e86\u5982\u4f55\u4ece\u5b9e\u9645\u7684\u95ee\u9898\u51fa\u53d1\uff0c\u7ed3\u5408\u4f7f\u7528 OpenAI API \u63d0\u4f9b\u7684\u6a21\u578b\u89e3\u51b3\u95ee\u9898\u3002\u5728\u8fd9\u4e2a\u57fa\u7840\u4e0a\uff0c\u6211\u4eec\u4e5f\u53ef\u4ee5\u53bb\u505a\u522b\u7684\u573a\u666f\u63a2\u7d22\uff0c\u8ba9 AI \u6210\u4e3a\u73b0\u5728\u6216\u8005\u5c06\u6765\u6211\u4eec\u89e3\u51b3\u95ee\u9898\u7684\u4e00\u79cd\u6280\u672f\u624b\u6bb5\u6216\u5c1d\u8bd5\uff0c\u800c\u4e0d\u662f\u4ec5\u4ec5\u5c40\u9650\u4e8e\u4f1a\u4f7f\u7528 ChatGPT \u63d0\u95ee\u548c\u83b7\u53d6\u7b54\u6848\u3002<\/p>\n\n
\u4e00\u3001Documate \u8fd0\u884c\u673a\u5236<\/h2>\n
upload<\/code> \u63a5\u53e3\uff0c\u800c\u670d\u52a1\u7aef\u4e3b\u8981\u804c\u8d23\u662f\u63d0\u4f9b upload<\/code> \u548c ask<\/code> \u63a5\u53e3\uff0c\u5b83\u4eec\u5206\u522b\u7684\u4f5c\u7528\uff1a<\/p>\n\n
upload<\/code> \u63a5\u53e3\uff0c\u63a5\u6536\u6765\u81ea CLI \u4e0a\u4f20\u7684\u6587\u6863\u6570\u636e\uff0c\u5bf9\u6570\u636e\u8fdb\u884c Token \u5316\u3001\u5206\u7247\u5165\u5e93\u7b49\u64cd\u4f5c<\/li>\nask<\/code> \u63a5\u53e3\uff0c\u63a5\u6536\u6765\u81ea\u6587\u6863\u7ad9\u70b9\u7684\u63d0\u95ee\u5185\u5bb9\uff0c\u6821\u9a8c\u5185\u5bb9\u5408\u6cd5\u6027\u3001\u751f\u6210\u5185\u5bb9\u7684\u77e2\u91cf\u5750\u6807\uff0c\u57fa\u4e8e\u6240\u6709\u6587\u6863\u8fdb\u884c\u77e2\u91cf\u641c\u7d22\u3001\u8fdb\u884c Chat \u63d0\u95ee\u83b7\u53d6\u7ed3\u679c\u5e76\u8fd4\u56de\u7b49\u64cd\u4f5c<\/li>\n<\/ul>\n<\/p>\n
init<\/code> \u548c upload<\/code> \u7b49 2 \u4e2a\u547d\u4ee4\uff0cinit<\/code> \u8d1f\u8d23\u5411\u6587\u6863\u5de5\u7a0b\u6ce8\u5165 Documate \u8fd0\u884c\u7684\u57fa\u7840\u5de5\u7a0b\u914d\u7f6e\uff0cupload<\/code> \u8d1f\u8d23\u4e0a\u4f20\u6587\u6863\u5de5\u7a0b\u7684\u6587\u6863\u5185\u5bb9\u5230\u670d\u52a1\u7aef\uff0c2 \u8005\u7684\u5b9e\u73b0\u5e76\u4e0d\u590d\u6742\uff0c\u6709\u5174\u8da3\u7684\u540c\u5b66\u53ef\u4ee5\u81ea\u884c\u4e86\u89e3\u3002<\/p>\n\u4e8c\u3001Documate \u670d\u52a1\u7aef<\/h2>\n
documate upload<\/code> \u547d\u4ee4\u4e0a\u4f20\u7684\u6587\u6863\u5185\u5bb9\u3001\u6839\u636e\u5bf9\u8bdd\u7684\u63d0\u95ee\u5185\u5bb9\u8fd4\u56de\u4e0e\u4e4b\u5173\u8054\u7684\u56de\u7b54\uff1a<\/p>\n<\/p>\n
2.1 OpenAI Text Embeddings<\/h3>\n
\n
[0.938293, 0.284951, 0.348264, 0.948276, 0.564720]<\/code>\u30022 \u4e2a\u77e2\u91cf\u4e4b\u95f4\u7684\u8ddd\u79bb\u8868\u793a\u5b83\u4eec\u7684\u5173\u8054\u6027\u3002\u8ddd\u79bb\u5c0f\u8868\u793a\u5b83\u4eec\u4e4b\u95f4\u7684\u5173\u8054\u6027\u9ad8\uff0c\u53cd\u4e4b\u5173\u8054\u6027\u4f4e\u3002<\/p>\n2.2 \u6587\u6863\u5185\u5bb9\u5b58\u50a8<\/h3>\n
text-embedding-ada-002<\/code> \u6a21\u578b\u53ef\u63a5\u6536\u7684 Token \u6700\u5927\u957f\u5ea6\u662f 1536\u3002\u6240\u4ee5\uff0c\u5728\u63a5\u6536\u5230 CLI \u4e0a\u4f20\u7684\u6587\u6863\u5185\u5bb9\u540e\uff0c\u9700\u8981\u6839\u636e Token \u7684\u6700\u5927\u957f\u5ea6 1536 \u6765\u5bf9\u6587\u6863\u5185\u5bb9\u8fdb\u884c\u5206\u7247\uff1a<\/p>\nconst tokenizer = require('gpt-3-encoder');\n\/\/ Split the page content into chunks base on the MAX_TOKEN_PER_CHUNK\nfunction getContentChunks(content) {\n \/\/ GPT-2 and GPT-3 use byte pair encoding to turn text into a series of integers to feed into the model.\n const encoded = tokenizer.encode(content);\n const tokenChunks = encoded.reduce(\n (acc, token) => (\n acc[acc.length - 1].length < MAX_TOKEN_PER_CHUNK\n ? acc[acc.length - 1].push(token)\n : acc.push([token]),\n acc\n ),\n [[]],\n );\n return tokenChunks.map(tokens => tokenizer.decode(tokens));\n}<\/code><\/pre>\n\n
encoded<\/code> \u662f\u4e00\u4e2a Token \u6570\u7ec4\uff0c\u4e14\u6a21\u578b\u6bcf\u6b21\u80fd\u6295\u5582\u662f\u6709\u6700\u5927\u957f\u5ea6\u7684\u9650\u5236\uff0c\u6240\u4ee5\u6839\u636e Token \u6700\u5927\u957f\u5ea6\u8fdb\u884c\u5206\u7247\uff0c\u4e5f\u5c31\u662f\u4ee3\u7801\u4e2d\u7684 acc<\/code>\uff0cacc<\/code> \u521d\u59cb\u503c\u662f\u4e00\u4e2a\u4e8c\u7ef4\u6570\u7ec4\uff0c\u6bcf\u4e2a\u503c\u662f\u4e00\u4e2a Token\uff0c\u6bcf\u4e2a\u5143\u7d20\u6570\u7ec4\u4e3b\u8981\u7528\u4e8e\u5b58\u50a8\u6a21\u578b\u6700\u5927 Token \u9650\u5236\u4e0b\u7684\u6570\u636e\uff0c\u5373\u5c06\u4e00\u4e2a\u5927\u7684 Token \u5206\u7247\u6210\u6a21\u578b\u5141\u8bb8\u4f20\u5165\u7684\u5c0f Token\u3002<\/p>\nChunkItem<\/code> \u5b58\u5165\u6570\u636e\u5e93\u4e2d\uff0cChunkItem<\/code> \u6570\u636e\u7ed3\u6784<\/strong><\/p>\nconst aircode = require('aircode');\nconst PagesTable = aircode.db.table('pages');\n\n\/\/ \u6839\u636e BPE \u548c\u6a21\u578b\u7684 Token \u4e0a\u9650\u9650\u5236\u53bb\u5212\u5206 chunk\nconst chunks = getContentChunks(content);\n\/\/ \u6784\u9020\u51fa\u5b58\u5230\u6570\u636e\u5e93\u4e2d\u7684\u6570\u636e\u7ed3\u6784\nconst pagesToSave = chunks.map((chunk, index) => ({\n project,\n \/\/ \u6587\u6863\u6587\u4ef6\u8def\u5f84\n path,\n title,\n \/\/ \u6587\u4ef6\u5185\u5bb9\u751f\u6210\u7684 hash \u503c\n checksum,\n chunkIndex: index,\n \/\/ \u5185\u5bb9\n content: chunk,\n embedding: null,\n}))\n\n\/\/ Save the result to database\nfor (let i = 0; i < pagesToSave.length; i += 100) {\n await PagesTable.save(pagesToSave.slice(i, i + 100));\n}<\/code><\/pre>\nPagesTable.save<\/code> API\uff0c\u7528\u4e8e\u5c06\u6784\u9020\u597d\u7684\u6570\u636e\u5165\u5e93\u3002<\/p>\n2.3 \u6839\u636e\u63d0\u95ee\u8fdb\u884c AI \u5bf9\u8bdd<\/h3>\n
const OpenAI = require('openai');\n\n\/\/ \u521b\u5efa OpenAI \u7684\u5b9e\u4f8b\nconst openai = new OpenAI({\n apiKey: process.env.OPENAI_API_KEY,\n});\n\n\/\/ \u63d0\u95ee\u5185\u5bb9\nconst question = params.question.trim();\n\/\/ https:\/\/platform.openai.com\/docs\/api-reference\/moderations\nconst { results: moderationRes } = await openai.moderations.create({\n input: question,\n});\nif (moderationRes[0].flagged) {\n console.log('The user input contains flagged content.', moderationRes[0].categories);\n context.status(403);\n return {\n error: 'Question input didn\/'t meet the moderation criteria.',\n categories: moderationRes[0].categories,\n };\n}<\/code><\/pre>\nmoderationsRes[0].flagged<\/code> \u5219\u89c6\u4e3a\u4e0d\u7b26\u5408\uff0c\u6807\u8bc6\u4e3a\u9519\u8bef\u7684\u8bf7\u6c42\u3002\u53cd\u4e4b\u7b26\u5408\uff0c\u63a5\u7740\u4f7f\u7528 Embeddings \u6765\u83b7\u53d6\u63d0\u95ee\u5185\u5bb9\u6240\u5bf9\u5e94\u7684\u77e2\u91cf\u5750\u6807\uff1a<\/p>\n\/\/ https:\/\/platform.openai.com\/docs\/api-reference\/embeddings\/object\nconst { data: [ { embedding }] } = await openai.embeddings.create({\n model: 'text-embedding-ada-002',\n input: question.replace(\/\/n\/g, ' '),\n});<\/code><\/pre>\nconst { project = 'default' } = params;\nconst pages = await PagesTable\n .where({ project })\n .projection({ path: 1, title: 1, content: 1, embedding: 1, _id: 0 })\n .find();<\/code><\/pre>\ncreate<\/code> \u65b9\u6cd5\u521d\u59cb\u5316\u4e00\u4e2a\u77e2\u91cf\u6570\u636e\u5e93 memDB<\/code>\uff0c\u5e76\u4e14\u5c06\u6587\u6863\u5185\u5bb9 pages<\/code> \u63d2\u5165\u5230\u6570\u636e\u5e93\u4e2d\uff1a<\/p>\nconst memDB = await create({\n \/\/ \u5efa\u7acb\u597d\u7d22\u5f15\u7684 `schema`\n schema: {\n path: 'string',\n title: 'string',\n content: 'string',\n embedding: 'vector[1536]',\n },\n});\nawait insertMultiple(memDB, pages);<\/code><\/pre>\nsearchVector<\/code> \u8fdb\u884c\u641c\u7d22\uff1a<\/p>\nconst { hits } = await searchVector(memDB, {\n vector: embedding,\n property: 'embedding',\n similarity: 0.8, \/\/ Minimum similarity. Defaults to `0.8`\n limit: 10, \/\/ Defaults to `10`\n offset: 0, \/\/ Defaults to `0`\n});<\/code><\/pre>\nhits<\/code> \u7684\u6570\u636e\u7ed3\u6784\uff1a<\/p>\n{\n count: 1,\n elapsed: {\n raw: 25000,\n formatted: '25ms',\n },\n hits: [\n {\n id: '1-19238',\n score: 0.812383129,\n document: {\n title: 'The Prestige',\n embedding: [0.938293, 0.284951, 0.348264, 0.948276, 0.564720],\n }\n }\n ]\n}<\/code><\/pre>\nhits<\/code> \u4e2d\u7684\u6570\u636e\u83b7\u53d6\u7684\u5185\u5bb9\u7ec4\u5408\u8d77\u6765\uff1a<\/p>\nlet tokenCount = 0;\nlet contextSections = '';\n\nfor (let i = 0; i < hits.length; i += 1) {\n const { content } = hits[i].document;\n \/\/ \u6ce8\u610f encode\uff0c\u7528\u4e8e\u7ec4\u5408\u5206\u7247\n const encoded = tokenizer.encode(content);\n tokenCount += encoded.length;\n \n \/\/ \u5224\u65ad\u662f\u5426\u8fbe\u5230 token \u4e0a\u9650\n if (tokenCount >= MAX_CONTEXT_TOKEN && contextSections !== '') {\n break;\n }\n\n contextSections += `${content.trim()}\/n---\/n`;\n}<\/code><\/pre>\nconst prompt = `You are a very kindly assistant who loves to help people. Given the following sections from documatation, answer the question using only that information, outputted in markdown format. If you are unsure and the answer is not explicitly written in the documentation, say "Sorry, I don't know how to help with that." Always trying to anwser in the spoken language of the questioner.\n\nContext sections:\n${contextSections}\n\nQuestion:\n${question}\n\nAnswer as markdown (including related code snippets if available):`<\/code><\/pre>\nconst messages = [{\n role: 'user',\n content: prompt,\n}];\n\nconst response = await openai.chat.completions.create({\n messages,\n model: 'gpt-3.5-turbo',\n max_tokens: 512,\n temperature: 0.4,\n stream: true,\n})<\/code><\/pre>\nresponse<\/code> \u662f\u4e00\u4e2a OpenAI API \u8c03\u7528\u8fd4\u56de\u7684\u81ea\u5b9a\u4e49\u6570\u636e\u7ed3\u6784\u7684 Streaming Responses\uff0c\u76f4\u63a5\u5c06 response<\/code> \u8fd4\u56de\u7ed9 ask<\/code> \u63a5\u53e3\u8bf7\u6c42\u65b9\u80af\u5b9a\u662f\u4e0d\u5408\u7406\u7684\uff08\u8bf7\u6c42\u65b9\u53ea\u9700\u8981\u62ff\u5230\u7b54\u6848\uff09\u3002\u90a3\u4e48\uff0c\u8fd9\u91cc\u53ef\u4ee5\u4f7f\u7528\u8fd9\u91cc\u53ef\u4ee5\u4f7f\u7528 Vercel \u56e2\u961f\u5b9e\u73b0\u7684 ai<\/a> \u63d0\u4f9b\u7684 OpenAIStream<\/code> \u51fd\u6570\u6765\u5b8c\u6210\uff1a<\/p>\nconst { OpenAIStream } = require('ai');\n\nconst stream = OpenAIStream(response);\nreturn stream;<\/code><\/pre>\nOpenAIStream<\/code> \u4f1a\u81ea\u52a8\u5c06 OpenAI Completions \u8fd4\u56de\u7684\u7ed3\u679c\u89e3\u6790\u6210\u53ef\u4ee5\u6b63\u5e38\u8bfb\u53d6\u7684 Streaming Resonsese\uff0c\u5982\u679c\u4f7f\u7528\u7684\u662f AirCode \u5219\u53ef\u4ee5\u76f4\u63a5\u8fd4\u56de stream<\/code>\uff0c\u5982\u679c\u4f7f\u7528\u7684\u662f\u666e\u901a\u7684 Node Server\uff0c\u53ef\u4ee5\u8fdb\u4e00\u6b65\u4f7f\u7528 ai<\/code> \u63d0\u4f9b\u7684 streamToResponse<\/code> \u51fd\u6570\u6765\u5c06 stream<\/code> \u8f6c\u4e3a ServerResponse \u5bf9\u8c61\uff1a<\/p>\nconst { OpenAIStream, streamToResponse } = require('ai');\n\nconst stream = OpenAIStream(response);\nstreamToResponse(stream);<\/code><\/pre>\n\u7ed3\u8bed<\/h2>\n