\u6211\u90e8\u7f72\u4e862\u4e2apod\uff0c\u6bcf\u4e2apod\u7684\u8d44\u6e90\uff1aCPU\uff084\u6838\uff09\u3001\u5185\u5b58\uff0830G\uff09\u30011\u5f20P40\u663e\u5361\uff08\u663e\u5b5824G\uff09\u3002<\/p>\n
\u4e09.\u90e8\u7f72\u5b9e\u8df5<\/h2>\n
Triton\u9ed8\u8ba4\u652f\u6301\u7684PyTorch\u6a21\u578b\u683c\u5f0f\u4e3aTorchScript\uff0c\u7531\u4e8eChatGLM2-6B\u6a21\u578b\u8f6c\u6362\u6210TorchScript\u683c\u5f0f\u4f1a\u62a5\u9519\uff0c\u672c\u6587\u5c06\u4ee5Python Backend\u7684\u65b9\u5f0f\u8fdb\u884c\u90e8\u7f72\u3002<\/p>\n
1. \u6a21\u578b\u76ee\u5f55\u7ed3\u6784<\/h4>\n
<\/p>\n
9N-Triton\u4f7f\u7528\u96c6\u6210\u6a21\u578b\uff0c\u5982\u4e0a\u56fe\u6240\u793a\u6a21\u578b\u4ed3\u5e93(model_repository), \u5b83\u5185\u90e8\u53ef\u4ee5\u5305\u542b\u4e00\u4e2a\u6216\u591a\u4e2a\u5b50\u6a21\u578b\uff08\u5982chatglm2-6b)\u3002\u4e0b\u9762\u5bf9\u5404\u4e2a\u90e8\u5206\u8fdb\u884c\u5c55\u5f00\u4ecb\u7ecd\uff1a<\/p>\n
2. python\u6267\u884c\u73af\u5883<\/h4>\n
\u8be5\u90e8\u5206\u4e3a\u6a21\u578b\u63a8\u7406\u65f6\u9700\u8981\u7684\u76f8\u5173python\u4f9d\u8d56\u5305\uff0c\u53ef\u4ee5\u4f7f\u7528conda-pack\u5c06conda\u865a\u62df\u73af\u5883\u6253\u5305\uff0c\u5982python-3-8.tar.gz\u3002\u5982\u5bf9\u6253\u5305conda\u73af\u5883\u4e0d\u719f\u6089\u7684\uff0c\u53ef\u4ee5\u53c2\u8003 https:\/\/conda.github.io\/conda-pack\/<\/a>\u3002\u7136\u540e\u5728config.pbtxt\u4e2d\u914d\u7f6e\u6267\u884c\u73af\u5883\u8def\u5f84\uff1a<\/p>\n \u5728\u5f53\u524d\u793a\u4f8b\u4e2d\uff0c$$TRITON\/_MODEL\/_DIRECTORY="$pwd\/model_repository\/chatglm2-6b”\u3002<\/p>\n \u6ce8\u610f\uff1a\u5f53\u524dpython\u6267\u884c\u73af\u5883\u4e3a\u6240\u6709\u5b50\u6a21\u578b\u5171\u4eab\uff0c\u5982\u679c\u60f3\u7ed9\u4e0d\u540c\u5b50\u6a21\u578b\u6307\u5b9a\u4e0d\u540c\u7684\u6267\u884c\u73af\u5883\uff0c\u5219\u5e94\u8be5\u5c06tar.gz\u6587\u4ef6\u653e\u5728\u5b50\u6a21\u578b\u76ee\u5f55\u4e0b\uff0c\u5982\u4e0b\u6240\u793a\uff1a<\/p>\n \u540c\u65f6\uff0c\u5728config.pbtxt\u4e2d\u914d\u7f6e\u6267\u884c\u73af\u5883\u8def\u5f84\u5982\u4e0b\uff1a<\/p>\n \u6a21\u578b\u4ed3\u5e93\u5e93\u4e2d\u7684\u6bcf\u4e2a\u6a21\u578b\u90fd\u5fc5\u987b\u5305\u542b\u4e00\u4e2a\u6a21\u578b\u914d\u7f6e\u6587\u4ef6config.pbtxt\uff0c\u7528\u4e8e\u6307\u5b9a\u5e73\u53f0\u548c\u6216\u540e\u7aef\u5c5e\u6027\u3001max\/_batch\/_size \u5c5e\u6027\u4ee5\u53ca\u6a21\u578b\u7684\u8f93\u5165\u548c\u8f93\u51fa\u5f20\u91cf\u7b49\u3002ChatGLM2-6B\u7684\u914d\u7f6e\u6587\u4ef6\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\uff1a<\/p>\n \u5176\u4e2d\u5fc5\u586b\u9879\u4e3a\u6700\u5c0f\u6a21\u578b\u914d\u7f6e\uff0c\u6a21\u578b\u914d\u7f6e\u6587\u4ef6\u66f4\u591a\u4fe1\u606f\u53ef\u4ee5\u53c2\u8003\uff1a https:\/\/github.com\/triton-inference-server\/server\/blob\/r22.04\/docs\/model_configuration.md<\/a><\/p>\n \u4e3b\u8981\u9700\u8981\u5b9e\u73b0model.py<\/a> \u4e2d\u63d0\u4f9b\u7684\u4e09\u4e2a\u63a5\u53e3\uff1a<\/p>\n \u2460. initialize: \u521d\u59cb\u5316\u8be5Python\u6a21\u578b\u65f6\u4f1a\u8fdb\u884c\u8c03\u7528\uff0c\u4e00\u822c\u6267\u884c\u83b7\u53d6\u8f93\u51fa\u4fe1\u606f\u53ca\u521b\u5efa\u6a21\u578b\u7684\u64cd\u4f5c<\/p>\n \u2461. execute: python\u6a21\u578b\u63a5\u6536\u8bf7\u6c42\u65f6\u7684\u6267\u884c\u51fd\u6570\uff1b<\/p>\n \u2462. finalize: \u5220\u9664\u6a21\u578b\u65f6\u4f1a\u8fdb\u884c\u8c03\u7528\uff1b<\/p>\n \u5982\u679c\u6709 n \u4e2a\u6a21\u578b\u5b9e\u4f8b\uff0c\u90a3\u4e48\u4f1a\u8c03\u7528 n \u6b21initialize \u548c finalize\u8fd9\u4e24\u4e2a\u51fd\u6570\u3002<\/p>\n ChatGLM2-6B\u7684model.py\u6587\u4ef6\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\uff1a<\/p>\n \u2460 \u9009\u62e99n-triton-devel-gpu-v0.3\u955c\u50cf\u521b\u5efanotebook\u6d4b\u8bd5\u5b9e\u4f8b\uff1b<\/p>\n \u2461 \u628a\u6a21\u578b\u653e\u5728\/9n-triton-devel\/model_repository\u76ee\u5f55\u4e0b\uff0c\u6a21\u578b\u76ee\u5f55\u7ed3\u6784\u53c2\u80033.1\uff1b<\/p>\n \u2462 \u8fdb\u5165\/9n-triton-devel\/server\/\u76ee\u5f55\uff0c\u62c9\u53d6\u6700\u65b0\u7248\u672c\u7684bin\u5e76\u89e3\u538b\uff1awget http:\/\/storage.jd.local\/com.bamboo.server.product\/7196560\/9n\/_predictor\/_server.tgz<\/a><\/p>\n \u2463 \u4fee\u6539\/9n-triton-devel\/server\/start.sh \u4e3a\u5982\u4e0b\uff1a<\/p>\n \u2464 \u8fd0\u884c \/9n-triton-devel\/server\/start.sh \u811a\u672c<\/p>\n \u2465 \u68c0\u67e5\u670d\u52a1\u542f\u52a8\u6210\u529f\uff08ChatGLM2-6B\u6a21\u578b\u542f\u52a8\uff0c\u5dee\u4e0d\u591a13\u5206\u949f\u5de6\u53f3\uff09<\/p>\n \u65b9\u6cd51:\u67e5\u770b8010\u7aef\u53e3\u662f\u5426\u542f\u52a8\uff1anetstat -natp | grep 8010<\/p>\n \u65b9\u6cd52:\u67e5\u770b\u65e5\u5fd7\uff1acat \/9n-triton-devel\/server\/logs\/predictor_core.INFO<\/p>\n \u2466 \u7f16\u5199python grpc client\u8bbf\u95ee\u6d4b\u8bd5\u670d\u52a1\u811a\u672c\uff0c\u653e\u4e8e\/9n-triton-devel\/client\/\u76ee\u5f55\u4e0b\uff0c\u8bbf\u95ee\u7aef\u53e3\u4e3a8010\uff0cip\u4e3a127.0.0.1\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\uff1a<\/p>\n \u901a\u7528predictor\u8bf7\u6c42\u683c\u5f0f\u53ef\u4ee5\u53c2\u8003\uff1a https:\/\/github.com\/kserve\/kserve\/blob\/master\/docs\/predict-api\/v2\/grpc\/_predict\/_v2.proto<\/a><\/p>\n \u4e5d\u6570\u7b97\u6cd5\u4e2d\u53f0\u63d0\u4f9b\u4e86\u4e24\u79cd\u90e8\u7f72\u6a21\u578b\u670d\u52a1\u65b9\u5f0f\uff0c\u5206\u522b\u4e3a\u754c\u9762\u90e8\u7f72\u548cSDK\u90e8\u7f72\u3002\u5229\u7528\u754c\u9762\u4e2d\u7684\u6a21\u578b\u90e8\u7f72\u53ea\u652f\u6301JSF\u534f\u8bae\u63a5\u53e3\uff0c\u82e5\u8981\u63d0\u4f9bJSF\u670d\u52a1\u63a5\u53e3\uff0c\u5219\u53ef\u4ee5\u53c2\u8003 http:\/\/easyalgo.jd.com\/help\/%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97\/%E6%A8%A1%E5%9E%8B%E8%AE%A1%E7%AE%97\/%E6%A8%A1%E5%9E%8B%E9%83%A8%E7%BD%B2.html<\/a> \u76f4\u63a5\u90e8\u7f72\u3002<\/p>\n \u7531\u4e8e\u6211\u540e\u7eed\u9700\u8981\u5c06ChatGLM2-6B\u6a21\u578b\u96c6\u6210\u81f3langchain\u4e2d\u4f7f\u7528\uff0c\u6240\u4ee5\u5bf9\u5916\u63d0\u4f9bhttp\u534f\u8bae\u63a5\u53e3\u6bd4\u8f83\u4fbf\u5229\uff0c\u7ecf\u4e0e\u7b97\u6cd5\u4e2d\u53f0\u540c\u5b66\u8bf7\u6559\u540e\u4f7f\u7528SDK\u65b9\u5f0f\u90e8\u7f72\u53ef\u4ee5\u6ee1\u8db3\u3002\u7531\u4e8e\u754c\u9762\u90e8\u7f72\u548cSDK\u90e8\u7f72\u76ee\u524d\u7814\u53d1\u6ca1\u6709\u5bf9\u9f50\uff0c\u7528\u754c\u9762\u90e8\u7f72\u65f6\u76f4\u63a5\u53ef\u4ee5\u4f7f\u75283.1\u4e2d\u7684\u6a21\u578b\u7ed3\u6784\uff0c\u4f7f\u7528SDK\u90e8\u7f72\u5219\u9700\u8981\u8c03\u6574\u6a21\u578b\u7ed3\u6784\u5982\u4e0b\uff1a<\/p>\n \u540c\u65f6\u9700\u8981\u5728config.pbtxt\u4e2d\u5c06\u6267\u884c\u73af\u5883\u8def\u5f84\u8bbe\u7f6e\u5982\u4e0b\uff1a<\/p>\n \u6a21\u578b\u90e8\u7f72\u4ee3\u7801\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\uff1a<\/p>\n \u4f7f\u7528langchain\u53ef\u4ee5\u5feb\u901f\u57fa\u4e8eLLM\u6a21\u578b\u5f00\u53d1\u4e00\u4e9b\u5e94\u7528\u3002\u4f7f\u7528LLMs\u6a21\u5757\u5c01\u88c5ChatGLM2-6B\uff0c\u8bf7\u6c42\u6211\u4eec\u7684\u6a21\u578b\u670d\u52a1\uff0c\u4e3b\u8981\u5b9e\u73b0_call\u51fd\u6570\uff0c\u53ef\u4ee5\u53c2\u8003\u5982\u4e0b\u4ee3\u7801\uff1a<\/p>\n \u6ce8\u610f\uff1a\u6a21\u578b\u670d\u52a1\u8c03\u7528url\u7b49\u4e8e\u5728\u6a21\u578b\u90e8\u7f72\u9875\u9762\u8c03\u7528\u4fe1\u606fURL\u540e\u52a0\u4e0a” MutilBackendService\/Predict<\/a> “<\/p>\n \u672c\u6587\u8be6\u7ec6\u4ecb\u7ecd\u4e86\u5728\u96c6\u56e29n-triton\u5de5\u5177\u4e0a\u90e8\u7f72ChatGLM2-6B\u8fc7\u7a0b\uff0c\u5e0c\u671b\u53ef\u4ee5\u4e3a\u6709\u90e8\u7f72\u9700\u6c42\u7684\u540c\u5b66\u63d0\u4f9b\u4e00\u4e9b\u5e2e\u52a9\u3002<\/p>\n \u4f5c\u8005\uff1a\u4eac\u4e1c\u4fdd\u9669\u00a0\u8d75\u98ce\u9f99<\/p>\n \u6765\u6e90\uff1a\u4eac\u4e1c\u4e91\u5f00\u53d1\u8005\u793e\u533a \u8f6c\u8f7d\u8bf7\u6ce8\u660e\u51fa\u5904<\/p>\n<\/blockquote>\nparameters: {\n key: "EXECUTION_ENV_PATH",\n value: {string_value: "$$TRITON_MODEL_DIRECTORY\/..\/python-3-8.tar.gz"}\n}\n<\/code><\/pre>\n<\/p>\n
parameters: {\n key: "EXECUTION_ENV_PATH",\n value: {string_value: "$$TRITON_MODEL_DIRECTORY\/python-3-8.tar.gz"}\n}\n<\/code><\/pre>\n3. \u6a21\u578b\u914d\u7f6e\u6587\u4ef6<\/h4>\n
name: "chatglm2-6b" \/\/ \u5fc5\u586b\uff0c\u6a21\u578b\u540d\uff0c\u9700\u4e0e\u8be5\u5b50\u6a21\u578b\u7684\u6587\u4ef6\u5939\u540d\u5b57\u76f8\u540c\nbackend: "python" \/\/ \u5fc5\u586b\uff0c\u6a21\u578b\u6240\u4f7f\u7528\u7684\u540e\u7aef\u5f15\u64ce\n\nmax_batch_size: 0 \/\/ \u6a21\u578b\u6bcf\u6b21\u8bf7\u6c42\u6700\u5927\u7684\u6279\u6570\u636e\u91cf\uff0c\u5f20\u91cfshape\u7531max_batch_size\u548cdims\u7ec4\u5408\u6307\u5b9a\uff0c\u5bf9\u4e8e max_batch_size \u5927\u4e8e 0 \u7684\u6a21\u578b\uff0c\u5b8c\u6574\u5f62\u72b6\u5f62\u6210\u4e3a [ -1 ] + dims\u3002 \u5bf9\u4e8e max_batch_size \u7b49\u4e8e 0 \u7684\u6a21\u578b\uff0c\u5b8c\u6574\u5f62\u72b6\u5f62\u6210\u4e3a dims\u3002\ninput [ \/\/ \u5fc5\u586b\uff0c\u8f93\u5165\u5b9a\u4e49\n {\n name: "prompt" \/\/\u5fc5\u586b\uff0c\u540d\u79f0\n data_type: TYPE_STRING \/\/\u5fc5\u586b\uff0c\u6570\u636e\u7c7b\u578b\n dims: [ -1 ] \/\/\u5fc5\u586b\uff0c\u6570\u636e\u7ef4\u5ea6\uff0c-1 \u8868\u793a\u53ef\u53d8\u7ef4\u5ea6\n },\n {\n name: "history"\n data_type: TYPE_STRING\n dims: [ -1 ]\n },\n {\n name: "temperature"\n data_type: TYPE_STRING\n dims: [ -1 ]\n },\n {\n name: "max_token"\n data_type: TYPE_STRING\n dims: [ -1 ]\n },\n {\n name: "history_len"\n data_type: TYPE_STRING\n dims: [ -1 ]\n }\n]\noutput [ \/\/\u5fc5\u586b\uff0c\u8f93\u51fa\u5b9a\u4e49\n {\n name: "response"\n data_type: TYPE_STRING\n dims: [ -1 ]\n },\n {\n name: "history"\n data_type: TYPE_STRING\n dims: [ -1 ]\n }\n]\nparameters: { \/\/\u6307\u5b9apython\u6267\u884c\u73af\u5883\n key: "EXECUTION_ENV_PATH",\n value: {string_value: "$$TRITON_MODEL_DIRECTORY\/..\/python-3-8.tar.gz"}\n}\ninstance_group [ \/\/\u6a21\u578b\u5b9e\u4f8b\u7ec4\n { \n count: 1 \/\/\u5b9e\u4f8b\u6570\u91cf\n kind: KIND_GPU \/\/\u5b9e\u4f8b\u7c7b\u578b\n gpus: [ 0 ] \/\/\u6307\u5b9a\u5b9e\u4f8b\u53ef\u7528\u7684GPU\u7d22\u5f15\n }\n]\n<\/code><\/pre>\n4. \u81ea\u5b9a\u4e49python backend<\/h4>\n
import os\n# \u8bbe\u7f6e\u663e\u5b58\u7a7a\u95f2block\u6700\u5927\u5206\u5272\u9608\u503c\nos.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:32'\n# \u8bbe\u7f6ework\u76ee\u5f55\n\nos.environ['TRANSFORMERS_CACHE'] = os.path.dirname(os.path.abspath(__file__))+"\/work\/"\nos.environ['HF_MODULES_CACHE'] = os.path.dirname(os.path.abspath(__file__))+"\/work\/"\n\nimport json\n\n# triton_python_backend_utils is available in every Triton Python model. You\n# need to use this module to create inference requests and responses. It also\n# contains some utility functions for extracting information from model_config\n# and converting Triton input\/output types to numpy types.\nimport triton_python_backend_utils as pb_utils\nimport sys\nimport gc\nimport time\nimport logging\nimport torch\nfrom transformers import AutoTokenizer, AutoModel\nimport numpy as np\n\ngc.collect()\ntorch.cuda.empty_cache()\n\nlogging.basicConfig(format='%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s',\n level=logging.INFO)\n\nclass TritonPythonModel:\n """Your Python model must use the same class name. Every Python model\n that is created must have "TritonPythonModel" as the class name.\n """\n\n def initialize(self, args):\n """`initialize` is called only once when the model is being loaded.\n Implementing `initialize` function is optional. This function allows\n the model to intialize any state associated with this model.\n\n Parameters\n ----------\n args : dict\n Both keys and values are strings. The dictionary keys and values are:\n * model_config: A JSON string containing the model configuration\n * model_instance_kind: A string containing model instance kind\n * model_instance_device_id: A string containing model instance device ID\n * model_repository: Model repository path\n * model_version: Model version\n * model_name: Model name\n """\n # You must parse model_config. JSON string is not parsed here\n self.model_config = json.loads(args['model_config'])\n \n output_response_config = pb_utils.get_output_config_by_name(self.model_config, "response")\n output_history_config = pb_utils.get_output_config_by_name(self.model_config, "history")\n\n # Convert Triton types to numpy types\n self.output_response_dtype = pb_utils.triton_string_to_numpy(output_response_config['data_type'])\n self.output_history_dtype = pb_utils.triton_string_to_numpy(output_history_config['data_type'])\n \n ChatGLM_path = os.path.dirname(os.path.abspath(__file__))+"\/ChatGLM2_6B"\n self.tokenizer = AutoTokenizer.from_pretrained(ChatGLM_path, trust_remote_code=True)\n model = AutoModel.from_pretrained(ChatGLM_path,\n torch_dtype=torch.bfloat16,\n trust_remote_code=True).half().cuda()\n self.model = model.eval()\n logging.info("model init success")\n \n def execute(self, requests):\n """`execute` MUST be implemented in every Python model. `execute`\n function receives a list of pb_utils.InferenceRequest as the only\n argument. This function is called when an inference request is made\n for this model. Depending on the batching configuration (e.g. Dynamic\n Batching) used, `requests` may contain multiple requests. Every\n Python model, must create one pb_utils.InferenceResponse for every\n pb_utils.InferenceRequest in `requests`. If there is an error, you can\n set the error argument when creating a pb_utils.InferenceResponse\n\n Parameters\n ----------\n requests : list\n A list of pb_utils.InferenceRequest\n\n Returns\n -------\n list\n A list of pb_utils.InferenceResponse. The length of this list must\n be the same as `requests`\n \n """\n output_response_dtype = self.output_response_dtype\n output_history_dtype = self.output_history_dtype\n\n # output_dtype = self.output_dtype\n responses = []\n # Every Python backend must iterate over everyone of the requests\n # and create a pb_utils.InferenceResponse for each of them.\n for request in requests:\n prompt = pb_utils.get_input_tensor_by_name(request, "prompt").as_numpy()[0]\n prompt = prompt.decode('utf-8')\n history_origin = pb_utils.get_input_tensor_by_name(request, "history").as_numpy()\n if len(history_origin) > 0:\n history = np.array([item.decode('utf-8') for item in history_origin]).reshape((-1,2)).tolist()\n else:\n history = []\n temperature = pb_utils.get_input_tensor_by_name(request, "temperature").as_numpy()[0]\n temperature = float(temperature.decode('utf-8'))\n max_token = pb_utils.get_input_tensor_by_name(request, "max_token").as_numpy()[0]\n max_token = int(max_token.decode('utf-8'))\n history_len = pb_utils.get_input_tensor_by_name(request, "history_len").as_numpy()[0]\n history_len = int(history_len.decode('utf-8'))\n \n # \u65e5\u5fd7\u8f93\u51fa\u4f20\u5165\u4fe1\u606f\n in_log_info = {\n "in_prompt":prompt,\n "in_history":history,\n "in_temperature":temperature,\n "in_max_token":max_token,\n "in_history_len":history_len\n }\n logging.info(in_log_info)\n response,history = self.model.chat(self.tokenizer,\n prompt,\n history=history[-history_len:] if history_len > 0 else [],\n max_length=max_token,\n temperature=temperature)\n # \u65e5\u5fd7\u8f93\u51fa\u5904\u7406\u540e\u7684\u4fe1\u606f\n out_log_info = {\n "out_response":response,\n "out_history":history\n }\n logging.info(out_log_info)\n response = np.array(response)\n history = np.array(history)\n \n response_output_tensor = pb_utils.Tensor("response",response.astype(self.output_response_dtype))\n history_output_tensor = pb_utils.Tensor("history",history.astype(self.output_history_dtype))\n\n final_inference_response = pb_utils.InferenceResponse(output_tensors=[response_output_tensor,history_output_tensor])\n responses.append(final_inference_response)\n # Create InferenceResponse. You can set an error here in case\n # there was a problem with handling this inference request.\n # Below is an example of how you can set errors in inference\n # response:\n #\n # pb_utils.InferenceResponse(\n # output_tensors=..., TritonError("An error occured"))\n\n # You should return a list of pb_utils.InferenceResponse. Length\n # of this list must match the length of `requests` list.\n return responses\n\n def finalize(self):\n """`finalize` is called only once when the model is being unloaded.\n Implementing `finalize` function is OPTIONAL. This function allows\n the model to perform any necessary clean ups before exit.\n """\n print('Cleaning up...')\n<\/code><\/pre>\n5. \u90e8\u7f72\u6d4b\u8bd5<\/h4>\n
mkdir logs\n\/rm -rf \/9n-triton-devel\/server\/logs\/*\n\/rm -rf \/tmp\/python_env_*\nexport LD_LIBRARY_PATH=\/9n-triton-devel\/server\/lib\/:$LD_LIBRARY_PATH\nnohup .\/bin\/9n_predictor_server --flagfile=.\/conf\/server.gflags 2>&1 >\/dev\/null &\nsleep 2\npid=`ps x |grep "9n_predictor_server" | grep -v "grep" | grep -v "ldd" | grep -v "stat" | awk '{print $1}'`\necho $pid\n\n\n<\/code><\/pre>\n#!\/usr\/bin\/python3\n# -*- coding: utf-8 -*-\nimport sys\nsys.path.append('.\/base')\nfrom multi_backend_client import MultiBackendClient\nimport triton_python_backend_utils as python_backend_utils\nimport multi_backend_message_pb2\n\nimport time\nimport argparse\nimport io\nimport os\nimport numpy as np\nimport json\nimport struct\n\ndef print_result(response, batch_size ):\n print("outputs len:" + str(len(response.outputs)))\n\n if (response.error_code == 0):\n print("response : ", response)\n\n print(f'res shape: {response.outputs[0].shape}')\n res = python_backend_utils.deserialize_bytes_tensor(response.raw_output_contents[0])\n for i in res:\n print(i.decode())\n\n print(f'history shape: {response.outputs[1].shape}')\n history = python_backend_utils.deserialize_bytes_tensor(response.raw_output_contents[1])\n for i in history:\n print(i.decode())\n\n\ndef send_one_request(sender, request_pb, batch_size):\n succ, response = sender.send_req(request_pb)\n if succ:\n print_result(response, batch_size)\n else:\n print('send_one_request fail ', response)\n\ndef send_request(ip, port, temperature, max_token, history_len, batch_size=1, send_cnt=1):\n request_sender = MultiBackendClient(ip, port)\n\n request = multi_backend_message_pb2.ModelInferRequest()\n request.model_name = "chatglm2-6b"\n\n # \u8f93\u5165\u5360\u4f4d\n input0 = multi_backend_message_pb2.ModelInferRequest().InferInputTensor()\n input0.name = "prompt"\n input0.datatype = "BYTES"\n input0.shape.extend([1])\n\n input1 = multi_backend_message_pb2.ModelInferRequest().InferInputTensor()\n input1.name = "history"\n input1.datatype = "BYTES"\n input1.shape.extend([-1])\n \n input2 = multi_backend_message_pb2.ModelInferRequest().InferInputTensor()\n input2.name = "temperature"\n input2.datatype = "BYTES"\n input2.shape.extend([1])\n \n input3 = multi_backend_message_pb2.ModelInferRequest().InferInputTensor()\n input3.name = "max_token"\n input3.datatype = "BYTES"\n input3.shape.extend([1])\n \n input4 = multi_backend_message_pb2.ModelInferRequest().InferInputTensor()\n input4.name = "history_len"\n input4.datatype = "BYTES"\n input4.shape.extend([1])\n\n query = '\u8bf7\u7ed9\u51fa\u4e00\u4e2a\u5177\u4f53\u793a\u4f8b'\n input0.contents.bytes_contents.append(bytes(query, encoding="utf8"))\n request.inputs.extend([input0])\n\n history_origin = np.array([['\u4f60\u77e5\u9053\u9e21\u5154\u540c\u7b3c\u95ee\u9898\u4e48', '\u9e21\u5154\u540c\u7b3c\u95ee\u9898\u662f\u4e00\u4e2a\u7ecf\u5178\u7684\u6570\u5b66\u95ee\u9898\uff0c\u6d89\u53ca\u5230\u57fa\u672c\u7684\u4ee3\u6570\u65b9\u7a0b\u548c\u89e3\u9898\u65b9\u6cd5\u3002\u95ee\u9898\u63cf\u8ff0\u4e3a\uff1a\u5728\u4e00\u4e2a\u7b3c\u5b50\u91cc\u9762\uff0c\u6709\u82e5\u5e72\u53ea\u9e21\u548c\u5154\u5b50\uff0c\u5df2\u77e5\u5b83\u4eec\u7684\u603b\u6570\u548c\u603b\u817f\u6570\uff0c\u95ee\u9e21\u548c\u5154\u5b50\u7684\u6570\u91cf\u5404\u662f\u591a\u5c11\uff1f\/n\/n\u89e3\u6cd5\u5982\u4e0b\uff1a\u5047\u8bbe\u9e21\u7684\u6570\u91cf\u4e3ax\uff0c\u5154\u5b50\u7684\u6570\u91cf\u4e3ay\uff0c\u5219\u603b\u817f\u6570\u4e3a2x+4y\u3002\u6839\u636e\u9898\u610f\uff0c\u53ef\u4ee5\u5217\u51fa\u65b9\u7a0b\u7ec4\uff1a\/n\/nx + y = \u603b\u6570\/n2x + 4y = \u603b\u817f\u6570\/n\/n\u901a\u8fc7\u89e3\u65b9\u7a0b\u7ec4\uff0c\u53ef\u4ee5\u6c42\u5f97x\u548cy\u7684\u503c\uff0c\u4ece\u800c\u786e\u5b9a\u9e21\u548c\u5154\u5b50\u7684\u6570\u91cf\u3002']]).reshape((-1,))\n history = [bytes(item, encoding="utf8") for item in history_origin]\n input1.contents.bytes_contents.extend(history)\n request.inputs.extend([input1])\n \n input2.contents.bytes_contents.append(bytes(temperature, encoding="utf8"))\n request.inputs.extend([input2])\n \n input3.contents.bytes_contents.append(bytes(max_token, encoding="utf8"))\n request.inputs.extend([input3])\n \n input4.contents.bytes_contents.append(bytes(history_len, encoding="utf8"))\n request.inputs.extend([input4])\n\n # \u8f93\u51fa\u5360\u4f4d\n output_tensor0 = multi_backend_message_pb2.ModelInferRequest().InferRequestedOutputTensor()\n output_tensor0.name = "response"\n request.outputs.extend([output_tensor0])\n\n output_tensor1 = multi_backend_message_pb2.ModelInferRequest().InferRequestedOutputTensor()\n output_tensor1.name = "history"\n request.outputs.extend([output_tensor1])\n \n min_ms = 0\n max_ms = 0\n avg_ms = 0\n for i in range(send_cnt):\n start = time.time_ns()\n send_one_request(request_sender, request, batch_size)\n cost = (time.time_ns()-start)\/1000000\n print ("idx:%d cost ms:%d" % (i, cost))\n if cost > max_ms:\n max_ms = cost\n if cost < min_ms or min_ms==0:\n min_ms = cost\n avg_ms += cost\n avg_ms \/= send_cnt\n print("cnt=%d max=%dms min=%dms avg=%dms" % (send_cnt, max_ms, min_ms, avg_ms))\n\nif __name__ == '__main__':\n parser = argparse.ArgumentParser()\n parser.add_argument( '-ip', '--ip_address', help = 'ip address', default='127.0.0.1', required=False)\n parser.add_argument( '-p', '--port', help = 'port', default='8010', required=False)\n parser.add_argument( '-t', '--temperature', help = 'temperature', default='0.01', required=False)\n parser.add_argument( '-m', '--max_token', help = 'max_token', default='16000', required=False)\n parser.add_argument( '-hl', '--history_len', help = 'history_len', default='10', required=False)\n parser.add_argument( '-b', '--batch_size', help = 'batch size', default=1, required=False, type = int)\n parser.add_argument( '-c', '--send_count', help = 'send count', default=1, required=False, type = int)\n args = parser.parse_args()\n send_request(args.ip_address, args.port, args.temperature, args.max_token, args.history_len, args.batch_size, args.send_count)\n<\/code><\/pre>\n6. \u6a21\u578b\u90e8\u7f72<\/h4>\n
<\/p>\n
parameters: {\n key: "EXECUTION_ENV_PATH",\n value: {string_value: "$$TRITON_MODEL_DIRECTORY\/1\/python-3-8.tar.gz"}\n}\n<\/code><\/pre>\nfrom das.triton.model import TritonModel\n\nmodel = TritonModel("chatglm2-6b")\n\npredictor = model.deploy(\n path="$pwd\/model_repository\/chatglm2-6b", # \u6a21\u578b\u6587\u4ef6\u6240\u5728\u7684\u76ee\u5f55\n protocol='http',\n endpoint = "9n-das-serving-lf2.jd.local",\n cpu=4,\n memory=30,\n use_gpu=True, # \u6839\u636e\u662f\u5426\u9700\u8981gpu\u52a0\u901f\u63a8\u7406\u6765\u914d\u7f6e\n override = True,\n instances=2\n )\n<\/code><\/pre>\n\u56db.\u96c6\u6210\u81f3langchain<\/h2>\n
\nimport json\nimport time\nimport base64\nimport struct\nimport requests\nimport numpy as np\nfrom pathlib import Path\nfrom abc import ABC, abstractmethod\nfrom langchain.llms.base import LLM\nfrom langchain.llms import OpenAI\nfrom langchain.llms.utils import enforce_stop_tokens\nfrom typing import Dict, List, Optional, Tuple, Union, Mapping, Any\n\nimport warnings\nwarnings.filterwarnings("ignore")\n\nclass ChatGLM(LLM):\n max_token = 32000\n temperature = 0.01\n history_len = 10\n url = ""\n def __init__(self):\n super(ChatGLM, self).__init__()\n \n @property\n def _llm_type(self):\n return "ChatGLM2-6B"\n \n @property\n def _history_len(self) -> int:\n return self.history_len\n \n @property\n def _max_token(self) -> int:\n return self.max_token\n \n @property\n def _temperature(self) -> float:\n return self.temperature\n \n def _deserialize_bytes_tensor(self, encoded_tensor):\n """\n Deserializes an encoded bytes tensor into an\n numpy array of dtype of python objects\n Parameters\n ----------\n encoded_tensor : bytes\n The encoded bytes tensor where each element\n has its length in first 4 bytes followed by\n the content\n Returns\n -------\n string_tensor : np.array\n The 1-D numpy array of type object containing the\n deserialized bytes in 'C' order.\n """\n strs = list()\n offset = 0\n val_buf = encoded_tensor\n while offset < len(val_buf):\n l = struct.unpack_from("<I", val_buf, offset)[0]\n offset += 4\n sb = struct.unpack_from("<{}s".format(l), val_buf, offset)[0]\n offset += l\n strs.append(sb)\n return (np.array(strs, dtype=np.object_))\n \n @classmethod\n def _infer(cls, url, query, history, temperature, max_token, history_len):\n query = base64.b64encode(query.encode('utf-8')).decode('utf-8')\n history_origin = np.asarray(history).reshape((-1,))\n history = [base64.b64encode(item.encode('utf-8')).decode('utf-8') for item in history_origin]\n temperature = base64.b64encode(temperature.encode('utf-8')).decode('utf-8')\n max_token = base64.b64encode(max_token.encode('utf-8')).decode('utf-8')\n history_len = base64.b64encode(history_len.encode('utf-8')).decode('utf-8')\n data = {\n "model_name": "chatglm2-6b",\n "inputs": [\n {"name": "prompt", "datatype": "BYTES", "shape": [1], "contents": {"bytes_contents": [query]}},\n {"name": "history", "datatype": "BYTES", "shape": [-1], "contents": {"bytes_contents": history}},\n {"name": "temperature", "datatype": "BYTES", "shape": [1], "contents": {"bytes_contents": [temperature]}},\n {"name": "max_token", "datatype": "BYTES", "shape": [1], "contents": {"bytes_contents": [max_token]}},\n {"name": "history_len", "datatype": "BYTES", "shape": [1], "contents": {"bytes_contents": [history_len]}}\n ],\n "outputs": [{"name": "response"},\n {"name": "history"}]\n }\n response = requests.post(url = url, \n data = json.dumps(data, ensure_ascii=True), \n headers = {"Content_Type": "application\/json"}, \n timeout=120)\n return response \n \n def _call(self, \n query: str, \n history: List[List[str]] =[], \n stop: Optional[List[str]] =None):\n temperature = str(self.temperature)\n max_token = str(self.max_token)\n history_len = str(self.history_len)\n url = self.url\n response = self._infer(url, query, history, temperature, max_token, history_len)\n if response.status_code!=200:\n return "\u67e5\u8be2\u7ed3\u679c\u9519\u8bef"\n if stop is not None:\n response = enforce_stop_tokens(response, stop)\n result = json.loads(response.text)\n # \u5904\u7406response\n res = base64.b64decode(result['raw_output_contents'][0].encode('utf-8'))\n res_response = self._deserialize_bytes_tensor(res)[0].decode()\n return res_response\n \n def chat(self, \n query: str, \n history: List[List[str]] =[], \n stop: Optional[List[str]] =None):\n temperature = str(self.temperature)\n max_token = str(self.max_token)\n history_len = str(self.history_len)\n url = self.url\n response = self._infer(url, query, history, temperature, max_token, history_len)\n if response.status_code!=200:\n return "\u67e5\u8be2\u7ed3\u679c\u9519\u8bef"\n if stop is not None:\n response = enforce_stop_tokens(response, stop)\n result = json.loads(response.text)\n # \u5904\u7406response\n res = base64.b64decode(result['raw_output_contents'][0].encode('utf-8'))\n res_response = self._deserialize_bytes_tensor(res)[0].decode()\n # \u5904\u7406history\n history_shape = result['outputs'][1]["shape"]\n history_enc = base64.b64decode(result['raw_output_contents'][1].encode('utf-8'))\n res_history = np.array([i.decode() for i in self._deserialize_bytes_tensor(history_enc)]).reshape(history_shape).tolist()\n return res_response, res_history\n \n @property\n def _identifying_params(self) -> Mapping[str, Any]:\n """Get the identifying parameters.\n """\n _param_dict = {\n "url": self.url\n }\n return _param_dict\n<\/code><\/pre>\n\u4e94.\u603b\u7ed3<\/h2>\n
\n