examples\/offline_inference\/lora_with_quantization_inference.py<\/a><\/p>\n# SPDX-License-Identifier: Apache-2.0\n\n"""\n\u6b64\u793a\u4f8b\u663e\u793a\u4e86\u5982\u4f55\u4f7f\u7528\u4e0d\u540c\u7684\u91cf\u5316\u6280\u672f\u4f7f\u7528 LoRA\n\u7528\u4e8e\u79bb\u7ebf\u63a8\u7406\u3002\n\u9700\u8981 HuggingFace \u51ed\u8bc1\u4ee5\u8bbf\u95ee\u3002\n"""\n\nimport gc\nfrom typing import Optional\n\nimport torch\nfrom huggingface_hub import snapshot_download\n\nfrom vllm import EngineArgs, LLMEngine, RequestOutput, SamplingParams\nfrom vllm.lora.request import LoRARequest\n\n\ndef create_test_prompts(\n lora_path: str\n) -> list[tuple[str, SamplingParams, Optional[LoRARequest]]]:\n return [\n # \u8fd9\u662f\u4e0d\u4f7f\u7528 LoRA \u7684\u91cf\u5316\u7684\u793a\u4f8b\n ("My name is",\n SamplingParams(temperature=0.0,\n logprobs=1,\n prompt_logprobs=1,\n max_tokens=128), None),\n # \u63a5\u4e0b\u6765\u7684\u4e09\u4e2a\u793a\u4f8b\u4f7f\u7528 LoRA \u7684\u91cf\u5316\u793a\u4f8b\n ("my name is",\n SamplingParams(temperature=0.0,\n logprobs=1,\n prompt_logprobs=1,\n max_tokens=128),\n LoRARequest("lora-test-1", 1, lora_path)),\n ("The capital of USA is",\n SamplingParams(temperature=0.0,\n logprobs=1,\n prompt_logprobs=1,\n max_tokens=128),\n LoRARequest("lora-test-2", 1, lora_path)),\n ("The capital of France is",\n SamplingParams(temperature=0.0,\n logprobs=1,\n prompt_logprobs=1,\n max_tokens=128),\n LoRARequest("lora-test-3", 1, lora_path)),\n ]\n\n\ndef process_requests(engine: LLMEngine,\n test_prompts: list[tuple[str, SamplingParams,\n Optional[LoRARequest]]]):\n "\u6301\u7eed\u5904\u7406\u63d0\u793a\u5217\u8868\u5e76\u5904\u7406\u8f93\u51fa"\n request_id = 0\n\n while test_prompts or engine.has_unfinished_requests():\n if test_prompts:\n prompt, sampling_params, lora_request = test_prompts.pop(0)\n engine.add_request(str(request_id),\n prompt,\n sampling_params,\n lora_request=lora_request)\n request_id += 1\n\n request_outputs: list[RequestOutput] = engine.step()\n for request_output in request_outputs:\n if request_output.finished:\n print("----------------------------------------------------")\n print(f"Prompt: {request_output.prompt}")\n print(f"Output: {request_output.outputs[0].text}")\n\n\ndef initialize_engine(model: str, quantization: str,\n lora_repo: Optional[str]) -> LLMEngine:\n """Initialize the LLMEngine."""\n\n if quantization == "bitsandbytes":\n # Qlora (https:\/\/arxiv.org\/abs\/2305.14314) \u662f\u4e00\u79cd\u91cf\u5316\u6280\u672f\u3002\n # \u5b83\u5728\u52a0\u8f7d\u65f6\u91cf\u5316\u6a21\u578b\uff0c\u5e76\u5e26\u6709\u4e00\u4e9b\u914d\u7f6e\u4fe1\u606f\n # LoRA \u9002\u914d\u5668\u5b58\u50a8\u5e93\u3002\u56e0\u6b64\uff0c\u9700\u8981\u8bbe\u7f6e load_format \u7684\u53c2\u6570\u548c\n # qlora_adapter_name_or_path \u5982\u4e0b\u3002\n engine_args = EngineArgs(model=model,\n quantization=quantization,\n qlora_adapter_name_or_path=lora_repo,\n enable_lora=True,\n max_lora_rank=64)\n else:\n engine_args = EngineArgs(model=model,\n quantization=quantization,\n enable_lora=True,\n max_loras=4)\n return LLMEngine.from_engine_args(engine_args)\n\n\ndef main():\n """Main function that sets up and runs the prompt processing."""\n\n test_configs = [{\n "name": "qlora_inference_example",\n 'model': "huggyllama\/llama-7b",\n 'quantization': "bitsandbytes",\n 'lora_repo': 'timdettmers\/qlora-flan-7b'\n }, {\n "name": "AWQ_inference_with_lora_example",\n 'model': 'TheBloke\/TinyLlama-1.1B-Chat-v0.3-AWQ',\n 'quantization': "awq",\n 'lora_repo': 'jashing\/tinyllama-colorist-lora'\n }, {\n "name": "GPTQ_inference_with_lora_example",\n 'model': 'TheBloke\/TinyLlama-1.1B-Chat-v0.3-GPTQ',\n 'quantization': "gptq",\n 'lora_repo': 'jashing\/tinyllama-colorist-lora'\n }]\n\n for test_config in test_configs:\n print(\n f"~~~~~~~~~~~~~~~~ Running: {test_config['name']} ~~~~~~~~~~~~~~~~"\n )\n engine = initialize_engine(test_config['model'],\n test_config['quantization'],\n test_config['lora_repo'])\n lora_path = snapshot_download(repo_id=test_config['lora_repo'])\n test_prompts = create_test_prompts(lora_path)\n process_requests(engine, test_prompts)\n\n # \u6e05\u7406\u4e0b\u4e00\u4e2a\u6d4b\u8bd5\u7684 GPU \u5185\u5b58\n del engine\n gc.collect()\n torch.cuda.empty_cache()\n\n\nif __name__ == '__main__':\n main()<\/code><\/pre>\nhttps:\/\/segmentfault.com\/a\/1190000047181862<\/a><\/p>\n","protected":false},"excerpt":{"rendered":"vLLM \u662f\u4e00\u6b3e\u4e13\u4e3a\u5927\u8bed\u8a00\u6a21\u578b\u63a8\u7406\u52a0\u901f\u800c\u8bbe\u8ba1\u7684\u6846\u67b6\uff0c\u5b9e\u73b0\u4e86 KV \u7f13\u5b58\u5185\u5b58\u51e0\u4e4e\u96f6\u6d6a\u8d39\uff0c\u89e3\u51b3\u4e86\u5185\u5b58\u7ba1\u7406\u74f6\u9888\u95ee\u9898\u3002 […]<\/p>\n","protected":false},"author":3,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"_jetpack_memberships_contains_paid_content":false,"footnotes":""},"categories":[4],"tags":[],"class_list":["post-10020958","post","type-post","status-publish","format-standard","hentry","category-share"],"jetpack_sharing_enabled":true,"jetpack_featured_media_url":"","_links":{"self":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10020958","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/users\/3"}],"replies":[{"embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=10020958"}],"version-history":[{"count":0,"href":"https:\/\/itmax.cn\/index.php?rest_route=\/wp\/v2\/posts\/10020958\/revisions"}],"wp:attachment":[{"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=10020958"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=10020958"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/itmax.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=10020958"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}