{"id":123895,"date":"2026-11-08T12:56:41","date_gmt":"2026-11-08T04:56:41","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/123895.html"},"modified":"2026-11-08T12:56:41","modified_gmt":"2026-11-08T04:56:41","slug":"%e3%80%90llm%e6%8e%a8%e7%90%86%e4%b8%8eserving%e2%91%a6%e3%80%91%e4%b8%bb%e6%b5%81%e6%8e%a8%e7%90%86%e6%a1%86%e6%9e%b6%e5%af%b9%e6%af%94%ef%bc%9avllm-tgi-tensorrt-llm-sglang-lmdeploy%e9%80%89%e5%9e%8b","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/123895.html","title":{"rendered":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357"},"content":{"rendered":"<h2>\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4&#xff1a;vLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357<\/h2>\n<p>\u7cfb\u5217\u6587\u7ae0\u7d22\u5f15&#xff1a;\u672c\u7cfb\u5217\u4ece\u57fa\u7840\u6982\u5ff5\u5230\u5de5\u7a0b\u5b9e\u8df5&#xff0c;\u7cfb\u7edf\u68b3\u7406LLM\u63a8\u7406\u4e0eServing\u7684\u6838\u5fc3\u77e5\u8bc6\u3002\u672c\u6587\u4e3a\u7b2c7\u7bc7&#xff0c;\u805a\u7126\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u7684\u6a2a\u5411\u5bf9\u6bd4\u4e0e\u9009\u578b\u65b9\u6cd5\u8bba\u3002<\/p>\n<h3>\u4e00\u3001\u5f00\u7bc7&#xff1a;\u9009\u63a8\u7406\u6846\u67b6\u4e0d\u662f\u9009&#034;\u6700\u5feb\u7684&#034;&#xff0c;\u800c\u662f\u9009\u6700\u5339\u914d\u4e1a\u52a1\u7684<\/h3>\n<p>\u521a\u63a5\u89e6LLM Serving\u7684\u540c\u5b66&#xff0c;\u7ecf\u5e38\u4f1a\u95ee\u4e00\u4e2a\u95ee\u9898&#xff1a;\u201c\u54ea\u4e2a\u63a8\u7406\u6846\u67b6\u6700\u5feb&#xff1f;\u201d \u6211\u7684\u56de\u7b54\u901a\u5e38\u662f&#xff1a;\u201c\u8fd9\u53d6\u51b3\u4e8e\u4f60\u7684\u4e1a\u52a1\u662f\u4ec0\u4e48\u3002\u201d<\/p>\n<p>\u5982\u679c\u4f60\u662f\u505a\u4e00\u4e2a\u7b80\u5355\u7684\u804a\u5929\u673a\u5668\u4ebaDemo&#xff0c;vLLM\u53ef\u80fd3\u5206\u949f\u5c31\u80fd\u8dd1\u8d77\u6765&#xff1b;\u5982\u679c\u4f60\u8981\u5728A100\u96c6\u7fa4\u4e0a\u8dd1\u5343\u4ebf\u53c2\u6570\u6a21\u578b\u8ffd\u6c42\u6781\u81f4\u541e\u5410&#xff0c;TensorRT-LLM\u53ef\u80fd\u662f\u66f4\u597d\u7684\u9009\u62e9&#xff1b;\u5982\u679c\u4f60\u56e2\u961f\u4ee5Python\u4e3a\u4e3b\u3001\u53c8\u9700\u8981\u7ed3\u6784\u5316\u8f93\u51fa\u80fd\u529b&#xff0c;SGLang\u53ef\u80fd\u8ba9\u4f60\u4e8b\u534a\u529f\u500d\u3002<\/p>\n<p>\u63a8\u7406\u6846\u67b6\u7684\u9009\u578b&#xff0c;\u672c\u8d28\u4e0a\u662f\u4e00\u4e2a\u591a\u76ee\u6807\u4f18\u5316\u95ee\u9898&#xff1a;\u541e\u5410\u8981\u9ad8\u3001\u5ef6\u8fdf\u8981\u4f4e\u3001\u663e\u5b58\u8981\u7701\u3001API\u8981\u517c\u5bb9\u3001\u8fd0\u7ef4\u8981\u7b80\u5355\u3001\u751f\u6001\u8981\u5b8c\u5584\u2026\u2026\u8fd9\u4e9b\u76ee\u6807\u4e4b\u95f4\u5f80\u5f80\u5b58\u5728\u51b2\u7a81&#xff0c;\u6211\u4eec\u9700\u8981\u6839\u636e\u4e1a\u52a1\u573a\u666f\u505a\u6743\u8861\u3002<\/p>\n<p>\u8fd9\u7bc7\u6587\u7ae0\u4e0d\u505a&#034;\u8c01\u662f\u7b2c\u4e00&#034;\u7684\u6392\u884c&#xff0c;\u800c\u662f\u5e26\u4f60\u5efa\u7acb\u4e00\u5957\u9009\u578b\u65b9\u6cd5\u8bba&#xff0c;\u8ba9\u4f60\u80fd\u6839\u636e\u81ea\u5df1\u7684\u4e1a\u52a1\u7279\u70b9&#xff0c;\u5728vLLM\u3001TGI\u3001TensorRT-LLM\u3001SGLang\u3001LMDeploy\u7b49\u4e3b\u6d41\u6846\u67b6\u4e2d\u505a\u51fa\u6700\u5408\u9002\u7684\u9009\u62e9\u3002<\/p>\n<h3>\u4e8c\u3001\u9009\u578b\u65f6\u5e94\u8be5\u770b\u54ea\u4e9b\u7ef4\u5ea6<\/h3>\n<p>\u5728\u5bf9\u6bd4\u6846\u67b6\u4e4b\u524d&#xff0c;\u6211\u4eec\u5148\u5efa\u7acb\u4e00\u4e2a\u8bc4\u4f30\u7ef4\u5ea6\u6e05\u5355\u3002\u4f60\u53ef\u4ee5\u628a\u5b83\u5f53\u4f5c\u4e00\u4e2a&#034;\u6253\u5206\u8868&#034;&#xff0c;\u6bcf\u4e00\u9879\u6839\u636e\u4e1a\u52a1\u91cd\u8981\u6027\u8d4b\u4e88\u4e0d\u540c\u6743\u91cd\u3002<\/p>\n<h4>2.1 \u6a21\u578b\u652f\u6301\u5e7f\u5ea6<\/h4>\n<ul>\n<li>\u652f\u6301\u7684\u6a21\u578b\u67b6\u6784&#xff1a;LLaMA\u3001Qwen\u3001Baichuan\u3001ChatGLM\u3001Mistral\u3001Gemma\u2026\u2026\u4f60\u7684\u76ee\u6807\u6a21\u578b\u662f\u5426\u5728\u652f\u6301\u5217\u8868\u4e2d&#xff1f;<\/li>\n<li>\u65b0\u6a21\u578b\u8ddf\u8fdb\u901f\u5ea6&#xff1a;SOTA\u6a21\u578b\u53d1\u5e03\u540e&#xff0c;\u6846\u67b6\u591a\u4e45\u80fd\u8ddf\u4e0a&#xff1f;<\/li>\n<li>\u81ea\u5b9a\u4e49\u6a21\u578b\u9002\u914d\u6210\u672c&#xff1a;\u5982\u679c\u662f\u81ea\u7814\u6a21\u578b&#xff0c;\u63a5\u5165\u9700\u8981\u6539\u591a\u5c11\u4ee3\u7801&#xff1f;<\/li>\n<\/ul>\n<h4>2.2 API\u517c\u5bb9\u6027<\/h4>\n<ul>\n<li>\u662f\u5426\u517c\u5bb9OpenAI API\u683c\u5f0f&#xff1a;\u8fd9\u662f\u4e8b\u5b9e\u6807\u51c6&#xff0c;\u517c\u5bb9\u610f\u5473\u7740\u4e0a\u5c42\u5e94\u7528\u51e0\u4e4e\u4e0d\u7528\u6539<\/li>\n<li>\u662f\u5426\u652f\u6301\u6d41\u5f0f\u8f93\u51fa&#xff08;SSE&#xff09;&#xff1a;\u5bf9\u8bdd\u573a\u666f\u7684\u521a\u9700<\/li>\n<li>\u662f\u5426\u652f\u6301Function Calling \/ \u5de5\u5177\u8c03\u7528&#xff1a;Agent\u573a\u666f\u5fc5\u5907<\/li>\n<\/ul>\n<h4>2.3 \u6027\u80fd\u6307\u6807<\/h4>\n<ul>\n<li>\u541e\u5410&#xff08;Throughput&#xff09;&#xff1a;\u5355\u4f4d\u65f6\u95f4\u80fd\u5904\u7406\u591a\u5c11token&#xff0c;\u51b3\u5b9a\u4e86\u670d\u52a1\u5bb9\u91cf\u548c\u6210\u672c<\/li>\n<li>\u9996Token\u5ef6\u8fdf&#xff08;TTFT&#xff09;&#xff1a;\u7528\u6237\u4ece\u53d1\u8bf7\u6c42\u5230\u770b\u5230\u7b2c\u4e00\u4e2a\u5b57\u7684\u65f6\u95f4&#xff0c;\u5f71\u54cd\u4f53\u9a8c<\/li>\n<li>\u6bcfToken\u5ef6\u8fdf&#xff08;TPOT&#xff09;&#xff1a;\u540e\u7eed\u6bcf\u4e2atoken\u7684\u751f\u6210\u901f\u5ea6&#xff0c;\u5f71\u54cd\u6253\u5b57\u6d41\u7545\u5ea6<\/li>\n<li>\u663e\u5b58\u5360\u7528&#xff1a;\u540c\u6837\u7684\u6a21\u578b&#xff0c;\u8c01\u7528\u7684\u663e\u5b58\u5c11&#xff0c;\u8c01\u5c31\u80fd\u585e\u4e0b\u66f4\u591a\u5e76\u53d1<\/li>\n<\/ul>\n<h4>2.4 KV Cache\u4e0ePrefix\u4f18\u5316<\/h4>\n<ul>\n<li>PagedAttention \/ Continuous Batching&#xff1a;\u662f\u5426\u652f\u6301\u52a8\u6001\u6279\u5904\u7406&#xff0c;\u8fd9\u662f\u9ad8\u541e\u5410\u7684\u57fa\u77f3<\/li>\n<li>Prefix Caching \/ \u5171\u4eab\u524d\u7f00&#xff1a;\u7cfb\u7edf\u63d0\u793a\u8bcd&#xff08;System Prompt&#xff09;\u80fd\u5426\u590d\u7528KV Cache&#xff0c;\u5bf9RAG\u548cAgent\u573a\u666f\u5f88\u91cd\u8981<\/li>\n<li>KV Cache\u91cf\u5316&#xff1a;FP8\/INT8\/INT4\u91cf\u5316KV&#xff0c;\u8fdb\u4e00\u6b65\u7701\u663e\u5b58<\/li>\n<\/ul>\n<h4>2.5 \u9ad8\u7ea7\u7279\u6027<\/h4>\n<ul>\n<li>\u91cf\u5316\u652f\u6301&#xff1a;GPTQ\u3001AWQ\u3001FP8\u3001SmoothQuant\u2026\u2026\u652f\u6301\u54ea\u4e9b\u91cf\u5316\u65b9\u6848&#xff1f;<\/li>\n<li>\u7ed3\u6784\u5316\u8f93\u51fa&#xff1a;JSON Schema\u3001Regex\u3001Grammar\u7ea6\u675f\u8f93\u51fa\u683c\u5f0f<\/li>\n<li>Speculative Decoding&#xff1a;\u6295\u673a\u89e3\u7801&#xff0c;\u7528\u5c0f\u6a21\u578b\u52a0\u901f\u5927\u6a21\u578b<\/li>\n<li>LoRA\u52a8\u6001\u52a0\u8f7d&#xff1a;\u591a\u79df\u6237\u573a\u666f\u4e0b\u80fd\u5426\u6309\u9700\u52a0\u8f7d\u4e0d\u540c\u7684LoRA\u9002\u914d\u5668<\/li>\n<\/ul>\n<h4>2.6 \u5206\u5e03\u5f0f\u4e0e\u90e8\u7f72<\/h4>\n<ul>\n<li>Tensor Parallel&#xff08;\u5f20\u91cf\u5e76\u884c&#xff09;&#xff1a;\u5355\u6a21\u578b\u8de8\u591a\u5f20\u5361<\/li>\n<li>Pipeline Parallel&#xff08;\u6d41\u6c34\u7ebf\u5e76\u884c&#xff09;&#xff1a;\u8d85\u5927\u6a21\u578b\u5206\u5c42\u90e8\u7f72<\/li>\n<li>\u591a\u8282\u70b9\u652f\u6301&#xff1a;\u80fd\u5426\u8de8\u673a\u5668\u5206\u5e03\u5f0f\u63a8\u7406<\/li>\n<li>\u5bb9\u5668\u5316\u90e8\u7f72&#xff1a;\u662f\u5426\u6709\u5b98\u65b9Docker\u955c\u50cf&#xff0c;K8s\u9002\u914d\u5982\u4f55<\/li>\n<\/ul>\n<h4>2.7 \u8fd0\u7ef4\u4e0e\u751f\u6001<\/h4>\n<ul>\n<li>\u76d1\u63a7\u6307\u6807&#xff1a;Prometheus metrics\u3001\u65e5\u5fd7\u5b8c\u5584\u7a0b\u5ea6<\/li>\n<li>\u793e\u533a\u6d3b\u8dc3\u5ea6&#xff1a;GitHub stars\u3001issue\u54cd\u5e94\u901f\u5ea6\u3001\u8fed\u4ee3\u9891\u7387<\/li>\n<li>\u6587\u6863\u8d28\u91cf&#xff1a;\u65b0\u624b\u53cb\u597d\u5ea6\u3001\u793a\u4f8b\u4e30\u5bcc\u5ea6<\/li>\n<li>\u5546\u4e1a\u652f\u6301&#xff1a;\u51fa\u4e86\u95ee\u9898\u6709\u6ca1\u6709\u4eba\u515c\u5e95<\/li>\n<\/ul>\n<h3>\u4e09\u3001\u4e3b\u6d41\u6846\u67b6\u901f\u89c8<\/h3>\n<p>\u4e0b\u9762\u6211\u4eec\u9010\u4e2a\u8ba4\u8bc6\u8fd9\u4e9b\u4e3b\u6d41\u6846\u67b6&#xff0c;\u4e86\u89e3\u5b83\u4eec\u7684&#034;\u6027\u683c\u7279\u70b9&#034;\u3002<\/p>\n<h4>3.1 vLLM&#xff1a;Python\u53cb\u597d\u7684\u9ad8\u541e\u5410\u5f15\u64ce<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;UC Berkeley\u51fa\u54c1&#xff0c;\u7528PagedAttention\u5f7b\u5e95\u6539\u53d8\u4e86LLM\u63a8\u7406\u7684\u541e\u5410\u6e38\u620f\u89c4\u5219\u3002<\/p>\n<p>vLLM\u6700\u5927\u7684\u8d21\u732e\u662f\u63d0\u51fa\u4e86PagedAttention\u2014\u2014\u628a\u64cd\u4f5c\u7cfb\u7edf\u4e2d\u865a\u62df\u5185\u5b58\u7684\u5206\u9875\u601d\u60f3\u7528\u5230\u4e86KV Cache\u7ba1\u7406\u4e0a\u3002\u4f20\u7edf\u65b9\u5f0f\u4e0bKV Cache\u662f\u8fde\u7eed\u5206\u914d\u7684&#xff0c;\u5bb9\u6613\u4ea7\u751f\u788e\u7247&#xff1b;PagedAttention\u628aKV Cache\u5206\u6210\u56fa\u5b9a\u5927\u5c0f\u7684&#034;\u9875&#034;&#xff0c;\u53ef\u4ee5\u975e\u8fde\u7eed\u5b58\u50a8&#xff0c;\u5927\u5927\u63d0\u5347\u4e86\u663e\u5b58\u5229\u7528\u7387\u548c\u5e76\u53d1\u6570\u3002<\/p>\n<p><span class=\"token comment\"># vLLM \u542f\u52a8\u793a\u4f8b&#xff08;OpenAI\u517c\u5bb9API&#xff09;<\/span><br \/>\n<span class=\"token comment\"># \u547d\u4ee4\u884c\u542f\u52a8<\/span><br \/>\n<span class=\"token comment\"># vllm serve meta-llama\/Llama-2-7b-chat-hf \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;tensor-parallel-size 1 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;gpu-memory-utilization 0.9 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;max-model-len 4096<\/span><\/p>\n<p><span class=\"token comment\"># Python \u5ba2\u6237\u7aef\u8c03\u7528&#xff08;OpenAI SDK\u517c\u5bb9&#xff09;<\/span><br \/>\n<span class=\"token keyword\">from<\/span> openai <span class=\"token keyword\">import<\/span> OpenAI<\/p>\n<p>client <span class=\"token operator\">&#061;<\/span> OpenAI<span class=\"token punctuation\">(<\/span><br \/>\n    base_url<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;http:\/\/localhost:8000\/v1&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    api_key<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;token-abc123&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p>response <span class=\"token operator\">&#061;<\/span> client<span class=\"token punctuation\">.<\/span>chat<span class=\"token punctuation\">.<\/span>completions<span class=\"token punctuation\">.<\/span>create<span class=\"token punctuation\">(<\/span><br \/>\n    model<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;meta-llama\/Llama-2-7b-chat-hf&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    messages<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span><br \/>\n        <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;system&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u4f60\u662f\u4e00\u4e2a\u4e50\u4e8e\u52a9\u4eba\u7684\u52a9\u624b\u3002&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;user&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u8bf7\u89e3\u91ca\u4ec0\u4e48\u662fPagedAttention\u3002&#034;<\/span><span class=\"token punctuation\">}<\/span><br \/>\n    <span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    stream<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">,<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">for<\/span> chunk <span class=\"token keyword\">in<\/span> response<span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">if<\/span> chunk<span class=\"token punctuation\">.<\/span>choices<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>delta<span class=\"token punctuation\">.<\/span>content<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>chunk<span class=\"token punctuation\">.<\/span>choices<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>delta<span class=\"token punctuation\">.<\/span>content<span class=\"token punctuation\">,<\/span> end<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>\u5b89\u88c5\u7b80\u5355&#xff0c;pip install vllm \u57fa\u672c\u641e\u5b9a<\/li>\n<li>\u539f\u751f\u652f\u6301OpenAI\u517c\u5bb9API&#xff0c;\u5f00\u7bb1\u5373\u7528<\/li>\n<li>Continuous Batching &#043; PagedAttention \u5e26\u6765\u6781\u9ad8\u541e\u5410<\/li>\n<li>\u652f\u6301\u591a\u79cd\u91cf\u5316&#xff1a;AWQ\u3001GPTQ\u3001FP8\u3001SqueezeLLM<\/li>\n<li>\u652f\u6301Speculative Decoding\u3001LoRA\u3001Prefix Caching<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;\u7edd\u5927\u591a\u6570\u751f\u4ea7\u573a\u666f\u7684\u9996\u9009&#xff0c;\u5c24\u5176\u662fPython\u6280\u672f\u6808\u56e2\u961f\u3002<\/p>\n<h4>3.2 TGI&#xff1a;Hugging Face\u5b98\u65b9\u6846\u67b6<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;Hugging Face\u4eb2\u513f\u5b50&#xff0c;transformers\u751f\u6001\u65e0\u7f1d\u8854\u63a5\u3002<\/p>\n<p>TGI&#xff08;Text Generation Inference&#xff09;\u662fHugging Face\u5b98\u65b9\u63a8\u51fa\u7684\u63a8\u7406\u670d\u52a1\u6846\u67b6\u3002\u5982\u679c\u4f60\u5df2\u7ecf\u5728\u7528transformers\u5e93&#xff0c;TGI\u7684\u5b66\u4e60\u6210\u672c\u51e0\u4e4e\u4e3a\u96f6\u2014\u2014\u56e0\u4e3a\u5b83\u5e95\u5c42\u5c31\u662ftransformers &#043; \u4e00\u4e9b\u4f18\u5316\u3002<\/p>\n<p><span class=\"token comment\"># TGI Docker \u542f\u52a8\u793a\u4f8b<\/span><br \/>\n<span class=\"token comment\"># docker run -p 8080:80 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;gpus all \\\\<\/span><br \/>\n<span class=\"token comment\">#   -e MODEL_ID&#061;meta-llama\/Llama-2-7b-chat-hf \\\\<\/span><br \/>\n<span class=\"token comment\">#   -e MAX_BATCH_PREFILL_TOKENS&#061;4096 \\\\<\/span><br \/>\n<span class=\"token comment\">#   -e MAX_INPUT_LENGTH&#061;1024 \\\\<\/span><br \/>\n<span class=\"token comment\">#   -e MAX_TOTAL_TOKENS&#061;2048 \\\\<\/span><br \/>\n<span class=\"token comment\">#   ghcr.io\/huggingface\/text-generation-inference:latest<\/span><\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>\u4e0eHugging Face\u751f\u6001\u6df1\u5ea6\u6574\u5408&#xff0c;\u6a21\u578b\u52a0\u8f7d\u65b9\u5f0f\u4e00\u81f4<\/li>\n<li>\u652f\u6301Flash Attention\u3001Paged Attention&#xff08;v2.0&#043;\u5f15\u5165&#xff09;<\/li>\n<li>\u5185\u7f6eProteus&#xff08;\u52a8\u6001\u6279\u5904\u7406\u8c03\u5ea6\u5668&#xff09;<\/li>\n<li>\u652f\u6301Tensor Parallel\u591a\u5361\u63a8\u7406<\/li>\n<li>\u5b8c\u5584\u7684\u65e5\u5fd7\u548cPrometheus\u76d1\u63a7<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;\u5df2\u7ecf\u6df1\u5ea6\u4f7f\u7528Hugging Face\u751f\u6001\u7684\u56e2\u961f&#xff0c;\u6216\u8005\u5bf9\u6a21\u578b\u53ef\u89e3\u91ca\u6027\u3001\u8c03\u8bd5\u9700\u6c42\u9ad8\u7684\u573a\u666f\u3002<\/p>\n<h4>3.3 TensorRT-LLM&#xff1a;NVIDIA\u6781\u9650\u6027\u80fd\u6846\u67b6<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;NVIDIA\u4eb2\u751f\u7684&#xff0c;\u8ffd\u6c42\u6781\u81f4\u6027\u80fd\u5c31\u9009\u5b83\u3002<\/p>\n<p>TensorRT-LLM\u662fNVIDIA\u57fa\u4e8eTensorRT\u6253\u9020\u7684LLM\u63a8\u7406\u52a0\u901f\u5e93\u3002\u5b83\u7684\u6838\u5fc3\u4f18\u52bf\u5728\u4e8e\u6781\u81f4\u7684kernel\u4f18\u5316\u548cFP8\u7cbe\u5ea6\u652f\u6301\u2014\u2014\u6bd5\u7adfNVIDIA\u6700\u61c2\u81ea\u5df1\u7684\u786c\u4ef6\u3002<\/p>\n<p>\u4ee3\u4ef7\u662f&#xff1a;\u4f7f\u7528\u95e8\u69db\u76f8\u5bf9\u8f83\u9ad8&#xff0c;\u9700\u8981\u7f16\u8bd1\u3001\u6784\u5efa\u5f15\u64ce&#xff0c;Python\u63a5\u53e3\u76f8\u5bf9\u5e95\u5c42\u3002<\/p>\n<p><span class=\"token comment\"># TensorRT-LLM \u7b80\u5316\u793a\u4f8b&#xff08;\u6784\u5efa\u5f15\u64ce&#043;\u63a8\u7406&#xff09;<\/span><br \/>\n<span class=\"token comment\"># \u5b9e\u9645\u4f7f\u7528\u901a\u5e38\u9700\u8981\u5148\u7f16\u8bd1\u3001\u6784\u5efaengine&#xff0c;\u518d\u901a\u8fc7Python\u6216C&#043;&#043;\u8c03\u7528<\/span><\/p>\n<p><span class=\"token comment\"># 1. \u6784\u5efa\u5f15\u64ce&#xff08;\u547d\u4ee4\u884c\u5de5\u5177&#xff09;<\/span><br \/>\n<span class=\"token comment\"># python build.py \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;model_dir .\/Llama-2-7b-hf \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;dtype float16 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;max_batch_size 64 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;max_input_len 2048 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;max_output_len 512 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;output_dir .\/trt_engines\/llama-2-7b<\/span><\/p>\n<p><span class=\"token comment\"># 2. Python \u63a8\u7406\u793a\u4f8b&#xff08;\u7b80\u5316\u7248&#xff09;<\/span><br \/>\n<span class=\"token keyword\">from<\/span> tensorrt_llm<span class=\"token punctuation\">.<\/span>runtime <span class=\"token keyword\">import<\/span> ModelRunner<\/p>\n<p>runner <span class=\"token operator\">&#061;<\/span> ModelRunner<span class=\"token punctuation\">.<\/span>from_dir<span class=\"token punctuation\">(<\/span><br \/>\n    engine_dir<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;.\/trt_engines\/llama-2-7b&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    max_batch_size<span class=\"token operator\">&#061;<\/span><span class=\"token number\">64<\/span><span class=\"token punctuation\">,<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><\/p>\n<p>output <span class=\"token operator\">&#061;<\/span> runner<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span><br \/>\n    prompt<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\u89e3\u91ca\u4e00\u4e0b\u4ec0\u4e48\u662fTensorRT-LLM&#xff1a;&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    max_new_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    temperature<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.7<\/span><span class=\"token punctuation\">,<\/span><br \/>\n<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>output<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>\u6781\u81f4\u7684kernel\u4f18\u5316&#xff0c;FP8\u8bad\u7ec3\/\u63a8\u7406\u5168\u94fe\u8def\u652f\u6301<\/li>\n<li>\u652f\u6301In-flight Batching&#xff08;\u8fde\u7eed\u6279\u5904\u7406&#xff09;<\/li>\n<li>\u652f\u6301TP\u3001PP\u591a\u79cd\u5e76\u884c\u7b56\u7565<\/li>\n<li>\u652f\u6301SmoothQuant\u3001AWQ\u3001GPTQ\u7b49\u91cf\u5316<\/li>\n<li>\u591aGPU\u3001\u591a\u8282\u70b9\u6269\u5c55\u6027\u5f3a<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;\u5bf9\u6027\u80fd\u8981\u6c42\u6781\u81f4\u3001\u6709\u4e13\u95e8\u7684\u63a8\u7406\u4f18\u5316\u56e2\u961f\u3001\u5927\u89c4\u6a21\u90e8\u7f72\u7684\u573a\u666f\u3002<\/p>\n<h4>3.4 SGLang&#xff1a;\u7ed3\u6784\u5316\u8f93\u51fa\u5f3a&#xff0c;RadixAttention\u524d\u7f00\u5171\u4eab<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;\u7ed3\u6784\u5316\u8f93\u51fa\u7684\u738b\u8005&#xff0c;\u524d\u7f00\u7f13\u5b58\u505a\u5f97\u975e\u5e38\u51fa\u8272\u3002<\/p>\n<p>SGLang&#xff08;Structured Generation Language&#xff09;\u662f\u4e00\u4e2a\u76f8\u5bf9\u65b0\u7684\u6846\u67b6&#xff0c;\u4f46\u5b83\u6709\u4e24\u4e2a\u6740\u624b\u7ea7\u7279\u6027&#xff1a;<\/p>\n<li>\n<p>RadixAttention&#xff1a;\u57fa\u4e8e\u524d\u7f00\u6811&#xff08;Trie&#xff09;\u7684KV Cache\u5171\u4eab&#xff0c;\u76f8\u540c\u524d\u7f00\u7684\u8bf7\u6c42\u53ef\u4ee5\u5171\u4eabKV Cache\u3002\u8fd9\u5bf9\u4e8eRAG\u3001Agent\u3001\u591a\u8f6e\u5bf9\u8bdd\u573a\u666f\u6548\u679c\u62d4\u7fa4\u2014\u2014\u7cfb\u7edf\u63d0\u793a\u8bcd\u53ea\u9700\u8981\u7b97\u4e00\u6b21\u3002<\/p>\n<\/li>\n<li>\n<p>\u7ed3\u6784\u5316\u751f\u6210&#xff1a;\u539f\u751f\u652f\u6301JSON Schema\u3001Regex\u3001Grammar\u7ea6\u675f&#xff0c;\u8f93\u51fa\u683c\u5f0f100%\u53ef\u63a7&#xff0c;\u4e0d\u9700\u8981\u4e8b\u540e\u6821\u9a8c\u548c\u91cd\u8bd5\u3002<\/p>\n<\/li>\n<p><span class=\"token comment\"># SGLang \u7ed3\u6784\u5316\u8f93\u51fa\u793a\u4f8b<\/span><br \/>\n<span class=\"token keyword\">import<\/span> sglang <span class=\"token keyword\">as<\/span> sgl<br \/>\n<span class=\"token keyword\">from<\/span> pydantic <span class=\"token keyword\">import<\/span> BaseModel<\/p>\n<p><span class=\"token comment\"># \u5b9a\u4e49\u8f93\u51fa\u7ed3\u6784<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">UserInfo<\/span><span class=\"token punctuation\">(<\/span>BaseModel<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    name<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><br \/>\n    age<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><br \/>\n    hobbies<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">[<\/span><span class=\"token builtin\">str<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p><span class=\"token decorator annotation punctuation\">&#064;sgl<span class=\"token punctuation\">.<\/span>function<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">extract_user_info<\/span><span class=\"token punctuation\">(<\/span>text<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    user_info <span class=\"token operator\">&#061;<\/span> text <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;\\\\n\u8bf7\u4ece\u4e0a\u8ff0\u6587\u672c\u4e2d\u63d0\u53d6\u7528\u6237\u4fe1\u606f&#xff1a;&#034;<\/span><br \/>\n    result <span class=\"token operator\">&#061;<\/span> sgl<span class=\"token punctuation\">.<\/span>gen<span class=\"token punctuation\">(<\/span><br \/>\n        <span class=\"token string\">&#034;json_output&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        response_format<span class=\"token operator\">&#061;<\/span>UserInfo<span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># \u76f4\u63a5\u7528Pydantic\u6a21\u578b\u7ea6\u675f<\/span><br \/>\n    <span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> result<\/p>\n<p><span class=\"token comment\"># \u8fd0\u884c<\/span><br \/>\n<span class=\"token keyword\">with<\/span> sgl<span class=\"token punctuation\">.<\/span>Endpoint<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;http:\/\/localhost:30000&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    state <span class=\"token operator\">&#061;<\/span> extract_user_info<span class=\"token punctuation\">.<\/span>run<span class=\"token punctuation\">(<\/span><br \/>\n        text<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;\u674e\u660e&#xff0c;28\u5c81&#xff0c;\u559c\u6b22\u7f16\u7a0b\u3001\u9605\u8bfb\u548c\u6253\u7bee\u7403\u3002&#034;<\/span><br \/>\n    <span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>state<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;json_output&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token comment\"># \u8f93\u51fa&#xff1a;{&#034;name&#034;: &#034;\u674e\u660e&#034;, &#034;age&#034;: 28, &#034;hobbies&#034;: [&#034;\u7f16\u7a0b&#034;, &#034;\u9605\u8bfb&#034;, &#034;\u6253\u7bee\u7403&#034;]}<\/span><\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>RadixAttention\u524d\u7f00\u5171\u4eab&#xff0c;\u7cfb\u7edf\u63d0\u793a\u8bcd\/\u957f\u4e0a\u4e0b\u6587\u590d\u7528\u7387\u9ad8<\/li>\n<li>\u6781\u5f3a\u7684\u7ed3\u6784\u5316\u751f\u6210\u80fd\u529b&#xff0c;\u652f\u6301JSON\/Regex\/Grammar<\/li>\n<li>\u652f\u6301Function Calling\u3001\u591a\u6a21\u6001&#xff08;LLaVA&#xff09;<\/li>\n<li>\u8fde\u7eed\u6279\u5904\u7406\u3001PagedAttention<\/li>\n<li>\u6027\u80fd\u63a5\u8fd1vLLM&#xff0c;\u67d0\u4e9b\u573a\u666f\u751a\u81f3\u66f4\u5feb<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;Agent\u3001RAG\u3001\u7ed3\u6784\u5316\u6570\u636e\u63d0\u53d6\u7b49\u5927\u91cf\u5171\u4eab\u524d\u7f00\u6216\u9700\u8981\u4e25\u683c\u8f93\u51fa\u683c\u5f0f\u7684\u573a\u666f\u3002<\/p>\n<h4>3.5 LMDeploy&#xff1a;\u56fd\u4ea7\u6a21\u578b\u53cb\u597d&#xff0c;\u90e8\u7f72\u91cf\u5316\u4e00\u4f53\u5316<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;\u4e0a\u6d77AI\u5b9e\u9a8c\u5ba4\u51fa\u54c1&#xff0c;\u56fd\u4ea7\u6a21\u578b\u9002\u914d\u505a\u5f97\u6700\u597d\u7684\u6846\u67b6\u4e4b\u4e00\u3002<\/p>\n<p>LMDeploy\u662f\u4e0a\u6d77AI\u5b9e\u9a8c\u5ba4\u5f00\u53d1\u7684LLM\u90e8\u7f72\u5de5\u5177\u5305\u3002\u5b83\u6700\u5927\u7684\u4f18\u52bf\u662f\u5bf9\u56fd\u4ea7\u5927\u6a21\u578b&#xff08;Qwen\u3001Baichuan\u3001InternLM\u3001ChatGLM\u7b49&#xff09;\u7684\u652f\u6301\u975e\u5e38\u5b8c\u5584&#xff0c;\u91cf\u5316\u65b9\u6848&#xff08;W4A16\u3001W8A8&#xff09;\u4e5f\u505a\u5f97\u6bd4\u8f83\u6210\u719f\u3002<\/p>\n<p><span class=\"token comment\"># LMDeploy \u542f\u52a8\u793a\u4f8b<\/span><br \/>\n<span class=\"token comment\"># pip install lmdeploy<\/span><\/p>\n<p><span class=\"token comment\"># 1. \u79bb\u7ebf\u63a8\u7406<\/span><br \/>\n<span class=\"token comment\"># lmdeploy chat turbomind \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;model-path Qwen\/Qwen-7B-Chat \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;tp 1<\/span><\/p>\n<p><span class=\"token comment\"># 2. \u542f\u52a8API\u670d\u52a1&#xff08;OpenAI\u517c\u5bb9&#xff09;<\/span><br \/>\n<span class=\"token comment\"># lmdeploy serve api_server \\\\<\/span><br \/>\n<span class=\"token comment\">#   Qwen\/Qwen-7B-Chat \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;server-port 23333 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;tp 1 \\\\<\/span><br \/>\n<span class=\"token comment\">#   &#8211;quant-policy 4  # W4A16\u91cf\u5316<\/span><\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>\u56fd\u4ea7\u6a21\u578b\u652f\u6301\u5168\u9762&#xff0c;\u6587\u6863\u5bf9\u4e2d\u6587\u7528\u6237\u53cb\u597d<\/li>\n<li>TurboMind\u63a8\u7406\u5f15\u64ce&#xff0c;\u6027\u80fd\u4f18\u79c0<\/li>\n<li>\u652f\u6301W4A16\u3001W8A8\u91cf\u5316&#xff0c;\u663e\u5b58\u538b\u7f29\u6bd4\u9ad8<\/li>\n<li>\u652f\u6301\u8fde\u7eed\u6279\u5904\u7406\u3001PagedAttention<\/li>\n<li>\u591a\u6a21\u6001\u652f\u6301&#xff08;LLaVA\u3001Qwen-VL\u7b49&#xff09;<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;\u4ee5\u56fd\u4ea7\u6a21\u578b\u4e3a\u4e3b\u7684\u56e2\u961f&#xff0c;\u5c24\u5176\u662f\u9700\u8981\u91cf\u5316\u90e8\u7f72\u964d\u4f4e\u6210\u672c\u7684\u573a\u666f\u3002<\/p>\n<h4>3.6 DeepSpeed-MII&#xff1a;DeepSpeed\u751f\u6001\u5ef6\u7eed<\/h4>\n<p>\u4e00\u53e5\u8bdd\u5b9a\u4f4d&#xff1a;\u5fae\u8f6fDeepSpeed\u5bb6\u65cf\u7684\u63a8\u7406\u4ea7\u54c1&#xff0c;\u8bad\u7ec3\u7528DeepSpeed\u7684\u56e2\u961f\u53ef\u4ee5\u65e0\u7f1d\u8854\u63a5\u3002<\/p>\n<p>DeepSpeed-MII&#xff08;Microsoft Inferencing for Inference&#xff09;\u662fDeepSpeed\u5728\u63a8\u7406\u4fa7\u7684\u5ef6\u4f38\u3002\u5982\u679c\u4f60\u8bad\u7ec3\u65f6\u7528\u4e86DeepSpeed\u505aZeRO\u4f18\u5316&#xff0c;\u63a8\u7406\u7528MII\u53ef\u4ee5\u5f62\u6210\u5b8c\u6574\u95ed\u73af\u3002<\/p>\n<p>\u6838\u5fc3\u7279\u70b9&#xff1a;<\/p>\n<ul>\n<li>DeepSpeed\u751f\u6001\u4e00\u81f4&#xff0c;\u5b66\u4e60\u6210\u672c\u4f4e<\/li>\n<li>\u652f\u6301DeepSpeed-Inference\u7684\u591a\u79cd\u4f18\u5316<\/li>\n<li>\u652f\u6301\u5f20\u91cf\u5e76\u884c\u3001\u6d41\u6c34\u7ebf\u5e76\u884c<\/li>\n<li>\u4e0eAzure\u7b49\u5fae\u8f6f\u4e91\u670d\u52a1\u96c6\u6210\u597d<\/li>\n<\/ul>\n<p>\u9002\u5408\u573a\u666f&#xff1a;\u5df2\u7ecf\u5728\u7528DeepSpeed\u8bad\u7ec3\u3001\u5e0c\u671b\u7edf\u4e00\u6280\u672f\u6808\u7684\u56e2\u961f\u3002<\/p>\n<h3>\u56db\u3001\u5b9a\u6027\u5bf9\u6bd4\u4e0e\u9009\u578b\u5efa\u8bae<\/h3>\n<p>\u770b\u5b8c\u4e86\u6bcf\u4e2a\u6846\u67b6\u7684\u7279\u70b9&#xff0c;\u6211\u4eec\u6765\u505a\u4e00\u4e2a\u6a2a\u5411\u5bf9\u6bd4&#xff0c;\u5e76\u7ed9\u51fa\u4e0d\u540c\u573a\u666f\u4e0b\u7684\u9009\u578b\u5efa\u8bae\u3002<\/p>\n<h4>4.1 \u5173\u952e\u7279\u6027\u5bf9\u6bd4\u8868<\/h4>\n<table>\n<tr>\u7279\u6027\u7ef4\u5ea6vLLMTGITensorRT-LLMSGLangLMDeploy<\/tr>\n<tbody>\n<tr>\n<td>\u5b89\u88c5\u96be\u5ea6<\/td>\n<td>\u7b80\u5355<\/td>\n<td>\u7b80\u5355<\/td>\n<td>\u8f83\u590d\u6742<\/td>\n<td>\u7b80\u5355<\/td>\n<td>\u7b80\u5355<\/td>\n<\/tr>\n<tr>\n<td>Python\u53cb\u597d\u5ea6<\/td>\n<td>\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<td>\u4e2d<\/td>\n<td>\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<\/tr>\n<tr>\n<td>\u541e\u5410\u6027\u80fd<\/td>\n<td>\u9ad8<\/td>\n<td>\u4e2d\u9ad8<\/td>\n<td>\u6781\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<\/tr>\n<tr>\n<td>\u5ef6\u8fdf\u8868\u73b0<\/td>\n<td>\u597d<\/td>\n<td>\u597d<\/td>\n<td>\u6781\u597d<\/td>\n<td>\u597d<\/td>\n<td>\u597d<\/td>\n<\/tr>\n<tr>\n<td>\u6a21\u578b\u652f\u6301<\/td>\n<td>\u5e7f<\/td>\n<td>\u5e7f<\/td>\n<td>\u4e2d<\/td>\n<td>\u4e2d<\/td>\n<td>\u5e7f&#xff08;\u56fd\u4ea7\u4f18&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>OpenAI API\u517c\u5bb9<\/td>\n<td>\u539f\u751f<\/td>\n<td>\u662f<\/td>\n<td>\u9700\u5305\u88c5<\/td>\n<td>\u662f<\/td>\n<td>\u662f<\/td>\n<\/tr>\n<tr>\n<td>\u7ed3\u6784\u5316\u8f93\u51fa<\/td>\n<td>\u4e00\u822c<\/td>\n<td>\u4e00\u822c<\/td>\n<td>\u4e00\u822c<\/td>\n<td>\u6781\u5f3a<\/td>\n<td>\u4e00\u822c<\/td>\n<\/tr>\n<tr>\n<td>Prefix\u7f13\u5b58<\/td>\n<td>\u6709<\/td>\n<td>\u6709<\/td>\n<td>\u6709<\/td>\n<td>Radix&#xff08;\u6781\u5f3a&#xff09;<\/td>\n<td>\u6709<\/td>\n<\/tr>\n<tr>\n<td>\u91cf\u5316\u652f\u6301<\/td>\n<td>\u591a<\/td>\n<td>\u4e2d<\/td>\n<td>\u591a<\/td>\n<td>\u4e2d<\/td>\n<td>\u591a&#xff08;W4A16\u4f18&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5206\u5e03\u5f0f\u80fd\u529b<\/td>\n<td>TP&#043;PP<\/td>\n<td>TP<\/td>\n<td>TP&#043;PP&#043;EP<\/td>\n<td>TP<\/td>\n<td>TP<\/td>\n<\/tr>\n<tr>\n<td>\u793e\u533a\u6d3b\u8dc3\u5ea6<\/td>\n<td>\u6781\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<td>\u9ad8<\/td>\n<td>\u5feb\u901f\u589e\u957f<\/td>\n<td>\u9ad8<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>4.2 \u4e0d\u540c\u573a\u666f\u7684\u9009\u578b\u5efa\u8bae<\/h4>\n<p>\u573a\u666f1&#xff1a;\u5feb\u901f\u4e0a\u7ebf \/ \u521d\u521b\u56e2\u961f \/ Python\u6808<br \/>\n\u2192 \u9996\u9009 vLLM\u3002\u7406\u7531&#xff1a;\u751f\u6001\u6700\u6210\u719f\u3001\u6587\u6863\u6700\u4e30\u5bcc\u3001\u5751\u5df2\u7ecf\u88ab\u8e29\u5f97\u5dee\u4e0d\u591a\u4e86&#xff0c;\u793e\u533a\u54cd\u5e94\u5feb\u3002<\/p>\n<p>\u573a\u666f2&#xff1a;Agent \/ RAG \/ \u5927\u91cf\u5171\u4eab\u524d\u7f00<br \/>\n\u2192 \u9996\u9009 SGLang\u3002\u7406\u7531&#xff1a;RadixAttention\u80fd\u8ba9\u7cfb\u7edf\u63d0\u793a\u8bcd\u7684KV Cache\u51e0\u4e4e\u96f6\u6210\u672c\u590d\u7528&#xff0c;\u7ed3\u6784\u5316\u8f93\u51fa\u7701\u53bb\u5927\u91cf\u540e\u5904\u7406\u903b\u8f91\u3002<\/p>\n<p>\u573a\u666f3&#xff1a;\u6781\u81f4\u6027\u80fd \/ \u5927\u89c4\u6a21\u96c6\u7fa4 \/ \u6709\u4e13\u95e8\u4f18\u5316\u56e2\u961f<br \/>\n\u2192 \u9996\u9009 TensorRT-LLM\u3002\u7406\u7531&#xff1a;\u5728NVIDIA\u786c\u4ef6\u4e0a\u5b83\u7684\u6027\u80fd\u5929\u82b1\u677f\u6700\u9ad8&#xff0c;FP8\u80fd\u5e26\u6765\u663e\u8457\u7684\u541e\u5410\u63d0\u5347\u548c\u663e\u5b58\u8282\u7701\u3002<\/p>\n<p>\u573a\u666f4&#xff1a;\u56fd\u4ea7\u6a21\u578b\u4e3a\u4e3b \/ \u91cf\u5316\u964d\u672c\u9700\u6c42\u5f3a<br \/>\n\u2192 \u9996\u9009 LMDeploy\u3002\u7406\u7531&#xff1a;\u56fd\u4ea7\u6a21\u578b\u9002\u914d\u5b8c\u5584&#xff0c;W4A16\u91cf\u5316\u6210\u719f&#xff0c;\u4e2d\u6587\u6587\u6863\u53cb\u597d\u3002<\/p>\n<p>\u573a\u666f5&#xff1a;Hugging Face\u91cd\u5ea6\u7528\u6237 \/ \u91cd\u89c6\u53ef\u8c03\u8bd5\u6027<br \/>\n\u2192 \u9996\u9009 TGI\u3002\u7406\u7531&#xff1a;\u548ctransformers\u5b8c\u5168\u4e00\u81f4\u7684\u6a21\u578b\u52a0\u8f7d\u65b9\u5f0f&#xff0c;\u8c03\u8bd5\u548c\u81ea\u5b9a\u4e49\u6700\u65b9\u4fbf\u3002<\/p>\n<p>\u4e00\u4e2a\u5b9e\u7528\u5efa\u8bae&#xff1a;\u5148\u7528vLLM\u8dd1\u901a\u4e1a\u52a1&#xff0c;\u9a8c\u8bc1PMF&#xff08;\u4ea7\u54c1\u5e02\u573a\u5951\u5408\u5ea6&#xff09;&#xff1b;\u4e1a\u52a1\u91cf\u4e0a\u6765\u540e&#xff0c;\u518d\u6839\u636e\u74f6\u9888\u70b9\u8003\u8651\u5207\u6362\u5230\u66f4\u5408\u9002\u7684\u6846\u67b6\u3002<\/p>\n<h3>\u4e94\u3001\u7528OpenAI-compatible API\u505a\u7edf\u4e00\u5192\u70df\u6d4b\u8bd5<\/h3>\n<p>\u597d\u6d88\u606f\u662f&#xff1a;\u4e3b\u6d41\u6846\u67b6\u51e0\u4e4e\u90fd\u652f\u6301OpenAI\u517c\u5bb9API\u3002\u8fd9\u610f\u5473\u7740\u6211\u4eec\u53ef\u4ee5\u7528\u540c\u4e00\u5957\u6d4b\u8bd5\u4ee3\u7801\u5bf9\u4e0d\u540c\u6846\u67b6\u505a\u5192\u70df\u6d4b\u8bd5&#xff0c;\u5feb\u901f\u9a8c\u8bc1\u529f\u80fd\u6b63\u786e\u6027\u548c\u57fa\u7840\u6027\u80fd\u3002<\/p>\n<p><span class=\"token comment\"># unified_smoke_test.py \u2014\u2014 \u7edf\u4e00\u5192\u70df\u6d4b\u8bd5\u811a\u672c<\/span><br \/>\n<span class=\"token keyword\">import<\/span> time<br \/>\n<span class=\"token keyword\">import<\/span> json<br \/>\n<span class=\"token keyword\">from<\/span> openai <span class=\"token keyword\">import<\/span> OpenAI<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">smoke_test<\/span><span class=\"token punctuation\">(<\/span>base_url<span class=\"token punctuation\">,<\/span> model_name<span class=\"token punctuation\">,<\/span> test_cases<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7edf\u4e00\u5192\u70df\u6d4b\u8bd5&#xff1a;\u529f\u80fd\u9a8c\u8bc1 &#043; \u57fa\u7840\u6027\u80fd\u6307\u6807&#034;&#034;&#034;<\/span><br \/>\n    client <span class=\"token operator\">&#061;<\/span> OpenAI<span class=\"token punctuation\">(<\/span>base_url<span class=\"token operator\">&#061;<\/span>base_url <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;\/v1&#034;<\/span><span class=\"token punctuation\">,<\/span> api_key<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;test&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i<span class=\"token punctuation\">,<\/span> tc <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">enumerate<\/span><span class=\"token punctuation\">(<\/span>test_cases<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n&#061;&#061;&#061; Test Case <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>i<span class=\"token operator\">&#043;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>tc<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;name&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> &#061;&#061;&#061;&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n        start <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        first_token_time <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><br \/>\n        total_tokens <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0<\/span><\/p>\n<p>        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            stream <span class=\"token operator\">&#061;<\/span> client<span class=\"token punctuation\">.<\/span>chat<span class=\"token punctuation\">.<\/span>completions<span class=\"token punctuation\">.<\/span>create<span class=\"token punctuation\">(<\/span><br \/>\n                model<span class=\"token operator\">&#061;<\/span>model_name<span class=\"token punctuation\">,<\/span><br \/>\n                messages<span class=\"token operator\">&#061;<\/span>tc<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;messages&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                stream<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                max_tokens<span class=\"token operator\">&#061;<\/span>tc<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;max_tokens&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">500<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                temperature<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.7<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><\/p>\n<p>            <span class=\"token keyword\">for<\/span> chunk <span class=\"token keyword\">in<\/span> stream<span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">if<\/span> first_token_time <span class=\"token keyword\">is<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    first_token_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                    ttft <span class=\"token operator\">&#061;<\/span> first_token_time <span class=\"token operator\">&#8211;<\/span> start<br \/>\n                    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  \u9996Token\u5ef6\u8fdf (TTFT): <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>ttft<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.3f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">s&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>                content <span class=\"token operator\">&#061;<\/span> chunk<span class=\"token punctuation\">.<\/span>choices<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>delta<span class=\"token punctuation\">.<\/span>content<br \/>\n                <span class=\"token keyword\">if<\/span> content<span class=\"token punctuation\">:<\/span><br \/>\n                    total_tokens <span class=\"token operator\">&#043;&#061;<\/span> <span class=\"token number\">1<\/span><br \/>\n                    <span class=\"token comment\"># \u6253\u5370\u524d\u51e0\u4e2a\u5b57\u7b26&#xff0c;\u786e\u8ba4\u8f93\u51fa\u6b63\u5e38<\/span><br \/>\n                    <span class=\"token keyword\">if<\/span> total_tokens <span class=\"token operator\">&lt;&#061;<\/span> <span class=\"token number\">20<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>content<span class=\"token punctuation\">,<\/span> end<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">,<\/span> flush<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            total_time <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> start<br \/>\n            tpot <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>total_time <span class=\"token operator\">&#8211;<\/span> ttft<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>total_tokens <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n  \u603bToken\u6570: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>total_tokens<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  \u603b\u8017\u65f6: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>total_time<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.3f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">s&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  \u6bcfToken\u5ef6\u8fdf (TPOT): <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>tpot<span class=\"token operator\">*<\/span><span class=\"token number\">1000<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">ms&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  \u751f\u6210\u901f\u5ea6: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>total_tokens<span class=\"token operator\">\/<\/span>total_time<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> tokens\/s&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            results<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#034;case&#034;<\/span><span class=\"token punctuation\">:<\/span> tc<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#034;success&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">True<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#034;ttft&#034;<\/span><span class=\"token punctuation\">:<\/span> ttft<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#034;tpot&#034;<\/span><span class=\"token punctuation\">:<\/span> tpot<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#034;total_tokens&#034;<\/span><span class=\"token punctuation\">:<\/span> total_tokens<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#034;throughput&#034;<\/span><span class=\"token punctuation\">:<\/span> total_tokens <span class=\"token operator\">\/<\/span> total_time<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">except<\/span> Exception <span class=\"token keyword\">as<\/span> e<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  ERROR: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>e<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n            results<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;case&#034;<\/span><span class=\"token punctuation\">:<\/span> tc<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;success&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">False<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;error&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">(<\/span>e<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">return<\/span> results<\/p>\n<p><span class=\"token keyword\">if<\/span> __name__ <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;__main__&#034;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token comment\"># \u6d4b\u8bd5\u7528\u4f8b<\/span><br \/>\n    test_cases <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><br \/>\n        <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u77ed\u6587\u672c\u751f\u6210&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;messages&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;user&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u7528\u4e00\u53e5\u8bdd\u4ecb\u7ecdLLM\u63a8\u7406\u6846\u67b6\u3002&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;max_tokens&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token number\">100<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u957f\u6587\u672c\u751f\u6210&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;messages&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;user&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u8bf7\u8be6\u7ec6\u4ecb\u7ecdPagedAttention\u7684\u5de5\u4f5c\u539f\u7406&#xff0c;\u5305\u62ec\u5b83\u89e3\u51b3\u4e86\u4ec0\u4e48\u95ee\u9898\u3001\u6838\u5fc3\u6570\u636e\u7ed3\u6784\u3001\u4ee5\u53ca\u4f18\u7f3a\u70b9\u3002&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;max_tokens&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token number\">500<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#034;name&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u7cfb\u7edf\u63d0\u793a\u8bcd\u590d\u7528&#xff08;Prefix\u6d4b\u8bd5&#xff09;&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;messages&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><br \/>\n                <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;system&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u4f60\u662f\u4e00\u4f4d\u8d44\u6df1\u7684LLM\u63a8\u7406\u5de5\u7a0b\u5e08&#xff0c;\u56de\u7b54\u8981\u4e13\u4e1a\u3001\u51c6\u786e\u3001\u7b80\u6d01\u3002&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># \u957f\u7cfb\u7edf\u63d0\u793a\u8bcd<\/span><br \/>\n                <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#034;role&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;user&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;content&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#034;\u4ec0\u4e48\u662fKV Cache&#xff1f;&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#034;max_tokens&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token punctuation\">]<\/span><\/p>\n<p>    <span class=\"token comment\"># \u5206\u522b\u6d4b\u8bd5\u4e0d\u540c\u6846\u67b6&#xff08;\u6839\u636e\u5b9e\u9645\u90e8\u7f72\u8c03\u6574URL\u548c\u6a21\u578b\u540d&#xff09;<\/span><br \/>\n    frameworks <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n        <span class=\"token string\">&#034;vLLM&#034;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;http:\/\/localhost:8000&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;meta-llama\/Llama-2-7b-chat-hf&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token comment\"># &#034;SGLang&#034;: (&#034;http:\/\/localhost:30000&#034;, &#034;meta-llama\/Llama-2-7b-chat-hf&#034;),<\/span><br \/>\n        <span class=\"token comment\"># &#034;TGI&#034;: (&#034;http:\/\/localhost:8080&#034;, &#034;meta-llama\/Llama-2-7b-chat-hf&#034;),<\/span><br \/>\n    <span class=\"token punctuation\">}<\/span><\/p>\n<p>    all_results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> name<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">(<\/span>url<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">in<\/span> frameworks<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span><span class=\"token string\">&#039;&#061;&#039;<\/span><span class=\"token operator\">*<\/span><span class=\"token number\">50<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;Testing Framework: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>name<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span><span class=\"token string\">&#039;&#061;&#039;<\/span><span class=\"token operator\">*<\/span><span class=\"token number\">50<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n        all_results<span class=\"token punctuation\">[<\/span>name<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> smoke_test<span class=\"token punctuation\">(<\/span>url<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">,<\/span> test_cases<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\"># \u4fdd\u5b58\u7ed3\u679c<\/span><br \/>\n    <span class=\"token keyword\">with<\/span> <span class=\"token builtin\">open<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;smoke_test_results.json&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;w&#034;<\/span><span class=\"token punctuation\">,<\/span> encoding<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;utf-8&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">as<\/span> f<span class=\"token punctuation\">:<\/span><br \/>\n        json<span class=\"token punctuation\">.<\/span>dump<span class=\"token punctuation\">(<\/span>all_results<span class=\"token punctuation\">,<\/span> f<span class=\"token punctuation\">,<\/span> ensure_ascii<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">,<\/span> indent<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n\u6d4b\u8bd5\u7ed3\u679c\u5df2\u4fdd\u5b58\u5230 smoke_test_results.json&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u8fd9\u4e2a\u811a\u672c\u80fd\u5e2e\u4f60\u505a\u4ec0\u4e48&#xff1f;<\/p>\n<ul>\n<li>\u9a8c\u8bc1\u6846\u67b6\u662f\u5426\u6b63\u5e38\u5de5\u4f5c&#xff08;\u529f\u80fd\u6b63\u786e\u6027&#xff09;<\/li>\n<li>\u6d4b\u91cfTTFT\u3001TPOT\u3001\u751f\u6210\u901f\u5ea6\u7b49\u57fa\u7840\u6027\u80fd\u6307\u6807<\/li>\n<li>\u5bf9\u6bd4\u4e0d\u540c\u6846\u67b6\u5728\u76f8\u540c\u786c\u4ef6\u548c\u6a21\u578b\u4e0b\u7684\u8868\u73b0<\/li>\n<li>\u6d4b\u8bd5Prefix\u573a\u666f\u4e0b\u7684KV Cache\u590d\u7528\u6548\u679c<\/li>\n<\/ul>\n<h3>\u516d\u3001\u751f\u4ea7\u4e0a\u7ebfChecklist<\/h3>\n<p>\u9009\u578b\u53ea\u662f\u7b2c\u4e00\u6b65&#xff0c;\u771f\u6b63\u4e0a\u7ebf\u524d\u8fd8\u6709\u5f88\u591a\u4e8b\u60c5\u8981\u786e\u8ba4\u3002\u8fd9\u91cc\u7ed9\u4f60\u4e00\u4efd\u751f\u4ea7\u4e0a\u7ebfChecklist&#xff0c;\u6309\u56fe\u7d22\u9aa5&#xff0c;\u907f\u514d\u8e29\u5751\u3002<\/p>\n<h4>6.1 \u6b63\u786e\u6027\u9a8c\u8bc1<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u57fa\u7840\u751f\u6210\u6d4b\u8bd5&#xff1a;prompt\u80fd\u5426\u6b63\u5e38\u8f93\u51fa&#xff0c;\u8f93\u51fa\u8d28\u91cf\u662f\u5426\u7b26\u5408\u9884\u671f<\/li>\n<li class=\"task-list-item\"> \u957f\u6587\u672c\u6d4b\u8bd5&#xff1a;\u63a5\u8fd1max_seq_len\u7684\u8f93\u5165\u8f93\u51fa\u662f\u5426\u6b63\u5e38<\/li>\n<li class=\"task-list-item\"> \u8fb9\u754ccase&#xff1a;\u7a7a\u8f93\u5165\u3001\u8d85\u957f\u8f93\u5165\u3001\u7279\u6b8a\u5b57\u7b26\u662f\u5426\u6709\u4f18\u96c5\u964d\u7ea7<\/li>\n<li class=\"task-list-item\"> \u4e00\u81f4\u6027\u9a8c\u8bc1&#xff1a;\u76f8\u540c\u8f93\u5165&#043;\u56fa\u5b9aseed&#xff0c;\u8f93\u51fa\u662f\u5426\u4e00\u81f4&#xff08;\u5c24\u5176\u662f\u91cf\u5316\u540e&#xff09;<\/li>\n<li class=\"task-list-item\"> \u591a\u8f6e\u5bf9\u8bdd&#xff1a;\u591a\u8f6e\u4e0a\u4e0b\u6587\u662f\u5426\u6b63\u786e\u7ef4\u62a4<\/li>\n<\/ul>\n<h4>6.2 \u6027\u80fd\u538b\u6d4b<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u541e\u5410\u6d4b\u8bd5&#xff1a;\u4e0d\u540c\u5e76\u53d1\u4e0b\u7684tokens\/s&#xff0c;\u627e\u5230\u74f6\u9888\u70b9<\/li>\n<li class=\"task-list-item\"> \u5ef6\u8fdf\u6d4b\u8bd5&#xff1a;\u4e0d\u540c\u8d1f\u8f7d\u4e0b\u7684P50\/P95\/P99\u5ef6\u8fdf<\/li>\n<li class=\"task-list-item\"> \u7a33\u5b9a\u6027\u6d4b\u8bd5&#xff1a;\u8fde\u7eed\u8dd124\u5c0f\u65f6\u4ee5\u4e0a&#xff0c;\u6709\u6ca1\u6709\u5185\u5b58\u6cc4\u6f0f\u3001\u6027\u80fd\u8870\u51cf<\/li>\n<li class=\"task-list-item\"> \u5cf0\u503c\u6d4b\u8bd5&#xff1a;\u6253\u6ee1GPU\u540e\u670d\u52a1\u662f\u5426\u8fd8\u80fd\u6b63\u5e38\u54cd\u5e94&#xff0c;\u4f1a\u4e0d\u4f1aOOM<\/li>\n<\/ul>\n<h4>6.3 \u663e\u5b58\u4e0e\u8d44\u6e90<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u663e\u5b58\u5360\u7528&#xff1a;\u7a7a\u8f7d\u3001\u6ee1\u8f7d\u65f6\u7684\u663e\u5b58\u4f7f\u7528\u60c5\u51b5&#xff0c;\u662f\u5426\u7559\u6709\u5b89\u5168\u4f59\u91cf<\/li>\n<li class=\"task-list-item\"> \u663e\u5b58\u788e\u7247&#xff1a;\u957f\u65f6\u95f4\u8fd0\u884c\u540e\u663e\u5b58\u788e\u7247\u662f\u5426\u4e25\u91cd<\/li>\n<li class=\"task-list-item\"> CPU\/\u5185\u5b58&#xff1a;CPU\u662f\u5426\u6210\u4e3a\u74f6\u9888&#xff0c;\u5185\u5b58\u662f\u5426\u8db3\u591f<\/li>\n<li class=\"task-list-item\"> \u7f51\u7edc\u5e26\u5bbd&#xff1a;\u591a\u5361\/\u591a\u673a\u573a\u666f\u4e0b\u901a\u4fe1\u662f\u5426\u591f\u7528<\/li>\n<\/ul>\n<h4>6.4 \u9ad8\u7ea7\u7279\u6027\u9a8c\u8bc1<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u6d41\u5f0f\u8f93\u51fa&#xff1a;SSE\u662f\u5426\u6b63\u5e38&#xff0c;\u4e2d\u65ad\u540e\u80fd\u5426\u6b63\u786e\u5904\u7406<\/li>\n<li class=\"task-list-item\"> \u91cf\u5316\u9a8c\u8bc1&#xff1a;\u91cf\u5316\u540e\u7cbe\u5ea6\u635f\u5931\u662f\u5426\u5728\u53ef\u63a5\u53d7\u8303\u56f4\u5185<\/li>\n<li class=\"task-list-item\"> Prefix\u7f13\u5b58&#xff1a;\u5171\u4eab\u524d\u7f00\u7684\u8bf7\u6c42\u662f\u5426\u771f\u7684\u547d\u4e2d\u7f13\u5b58\u3001\u52a0\u901f\u660e\u663e<\/li>\n<li class=\"task-list-item\"> \u7ed3\u6784\u5316\u8f93\u51fa&#xff1a;JSON\/Function Calling\u683c\u5f0f\u662f\u5426100%\u6b63\u786e<\/li>\n<\/ul>\n<h4>6.5 API\u4e0e\u96c6\u6210<\/h4>\n<ul>\n<li class=\"task-list-item\"> API\u517c\u5bb9\u6027&#xff1a;\u548c\u73b0\u6709\u4e1a\u52a1\u4ee3\u7801\u7684\u5bf9\u63a5\u662f\u5426\u987a\u7545<\/li>\n<li class=\"task-list-item\"> \u9519\u8bef\u5904\u7406&#xff1a;\u5404\u79cd\u5f02\u5e38\u60c5\u51b5\u4e0b\u7684\u9519\u8bef\u7801\u548c\u9519\u8bef\u4fe1\u606f\u662f\u5426\u5408\u7406<\/li>\n<li class=\"task-list-item\"> \u8d85\u65f6\u673a\u5236&#xff1a;\u8bf7\u6c42\u8d85\u65f6\u662f\u5426\u6b63\u786e\u65ad\u5f00&#xff0c;\u8d44\u6e90\u662f\u5426\u91ca\u653e<\/li>\n<li class=\"task-list-item\"> \u9650\u6d41\u80fd\u529b&#xff1a;\u662f\u5426\u652f\u6301QPS\/\u5e76\u53d1\u9650\u5236<\/li>\n<\/ul>\n<h4>6.6 \u8fd0\u7ef4\u4e0e\u76d1\u63a7<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u5065\u5eb7\u68c0\u67e5&#xff1a;\/health \u7aef\u70b9\u662f\u5426\u6b63\u5e38\u5de5\u4f5c<\/li>\n<li class=\"task-list-item\"> \u76d1\u63a7\u6307\u6807&#xff1a;QPS\u3001\u5ef6\u8fdf\u3001\u663e\u5b58\u3001GPU\u5229\u7528\u7387\u7b49\u662f\u5426\u80fd\u63a5\u5165Prometheus<\/li>\n<li class=\"task-list-item\"> \u65e5\u5fd7&#xff1a;\u8bf7\u6c42\u65e5\u5fd7\u3001\u9519\u8bef\u65e5\u5fd7\u3001\u6162\u8bf7\u6c42\u65e5\u5fd7\u662f\u5426\u5b8c\u5584<\/li>\n<li class=\"task-list-item\"> \u7070\u5ea6\u53d1\u5e03&#xff1a;\u80fd\u5426\u505aA\/B\u6d4b\u8bd5\u3001\u7070\u5ea6\u5207\u6362<\/li>\n<li class=\"task-list-item\"> \u56de\u6eda\u80fd\u529b&#xff1a;\u51fa\u95ee\u9898\u80fd\u5426\u5feb\u901f\u56de\u9000\u5230\u4e0a\u4e00\u7248\u672c<\/li>\n<\/ul>\n<h4>6.7 \u5b89\u5168\u4e0e\u5408\u89c4<\/h4>\n<ul>\n<li class=\"task-list-item\"> \u8f93\u5165\u8fc7\u6ee4&#xff1a;\u662f\u5426\u6709Prompt\u6ce8\u5165\u9632\u62a4<\/li>\n<li class=\"task-list-item\"> \u8f93\u51fa\u5ba1\u6838&#xff1a;\u662f\u5426\u9700\u8981\u63a5\u5165\u5185\u5bb9\u5b89\u5168<\/li>\n<li class=\"task-list-item\"> \u6570\u636e\u9690\u79c1&#xff1a;\u7528\u6237\u6570\u636e\u662f\u5426\u4f1a\u6cc4\u9732\u3001\u662f\u5426\u843d\u76d8<\/li>\n<li class=\"task-list-item\"> \u9274\u6743\u673a\u5236&#xff1a;API Key \/ Token \u8ba4\u8bc1\u662f\u5426\u5230\u4f4d<\/li>\n<\/ul>\n<h3>\u4e03\u3001\u5c0f\u7ed3\u4e0e\u601d\u8003<\/h3>\n<p>\u8fd9\u7bc7\u6587\u7ae0\u6211\u4eec\u505a\u4e86\u4e09\u4ef6\u4e8b&#xff1a;<\/p>\n<li>\u5efa\u7acb\u4e86\u63a8\u7406\u6846\u67b6\u7684\u9009\u578b\u7ef4\u5ea6\u4f53\u7cfb\u2014\u2014\u4e0b\u6b21\u518d\u6709\u4eba\u95ee&#034;\u54ea\u4e2a\u6846\u67b6\u597d&#034;&#xff0c;\u4f60\u53ef\u4ee5\u5148\u53cd\u95ee&#034;\u4f60\u7684\u4e1a\u52a1\u573a\u666f\u662f\u4ec0\u4e48&#xff1f;&#034;<\/li>\n<li>\u68b3\u7406\u4e86\u4e94\u5927\u4e3b\u6d41\u6846\u67b6\u7684\u7279\u70b9\u548c\u9002\u7528\u573a\u666f\u2014\u2014vLLM\u7684\u901a\u7528\u6027\u3001SGLang\u7684\u7ed3\u6784\u5316\u8f93\u51fa\u3001TensorRT-LLM\u7684\u6781\u81f4\u6027\u80fd\u3001LMDeploy\u7684\u56fd\u4ea7\u6a21\u578b\u53cb\u597d\u3001TGI\u7684HF\u751f\u6001<\/li>\n<li>\u7ed9\u51fa\u4e86\u751f\u4ea7\u4e0a\u7ebfChecklist\u2014\u2014\u4ece\u6b63\u786e\u6027\u5230\u6027\u80fd&#xff0c;\u4ece\u5b89\u5168\u5230\u8fd0\u7ef4&#xff0c;\u5e2e\u4f60\u7cfb\u7edf\u6027\u5730\u89c4\u907f\u98ce\u9669<\/li>\n<p>\u6700\u540e\u5206\u4eab\u4e00\u4e2a\u89c2\u70b9&#xff1a;\u6846\u67b6\u9009\u578b\u662f\u624b\u6bb5&#xff0c;\u4e1a\u52a1\u6210\u529f\u624d\u662f\u76ee\u7684\u3002 \u4e0d\u8981\u9677\u5165&#034;\u6280\u672f\u9009\u578b\u5185\u5377&#034;\u2014\u2014\u4eca\u5929vLLM\u51fa\u4e86\u65b0\u7248\u672c\u5c31\u60f3\u6362&#xff0c;\u660e\u5929SGLang\u53c8\u6709\u65b0\u7279\u6027\u53c8\u60f3\u8bd5\u3002\u9009\u4e00\u4e2a\u591f\u7528\u7684\u6846\u67b6&#xff0c;\u628a\u4e1a\u52a1\u8dd1\u8d77\u6765&#xff0c;\u6bd4\u4ec0\u4e48\u90fd\u91cd\u8981\u3002\u7b49\u4e1a\u52a1\u91cf\u4e0a\u6765\u4e86&#xff0c;\u518d\u9488\u5bf9\u6027\u5730\u4f18\u5316\u6846\u67b6&#xff0c;\u90a3\u65f6\u5019\u6bcf\u4e00\u5206\u6027\u80fd\u63d0\u5347\u90fd\u80fd\u6362\u7b97\u6210\u5b9e\u5b9e\u5728\u5728\u7684\u6210\u672c\u8282\u7701\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4&#xff1a;vLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357\u7cfb\u5217\u6587\u7ae0\u7d22\u5f15&#xff1a;\u672c\u7cfb\u5217\u4ece\u57fa\u7840\u6982\u5ff5\u5230\u5de5\u7a0b\u5b9e\u8df5&#xff0c;\u7cfb\u7edf\u68b3\u7406LLM\u63a8\u7406\u4e0eServing\u7684\u6838\u5fc3\u77e5\u8bc6\u3002\u672c\u6587\u4e3a\u7b2c7\u7bc7&#xff0c;\u805a\u7126\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u7684\u6a2a\u5411\u5bf9\u6bd4\u4e0e\u9009\u578b\u65b9\u6cd5\u8bba\u3002\u4e00\u3001\u5f00\u7bc7&#xff1a;\u9009\u63a8\u7406\u6846\u67b6\u4e0d\u662f\u9009\\&#8221;\u6700\u5feb\u7684\\&#8221;&#xff0c;\u800c\u662f\u9009\u6700\u5339\u914d\u4e1a\u52a1\u7684<br \/>\n\u521a\u63a5\u89e6LLM Serving\u7684\u540c\u5b66&#xff0c;\u7ecf\u5e38\u4f1a\u95ee\u4e00\u4e2a\u95ee<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[1699,52,50,188,51],"topic":[],"class_list":["post-123895","post","type-post","status-publish","format-standard","hentry","category-server","tag-agi","tag-aigc","tag-50","tag-188","tag-51"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/123895.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4&#xff1a;vLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357\u7cfb\u5217\u6587\u7ae0\u7d22\u5f15&#xff1a;\u672c\u7cfb\u5217\u4ece\u57fa\u7840\u6982\u5ff5\u5230\u5de5\u7a0b\u5b9e\u8df5&#xff0c;\u7cfb\u7edf\u68b3\u7406LLM\u63a8\u7406\u4e0eServing\u7684\u6838\u5fc3\u77e5\u8bc6\u3002\u672c\u6587\u4e3a\u7b2c7\u7bc7&#xff0c;\u805a\u7126\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u7684\u6a2a\u5411\u5bf9\u6bd4\u4e0e\u9009\u578b\u65b9\u6cd5\u8bba\u3002\u4e00\u3001\u5f00\u7bc7&#xff1a;\u9009\u63a8\u7406\u6846\u67b6\u4e0d\u662f\u9009&quot;\u6700\u5feb\u7684&quot;&#xff0c;\u800c\u662f\u9009\u6700\u5339\u914d\u4e1a\u52a1\u7684 \u521a\u63a5\u89e6LLM Serving\u7684\u540c\u5b66&#xff0c;\u7ecf\u5e38\u4f1a\u95ee\u4e00\u4e2a\u95ee\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/123895.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-11-08T04:56:41+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"7 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/123895.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/123895.html\",\"name\":\"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-11-08T04:56:41+00:00\",\"dateModified\":\"2026-11-08T04:56:41+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/123895.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/123895.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/123895.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/123895.html","og_locale":"zh_CN","og_type":"article","og_title":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4&#xff1a;vLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357\u7cfb\u5217\u6587\u7ae0\u7d22\u5f15&#xff1a;\u672c\u7cfb\u5217\u4ece\u57fa\u7840\u6982\u5ff5\u5230\u5de5\u7a0b\u5b9e\u8df5&#xff0c;\u7cfb\u7edf\u68b3\u7406LLM\u63a8\u7406\u4e0eServing\u7684\u6838\u5fc3\u77e5\u8bc6\u3002\u672c\u6587\u4e3a\u7b2c7\u7bc7&#xff0c;\u805a\u7126\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u7684\u6a2a\u5411\u5bf9\u6bd4\u4e0e\u9009\u578b\u65b9\u6cd5\u8bba\u3002\u4e00\u3001\u5f00\u7bc7&#xff1a;\u9009\u63a8\u7406\u6846\u67b6\u4e0d\u662f\u9009\"\u6700\u5feb\u7684\"&#xff0c;\u800c\u662f\u9009\u6700\u5339\u914d\u4e1a\u52a1\u7684 \u521a\u63a5\u89e6LLM Serving\u7684\u540c\u5b66&#xff0c;\u7ecf\u5e38\u4f1a\u95ee\u4e00\u4e2a\u95ee","og_url":"https:\/\/www.wsisp.com\/helps\/123895.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-11-08T04:56:41+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"7 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/123895.html","url":"https:\/\/www.wsisp.com\/helps\/123895.html","name":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-11-08T04:56:41+00:00","dateModified":"2026-11-08T04:56:41+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/123895.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/123895.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/123895.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u3010LLM\u63a8\u7406\u4e0eServing\u2466\u3011\u4e3b\u6d41\u63a8\u7406\u6846\u67b6\u5bf9\u6bd4\uff1avLLM\/TGI\/TensorRT-LLM\/SGLang\/LMDeploy\u9009\u578b\u6307\u5357"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/123895","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=123895"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/123895\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=123895"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=123895"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=123895"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=123895"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}