![\"\u5c01\u9762\u4fe1\u606f\u56fe\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/09\/20260922184059-6ab2cbbb8fd08.webp\")
<\/p>\n<\/p>\n
\u5728\u5927\u8bed\u8a00\u6a21\u578b（LLM）\u5728\u7ebf\u63a8\u7406\u670d\u52a1（\u5982\u57fa\u4e8e vLLM\u3001SGLang \u6216 Triton \u642d\u5efa\u7684 API \u5fae\u670d\u52a1）\u4e0a\u7ebf\u4ea4\u4ed8\u524d，\u8bb8\u591a\u56e2\u961f\u5e38\u5e38\u8bef\u4ee5\u4e3a\u201c\u5e73\u5747\u54cd\u5e94\u65f6\u95f4（Average Latency）\u201d\u8fbe\u5230 500ms \u5c31\u4ee3\u8868\u7cfb\u7edf\u6027\u80fd\u8fbe\u6807\u3002<\/p>\n
\u7136\u800c，\u5728\u771f\u5b9e\u7684\u9ad8\u5e76\u53d1\u751f\u4ea7\u6d41\u91cf\u4e2d，\u51b3\u5b9a\u7528\u6237\u771f\u5b9e\u4f53\u9a8c\u3001SLA \u8fbe\u6807\u7387\u4e0e\u7cfb\u7edf\u53ef\u7528\u6027\u7684\u6838\u5fc3\u6307\u6807，\u662f\u6781\u5176\u6076\u52a3\u7684\u201c\u957f\u5c3e\u5ef6\u8fdf（P95 \/ P99 Latency）\u201d：<\/p>\n
\u8fd9\u79cd\u5267\u70c8\u7684\u957f\u5c3e\u5ef6\u8fdf\u6296\u52a8\u4f1a\u76f4\u63a5\u5bfc\u81f4\u524d\u7aef\u7528\u6237\u4f53\u9a8c\u4e25\u91cd\u5361\u987f，\u751a\u81f3\u89e6\u53d1 Nginx API \u7f51\u5173\u7684 HTTP 504 Gateway Timeout \u8d85\u65f6\u7194\u65ad\u3002<\/p>\n
\u5982\u4f55\u4f7f\u7528\u5206\u5e03\u5f0f\u538b\u6d4b\u6846\u67b6 Locust \u6784\u5efa\u652f\u6301 \u6d41\u5f0f SSE（Server-Sent Events）\u9996\u5b57\u5ef6\u8fdf（TTFT）\u4e0e Token \u541e\u5410\u7cbe\u786e\u8ba1\u65f6 \u7684\u4e13\u4e1a\u538b\u6d4b\u5ba2\u6237\u7aef？\u5982\u4f55\u7ed3\u5408\u6392\u961f\u8bba\u6cbb\u7406 P99 \u5ef6\u8fdf？<\/p>\n
\u672c\u6587\u8be6\u89e3\u751f\u4ea7\u7ea7 LLM \u538b\u6d4b\u4e0e\u957f\u5c3e\u5ef6\u8fdf\u8c03\u4f18\u5b9e\u6218\u3002<\/p>\n
\u6839\u636e\u6392\u961f\u8bba\u4e2d\u7684 $M\/M\/c$ \u961f\u5217\u6a21\u578b，\u5f53\u7cfb\u7edf\u5230\u8fbe\u7387 $\\\\lambda$ \u63a5\u8fd1\u670d\u52a1\u5904\u7406\u6781\u9650\u5bb9\u91cf $\\\\mu$（\u5373\u7cfb\u7edf\u5229\u7528\u7387 $\\\\rho = \\\\lambda \/ \\\\mu \\\\rightarrow 1.0$）\u65f6，\u8bf7\u6c42\u5728\u7b49\u5f85\u961f\u5217\u4e2d\u7684\u5e73\u5747\u6392\u961f\u65f6\u95f4 $W_q$ \u5448\u6307\u6570\u7ea7\u6025\u5267\u53d1\u6563：<\/p>\n
$$W_q \\\\propto \\\\frac{\\\\rho}{1 – \\\\rho} \\\\cdot \\\\frac{1}{\\\\mu}$$<\/p>\n
[\u5e76\u53d1\u8bf7\u6c42\u5230\u8fbe\u6d41 (Poisson Arrival)]
\n \u2502
\n \u25bc (\u82e5\u77ac\u65f6\u7a81\u53d1\u6d41\u91cf\u5bfc\u81f4 GPU \u5904\u4e8e\u6ee1\u8d1f\u8377\u72b6\u6001)
\n[Triton \/ vLLM \u5185\u90e8\u7b49\u5f85\u961f\u5217 (Waiting Queue)]
\n \u251c\u2500\u2500 \u8bf7\u6c42 1: \u6b63\u5728\u88ab GPU \u6267\u884c Prefill \u8ba1\u7b97 (\u5360\u7528 Tensor Core \u7b97\u529b)
\n \u251c\u2500\u2500 \u8bf7\u6c42 2: \u7b49\u5f85 Prefill \u5c31\u7eea…
\n \u251c\u2500\u2500 …
\n \u2514\u2500\u2500 \u8bf7\u6c42 99: \u906d\u9047\u4e25\u91cd\u7684\u961f\u5217\u79ef\u538b (Head-of-Line Blocking)！
\n \u2502
\n \u25bc
\n[\u8bf7\u6c42 99 \u7684\u9996\u5b57\u65f6\u95f4 TTFT = \u961f\u5217\u7b49\u5f85\u65f6\u95f4 (8.5\u79d2) + \u7eaf\u63a8\u7406\u8ba1\u7b97\u65f6\u95f4 (0.05\u79d2) = 8.55\u79d2！]<\/p>\n
\u56e0\u6b64，\u6cbb\u7406 P99 \u5ef6\u8fdf\u7684\u6838\u5fc3\u624b\u6bb5，\u662f\u4e25\u683c\u9650\u5236\u961f\u5217\u79ef\u538b\u6df1\u5ea6\u3001\u5f15\u5165\u52a8\u6001\u5206\u5757\u62a2\u5360（Chunked-Prefill）\u4e0e\u8fc7\u8f7d\u4fdd\u62a4（Load Shedding）\u3002<\/p>\n
import time
\nimport json
\nfrom locust import HttpUser, task, between, events<\/p>\n
class LLMStreamingLoadTestUser(HttpUser):
\n # \u6a21\u62df\u771f\u5b9e\u7528\u6237\u5728\u63d0\u95ee\u4e4b\u95f4\u7684\u601d\u8003\u505c\u987f\u65f6\u95f4 (1~3 \u79d2)
\n wait_time = between(1.0, 3.0)<\/p>\n
@task
\n def test_streaming_chat_completion(self):
\n payload = {
\n "model": "qwen2.5-7b-instruct",
\n "messages": [
\n {"role": "user", "content": "\u8bf7\u7528\u4e25\u8c28\u7684\u903b\u8f91\u6df1\u5165\u5206\u6790\u5206\u5e03\u5f0f\u6570\u636e\u5e93\u4e2d Paxos \u4e0e Raft \u5171\u8bc6\u7b97\u6cd5\u7684\u5f02\u540c\u3002"}
\n ],
\n "max_tokens": 256,
\n "temperature": 0.0,
\n "stream": True # \u5f00\u542f\u6d41\u5f0f\u54cd\u5e94 (SSE)
\n }<\/p>\n
start_time = time.perf_counter()
\n ttft_recorded = False
\n ttft_latency_ms = 0.0
\n total_tokens = 0<\/p>\n
# \u53d1\u8d77 HTTP POST \u6d41\u5f0f\u8bf7\u6c42
\n with self.client.post(
\n "\/v1\/chat\/completions",
\n json=payload,
\n stream=True,
\n catch_response=True
\n ) as response:
\n if response.status_code != 200:
\n response.failure(f"HTTP Status: {response.status_code}")
\n return<\/p>\n
# \u9010\u884c\u6d88\u8d39 SSE \u6570\u636e\u6d41
\n for line in response.iter_lines():
\n if line:
\n decoded_line = line.decode("utf-8")
\n if decoded_line.startswith("data: ") and decoded_line != "data: [DONE]":
\n total_tokens += 1<\/p>\n
# \u6838\u5fc3：\u7cbe\u51c6\u6355\u83b7\u9996\u5b57\u5230\u8fbe\u65f6\u95f4 (Time To First Token, TTFT)
\n if not ttft_recorded:
\n ttft_latency_ms = (time.perf_counter() – start_time) * 1000
\n ttft_recorded = True<\/p>\n
total_duration_ms = (time.perf_counter() – start_time) * 1000<\/p>\n
# \u4e0a\u62a5\u81ea\u5b9a\u4e49\u6307\u6807\u81f3 Locust \u5f15\u64ce
\n if ttft_recorded:
\n response.success()
\n events.request.fire(
\n request_type="LLM_TTFT",
\n name="Time_To_First_Token",
\n response_time=ttft_latency_ms,
\n response_length=total_tokens
\n )
\n else:
\n response.failure("\u6d41\u5f0f\u8f93\u51fa\u672a\u6536\u5230\u4efb\u4f55\u6709\u6548 Token！")<\/p>\n
\u6211\u4eec\u5728\u5355\u673a 8x A100 \u96c6\u7fa4\u4e0a，\u4f7f\u7528 Locust \u5bf9\u6bd4\u670d\u52a1\u5728\u4e0d\u540c\u8c03\u5ea6\u6cbb\u7406\u7b56\u7565\u4e0b\u7684\u5ef6\u8fdf\u5206\u5e03：<\/p>\n
| \u6734\u7d20\u9ed8\u8ba4\u8c03\u5ea6 (\u65e0\u5206\u5757Prefill)<\/td>\n | 2,150<\/td>\n | 450 ms<\/td>\n | 3,200 ms<\/td>\n | 9,450 ms (\u957f\u5c3e\u4e25\u91cd\u5d29\u584c)<\/td>\n | 8.4% (\u8d85\u65f6\u65ad\u8fde)<\/td>\n<\/tr>\n |
| \u5f00\u542f Chunked-Prefill (vLLM)<\/td>\n | 2,850<\/td>\n | 180 ms<\/td>\n | 680 ms<\/td>\n | 1,250 ms (\u538b\u964d 87%)<\/td>\n | 0.0% (\u7edd\u5bf9 0 \u8d85\u65f6)<\/td>\n<\/tr>\n |
| Chunked + \u52a8\u6001\u961f\u5217\u8d85\u65f6\u7194\u65ad (Ours)<\/td>\n | 3,150 (\u6700\u9ad8\u541e\u5410)<\/td>\n | 95 ms<\/td>\n | 280 ms<\/td>\n | 450 ms (\u6781\u81f4\u5e73\u7a33！)<\/td>\n | 0.0% (\u6781\u81f4\u7a33\u5065)<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n \u5b9e\u6d4b\u6570\u636e\u8868\u660e：\u901a\u8fc7\u5f15\u5165 Chunked-Prefill \u4e0e\u961f\u5217\u52a8\u6001\u9650\u6d41，P99 \u9996\u5b57\u5ef6\u8fdf\u76f4\u63a5\u4ece 9.45 \u79d2\u538b\u7f29\u81f3 450 \u6beb\u79d2（\u5ef6\u8fdf\u6296\u52a8\u538b\u964d\u8d85\u8fc7 95%），\u5f7b\u5e95\u6d88\u9664\u4e86\u5ba2\u6237\u7aef\u8bf7\u6c42\u8d85\u65f6\u7684\u987d\u75be\u3002<\/p>\n 4. \u957f\u5c3e\u5ef6\u8fdf\u6cbb\u7406\u56db\u5927\u5de5\u7a0b\u94c1\u5f8b<\/h3>\n\u5927\u6a21\u578b\u5728\u7ebf\u670d\u52a1\u9ad8\u5e76\u53d1\u538b\u6d4b：\u57fa\u4e8e Locust \u4e0e Prometheus \u7684 P99 \u5ef6\u8fdf\u6cbb\u7406\u5728\u5927\u8bed\u8a00\u6a21\u578b（LLM）\u5728\u7ebf\u63a8\u7406\u670d\u52a1（\u5982\u57fa\u4e8e vLLM\u3001SGLang \u6216 Triton \u642d\u5efa\u7684 API \u5fae\u670d\u52a1）\u4e0a\u7ebf\u4ea4\u4ed8\u524d，\u8bb8\u591a\u56e2\u961f\u5e38\u5e38\u8bef\u4ee5\u4e3a\u201c\u5e73\u5747\u54cd\u5e94\u65f6\u95f4（Average Latency）\u201d\u8fbe\u5230 500ms \u5c31\u4ee3\u8868\u7cfb\u7edf\u6027\u80fd\u8fbe\u6807\u3002 |