{"id":114265,"date":"2026-10-08T18:00:44","date_gmt":"2026-10-08T10:00:44","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/114265.html"},"modified":"2026-10-08T18:00:44","modified_gmt":"2026-10-08T10:00:44","slug":"%e4%b8%80%e5%9d%97-1050ti-%e5%ad%a6-ai-infra%ef%bc%9a%e6%88%91%e6%89%8b%e6%90%93%e4%ba%86%e4%b8%80%e4%b8%aa-llm-%e7%bd%91%e5%85%b3%ef%bc%8c%e4%bb%8e-redis-%e9%99%90%e6%b5%81%e4%b8%80%e8%b7%af%e5%81%9a","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/114265.html","title":{"rendered":"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861"},"content":{"rendered":"<h2>\u4e00\u5757 1050Ti \u5b66 AI Infra&#xff1a;\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173&#xff0c;\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861<\/h2>\n<p>4GB \u663e\u5b58\u3001i5-7300HQ\u300111GB \u5185\u5b58\u7684\u8001\u7b14\u8bb0\u672c&#xff0c;\u8dd1\u4e0d\u52a8 7B \u6a21\u578b&#xff0c;\u66f4\u522b\u8bf4\u5206\u5e03\u5f0f\u63a8\u7406\u3002 \u4f46\u6211\u7528\u5b83\u628a AI Infra \u91cc\u6700\u5e38\u88ab\u95ee\u5230\u7684\u4e1c\u897f\u90fd\u505a\u4e86\u4e00\u904d&#xff1a; \u9274\u6743\u3001\u9650\u6d41\u3001\u6a21\u578b\u8def\u7531\u3001SSE \u6d41\u5f0f\u3001Prometheus\/Grafana\u3001Redis\u3001PostgreSQL\u3001Kafka\u3001\u964d\u7ea7\u6f14\u7ec3\u3001 \u591a\u5b9e\u4f8b\u90e8\u7f72\u3001nginx \u8d1f\u8f7d\u5747\u8861\u3001\u6545\u969c\u8f6c\u79fb\u2014\u2014\u5168\u90e8\u6709\u5b9e\u6d4b\u6570\u636e&#xff0c;\u5168\u90e8\u80fd\u4e00\u952e\u590d\u73b0\u3002<\/p>\n<p>\u8fd9\u7bc7\u6587\u7ae0\u4e0d\u662f&#034;\u90e8\u7f72\u6559\u7a0b&#034;&#xff0c;\u800c\u662f\u6211\u7684\u5b9e\u9a8c\u8bb0\u5f55&#xff1a;\u6bcf\u4e2a\u9636\u6bb5\u505a\u4e86\u4ec0\u4e48\u3001\u6d4b\u51fa\u4ec0\u4e48\u6570\u5b57\u3001 \u88ab\u54ea\u4e2a\u5751\u5361\u4e86\u591a\u4e45\u3002\u5168\u6587 30&#043; \u7ec4\u5b9e\u6d4b\u6570\u636e&#xff0c;\u90fd\u5728\u4e00\u53f0 1050Ti \u4e0a\u8dd1\u51fa\u6765\u7684\u3002 github: https:\/\/github.com\/BumbleBee-ZDS\/AI_infra_study<\/p>\n<p><img decoding=\"async\" src=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/10\/20261008100042-6ac769cabae90.png\" alt=\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\" \/><\/p>\n<h3>\u76ee\u5f55<\/h3>\n<ul>\n<li>0. \u5148\u8bf4\u6e05\u695a&#xff1a;\u4e3a\u4ec0\u4e48\u7528 1050Ti \u5b66 AI Infra<\/li>\n<li>1. \u5b9e\u9a8c\u73af\u5883&#xff08;\u603b\u6210\u672c&#xff1a;0 \u5143&#xff09;<\/li>\n<li>2. \u8fd9\u4e2a\u9879\u76ee\u662f\u4ec0\u4e48&#xff1a;\u4e00\u4e2a OpenAI \u517c\u5bb9\u7684\u672c\u5730 LLM \u7f51\u5173<\/li>\n<li>3. \u5b66\u4e60\u8def\u7ebf&#xff1a;11 \u4e2a\u9636\u6bb5<\/li>\n<li>4. \u9636\u6bb5 1~3&#xff1a;\u8dd1\u901a\u3001\u8bfb\u61c2\u4e3b\u94fe\u8def\u3001\u6d41\u5f0f\u8f93\u51fa<\/li>\n<li>5. \u9636\u6bb5 4&#xff1a;\u53ef\u89c2\u6d4b&#xff08;Prometheus &#043; Grafana&#xff09;<\/li>\n<li>6. \u9636\u6bb5 5&#xff1a;\u5206\u5e03\u5f0f\u9650\u6d41\u2014\u2014\u672c\u9879\u76ee\u7684&#034;\u62db\u724c\u5b9e\u9a8c&#034;<\/li>\n<li>7. \u9636\u6bb5 6~7&#xff1a;\u8bf7\u6c42\u5386\u53f2&#xff08;SQLite\/PG&#xff09;\u4e0e Kafka \u5f02\u6b65\u4e8b\u4ef6<\/li>\n<li>8. \u9636\u6bb5 8&#xff1a;\u964d\u7ea7\u6f14\u7ec3&#xff08;\u6211\u8ba4\u4e3a\u6700\u503c\u94b1\u7684\u4e00\u7ae0&#xff09;<\/li>\n<li>9. \u9636\u6bb5 9&#xff1a;\u81ea\u52a8\u5316\u9a8c\u6536 &#043; 1050Ti \u53c2\u6570\u8c03\u4f18\u5b9e\u9a8c<\/li>\n<li>10. \u9636\u6bb5 11&#xff1a;\u591a\u5b9e\u4f8b\u90e8\u7f72 &#043; nginx \u8d1f\u8f7d\u5747\u8861&#xff08;\u5168\u6587\u91cd\u70b9&#xff09;<\/li>\n<li>11. \u8e29\u5751\u5408\u96c6&#xff1a;\u8fd9 10 \u4e2a\u5751&#xff0c;\u6211\u6bcf\u4e2a\u90fd\u82b1\u8fc7\u81f3\u5c11\u534a\u5c0f\u65f6<\/li>\n<li>12. \u4e00\u5757 1050Ti \u6559\u4f1a\u6211\u7684 5 \u4ef6\u4e8b<\/li>\n<li>13. \u60f3\u590d\u73b0&#xff1f;\u6700\u77ed\u8def\u5f84&#xff08;\u7ea6\u4e00\u4e2a\u4e0b\u5348&#xff09;<\/li>\n<li>14. \u5199\u5728\u6700\u540e<\/li>\n<\/ul>\n<hr \/>\n<h3>0. \u5148\u8bf4\u6e05\u695a&#xff1a;\u4e3a\u4ec0\u4e48\u7528 1050Ti \u5b66 AI Infra<\/h3>\n<p>\u6211\u4e00\u5f00\u59cb\u4e5f\u8d70\u4e86\u5f2f\u8def\u3002<\/p>\n<p>\u521a\u5165\u95e8 AI Infra \u7684\u65f6\u5019&#xff0c;\u6211\u4ee5\u4e3a\u8981\u5b66\u7684\u662f&#034;\u600e\u4e48\u628a\u6a21\u578b\u8dd1\u5f97\u66f4\u5feb&#034;&#xff1a;\u91cf\u5316\u3001KV cache \u4f18\u5316\u3001 PagedAttention\u3001\u5f20\u91cf\u5e76\u884c\u2026\u2026\u7136\u540e\u6211\u6253\u5f00 1050Ti \u7684 4GB \u663e\u5b58&#xff0c;\u53d1\u73b0\u8fde qwen2.5:7b \u7684 \u91cf\u5316\u7248\u90fd\u88c5\u4e0d\u8fdb\u53bb\u3002\u5361\u5728\u4e00\u5f00\u59cb&#xff0c;\u5f88\u5bb9\u6613\u5f97\u51fa&#034;\u786c\u4ef6\u4e0d\u591f&#xff0c;\u5b66\u4e0d\u4e86&#034;\u7684\u7ed3\u8bba\u3002<\/p>\n<p>\u540e\u6765\u6211\u60f3\u660e\u767d\u4e86\u4e00\u4ef6\u4e8b&#xff1a;\u771f\u6b63\u7684 AI Infra&#xff0c;90% \u7684\u5de5\u7a0b\u91cf\u4e0d\u5728\u6a21\u578b\u5185\u90e8&#xff0c;\u800c\u5728\u6a21\u578b\u5916\u9762\u90a3\u4e00\u5708\u3002<\/p>\n<p>\u628a Ollama \u5f53\u6210\u4e00\u4e2a\u9ed1\u76d2 HTTP \u670d\u52a1&#xff08;\u4e8b\u5b9e\u4e0a\u5b83\u786e\u5b9e\u662f&#xff1a;POST http:\/\/localhost:11434\/api\/chat&#xff09;&#xff0c; \u90a3\u4e48\u4e00\u4e2a LLM \u670d\u52a1\u4e0a\u7ebf\u8981\u9762\u5bf9\u7684\u95ee\u9898&#xff0c;\u5168\u90fd\u8ddf\u663e\u5b58\u65e0\u5173&#xff1a;<\/p>\n<table>\n<tr>\u751f\u4ea7\u95ee\u9898\u9700\u8981\u4ec0\u4e48\u5de5\u7a0b\u80fd\u529b1050Ti \u80fd\u7ec3\u5417<\/tr>\n<tbody>\n<tr>\n<td>\u8c01\u80fd\u8c03&#xff1f;\u8c03\u4e86\u591a\u5c11\u6b21&#xff1f;<\/td>\n<td>\u9274\u6743 &#043; \u8ba1\u91cf<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u4e00\u4e2a\u7528\u6237\u5237\u7206 GPU&#xff0c;\u522b\u4eba\u5168\u5361\u6b7b<\/td>\n<td>\u9650\u6d41&#xff08;\u5355\u673a \/ \u5206\u5e03\u5f0f&#xff09;<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u7528\u6237\u7b49 30 \u79d2\u6ca1\u53cd\u5e94\u5c31\u5173\u9875\u9762<\/td>\n<td>SSE \u6d41\u5f0f &#043; TTFT \u89c2\u6d4b<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u51fa\u4e8b\u53ea\u77e5\u9053&#034;\u5f88\u6162&#034;&#xff0c;\u4e0d\u77e5\u9053\u6162\u5728\u54ea<\/td>\n<td>\u6307\u6807\u3001\u65e5\u5fd7\u3001\u94fe\u8def\u8ffd\u8e2a<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u4e0a\u6e38\u6a21\u578b\u6362\u4e86&#xff0c;\u5ba2\u6237\u7aef\u8981\u6539\u4ee3\u7801<\/td>\n<td>\u6a21\u578b\u522b\u540d\u8def\u7531<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u4e0b\u6e38 Redis \/ \u5e93 \/ \u961f\u5217\u6302\u4e86&#xff0c;\u670d\u52a1\u8ddf\u7740\u6b7b<\/td>\n<td>\u964d\u7ea7\u8bbe\u8ba1<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<tr>\n<td>\u5355\u8fdb\u7a0b\u5403\u6ee1\u4e00\u4e2a\u6838\u3001\u8fdb\u7a0b\u5d29\u4e86\u670d\u52a1\u5c31\u6ca1\u4e86<\/td>\n<td>\u591a\u5b9e\u4f8b &#043; \u8d1f\u8f7d\u5747\u8861 &#043; \u6545\u969c\u8f6c\u79fb<\/td>\n<td>\u2705 \u5b8c\u5168\u80fd<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u660e\u786e\u4e00\u4e0b\u8fd9\u5757\u5361&#034;\u5b66\u4e0d\u5230&#034;\u7684\u4e1c\u897f&#xff1a;\u591a\u5361\u5e76\u884c\u3001\u663e\u5b58\u5e26\u5bbd\u4f18\u5316\u7684\u771f\u5b9e\u6536\u76ca\u3001vLLM\/SGLang \u90a3\u79cd continuous batching \u7684\u541e\u5410\u9b54\u6cd5\u3001\u771f\u6b63\u7684 GPU \u8c03\u5ea6&#xff08;MIG\/\u65f6\u95f4\u6bb5\u5171\u4eab&#xff09;\u3002\u8fd9\u4e9b\u8981\u771f\u5361\u3002 \u4f46\u8981\u5b66&#034;AI \u670d\u52a1\u5316&#034;\u7684\u5b8c\u6574\u94fe\u8def&#xff0c;\u4e00\u5f20 1050Ti \u5b8c\u5168\u591f\u2014\u2014\u800c\u4e14\u56e0\u4e3a\u6162&#xff0c;\u53cd\u800c\u66f4\u5bb9\u6613\u89c2\u5bdf\u5230\u73b0\u8c61&#xff1a; \u6bcf token \u5ef6\u8fdf\u88ab\u653e\u5927\u5230\u51e0\u5341\u6beb\u79d2\u7ea7&#xff0c;\u4e00\u4e2a\u5e76\u53d1\u4e0a\u5347&#xff0c;TTFT \u4ece 0.03s \u6da8\u5230 3.6s&#xff0c;\u8089\u773c\u53ef\u89c1\u3002<\/p>\n<p>\u6240\u4ee5\u6211\u7ed9\u81ea\u5df1\u5b9a\u4e86\u4e2a\u76ee\u6807&#xff1a;\u624b\u5199\u4e00\u4e2a OpenAI \u517c\u5bb9\u7684 LLM \u7f51\u5173&#xff0c;\u56f4\u7ed5\u5b83\u628a\u4e0a\u9762\u90a3\u5f20\u8868\u91cc\u7684 \u6bcf\u4e00\u884c\u90fd\u4eb2\u624b\u505a\u4e00\u904d\u3001\u6d4b\u4e00\u904d&#xff0c;\u7136\u540e\u628a\u6574\u4e2a\u8fc7\u7a0b\u5199\u6210\u4e86\u5b66\u4e60\u624b\u518c\u3002<\/p>\n<hr \/>\n<h3>1. \u5b9e\u9a8c\u73af\u5883&#xff08;\u603b\u6210\u672c&#xff1a;0 \u5143&#xff09;<\/h3>\n<p>\u5168\u90e8\u5728\u672c\u673a\u8dd1&#xff0c;\u6ca1\u6709\u4efb\u4f55\u4e91\u670d\u52a1&#xff1a;<\/p>\n<table>\n<tr>\u9879\u76ee\u914d\u7f6e<\/tr>\n<tbody>\n<tr>\n<td>CPU<\/td>\n<td>Intel Core i5-7300HQ &#064; 2.50GHz&#xff08;4 \u6838 4 \u7ebf\u7a0b&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5185\u5b58<\/td>\n<td>11 GiB<\/td>\n<\/tr>\n<tr>\n<td>GPU<\/td>\n<td>NVIDIA GTX 1050 Ti&#xff0c;4GB \u663e\u5b58&#xff0c;\u9a71\u52a8 580.178.04<\/td>\n<\/tr>\n<tr>\n<td>\u7cfb\u7edf<\/td>\n<td>Ubuntu 26.04<\/td>\n<\/tr>\n<tr>\n<td>Python<\/td>\n<td>3.12.14&#xff08;conda \u73af\u5883 aiinfra&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u6a21\u578b\u8fd0\u884c\u65f6<\/td>\n<td>Ollama 0.35.0<\/td>\n<\/tr>\n<tr>\n<td>\u6a21\u578b<\/td>\n<td>qwen2.5:0.5b(397MB) \/ qwen2.5:1.5b(986MB) \/ llama3.2:1b(1.3GB)<\/td>\n<\/tr>\n<tr>\n<td>\u76d1\u63a7<\/td>\n<td>Prometheus 3.15.0 &#043; Grafana 13.2.3<\/td>\n<\/tr>\n<tr>\n<td>\u57fa\u7840\u8bbe\u65bd<\/td>\n<td>Redis 8.0.5 \/ PostgreSQL 18.6 \/ Kafka&#xff08;OpenJDK 25&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u8d1f\u8f7d\u5747\u8861<\/td>\n<td>nginx 1.28.3<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5c0f\u6280\u5de7&#xff1a;\u8fd9\u4e9b\u7ec4\u4ef6\u6211\u4e00\u4e2a\u90fd\u6ca1\u7528 root \u88c5\u3002Redis\/PG\/Kafka\/nginx\/Prometheus\/Grafana \u5168\u90e8\u7528 apt-get download &#043; dpkg-deb -x&#xff08;\u6216\u5b98\u65b9 tar \u5305&#xff09;\u89e3\u5305\u5230 ~\/opt\/ \u4e0b&#xff0c; \u7528\u5404\u81ea\u7684\u914d\u7f6e\u6587\u4ef6\u6307\u5b9a\u7aef\u53e3\u4e0e\u6570\u636e\u76ee\u5f55\u3002\u8001\u7b14\u8bb0\u672c\u3001\u6ca1 sudo \u6743\u9650\u3001\u6015\u6c61\u67d3\u7cfb\u7edf\u73af\u5883\u7684\u90fd\u9002\u7528\u3002<\/p>\n<hr \/>\n<h3>2. \u8fd9\u4e2a\u9879\u76ee\u662f\u4ec0\u4e48&#xff1a;\u4e00\u4e2a OpenAI \u517c\u5bb9\u7684\u672c\u5730 LLM \u7f51\u5173<\/h3>\n<p>\u4e00\u53e5\u8bdd&#xff1a;\u628a\u4f60\u7684\u8bf7\u6c42\u7edf\u4e00\u6536\u53e3&#xff0c;\u505a\u300c\u9274\u6743 \u2192 \u9650\u6d41 \u2192 \u6a21\u578b\u8def\u7531 \u2192 \u8f6c\u53d1\u7ed9 Ollama \u2192 \u8bb0\u5f55\u6307\u6807\/\u65e5\u5fd7\/\u5386\u53f2\/\u4e8b\u4ef6\u300d&#xff0c;\u5bf9\u5916\u957f\u5f97\u548c OpenAI \u4e00\u6a21\u4e00\u6837\u3002<\/p>\n<p>\u6240\u4ee5\u4efb\u4f55\u652f\u6301 OpenAI \u534f\u8bae\u7684\u5ba2\u6237\u7aef&#xff08;\u5b98\u65b9 SDK\u3001LangChain\u3001Cherry Studio\u3001NextChat\u2026\u2026&#xff09; \u53ea\u8981\u628a base_url \u6539\u6210 http:\/\/localhost:8000\/v1 \u5c31\u80fd\u63a5\u4e0a&#xff0c;\u4e1a\u52a1\u4ee3\u7801\u4e00\u884c\u4e0d\u6539\u3002<\/p>\n<h4>2.1 \u67b6\u6784\u56fe<\/h4>\n<p>                  \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n  \u5ba2\u6237\u7aef curl \u2500\u2500\u2500\u25ba \u2502            FastAPI \u7f51\u5173 (main.py)            \u2502<br \/>\n                  \u2502                                              \u2502<br \/>\n                  \u2502 \u2460 \u9274\u6743 auth.py       \u2192 \u5931\u8d25 401               \u2502<br \/>\n                  \u2502 \u2461 \u9650\u6d41 ratelimit.py  \u2192 \u8d85\u9650 429               \u2502<br \/>\n                  \u2502 \u2462 \u53c2\u6570\u6821\u9a8c\/\u8def\u7531 router.py \u2192 \u975e\u6cd5 400           \u2502<br \/>\n                  \u2502 \u2463 \u8f6c\u53d1 httpx \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510              \u2502<br \/>\n                  \u2502 \u2464 \u8bb0\u5f55\u6307\u6807\/\u65e5\u5fd7\/\u5386\u53f2\/\u4e8b\u4ef6      \u2502              \u2502<br \/>\n                  \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u253c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<br \/>\n                                                  \u25bc<br \/>\n                                        \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n                                        \u2502  Ollama :11434   \u2502<br \/>\n                                        \u2502 qwen2.5:0.5b &#8230; \u2502<br \/>\n                                        \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<\/p>\n<p>  \u57fa\u7840\u8bbe\u65bd&#xff08;\u6bcf\u4e00\u4ef6\u90fd\u80fd\u72ec\u7acb\u964d\u7ea7&#xff0c;\u7f3a\u4e86\u4e5f\u4e0d\u5f71\u54cd\u63a8\u7406&#xff09;&#xff1a;<br \/>\n    \u2461 \u9650\u6d41\u8ba1\u6570 \u2500\u2500\u25ba Redis      :6379   &#xff08;\u6302\u4e86 \u2192 \u8fdb\u7a0b\u5185\u8ba1\u6570\u515c\u5e95&#xff09;<br \/>\n    \u2464 \u5386\u53f2\u660e\u7ec6 \u2500\u2500\u25ba SQLite\/PG  :5433   &#xff08;\u6302\u4e86 \u2192 \u53ea\u4e22\u5386\u53f2&#xff0c;\u8bf7\u6c42\u7167\u8fc7&#xff09;<br \/>\n    \u2464 \u4e8b\u4ef6\u6d41   \u2500\u2500\u25ba Kafka      :9092   &#xff08;\u6302\u4e86 \u2192 \u964d\u7ea7\u5199 logs\/events.jsonl&#xff09;<\/p>\n<p>  \u89c2\u6d4b&#xff1a;<br \/>\n    Prometheus :9090  \u2500\u2500\u6293\u53d6\u2500\u2500\u25ba \u7f51\u5173 \/metrics<br \/>\n    Grafana    :3000  \u2500\u2500\u67e5\u8be2\u2500\u2500\u25ba Prometheus&#xff08;5 \u884c \/ 35 \u4e2a\u9762\u677f&#xff09;<\/p>\n<p>\u6574\u5957\u4e1c\u897f\u7684\u771f\u5b9e\u7aef\u53e3\u8868&#xff08;\u540e\u9762\u6bcf\u4e2a\u5b9e\u9a8c\u90fd\u8981\u7528&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u670d\u52a1\u5730\u5740\u600e\u4e48\u8d77\u6570\u636e\u76ee\u5f55<\/tr>\n<tbody>\n<tr>\n<td>\u7f51\u5173&#xff08;\u5355\u5b9e\u4f8b&#xff09;<\/td>\n<td>http:\/\/localhost:8000<\/td>\n<td>python main.py<\/td>\n<td>logs\/<\/td>\n<\/tr>\n<tr>\n<td>\u7f51\u5173\u96c6\u7fa4\u5165\u53e3&#xff08;nginx&#xff09;<\/td>\n<td>http:\/\/localhost:8080<\/td>\n<td>bash scripts\/cluster.sh start<\/td>\n<td>~\/opt\/nginx\/logs<\/td>\n<\/tr>\n<tr>\n<td>Ollama<\/td>\n<td>http:\/\/localhost:11434<\/td>\n<td>\u7cfb\u7edf\u670d\u52a1 ollama serve<\/td>\n<td>~\/.ollama<\/td>\n<\/tr>\n<tr>\n<td>Prometheus<\/td>\n<td>http:\/\/localhost:9090<\/td>\n<td>bash scripts\/monitoring.sh start<\/td>\n<td>~\/opt\/monitoring\/prometheus\/data<\/td>\n<\/tr>\n<tr>\n<td>Grafana<\/td>\n<td>http:\/\/localhost:3000<\/td>\n<td>\u540c\u4e0a<\/td>\n<td>~\/opt\/monitoring\/grafana\/data<\/td>\n<\/tr>\n<tr>\n<td>Redis<\/td>\n<td>localhost:6379<\/td>\n<td>bash scripts\/infra.sh start<\/td>\n<td>\u65e0&#xff08;&#8211;save &#039;&#039; \u4e0d\u843d\u76d8&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>PostgreSQL<\/td>\n<td>localhost:5433<\/td>\n<td>\u540c\u4e0a<\/td>\n<td>~\/opt\/infra\/pgdata<\/td>\n<\/tr>\n<tr>\n<td>Kafka<\/td>\n<td>localhost:9092<\/td>\n<td>\u540c\u4e0a<\/td>\n<td>~\/opt\/infra\/kafka-data<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>2.2 \u6211\u7528\u5230\u7684 4 \u4e2a&#034;\u5b66\u4e60\u6280\u5de7&#034;<\/h4>\n<p>\u8fd9\u5957\u9879\u76ee\u662f\u8fb9\u5b66\u8fb9\u5199\u7684&#xff0c;\u6709\u51e0\u4e2a\u8ba9\u6211\u5c11\u8d70\u5f2f\u8def\u7684\u5c0f\u8bbe\u8ba1&#xff0c;\u503c\u5f97\u5148\u8bf4&#xff1a;<\/p>\n<li>\u6bcf\u4e00\u5c42\u90fd\u80fd\u5355\u72ec\u964d\u7ea7\u3002\u5199\u5b8c\u4efb\u4f55\u4e00\u4e2a\u6a21\u5757&#xff0c;\u6211\u90fd\u95ee\u81ea\u5df1\u4e00\u53e5&#xff1a;\u5b83\u6302\u4e86\u4f1a\u600e\u6837&#xff1f; Redis \u6302\u4e86 \u2192 \u81ea\u52a8\u5207\u8fdb\u7a0b\u5185\u8ba1\u6570&#xff1b;PG \u6302\u4e86 \u2192 \u53ea\u4e22\u5386\u53f2&#xff0c;\u8bf7\u6c42\u7167\u8fc7 200&#xff1b; Kafka \u6302\u4e86 \u2192 \u4e8b\u4ef6\u5199\u672c\u5730 JSONL\u3002\u63a8\u7406\u4e3b\u94fe\u8def\u6c38\u4e0d\u88ab\u89c2\u6d4b\u7ec4\u4ef6\u62d6\u6b7b\u2014\u2014\u8fd9\u662f\u6574\u4e2a\u9879\u76ee\u6700\u503c\u94b1\u7684\u4e00\u6761\u539f\u5219\u3002<\/li>\n<li>\u6bcf\u4e2a\u5224\u65ad\u90fd\u8981\u6709\u6570\u5b57\u3002\u4e0d\u5199&#034;\u5f88\u5feb&#034;\u201c\u5f88\u7a33\u201d&#xff0c;\u53ea\u5199&#034;\u9996\u5b57\u8282 0.041s \/ \u603b\u65f6\u957f 0.710s&#034;\u3002 \u540e\u9762\u4f60\u4f1a\u770b\u5230\u6211\u56e0\u4e3a&#034;\u770b\u7740\u50cf\u88ab\u7f13\u51b2\u4e86&#034;\u800c\u8bef\u5224\u8fc7\u4e00\u6b21&#xff0c;\u5c31\u662f\u9760\u91cd\u65b0\u6d4b\u6570\u636e\u624d\u627e\u5230\u771f\u56e0&#xff08;\u89c1\u7b2c 10 \u8282&#xff09;\u3002<\/li>\n<li>\u9a8c\u6536\u811a\u672c\u5316\u3002\u57fa\u7840\u8bbe\u65bd\u4e00\u5957 21 \u9879\u68c0\u67e5\u3001\u591a\u5b9e\u4f8b\u4e00\u5957 34 \u9879\u68c0\u67e5&#xff0c; \u6bcf\u6b21\u6539\u5b8c\u4ee3\u7801\u8dd1\u4e00\u904d&#xff0c;\u6bd4\u4eba\u8089\u70b9 30 \u6b21 curl \u9760\u8c31\u3002<\/li>\n<li>\u5b9e\u9a8c\u8981\u9694\u79bb\u3002\u8fd9\u6761\u662f\u6211\u8e29\u4e86\u5751\u624d\u660e\u767d\u7684&#xff1a;\u4e00\u8fb9\u538b\u6d4b\u4e00\u8fb9\u8dd1\u9a8c\u6536&#xff0c; \u4e24\u8fb9\u4f1a\u62a2\u540c\u4e00\u4e2a\u9650\u6d41\u989d\u5ea6\u3001\u62a2\u540c\u4e00\u5757 GPU&#xff0c;\u7ed3\u679c\u53cc\u53cc FAIL \u6216\u6570\u5b57\u5931\u771f&#xff08;\u7b2c 11 \u8282\u6709\u5b8c\u6574\u6848\u4f8b&#xff09;\u3002<\/li>\n<hr \/>\n<h3>3. \u5b66\u4e60\u8def\u7ebf&#xff1a;11 \u4e2a\u9636\u6bb5<\/h3>\n<p>\u6211\u628a\u6574\u4e2a\u8fc7\u7a0b\u62c6\u6210\u4e86 11 \u4e2a\u9636\u6bb5&#xff0c;\u6bcf\u4e2a\u9636\u6bb5\u90fd\u6709&#034;\u6982\u5ff5 \u2192 \u52a8\u624b \u2192 \u9a8c\u6536 \u2192 \u8e29\u5751&#034;\u56db\u6bb5&#xff1a;<\/p>\n<table>\n<tr>\u9636\u6bb5\u5b66\u4ec0\u4e48\u9a8c\u6536\u6807\u51c6<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u73af\u5883\u51c6\u5907\u4e0e\u8dd1\u901a\u7b2c\u4e00\u6761\u8bf7\u6c42<\/td>\n<td>curl \u62ff\u5230 200&#xff0c;\u65e5\u5fd7\u6709\u8bb0\u5f55<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u8bfb\u61c2\u4e3b\u94fe\u8def main.py<\/td>\n<td>\u80fd\u753b\u51fa 5 \u6b65\u6d41\u7a0b&#xff0c;\u80fd\u89e6\u53d1 4 \u79cd\u9519\u8bef\u7801<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>\u6d41\u5f0f SSE \u4e0e\u9996 token \u5ef6\u8fdf<\/td>\n<td>curl -N \u770b\u5230\u9010\u5b57\u8f93\u51fa &#043; [DONE]<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u6307\u6807\u4e0e\u76d1\u63a7&#xff08;Prometheus &#043; Grafana&#xff09;<\/td>\n<td>\/metrics \u6709\u6570\u636e&#xff0c;\u770b\u677f\u6709\u66f2\u7ebf<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u5206\u5e03\u5f0f\u9650\u6d41&#xff08;Redis&#xff09;<\/td>\n<td>\u4e24\u5b9e\u4f8b\u5171\u4eab 10 \u6b21\u989d\u5ea6&#xff0c;\u5408\u8ba1\u53ea\u653e\u884c 10<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u8bf7\u6c42\u5386\u53f2&#xff08;SQLite \/ PG&#xff09;<\/td>\n<td>\/history \u67e5\u5230\u81ea\u5df1\u7684\u8bf7\u6c42&#xff0c;\u4f1a\u76f4\u63a5\u67e5\u5e93<\/td>\n<\/tr>\n<tr>\n<td>7<\/td>\n<td>Kafka \u5f02\u6b65\u4e8b\u4ef6<\/td>\n<td>\u547d\u4ee4\u884c\u6d88\u8d39\u8005\u80fd\u770b\u5230 4 \u7c7b\u4e8b\u4ef6<\/td>\n<\/tr>\n<tr>\n<td>8<\/td>\n<td>\u964d\u7ea7\u6f14\u7ec3<\/td>\n<td>\u9010\u4e2a kill \u670d\u52a1&#xff0c;\u8bf7\u6c42\u4ecd\u7136 200<\/td>\n<\/tr>\n<tr>\n<td>9<\/td>\n<td>\u81ea\u52a8\u5316\u9a8c\u6536\u3001\u538b\u6d4b\u3001\u53c2\u6570\u8c03\u4f18<\/td>\n<td>21\/21 \u901a\u8fc7 &#043; \u4e00\u5f20\u53c2\u6570\u626b\u63cf\u8868<\/td>\n<\/tr>\n<tr>\n<td>10<\/td>\n<td>12 \u9053\u8fdb\u9636\u52a8\u624b\u9898<\/td>\n<td>\u81ea\u5df1\u52a8\u624b\u505a 3 \u4e2a\u4ee5\u4e0a<\/td>\n<\/tr>\n<tr>\n<td>11<\/td>\n<td>\u591a\u5b9e\u4f8b &#043; nginx \u8d1f\u8f7d\u5747\u8861<\/td>\n<td>\u6740\u6389\u4e00\u4e2a\u5b9e\u4f8b&#xff0c;\u8bf7\u6c42\u4ecd\u5168\u90e8 200<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e0b\u9762\u6309\u9636\u6bb5\u8bb2\u6211\u505a\u4e86\u4ec0\u4e48\u3001\u6d4b\u51fa\u4e86\u4ec0\u4e48\u3002<\/p>\n<hr \/>\n<h3>4. \u9636\u6bb5 1~3&#xff1a;\u8dd1\u901a\u3001\u8bfb\u61c2\u4e3b\u94fe\u8def\u3001\u6d41\u5f0f\u8f93\u51fa<\/h3>\n<h4>4.1 \u7b2c\u4e00\u6761\u8bf7\u6c42<\/h4>\n<p>python main.py     <span class=\"token comment\"># \u9ed8\u8ba4 0.0.0.0:8000<\/span><\/p>\n<p>\u542f\u52a8\u65e5\u5fd7\u4f1a\u76f4\u63a5\u544a\u8bc9\u4f60&#034;\u5f53\u524d\u751f\u6548\u7684\u540e\u7aef\u662f\u8c01&#034;\u2014\u2014\u8fd9\u4e2a\u8bbe\u8ba1\u5f88\u91cd\u8981&#xff0c;\u591a\u5b9e\u4f8b\u6392\u969c\u65f6\u4e00\u773c\u5c31\u80fd\u770b\u51fa \u67d0\u4e2a\u5b9e\u4f8b\u662f\u4e0d\u662f\u5077\u5077\u964d\u7ea7\u4e86&#xff1a;<\/p>\n<p>\u9650\u6d41\u540e\u7aef: Redis&#xff08;redis:\/\/localhost:6379\/0&#xff0c;\u7a97\u53e3 60s&#xff0c;\u591a\u5b9e\u4f8b\u5171\u4eab\u8ba1\u6570&#xff09;<br \/>\n\u8bf7\u6c42\u5386\u53f2: postgresql:\/\/gw:***&#064;localhost:5433\/gateway&#xff08;\u6279\u91cf 20 \u884c \/ 1.0s&#xff09;<br \/>\n\u4e8b\u4ef6\u603b\u7ebf: Kafka localhost:9092 -&gt; topic llm-gateway-events<br \/>\nUvicorn running on http:\/\/0.0.0.0:8000<\/p>\n<p>\u53d1\u7b2c\u4e00\u6761\u8bf7\u6c42&#xff1a;<\/p>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8000\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;\u7528\u4e00\u53e5\u8bdd\u8bf4\u660e\u4ec0\u4e48\u662fTTFT&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:40}}&#039;<\/span><\/p>\n<p><span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token string-property property\">&#034;id&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;chatcmpl-1791388386659&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;object&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;chat.completion&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;model&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;qwen2.5:0.5b&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;usage&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;prompt_tokens&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">35<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;completion_tokens&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">32<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;total_tokens&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">67<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;choices&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">{<\/span><br \/>\n        <span class=\"token string-property property\">&#034;index&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token string-property property\">&#034;message&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;role&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;assistant&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string-property property\">&#034;content&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;TTFT\u662fTaiShan Cloud BigData\u9ad8\u6027\u80fd\u8ba1\u7b97\u96c6\u7fa4\u5e73\u53f0&#xff0c;\u63d0\u4f9b\u5f3a\u5927\u7684\u8ba1\u7b97\u80fd\u529b\u3001\u9ad8\u53ef\u7528\u6027\u3001\u53ef\u6269\u5c55\u6027\u53ca\u81ea\u52a8\u5316\u8fd0\u7ef4\u80fd\u529b\u3002&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token string-property property\">&#034;finish_reason&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;stop&#034;<\/span><br \/>\n    <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">]<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p>\u987a\u4fbf\u8bf4\u4e2a\u5b9e\u8bdd&#xff1a;0.5B \u6a21\u578b\u7b54\u5f97\u79bb\u8c31&#xff08;TTFT \u660e\u660e\u662f Time To First Token&#xff09;\u3002 \u6211\u7528\u5b83\u505a\u5b9e\u9a8c\u662f\u56e0\u4e3a\u5b83\u5feb\u3001\u53ea\u5360 397MB \u663e\u5b58&#xff0c;\u53c2\u6570\u626b\u63cf\u8dd1 13 \u7ec4\u4e0d\u5fc3\u75bc\u3002 \u60f3\u8981\u56de\u7b54\u8d28\u91cf\u5c31\u6362 qwen1.5b \u6216 llama1b\u2014\u2014\u628a &#034;model&#034; \u6362\u6210\u522b\u540d\u5373\u53ef&#xff0c; \u7f51\u5173\u4f1a\u6620\u5c04\u5230\u771f\u5b9e\u7684 Ollama \u6a21\u578b\u540d&#xff08;\u8fd9\u5c31\u662f&#034;\u6a21\u578b\u8def\u7531&#034;&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u522b\u540d\u5b9e\u9645\u6a21\u578b<\/tr>\n<tbody>\n<tr>\n<td>qwen0.5b<\/td>\n<td>qwen2.5:0.5b<\/td>\n<\/tr>\n<tr>\n<td>qwen1.5b<\/td>\n<td>qwen2.5:1.5b<\/td>\n<\/tr>\n<tr>\n<td>llama1b<\/td>\n<td>llama3.2:1b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>4.2 \u4e00\u6b21\u8bf7\u6c42\u7684 5 \u6b65\u94fe\u8def&#xff08;\u8fd9\u662f\u7f51\u5173\u7684\u9aa8\u67b6&#xff09;<\/h4>\n<p>\u8bfb main.py \u65f6\u6211\u5efa\u8bae\u53ea\u6293\u8fd9 5 \u6b65&#xff1a;<\/p>\n<p>\u2460 \u9274\u6743&#xff1a;x-api-key \u2192 tier&#xff08;free \/ pro&#xff09;<br \/>\n\u2461 \u9650\u6d41&#xff1a;Redis \u56fa\u5b9a\u7a97\u53e3\u8ba1\u6570&#xff08;Lua \u539f\u5b50\u64cd\u4f5c&#xff09;&#xff0c;\u8d85\u9650 \u2192 429<br \/>\n\u2462 \u89e3\u6790\u4e0e\u8def\u7531&#xff1a;options \u767d\u540d\u5355\u6821\u9a8c&#xff08;\u9632\u6ce8\u5165&#xff09;\u2192 \u6a21\u578b\u522b\u540d\u6620\u5c04<br \/>\n\u2463 \u8f6c\u53d1&#xff1a;httpx \u6253 Ollama \/api\/chat&#xff1b;\u6d41\u5f0f\u5219\u8f6c\u6210 OpenAI SSE<br \/>\n\u2464 \u7559\u75d5&#xff1a;\u4e00\u6761\u8bb0\u5f55\u540c\u65f6\u5199\u4e09\u5904 \u2014\u2014 JSONL \u65e5\u5fd7 &#043; \u5386\u53f2\u5e93 &#043; Kafka \u4e8b\u4ef6<\/p>\n<p>\u7b2c \u2464 \u6b65\u662f\u6211\u89c9\u5f97\u6700\u503c\u5f97\u6284\u7684\u8bbe\u8ba1&#xff0c;\u5b83\u53ea\u6709 3 \u884c&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">_persist<\/span><span class=\"token punctuation\">(<\/span>record<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">dict<\/span><span class=\"token punctuation\">,<\/span> event<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;request.completed&#034;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u4e00\u6761\u8bf7\u6c42\u8bb0\u5f55\u540c\u65f6\u843d\u4e09\u5904&#xff1a;JSONL \u6587\u4ef6\u65e5\u5fd7 &#043; \u5386\u53f2\u5e93 &#043; Kafka \u4e8b\u4ef6\u3002<\/p>\n<p>    \u4e09\u5904\u90fd\u662f\u300c\u5c3d\u529b\u800c\u4e3a\u300d&#xff1a;history.submit \/ events.publish \u53ea\u505a\u5165\u961f&#xff0c;<br \/>\n    \u961f\u5217\u6ee1\u6216\u540e\u7aef\u4e0d\u53ef\u7528\u90fd\u53ea\u8ba1\u6570\u4e0d\u629b\u9519&#xff0c;\u7edd\u4e0d\u53cd\u538b\u63a8\u7406\u4e3b\u94fe\u8def\u3002<br \/>\n    &#034;&#034;&#034;<\/span><br \/>\n    log_request<span class=\"token punctuation\">(<\/span>record<span class=\"token punctuation\">)<\/span><br \/>\n    history<span class=\"token punctuation\">.<\/span>submit<span class=\"token punctuation\">(<\/span>record<span class=\"token punctuation\">)<\/span><br \/>\n    events<span class=\"token punctuation\">.<\/span>publish<span class=\"token punctuation\">(<\/span>event<span class=\"token punctuation\">,<\/span> record<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5173\u952e\u5728\u6700\u540e\u90a3\u53e5\u6ce8\u91ca&#xff1a;\u6240\u6709\u7559\u75d5\u52a8\u4f5c\u90fd\u53ea&#034;\u6295\u9012&#034;&#xff0c;\u4e0d\u7b49\u5f85\u3001\u4e0d\u629b\u5f02\u5e38\u3002 \u8fd9\u5c31\u662f\u7b2c 8 \u8282\u964d\u7ea7\u6f14\u7ec3\u80fd\u6210\u529f\u7684\u6839\u672c\u539f\u56e0\u3002<\/p>\n<h4>4.3 \u4eb2\u624b\u89e6\u53d1 4 \u79cd\u9519\u8bef\u7801<\/h4>\n<p>\u5149\u770b\u4ee3\u7801\u8bb0\u4e0d\u4f4f&#xff0c;\u6211\u7528\u4e00\u5f20\u8868\u628a 4 \u4e2a\u9519\u8bef\u7801\u90fd\u6253\u51fa\u6765&#xff08;\u5bf9\u540e\u9762\u6392\u969c\u5e2e\u52a9\u6781\u5927&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u600e\u4e48\u6253\u51fa\u6765\u72b6\u6001\u7801\u4ece\u54ea\u4e00\u5c42\u8fd4\u56de<\/tr>\n<tbody>\n<tr>\n<td>\u4e0d\u5e26 x-api-key<\/td>\n<td>401<\/td>\n<td>\u9274\u6743\u5c42<\/td>\n<\/tr>\n<tr>\n<td>\u540c\u4e00\u4e2a key \u77ed\u65f6\u95f4\u5185\u8d85\u8fc7\u914d\u989d<\/td>\n<td>429<\/td>\n<td>\u9650\u6d41\u5c42<\/td>\n<\/tr>\n<tr>\n<td>&#034;options&#034;: {&#034;num_ctx&#034;: 999999}<\/td>\n<td>400<\/td>\n<td>\u53c2\u6570\u6821\u9a8c&#xff08;\u8d85\u51fa\u767d\u540d\u5355\u8303\u56f4&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u628a Ollama \u505c\u6389\u518d\u8bf7\u6c42<\/td>\n<td>502<\/td>\n<td>\u4e0a\u6e38\u8f6c\u53d1\u5c42<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>4.4 SSE \u6d41\u5f0f&#xff1a;\u4e3a\u4ec0\u4e48\u5b83\u662f LLM \u670d\u52a1\u7684&#034;\u4f53\u9a8c\u547d\u95e8&#034;<\/h4>\n<p>\u975e\u6d41\u5f0f\u8981\u7b49\u6574\u6bb5\u751f\u6210\u5b8c\u624d\u8fd4\u56de&#xff1b;\u6d41\u5f0f&#xff08;Server-Sent Events&#xff09;\u5219\u662f\u4e00\u4e2a token \u4e00\u4e2a data: \u5206\u7247&#xff1a;<\/p>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-N<\/span> <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8000\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;stream&#034;:true,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;\u6570\u5230\u4e94&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:20}}&#039;<\/span><\/p>\n<p>data: {&#034;id&#034;: &#034;chatcmpl-1791388386688&#034;, &#034;object&#034;: &#034;chat.completion.chunk&#034;, &#034;choices&#034;: [{&#034;index&#034;: 0, &#034;delta&#034;: {&#034;content&#034;: &#034;\u597d\u7684&#034;}, &#034;finish_reason&#034;: null}]}<\/p>\n<p>data: {&#034;id&#034;: &#034;chatcmpl-1791388386688&#034;, &#034;object&#034;: &#034;chat.completion.chunk&#034;, &#034;choices&#034;: [{&#034;index&#034;: 0, &#034;delta&#034;: {&#034;content&#034;: &#034;&#xff0c;&#034;}, &#034;finish_reason&#034;: null}]}<\/p>\n<p>&#8230;<\/p>\n<p>data: [DONE]<\/p>\n<p>\u4ece\u6b64\u6211\u8111\u5b50\u91cc\u591a\u4e86\u4e00\u4e2a\u6307\u6807&#xff1a;TTFT&#xff08;Time To First Token&#xff0c;\u9996 token \u5ef6\u8fdf&#xff09;\u3002 \u5bf9\u7528\u6237\u4f53\u611f\u800c\u8a00&#xff0c;TTFT \u6bd4&#034;\u603b\u8017\u65f6&#034;\u91cd\u8981\u5f97\u591a\u2014\u20140.5s \u51fa\u7b2c\u4e00\u4e2a\u5b57&#xff0c;\u7528\u6237\u89c9\u5f97\u5f88\u5feb&#xff1b; 10s \u624d\u51fa\u7b2c\u4e00\u4e2a\u5b57&#xff0c;\u7528\u6237\u65e9\u5173\u9875\u9762\u4e86&#xff0c;\u54ea\u6015\u540e\u9762\u53ea\u82b1 2s\u3002<\/p>\n<p>\u6240\u4ee5\u7f51\u5173\u81ea\u5df1\u7ed9\u6bcf\u4e00\u6b21\u8bf7\u6c42\u90fd\u7b97\u4e86 TTFT \u4e0e TPS&#xff0c;\u5e76\u843d\u8fdb\u65e5\u5fd7\u3001\u5386\u53f2\u5e93\u3001Prometheus&#xff1a;<\/p>\n<p><span class=\"token punctuation\">{<\/span><br \/>\n  <span class=\"token string-property property\">&#034;api_key&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;sk-test-001&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;tier&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;free&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;model&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;qwen2.5:0.5b&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;status&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;wall_time&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0.057<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;stream&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">false<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;instance&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;gateway-8000&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;ttft&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0.012<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;tps&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">130.56<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;prompt_tokens&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">30<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;completion_tokens&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;gpu_util&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;gpu_mem_mb&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">472<\/span><span class=\"token punctuation\">,<\/span><br \/>\n  <span class=\"token string-property property\">&#034;time&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;2026-10-06T00:34:03&#034;<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<hr \/>\n<h3>5. \u9636\u6bb5 4&#xff1a;\u53ef\u89c2\u6d4b&#xff08;Prometheus &#043; Grafana&#xff09;&#xff0c;\u628a&#034;\u9ed1\u76d2&#034;\u53d8\u6210&#034;\u4eea\u8868\u76d8&#034;<\/h3>\n<h4>5.1 \u7f51\u5173\u66b4\u9732\u4e86\u4ec0\u4e48<\/h4>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> localhost:8000\/metrics <span class=\"token operator\">|<\/span> <span class=\"token function\">grep<\/span> <span class=\"token string\">&#039;^gateway_&#039;<\/span><\/p>\n<p>gateway_requests_total{model&#061;&#034;qwen2.5:0.5b&#034;,status&#061;&#034;200&#034;,stream&#061;&#034;false&#034;,tier&#061;&#034;pro&#034;} 5.0<br \/>\ngateway_requests_total{model&#061;&#034;unknown&#034;,status&#061;&#034;401&#034;,stream&#061;&#034;false&#034;,tier&#061;&#034;unknown&#034;} 427.0<br \/>\ngateway_request_duration_seconds_bucket{le&#061;&#034;1.0&#034;,model&#061;&#034;qwen2.5:0.5b&#034;,tier&#061;&#034;pro&#034;} 1.0<br \/>\ngateway_request_duration_seconds_bucket{le&#061;&#034;2.0&#034;,model&#061;&#034;qwen2.5:0.5b&#034;,tier&#061;&#034;pro&#034;} 4.0<br \/>\n&#8230;<\/p>\n<p>\u6211\u5728\u8fd9\u91cc\u7b2c\u4e00\u6b21\u771f\u6b63\u5206\u6e05\u4e86\u4e09\u7c7b\u6307\u6807&#xff08;\u5206\u6e05\u4e4b\u540e\u5c31\u4e0d\u4f1a\u4e71\u7528\u4e86&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u7c7b\u578b\u4f8b\u5b50\u80fd\u56de\u7b54\u4ec0\u4e48\u95ee\u9898<\/tr>\n<tbody>\n<tr>\n<td>Counter&#xff08;\u53ea\u589e&#xff09;<\/td>\n<td>gateway_requests_total<\/td>\n<td>\u4e00\u5171\u5904\u7406\u4e86\u591a\u5c11\u8bf7\u6c42&#xff1f;\u9519\u8bef\u7387\u591a\u5c11&#xff1f;<\/td>\n<\/tr>\n<tr>\n<td>Gauge&#xff08;\u53ef\u589e\u53ef\u51cf&#xff09;<\/td>\n<td>gateway_inflight_requests\u3001GPU \u5229\u7528\u7387<\/td>\n<td>\u73b0\u5728\u6709\u51e0\u4e2a\u8bf7\u6c42\u5728\u5904\u7406&#xff1f;<\/td>\n<\/tr>\n<tr>\n<td>Histogram&#xff08;\u76f4\u65b9\u56fe&#xff09;<\/td>\n<td>gateway_request_duration_seconds<\/td>\n<td>P50\/P95\/P99 \u5ef6\u8fdf\u662f\u591a\u5c11&#xff1f;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>5.2 \u4e09\u6761\u6211\u5929\u5929\u7528\u7684 PromQL<\/h4>\n<p># \u6bcf\u79d2\u8bf7\u6c42\u6570&#xff08;\u6309\u72b6\u6001\u7801&#xff09;<br \/>\nsum(rate(gateway_requests_total[1m])) by (status)<\/p>\n<p># \u6b63\u5728\u5904\u7406\u4e2d\u7684\u8bf7\u6c42\u6570&#xff08;\u770b\u6392\u961f\u60c5\u51b5&#xff09;<br \/>\ngateway_inflight_requests<\/p>\n<p># P95 \u5ef6\u8fdf&#xff08;\u7528\u76f4\u65b9\u56fe\u7b97\u5206\u4f4d\u6570&#xff09;<br \/>\nhistogram_quantile(0.95, sum(rate(gateway_request_duration_seconds_bucket[5m])) by (le))<\/p>\n<h4>5.3 \u770b\u677f&#xff1a;5 \u884c 35 \u4e2a\u9762\u677f<\/h4>\n<p>\u4eea\u8868\u76d8\u6309&#034;\u6211\u4f1a\u600e\u4e48\u6392\u969c&#034;\u5206\u884c&#xff0c;\u800c\u4e0d\u662f\u6309\u6307\u6807\u540d\u5b57\u6392&#xff1a;<\/p>\n<p>\u603b\u89c8<br \/>\n\u8def\u7531\u3001\u9650\u6d41\u4e0e\u9519\u8bef<br \/>\nGPU \u4e0e\u8fdb\u7a0b\u8d44\u6e90<br \/>\nOllama \u53c2\u6570\u8c03\u4f18 (num_ctx \/ num_gpu \/ num_batch)<br \/>\n\u5206\u5e03\u5f0f\u57fa\u7840\u8bbe\u65bd (Redis \u9650\u6d41 \/ SQLite\u00b7Postgres \u5386\u53f2 \/ Kafka \u4e8b\u4ef6)<\/p>\n<p>\u542f\u52a8\u53ea\u8981\u4e24\u6761\u547d\u4ee4&#xff1a;<\/p>\n<p><span class=\"token function\">bash<\/span> scripts\/install_monitoring.sh    <span class=\"token comment\"># \u4e0b\u8f7d Prometheus &#043; Grafana&#xff08;\u514d root&#xff09;<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/monitoring.sh start      <span class=\"token comment\"># \u8d77\u670d\u52a1&#xff0c;\u81ea\u52a8\u914d\u6570\u636e\u6e90\u4e0e\u770b\u677f<\/span><br \/>\n<span class=\"token comment\"># Prometheus: http:\/\/localhost:9090\/targets    Grafana: http:\/\/localhost:3000\/d\/llm-gateway<\/span><\/p>\n<p>Prometheus \u6bcf 5 \u79d2\u6293\u4e00\u6b21 \/metrics&#xff1b;GPU \u5229\u7528\u7387\u4e0e\u663e\u5b58\u5219\u662f\u7f51\u5173\u5361\u6bcf 5 \u79d2\u8c03\u4e00\u6b21 nvidia-smi \u8bb0\u4e0b\u6765\u7684 &#xff08;1050Ti \u4e0d\u652f\u6301 DCGM \u90a3\u5957&#xff0c;nvidia-smi \u662f\u552f\u4e00\u9009\u62e9&#xff0c;\u591f\u7528&#xff09;\u3002<\/p>\n<p>\u8fd9\u91cc\u57cb\u4e86\u540e\u9762\u7b2c 10 \u8282\u7684\u4e00\u4e2a\u5927\u5751&#xff1a;\u591a\u5b9e\u4f8b\u90e8\u7f72\u65f6&#xff0c;\/metrics \u5fc5\u987b\u76f4\u8fde\u6bcf\u4e2a\u5b9e\u4f8b\u53bb\u6293&#xff0c; \u4e0d\u80fd\u8d70\u8d1f\u8f7d\u5747\u8861\u2014\u2014\u5426\u5219\u6240\u6709\u5b9e\u4f8b\u7684\u6837\u672c instance \u6807\u7b7e\u90fd\u4f1a\u53d8\u6210 nginx \u7684\u5730\u5740&#xff0c;\u66f2\u7ebf\u5168\u7cca\u5728\u4e00\u8d77\u3002<\/p>\n<hr \/>\n<h3>6. \u9636\u6bb5 5&#xff1a;\u5206\u5e03\u5f0f\u9650\u6d41\u2014\u2014\u672c\u9879\u76ee\u7684&#034;\u62db\u724c\u5b9e\u9a8c&#034;<\/h3>\n<h4>6.1 \u4e3a\u4ec0\u4e48\u5355\u673a\u9650\u6d41\u4e0d\u591f<\/h4>\n<p>\u6211\u4e00\u5f00\u59cb\u7684\u9650\u6d41\u662f\u7eaf\u5185\u5b58\u5b57\u5178&#xff1a;\u6bcf\u4e2a key \u6bcf\u5206\u949f\u6700\u591a N \u6b21\u3002 \u5355\u5b9e\u4f8b\u6ca1\u95ee\u9898&#xff0c;\u4e00\u65e6\u8d77\u4e24\u4e2a\u5b9e\u4f8b\u5c31\u5e9f\u4e86\u2014\u2014\u7528\u6237\u5728\u6bcf\u4e2a\u5b9e\u4f8b\u5404\u62ff\u4e00\u4efd\u989d\u5ea6&#xff0c;\u603b\u989d\u5ea6\u88ab\u653e\u5927\u6210 2 \u500d\u3002 \u6240\u4ee5\u9650\u6d41\u72b6\u6001\u5fc5\u987b\u5916\u7f6e\u5230 Redis&#xff0c;\u4e24\u4e2a\u8fdb\u7a0b\u5171\u4eab\u540c\u4e00\u4e2a\u8ba1\u6570\u5668&#xff1a;<\/p>\n<p>\u5ba2\u6237\u7aef A \u2500\u2500\u2510<br \/>\n           \u251c\u2500\u25ba \u7f51\u5173\u5b9e\u4f8b :8000 \u2500\u2500\u2510<br \/>\n\u5ba2\u6237\u7aef B \u2500\u2500\u2518                    \u251c\u2500\u25ba Redis&#xff08;\u56fa\u5b9a\u7a97\u53e3 &#043; Lua \u539f\u5b50\u8ba1\u6570&#xff09;<br \/>\n           \u250c\u2500\u25ba \u7f51\u5173\u5b9e\u4f8b :8001 \u2500\u2500\u2518<br \/>\n\u5ba2\u6237\u7aef C \u2500\u2500\u2518<\/p>\n<p>\u8ba1\u6570\u7528 Lua \u811a\u672c\u5b9e\u73b0&#034;\u8bfb-\u5224\u65ad-\u81ea\u589e-\u9996\u6b21\u8bbe\u7f6e\u8fc7\u671f&#034;\u7684\u539f\u5b50\u64cd\u4f5c&#xff0c;\u907f\u514d\u5e76\u53d1\u7ade\u6001\u3002<\/p>\n<h4>6.2 \u62db\u724c\u5b9e\u9a8c&#xff1a;\u4e24\u53f0\u5b9e\u4f8b\u5171\u4eab 10 \u6b21\u989d\u5ea6<\/h4>\n<p>free \u6863\u914d\u989d 10 \u6b21\/\u5206\u949f\u3002\u540c\u65f6\u8d77\u4e24\u4e2a\u5b9e\u4f8b&#xff0c;\u7528\u540c\u4e00\u4e2a key \u5404\u53d1 6 \u6b21&#xff1a;<\/p>\n<p><span class=\"token comment\"># redis \u6a21\u5f0f&#xff1a;\u5148\u6e05\u7a7a\u8ba1\u6570<\/span><br \/>\nredis-cli <span class=\"token parameter variable\">-p<\/span> <span class=\"token number\">6379<\/span> FLUSHDB<br \/>\n<span class=\"token keyword\">for<\/span> <span class=\"token for-or-select variable\">i<\/span> <span class=\"token keyword\">in<\/span> <span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">seq<\/span> <span class=\"token number\">1<\/span> <span class=\"token number\">6<\/span><span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span> hit <span class=\"token number\">8000<\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">done<\/span>   <span class=\"token comment\"># \u5b9e\u4f8b A<\/span><br \/>\n<span class=\"token keyword\">for<\/span> <span class=\"token for-or-select variable\">i<\/span> <span class=\"token keyword\">in<\/span> <span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">seq<\/span> <span class=\"token number\">1<\/span> <span class=\"token number\">6<\/span><span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span> hit <span class=\"token number\">8001<\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">done<\/span>   <span class=\"token comment\"># \u5b9e\u4f8b B<\/span><\/p>\n<p>\u5b9e\u6d4b\u7ed3\u679c&#xff08;redis \u6a21\u5f0f&#xff09;&#xff1a;<\/p>\n<p>200 200 200 200 200 200   &lt;- \u5b9e\u4f8bA&#xff08;6 \u6b21\u5168\u8fc7&#xff0c;Redis \u8ba1\u6570\u5230 6&#xff09;<br \/>\n200 200 200 200 429 429   &lt;- \u5b9e\u4f8bB&#xff08;\u5168\u5c40\u53ea\u5269 4 \u4e2a\u540d\u989d&#xff0c;\u540e 2 \u6b21\u88ab\u62d2&#xff09;<br \/>\n                  \u2191 \u5408\u8ba1\u653e\u884c 10 \u6b21 &#061; \u914d\u989d<\/p>\n<p>\u53cd\u8bc1&#xff08;\u8fd9\u4e00\u6b65\u6700\u91cd\u8981&#xff09;&#xff1a;\u628a\u4e24\u4e2a\u5b9e\u4f8b\u90fd\u6539\u6210 RATE_LIMIT_BACKEND&#061;memory \u91cd\u8dd1&#xff1a;<\/p>\n<p>200 200 200 200 200 200   &lt;- \u5b9e\u4f8bA&#xff08;\u81ea\u5df1\u6570\u5230 6&#xff09;<br \/>\n200 200 200 200 200 200   &lt;- \u5b9e\u4f8bB&#xff08;\u81ea\u5df1\u6570\u5230 6&#xff09;<br \/>\n                  \u2191 \u5408\u8ba1 12 \u6b21&#xff0c;\u989d\u5ea6\u88ab\u653e\u5927 \u2192 \u53ea\u80fd\u5355\u5b9e\u4f8b\u7528<\/p>\n<p>\u4e00\u4e2a 10\u3001\u4e00\u4e2a 12&#xff0c;\u8fd9\u624d\u53eb&#034;\u8bc1\u660e\u4e86\u5171\u4eab\u786e\u5b9e\u751f\u6548&#034;&#xff0c;\u800c\u4e0d\u662f&#034;\u6570\u5b57\u78b0\u5de7\u5bf9\u4e0a\u4e86&#034;\u3002<\/p>\n<h4>6.3 \u6211\u5b9e\u6d4b\u7684\u54cd\u5e94\u5934\u4e0e Redis key<\/h4>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-sD<\/span> &#8211; <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8080\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-test-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;hi&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:2}}&#039;<\/span><\/p>\n<p>HTTP\/1.1 200 OK<br \/>\nx-ratelimit-limit: 10<br \/>\nx-ratelimit-remaining: 9<br \/>\nx-ratelimit-reset: 60<br \/>\nx-gateway-instance: gateway-8000        \u2190 \u591a\u5b9e\u4f8b\u65f6\u7528\u6765\u786e\u8ba4\u843d\u5728\u54ea\u4e2a\u5b9e\u4f8b<\/p>\n<p>redis-cli <span class=\"token parameter variable\">-p<\/span> <span class=\"token number\">6379<\/span> KEYS <span class=\"token string\">&#039;gw:rl:*&#039;<\/span><br \/>\n<span class=\"token comment\"># gw:rl:free:sk-test-001:29856467       \u2190 \u7a97\u53e3\u53f7 &#061; epoch \u5206\u949f&#xff0c;\u5929\u7136\u5e26\u8fc7\u671f\u8bed\u4e49<\/span><\/p>\n<h4>6.4 fail-open \u8fd8\u662f fail-closed&#xff1f;&#xff08;\u4e00\u4e2a\u771f\u5b9e\u7684\u5de5\u7a0b\u53d6\u820d&#xff09;<\/h4>\n<table>\n<tr>\u914d\u7f6eRedis \u6302\u6389\u65f6\u9002\u5408\u573a\u666f<\/tr>\n<tbody>\n<tr>\n<td>auto &#043; REDIS_FAIL_OPEN&#061;1&#xff08;\u6211\u7528\u7684\u9ed8\u8ba4&#xff09;<\/td>\n<td>\u8be5\u5b9e\u4f8b\u5207\u8fdb\u7a0b\u5185\u8ba1\u6570&#xff0c;\u8bf7\u6c42\u7ee7\u7eed\u653e\u884c<\/td>\n<td>\u53ef\u7528\u6027\u4f18\u5148<\/td>\n<\/tr>\n<tr>\n<td>redis &#043; REDIS_FAIL_OPEN&#061;0<\/td>\n<td>\u6bcf\u4e2a\u8bf7\u6c42\u76f4\u63a5 429<\/td>\n<td>\u914d\u989d\/\u6210\u672c\u4f18\u5148&#xff08;\u6309\u91cf\u8ba1\u8d39\u7684\u5bf9\u5916 API&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>memory<\/td>\n<td>\u6c38\u8fdc\u7528\u8fdb\u7a0b\u5185\u8ba1\u6570<\/td>\n<td>\u5355\u5b9e\u4f8b\u90e8\u7f72 \/ \u505a\u5bf9\u7167\u5b9e\u9a8c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6211\u7684\u9009\u62e9\u7406\u7531&#xff1a;\u9650\u6d41\u662f\u4fdd\u62a4\u63aa\u65bd&#xff0c;\u4e0d\u662f\u4e1a\u52a1\u529f\u80fd\u3002 \u5982\u679c\u56e0\u4e3a\u5b83\u6302\u6389\u5c31\u628a\u6240\u6709\u63a8\u7406\u8bf7\u6c42\u5168\u62d2\u4e86&#xff0c;\u7b49\u4e8e&#034;\u4e3a\u4e86\u9632\u6d2a\u6c34\u628a\u6c34\u9f99\u5934\u5168\u5173\u4e86&#034;\u3002 \u8fd9\u4e2a\u53d6\u820d\u540e\u6765\u5728\u7b2c 8 \u8282\u88ab\u9a8c\u8bc1\u662f\u5bf9\u7684\u3002<\/p>\n<hr \/>\n<h3>7. \u9636\u6bb5 6~7&#xff1a;\u8bf7\u6c42\u5386\u53f2&#xff08;SQLite\/PG&#xff09;\u4e0e Kafka \u5f02\u6b65\u4e8b\u4ef6<\/h3>\n<h4>7.1 \u4e3a\u4ec0\u4e48\u4e0d\u53ea\u7528\u65e5\u5fd7\u6587\u4ef6<\/h4>\n<p>\u4e00\u5f00\u59cb\u6211\u53ea\u5199 logs\/gateway.log&#xff08;\u4e00\u884c\u4e00\u4e2a JSON&#xff09;\u3002\u95ee\u9898\u5f88\u5feb\u5c31\u6765\u4e86&#xff1a;<\/p>\n<ul>\n<li>\u6211\u60f3\u77e5\u9053&#034;\u8fc7\u53bb\u4e00\u5c0f\u65f6 P95 TTFT \u662f\u591a\u5c11&#034;\u2014\u2014\u96be\u9053\u8981 grep &#043; awk \u7b97\u4e00\u904d&#xff1f;<\/li>\n<li>\u6211\u60f3\u77e5\u9053&#034;\u54ea\u4e2a key \u7528\u5f97\u6700\u591a&#034;\u2014\u2014\u65e5\u5fd7\u91cc\u641c\u5f97\u773c\u775b\u75bc\u3002<\/li>\n<\/ul>\n<p>\u6240\u4ee5\u52a0\u4e86\u4e00\u4e2a\u5386\u53f2\u5c42&#xff1a;\u6bcf\u6b21\u8bf7\u6c42\u4e00\u6761\u7ed3\u6784\u5316\u8bb0\u5f55\u5165\u5e93&#xff0c;\u540c\u65f6\u63d0\u4f9b \/history \u548c \/history\/stats \u4e24\u4e2a\u67e5\u8be2\u7aef\u70b9\u3002\u9ed8\u8ba4 SQLite&#xff08;\u96f6\u4f9d\u8d56&#xff0c;\u672c\u5730\u6587\u4ef6&#xff09;&#xff0c;\u8bbe\u4e86 DATABASE_URL&#061;postgresql:\/\/&#8230; \u5c31\u5207\u5230 PostgreSQL&#xff0c;\u4ee3\u7801\u4e00\u884c\u4e0d\u7528\u6539\u3002<\/p>\n<p>\u5199\u8def\u5f84\u662f\u5f02\u6b65\u6279\u91cf\u7684&#xff0c;\u8fd9\u662f\u6211\u7b2c\u4e00\u6b21\u8ba4\u771f\u505a&#034;\u5199\u653e\u5927&#034;\u7684\u4f18\u5316&#xff1a;<\/p>\n<p>\u8bf7\u6c42\u7ebf\u7a0b&#xff1a;record \u2192 \u5165\u961f&#xff08;\u4e0d\u963b\u585e&#xff0c;\u961f\u5217\u6ee1\u5c31\u4e22\u5f03\u5e76\u8ba1\u6570&#xff09;<br \/>\n\u540e\u53f0\u4efb\u52a1&#xff1a;\u6512\u591f 20 \u884c \u6216 \u7b49\u6ee1 1.0 \u79d2 \u2192 \u4e00\u6b21\u6279\u91cf INSERT<\/p>\n<p>\u8bf7\u6c42\u9ad8\u5cf0\u671f\u5982\u679c\u6bcf\u6761\u8bb0\u5f55\u90fd\u540c\u6b65\u5199\u5e93&#xff0c;\u5149\u662f fsync \u5c31\u80fd\u628a\u5ef6\u8fdf\u62d6\u57ae\u3002\u6279\u91cf\u4e4b\u540e\u5199\u5e93\u5b8c\u5168\u5728\u65c1\u8def\u3002<\/p>\n<p>\/infra \u7aef\u70b9&#xff08;\u6211\u6392\u969c\u65f6\u6700\u5e38\u6253\u5f00\u7684\u4e00\u4e2a&#xff09;\u80fd\u770b\u5230\u961f\u5217\u79ef\u538b\u548c\u5b9e\u9645\u751f\u6548\u7684\u540e\u7aef&#xff1a;<\/p>\n<p><span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token string-property property\">&#034;instance&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;gateway-8000&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;rate_limit&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;mode&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;auto&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;active&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;redis&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;redis_up&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">true<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                   <span class=\"token string-property property\">&#034;window_seconds&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">60<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;fail_open&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">true<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;error&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;history&#034;<\/span><span class=\"token operator\">:<\/span>    <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;enabled&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">true<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;backend&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;sqlite&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;queued&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;written&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">4322<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                   <span class=\"token string-property property\">&#034;batch_size&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">20<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;flush_interval&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;error&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;events&#034;<\/span><span class=\"token operator\">:<\/span>     <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;enabled&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">true<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;connected&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token boolean\">true<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;topic&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;llm-gateway-events&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                   <span class=\"token string-property property\">&#034;queued&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;published&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">4322<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;failed&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                   <span class=\"token string-property property\">&#034;fallback_written&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;dropped&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;error&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;&#034;<\/span><span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p>\u805a\u5408\u67e5\u8be2\u4e5f\u662f\u73b0\u6210\u7684&#xff1a;<\/p>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token string\">&#039;localhost:8000\/history\/stats&#039;<\/span><\/p>\n<p><span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token string-property property\">&#034;window_seconds&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">3600<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string-property property\">&#034;totals&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;requests&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">6507<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;avg_wall_time&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">1.97<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;avg_ttft&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">0.983<\/span><span class=\"token punctuation\">,<\/span><br \/>\n               <span class=\"token string-property property\">&#034;avg_tps&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">140.93<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;p95_wall_time&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">6.244<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;p95_ttft&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">5.69<\/span><span class=\"token punctuation\">}<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p>\u4e3a\u4ec0\u4e48 P95 \u548c\u5e73\u5747\u503c\u5dee\u8fd9\u4e48\u591a&#xff1f;\u56e0\u4e3a\u6211\u628a\u53c2\u6570\u626b\u63cf\u3001\u5e76\u53d1\u538b\u6d4b\u7684\u6570\u636e\u4e5f\u7b97\u8fdb\u53bb\u4e86&#xff0c; \u91cc\u9762\u6df7\u7740&#034;\u51b7\u542f\u52a8 2.1s&#034;\u548c&#034;8 \u5e76\u53d1\u6392\u961f&#034;\u7684\u6837\u672c\u3002\u8fd9\u4e5f\u63d0\u9192\u6211&#xff1a;\u505a\u6027\u80fd\u7ed3\u8bba\u524d\u4e00\u5b9a\u8981\u5148\u770b\u5206\u5e03&#xff0c; \u5e73\u5747\u503c\u4f1a\u9a97\u4eba\u3002<\/p>\n<p>\u5386\u53f2\u8868\u91cc\u8fd8\u4e13\u95e8\u5b58\u4e86 num_ctx \/ num_gpu \/ num_batch \u4e09\u4e2a\u53c2\u6570\u6807\u7b7e&#xff0c; \u8fd9\u6837&#034;\u540c\u4e00\u6761 prompt \u5728\u4e0d\u540c\u53c2\u6570\u4e0b TTFT \u662f\u591a\u5c11&#034;\u53ef\u4ee5\u6309\u6807\u7b7e\u5206\u7ec4\u67e5\u2014\u2014\u7b2c 9 \u8282\u7684\u8c03\u4f18\u8868\u5c31\u662f\u8fd9\u4e48\u6765\u7684\u3002<\/p>\n<h4>7.2 Kafka&#xff1a;\u4e3a\u4ec0\u4e48\u8fd8\u8981\u4e00\u6761&#034;\u4e8b\u4ef6\u6d41&#034;<\/h4>\n<p>\u6570\u636e\u5e93\u662f&#034;\u4e8b\u540e\u67e5&#034;&#xff0c;Kafka \u662f&#034;\u5b9e\u65f6\u63a8&#034;\u3002\u6211\u628a\u8bf7\u6c42\u751f\u547d\u5468\u671f\u505a\u6210 4 \u7c7b\u4e8b\u4ef6&#xff1a;<\/p>\n<p>request.completed     \u8bf7\u6c42\u6210\u529f<br \/>\nrequest.rate_limited  \u88ab\u9650\u6d41<br \/>\nrequest.auth_failed   \u9274\u6743\u5931\u8d25<br \/>\nrequest.error         \u4e0a\u6e38\/\u8bf7\u6c42\u9519\u8bef<\/p>\n<p>\u8fd9\u6837\u5c31\u80fd\u63a5\u51fa\u5f88\u591a\u73a9\u6cd5&#xff1a;\u5b9e\u65f6\u8ba1\u8d39\u3001\u5f02\u5e38\u544a\u8b66\u3001\u7528\u91cf\u770b\u677f\u3001\u628a\u8bf7\u6c42\u660e\u7ec6\u540c\u6b65\u5230\u6570\u636e\u4ed3\u5e93\u2026\u2026 \u800c\u4e14\u7f51\u5173\u4e0d\u9700\u8981\u77e5\u9053\u4e0b\u6e38\u662f\u8c01&#xff0c;\u53ea\u8d1f\u8d23\u5f80 topic \u91cc\u4e22\u4e8b\u4ef6&#xff0c;\u5f7b\u5e95\u89e3\u8026\u3002<\/p>\n<p>\u4e8b\u4ef6\u4fe1\u5c01\u957f\u8fd9\u6837&#xff08;instance \u5b57\u6bb5\u662f\u591a\u5b9e\u4f8b\u6392\u969c\u7684\u5173\u952e&#xff09;&#xff1a;<\/p>\n<p><span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;event&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;request.completed&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;ts&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">1791041576.094<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;ts_iso&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;2026-10-03T23:32:56&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;instance&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;gateway-degrade&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;host&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;dawson-RESCUER-R720-15IKBN&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;data&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;api_key&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;sk-prod-001&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;tier&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;pro&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;model&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;qwen2.5:0.5b&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n          <span class=\"token string-property property\">&#034;status&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;wall_time&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">2.893<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;ttft&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">2.832<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;tps&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">113.19<\/span><span class=\"token punctuation\">,<\/span><br \/>\n          <span class=\"token string-property property\">&#034;instance&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;gateway-degrade&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-property property\">&#034;gpu_mem_mb&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token number\">12<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;fallback_reason&#034;<\/span><span class=\"token operator\">:<\/span> <span class=\"token string\">&#034;UnknownTopicOrPartitionError: [Error3] UnknownTopicOrPartitionError&#034;<\/span><span class=\"token punctuation\">}<\/span><\/p>\n<p>\u6ce8\u610f\u6700\u540e\u90a3\u4e2a fallback_reason&#xff1a;\u8fd9\u6761\u4e8b\u4ef6\u5176\u5b9e\u662f\u964d\u7ea7\u843d\u76d8\u7684\u4ea7\u7269 &#xff08;Kafka \u91cc\u8fd8\u6ca1\u6709\u90a3\u4e2a topic&#xff0c;\u6295\u9012\u5931\u8d25 \u2192 \u5199\u4e86\u672c\u5730 logs\/events.jsonl&#xff09;\u3002 \u6211\u7279\u610f\u628a\u5931\u8d25\u539f\u56e0\u4e5f\u5199\u8fdb\u53bb&#xff0c;\u5c31\u662f\u4e3a\u4e86\u7b2c\u4e8c\u5929\u6392\u969c\u65f6\u4e0d\u7528\u731c\u3002<\/p>\n<hr \/>\n<h3>8. \u9636\u6bb5 8&#xff1a;\u964d\u7ea7\u6f14\u7ec3&#xff08;\u6211\u8ba4\u4e3a\u6700\u503c\u94b1\u7684\u4e00\u7ae0&#xff09;<\/h3>\n<p>\u524d\u9762\u6240\u6709\u6a21\u5757\u90fd\u5199\u4e86&#034;\u5c3d\u529b\u800c\u4e3a&#034;&#xff0c;\u4f46\u6ca1\u6f14\u7ec3\u8fc7\u5c31\u4e0d\u7b97\u4f1a\u3002\u6240\u4ee5\u6211\u505a\u4e86\u4e00\u8f6e&#034;\u9010\u4e2a\u6253\u6302&#034;&#xff1a;<\/p>\n<table>\n<tr>\u6f14\u7ec3\u64cd\u4f5c\u8bf7\u6c42\u8fd8\u80fd 200 \u5417\u89c2\u6d4b\u5230\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>\u6253\u6302 Kafka<\/td>\n<td>\u505c broker<\/td>\n<td>\u2705 \u5168\u90e8 200<\/td>\n<td>\u4e8b\u4ef6\u6539\u5199 logs\/events.jsonl&#xff0c;\/infra \u91cc connected:false\u3001fallback_written \u589e\u957f<\/td>\n<\/tr>\n<tr>\n<td>\u6062\u590d Kafka<\/td>\n<td>\u91cd\u542f broker<\/td>\n<td>\u2705<\/td>\n<td>\u7ea6 10 \u79d2\u5185\u81ea\u52a8\u91cd\u8fde&#xff0c;published \u7ee7\u7eed\u589e\u957f<\/td>\n<\/tr>\n<tr>\n<td>\u6253\u6302 Redis<\/td>\n<td>kill $(cat redis.pid)<\/td>\n<td>\u2705 \u5168\u90e8 200<\/td>\n<td>\/health \u7684 rate_limit_backend \u4ece redis \u53d8 memory&#xff1b;gateway_ratelimit_backend_up 0<\/td>\n<\/tr>\n<tr>\n<td>\u6062\u590d Redis<\/td>\n<td>\u91cd\u542f<\/td>\n<td>\u2705<\/td>\n<td>5 \u79d2\u5185\u81ea\u52a8\u5207\u56de redis&#xff0c;\u989d\u5ea6\u91cd\u65b0\u5168\u5c40\u5171\u4eab<\/td>\n<\/tr>\n<tr>\n<td>\u6253\u6302 PostgreSQL<\/td>\n<td>pg_ctl stop<\/td>\n<td>\u2705 \u5168\u90e8 200<\/td>\n<td>gateway_history_up 0\u3001gateway_history_write_errors_total \u589e\u957f&#xff0c;\u53ea\u4e22\u5386\u53f2<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5176\u4e2d&#034;\u6253\u6302 Redis \u540e\u8fde\u53d1 12 \u6b21&#034;\u8fd9\u4e2a\u5b9e\u9a8c\u6700\u76f4\u89c2&#xff1a;<\/p>\n<p>200 200 200 200 200 200 200 200 200 200 429 429<br \/>\n                                       \u2191 \u9650\u6d41\u4ecd\u7136\u751f\u6548&#xff0c;\u53ea\u662f\u989d\u5ea6\u4e0d\u518d\u5168\u5c40\u5171\u4eab<\/p>\n<p>\u964d\u7ea7 \u2260 \u5931\u6548\u3002Redis \u6ca1\u4e86&#xff0c;\u9650\u6d41\u9000\u5316\u6210&#034;\u5355\u5b9e\u4f8b\u9650\u6d41&#034;&#xff0c;\u4ecd\u7136\u5728\u4fdd\u62a4 GPU&#xff1b; \u5982\u679c\u5f53\u521d\u9009\u4e86 fail-closed&#xff0c;\u8fd9\u91cc\u5c31\u4f1a\u53d8\u6210 12 \u4e2a 429&#xff0c;\u7528\u6237\u76f4\u63a5\u9a82\u4eba\u3002<\/p>\n<p>\u8fd8\u6709\u4e00\u4e2a\u5c0f\u7ec6\u8282\u6211\u633a\u5f97\u610f&#xff1a;\u5386\u53f2\u5e93\u5199\u5931\u8d25\u540e\u4e0d\u9700\u8981\u91cd\u542f\u7f51\u5173\u3002 \u4ee3\u7801\u5728\u5199\u5931\u8d25\u65f6\u4f1a\u628a\u574f\u8fde\u63a5\u5173\u6389&#xff0c;\u4e0b\u4e00\u6279 INSERT \u4f1a\u7528\u65b0\u8fde\u63a5\u91cd\u8bd5&#xff0c;\u6240\u4ee5 PG \u6062\u590d\u540e\u81ea\u52a8\u7eed\u4e0a\u3002<\/p>\n<p>\u8fd9\u7ae0\u8ba9\u6211\u5efa\u7acb\u4e86\u6574\u5957\u9879\u76ee\u7684\u6838\u5fc3\u539f\u5219&#xff1a;<\/p>\n<p>\u89c2\u6d4b\u662f\u589e\u5f3a&#xff0c;\u4e0d\u662f\u4f9d\u8d56&#xff1b;\u63a8\u7406\u4e3b\u94fe\u8def\u6c38\u8fdc\u4f18\u5148\u3002<\/p>\n<hr \/>\n<h3>9. \u9636\u6bb5 9&#xff1a;\u81ea\u52a8\u5316\u9a8c\u6536 &#043; 1050Ti \u53c2\u6570\u8c03\u4f18\u5b9e\u9a8c<\/h3>\n<h4>9.1 \u4e00\u952e\u9a8c\u6536&#xff1a;21 \u9879 &#043; 34 \u9879<\/h4>\n<p>\u624b\u52a8\u70b9 30 \u6b21 curl \u592a\u8822&#xff0c;\u6240\u4ee5\u6211\u5199\u4e86\u4e24\u4e2a\u9a8c\u6536\u811a\u672c&#xff1a;<\/p>\n<p>python verify_infra.py                    <span class=\"token comment\"># \u57fa\u7840\u8bbe\u65bd 21 \u9879&#xff08;\u8981\u4e0d\u8981 Redis\/PG\/Kafka&#xff09;<\/span><br \/>\npython verify_cluster.py                  <span class=\"token comment\"># \u591a\u5b9e\u4f8b &#043; nginx 34 \u9879<\/span><\/p>\n<p>verify_infra.py \u4f1a\u81ea\u5df1\u62c9\u8d77\u4e24\u4e2a\u7f51\u5173\u5b9e\u4f8b&#xff0c;\u628a&#034;\u5171\u4eab\u9650\u6d41 \/ \u53cc\u540e\u7aef\u5386\u53f2 \/ Kafka \u56db\u7c7b\u4e8b\u4ef6 \/ \u964d\u7ea7\u884c\u4e3a&#034;\u5168\u8dd1\u4e00\u904d&#xff0c;\u6700\u540e\u6253\u5370 21\/21 \u9879\u901a\u8fc7&#xff1b; verify_cluster.py \u4f1a\u4e34\u65f6\u5728 :8100~8102 \u8d77 3 \u4e2a\u5b9e\u4f8b &#043; \u4e00\u4e2a\u72ec\u7acb\u7aef\u53e3\u7684 nginx&#xff08;\u4e0d\u6253\u6270\u4f60\u6b63\u5728\u8dd1\u7684 8080&#xff09;&#xff0c; \u628a\u8d1f\u8f7d\u5747\u8861\u3001SSE\u3001\u6545\u969c\u8f6c\u79fb\u5168\u9a8c\u4e00\u904d\u3002<\/p>\n<p>\u8fd9\u662f\u6211\u7684\u7b2c\u4e00\u6b21&#034;\u57fa\u7840\u8bbe\u65bd\u5373\u4ee3\u7801&#034;\u4f53\u9a8c&#xff1a;\u628a&#034;\u6211\u4ee5\u4e3a\u662f\u5bf9\u7684&#034;\u53d8\u6210&#034;\u811a\u672c\u8bc1\u660e\u662f\u5bf9\u7684&#034;\u3002<\/p>\n<h4>9.2 \u53c2\u6570\u8c03\u4f18\u5b9e\u9a8c&#xff1a;13 \u7ec4\u914d\u7f6e\u626b\u51fa\u4e00\u5f20\u8868<\/h4>\n<p>\u8fd9\u662f\u6211\u89c9\u5f97\u6700\u597d\u73a9\u3001\u4e5f\u6700\u6709\u6536\u83b7\u7684\u4e00\u6b65\u3002\u5199\u4e86\u4e2a tuning_experiment.py&#xff0c; \u901a\u8fc7\u7f51\u5173\u5bf9 Ollama \u7684\u4e09\u4e2a\u5173\u952e\u53c2\u6570\u505a\u53d7\u63a7\u626b\u63cf&#xff1a;<\/p>\n<ul>\n<li>num_ctx&#xff1a;\u4e0a\u4e0b\u6587\u7a97\u53e3\u5927\u5c0f&#xff08;\u51b3\u5b9a KV cache \u5360\u591a\u5c11\u663e\u5b58&#xff09;<\/li>\n<li>num_gpu&#xff1a;\u591a\u5c11\u5c42\u653e\u5230 GPU&#xff08;1050Ti \u53ea\u6709 4GB&#xff0c;\u653e\u4e0d\u4e0b\u5c31\u5f97\u5f80 CPU \u4e22&#xff09;<\/li>\n<li>num_batch&#xff1a;\u6279\u5904\u7406\u5927\u5c0f&#xff08;prompt \u5904\u7406\u9636\u6bb5\u7684\u5e76\u884c\u5ea6&#xff09;<\/li>\n<\/ul>\n<p>\u5b9e\u9a8c\u8bbe\u8ba1\u5f88\u8bb2\u7a76&#xff1a;\u56fa\u5b9a 208 token \u7684\u82f1\u6587 prompt\u3001num_predict&#061;64\u3001temperature&#061;0&#xff0c; \u6bcf\u7ec4\u5148\u8dd1 1 \u6b21 warmup&#xff08;\u51b7\u52a0\u8f7d&#xff0c;\u4e0d\u8ba1\u5165\u7edf\u8ba1&#xff09;\u518d\u6d4b 3 \u6b21&#xff0c;\u540c\u65f6\u8bb0\u5f55\u7aef\u5230\u7aef TTFT\u3001 \u7f51\u5173\u5185\u90e8 TTFT\u3001tokens\/s \u548c\u663e\u5b58\u3002\u603b\u5171 52 \u4e2a\u8bf7\u6c42&#xff0c;\u5168\u90e8\u8dd1\u5b8c\u7ea6 109 \u79d2\u3002<\/p>\n<p>\u5b9e\u6d4b\u7ed3\u679c&#xff08;qwen2.5:0.5b&#xff0c;GTX 1050 Ti 4GB&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u914d\u7f6e\u7f51\u5173 TTFT (s)tokens\/s\u663e\u5b58 (MiB)<\/tr>\n<tbody>\n<tr>\n<td>num_ctx&#061;512<\/td>\n<td align=\"right\">0.014<\/td>\n<td align=\"right\">192.88<\/td>\n<td align=\"right\">472<\/td>\n<\/tr>\n<tr>\n<td>num_ctx&#061;1024<\/td>\n<td align=\"right\">0.017<\/td>\n<td align=\"right\">184.29<\/td>\n<td align=\"right\">478<\/td>\n<\/tr>\n<tr>\n<td>num_ctx&#061;2048<\/td>\n<td align=\"right\">0.022<\/td>\n<td align=\"right\">182.04<\/td>\n<td align=\"right\">490<\/td>\n<\/tr>\n<tr>\n<td>num_ctx&#061;4096<\/td>\n<td align=\"right\">0.013<\/td>\n<td align=\"right\">183.92<\/td>\n<td align=\"right\">516<\/td>\n<\/tr>\n<tr>\n<td>num_ctx&#061;8192<\/td>\n<td align=\"right\">0.014<\/td>\n<td align=\"right\">171.66<\/td>\n<td align=\"right\">568<\/td>\n<\/tr>\n<tr>\n<td>num_gpu&#061;0&#xff08;\u7eaf CPU&#xff09;<\/td>\n<td align=\"right\">0.079<\/td>\n<td align=\"right\">71.17<\/td>\n<td align=\"right\">2<\/td>\n<\/tr>\n<tr>\n<td>num_gpu&#061;12&#xff08;24 \u5c42\u91cc\u7684 12 \u5c42&#xff09;<\/td>\n<td align=\"right\">0.024<\/td>\n<td align=\"right\">90.54<\/td>\n<td align=\"right\">354<\/td>\n<\/tr>\n<tr>\n<td>num_gpu&#061;-1&#xff08;\u5168\u91cf GPU&#xff09;<\/td>\n<td align=\"right\">0.016<\/td>\n<td align=\"right\">176.27<\/td>\n<td align=\"right\">490<\/td>\n<\/tr>\n<tr>\n<td>num_batch&#061;64<\/td>\n<td align=\"right\">0.013<\/td>\n<td align=\"right\">187.96<\/td>\n<td align=\"right\">460<\/td>\n<\/tr>\n<tr>\n<td>num_batch&#061;128<\/td>\n<td align=\"right\">0.013<\/td>\n<td align=\"right\">178.25<\/td>\n<td align=\"right\">464<\/td>\n<\/tr>\n<tr>\n<td>num_batch&#061;256<\/td>\n<td align=\"right\">0.014<\/td>\n<td align=\"right\">169.80<\/td>\n<td align=\"right\">472<\/td>\n<\/tr>\n<tr>\n<td>num_batch&#061;512<\/td>\n<td align=\"right\">0.015<\/td>\n<td align=\"right\">186.14<\/td>\n<td align=\"right\">490<\/td>\n<\/tr>\n<tr>\n<td>num_batch&#061;1024<\/td>\n<td align=\"right\">0.014<\/td>\n<td align=\"right\">192.78<\/td>\n<td align=\"right\">526<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u6211\u8bfb\u51fa\u6765\u7684\u4e09\u6761\u7ed3\u8bba&#xff1a;<\/p>\n<li>num_ctx \u662f&#034;\u663e\u5b58\u6362\u957f\u5ea6&#034;\u7684\u5f00\u5173\u3002\u77ed prompt \u4e0b TTFT \/ TPS \u57fa\u672c\u65e0\u611f&#xff0c;\u4f46\u663e\u5b58\u4ece 472MiB \u6da8\u5230 568MiB&#xff1a;(568-472) \/ (8192-512) \u2248 12.8 KiB\/token&#xff0c;\u6b63\u662f KV cache \u7684\u5f00\u9500\u3002 \u6240\u4ee5\u957f\u5bf9\u8bdd\u624d\u9700\u8981\u5927 num_ctx&#xff0c;\u5426\u5219\u767d\u767d\u591a\u5360\u663e\u5b58\u2014\u20144GB \u5361\u4e0a\u8fd9\u6761\u5c24\u5176\u81f4\u547d\u3002<\/li>\n<li>num_gpu \u5728 4GB \u5361\u4e0a&#034;\u8981\u4e48\u5168\u653e\u3001\u8981\u4e48\u522b\u653e&#034;\u3002\u5168\u91cf offload 176 tok\/s&#xff0c; \u653e\u4e00\u534a\u5c42&#xff08;12\/24&#xff09;\u53ea\u6709 90.5 tok\/s&#xff0c;\u7eaf CPU 71.2 tok\/s\u3002 \u6211\u539f\u672c\u4ee5\u4e3a&#034;\u653e\u4e00\u534a\u603b\u6bd4\u4e0d\u653e\u5feb\u4e00\u70b9&#034;&#xff0c;\u5b9e\u6d4b\u53cd\u800c\u56e0\u4e3a GPU\u2194CPU \u4e4b\u95f4\u6765\u56de\u642c\u8fd0\u62d6\u6162\u4e86\u8fd1\u4e00\u534a\u3002<\/li>\n<li>num_batch \u5bf9\u89e3\u7801\u901f\u5ea6\u51e0\u4e4e\u6ca1\u6709\u5f71\u54cd&#xff08;64\u21921024 \u7684\u5dee\u5f02\u5728\u6d4b\u91cf\u566a\u58f0\u5185&#xff09;&#xff0c; \u4f46\u663e\u5b58\u968f\u6279\u589e\u5927&#xff08;460\u2192526MiB&#xff09;\u3002\u7ed3\u8bba&#xff1a;\u4fdd\u6301\u9ed8\u8ba4 512 \u5c31\u597d&#xff0c;\u522b\u4e71\u8c03\u3002<\/li>\n<p>\u8fd8\u6709\u4e00\u4e2a&#034;\u53cd\u5e38\u8bc6&#034;\u7684\u5751&#xff1a;num_ctx \/ num_gpu \/ num_batch \u4efb\u610f\u4e00\u4e2a\u53d8\u5316&#xff0c; Ollama \u90fd\u4f1a\u91cd\u8f7d\u6a21\u578b&#xff08;\u51b7\u542f\u52a8 TTFT \u2248 2.1s&#xff09;\u3002\u5982\u679c\u4f60\u5728\u6d41\u91cf\u9ad8\u5cf0\u671f\u52a8\u6001\u6362\u53c2&#xff0c; \u7528\u6237\u4f1a\u770b\u5230&#034;\u7a81\u7136\u5361 2 \u79d2&#034;\u3002\u800c\u4e14 Ollama 0.35 \u6709\u4e2a bug&#xff1a;\u4ece\u663e\u5f0f num_gpu&#061;0\/12 \u5207\u56de -1 \u65f6 \u5b83\u5224\u5b9a&#034;\u914d\u7f6e\u6ca1\u53d8&#034;&#xff0c;\u4e0d\u6362 runner\u2014\u2014\u6211\u7684\u811a\u672c\u53ea\u80fd\u5148\u7528 keep_alive&#061;0 \u5f3a\u5236\u5378\u8f7d\u6765\u89c4\u907f\u3002<\/p>\n<p>\u8bf4\u5b9e\u8bdd&#xff0c;\u8fd9\u5f20\u8868\u672c\u8eab\u5bf9\u751f\u4ea7\u6ca1\u4ec0\u4e48\u6307\u5bfc\u610f\u4e49&#xff08;0.5B \u6a21\u578b\u8c01\u4e5f\u4e0d\u4f1a\u62ff\u53bb\u4e0a\u7ebf&#xff09;&#xff0c; \u4f46**&#034;\u600e\u4e48\u505a\u4e00\u6b21\u53ef\u4fe1\u7684\u6027\u80fd\u5b9e\u9a8c&#034;\u8fd9\u5957\u65b9\u6cd5\u8bba**\u5f88\u503c\u94b1&#xff1a; \u56fa\u5b9a\u53d8\u91cf\u3001\u5148 warmup\u3001\u591a\u8f6e\u53d6\u5747\u503c\u3001\u8bb0\u5f55\u73af\u5883\u3001\u5254\u9664\u5f02\u5e38\u6837\u672c\u3001\u6c38\u8fdc\u770b\u5206\u5e03\u4e0d\u770b\u5355\u4e2a\u6570\u5b57\u3002<\/p>\n<p>\u8fd9\u4e00\u884c JSON \u662f\u6211\u540e\u6765\u6240\u6709\u4f18\u5316\u51b3\u7b56\u7684\u4f9d\u636e&#xff1a;\u6ca1\u6709 TTFT&#xff0c;\u4f60\u6839\u672c\u4e0d\u77e5\u9053\u6162\u5728\u54ea\u3002<\/p>\n<hr \/>\n<h3>10. \u9636\u6bb5 11&#xff1a;\u591a\u5b9e\u4f8b\u90e8\u7f72 &#043; nginx \u8d1f\u8f7d\u5747\u8861&#xff08;\u5168\u6587\u91cd\u70b9&#xff09;<\/h3>\n<p>\u524d\u9762 10 \u4e2a\u9636\u6bb5\u90fd\u5728\u89e3\u51b3&#034;\u4e00\u4e2a\u8fdb\u7a0b\u600e\u4e48\u628a\u4e8b\u505a\u5bf9&#034;\u3002\u8fd9\u4e00\u9636\u6bb5\u56de\u7b54\u53e6\u4e00\u4e2a\u95ee\u9898&#xff1a; \u4e00\u4e2a\u8fdb\u7a0b\u5d29\u4e86\u600e\u4e48\u529e&#xff1f;\u4e00\u4e2a\u6838\u4e0d\u591f\u7528\u4e86\u600e\u4e48\u529e&#xff1f;<\/p>\n<h4>10.1 \u4e3a\u4ec0\u4e48&#034;\u591a\u8d77\u4e00\u4e2a\u8fdb\u7a0b&#034;\u5c31\u7b97\u6269\u5bb9<\/h4>\n<p>\u8fd9\u662f\u6211\u8fd9\u9636\u6bb5\u6700\u5927\u7684\u8ba4\u77e5\u6536\u83b7\u3002\u80fd\u6c34\u5e73\u6269\u5bb9\u7684\u524d\u63d0\u662f&#xff1a;\u7f51\u5173\u8fdb\u7a0b\u672c\u8eab\u662f\u65e0\u72b6\u6001\u7684\u3002<\/p>\n<p>\u56de\u5934\u770b\u4e00\u904d\u524d\u9762\u7684\u8bbe\u8ba1&#xff0c;\u4f1a\u53d1\u73b0\u6211\u628a\u6240\u6709\u72b6\u6001\u90fd\u642c\u8d70\u4e86&#xff1a;<\/p>\n<table>\n<tr>\u72b6\u6001\u653e\u5728\u54ea\u6240\u4ee5\u7f51\u5173\u8fdb\u7a0b\u5d29\u6e83\u65f6<\/tr>\n<tbody>\n<tr>\n<td>\u9650\u6d41\u8ba1\u6570<\/td>\n<td>Redis<\/td>\n<td>\u8ba1\u6570\u4e0d\u4e22&#xff0c;\u65b0\u5b9e\u4f8b\u63a5\u7740\u6570<\/td>\n<\/tr>\n<tr>\n<td>\u8bf7\u6c42\u5386\u53f2<\/td>\n<td>SQLite \/ PostgreSQL<\/td>\n<td>\u8bb0\u5f55\u4e0d\u4e22<\/td>\n<\/tr>\n<tr>\n<td>\u4e8b\u4ef6\u7f13\u51b2<\/td>\n<td>Kafka<\/td>\n<td>\u4e8b\u4ef6\u4e0d\u4e22<\/td>\n<\/tr>\n<tr>\n<td>\u914d\u7f6e<\/td>\n<td>\u73af\u5883\u53d8\u91cf&#xff08;\u7aef\u53e3 \/ \u5b9e\u4f8b\u540d \/ \u540e\u7aef\u5730\u5740&#xff09;<\/td>\n<td>\u65b0\u8fdb\u7a0b\u6362\u4e2a\u7aef\u53e3\u5c31\u662f\u65b0\u5b9e\u4f8b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7f51\u5173\u81ea\u5df1\u53ea\u5269\u7eaf\u8ba1\u7b97&#xff0c;\u4e8e\u662f&#034;\u6269\u5bb9&#034;\u9000\u5316\u6210\u4e00\u4ef6\u5f88\u6734\u7d20\u7684\u4e8b&#xff1a;\u518d\u8d77\u4e00\u4e2a\u8fdb\u7a0b&#xff0c;\u6362\u4e2a\u7aef\u53e3&#xff0c;\u628a\u5b83\u52a0\u8fdb\u8d1f\u8f7d\u5747\u8861\u7684\u4e0a\u6e38\u5217\u8868\u3002<\/p>\n<p><span class=\"token function\">bash<\/span> scripts\/nginx.sh <span class=\"token function\">install<\/span>     <span class=\"token comment\"># \u9996\u6b21&#xff1a;\u514d root \u89e3\u5305 nginx \u5230 ~\/opt\/nginx<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/cluster.sh start     <span class=\"token comment\"># \u8d77 3 \u4e2a\u5b9e\u4f8b(8000\/8001\/8002) &#043; nginx \u5165\u53e3 :8080<\/span><\/p>\n<p>\u811a\u672c\u8dd1\u5b8c\u4f1a\u76f4\u63a5\u6253\u5370\u5206\u53d1\u9a8c\u8bc1&#xff0c;\u6211\u7b2c\u4e00\u6b21\u770b\u5230\u8fd9\u4e2a\u8f93\u51fa\u7684\u65f6\u5019\u8fd8\u633a\u6fc0\u52a8&#xff1a;<\/p>\n<p>&#8212; \u8d1f\u8f7d\u5747\u8861\u5165\u53e3 &#8212;<br \/>\nnginx lb ok<br \/>\n&#8212; \u7ecf LB \u8bf7\u6c42 6 \u6b21\u7684\u5b9e\u4f8b\u5206\u5e03 &#8212;<br \/>\n  #1 -&gt; gateway-8000<br \/>\n  #2 -&gt; gateway-8001<br \/>\n  #3 -&gt; gateway-8002<br \/>\n  #4 -&gt; gateway-8000<br \/>\n  #5 -&gt; gateway-8001<br \/>\n  #6 -&gt; gateway-8002<br \/>\n&#8212; \u65e5\u5fd7 &#8212;<br \/>\n  \u8bbf\u95ee\u65e5\u5fd7: \/home\/dawson\/opt\/nginx\/logs\/nginx-access.log<br \/>\n  \u9519\u8bef\u65e5\u5fd7: \/home\/dawson\/opt\/nginx\/logs\/nginx-error.log<\/p>\n<h4>10.2 \u9a8c\u8bc1\u5206\u53d1&#xff1a;12 \u6b21\u8bf7\u6c42&#xff0c;3 \u4e2a\u5b9e\u4f8b\u5404 4 \u6b21<\/h4>\n<p>python3 &#8211; <span class=\"token operator\">&lt;&lt;<\/span><span class=\"token string\">&#039;PY&#039;<br \/>\nimport requests<br \/>\nfor i in range(12):<br \/>\n    r &#061; requests.get(&#034;http:\/\/localhost:8080\/health&#034;)<br \/>\n    print(r.json()[&#034;instance&#034;])          # \u7ecf LB \u7684 \/health \u4f1a\u544a\u8bc9\u4f60\u843d\u70b9<br \/>\nPY<\/span><\/p>\n<p>\u5b9e\u6d4b\u5206\u5e03&#xff1a;{gateway-8000: 4, gateway-8001: 4, gateway-8002: 4}&#xff08;12 \u6b21\u5747\u5206&#xff09;\u3002<\/p>\n<p>\u804a\u5929\u63a5\u53e3\u5219\u53ef\u4ee5\u76f4\u63a5\u770b\u54cd\u5e94\u5934&#xff08;\/health \u6ca1\u6709\u8fd9\u4e2a\u5934&#xff0c;\u5b83\u7684 instance \u5728 JSON body \u91cc&#xff09;&#xff1a;<\/p>\n<p><span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-sD<\/span> &#8211; <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8080\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;hi&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:2}}&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token operator\">|<\/span> <span class=\"token function\">grep<\/span> <span class=\"token parameter variable\">-i<\/span> x-gateway-instance<br \/>\n<span class=\"token comment\"># x-gateway-instance: gateway-8000<\/span><\/p>\n<p>\u540c\u4e00\u65f6\u95f4\u5728 Prometheus \u91cc\u67e5 sum(gateway_requests_total) by (instance)&#xff0c; \u80fd\u770b\u5230 3 \u6761\u72ec\u7acb\u66f2\u7ebf\u2014\u2014\u6bcf\u4e2a\u5b9e\u4f8b\u7684\u6307\u6807\u662f\u5206\u5f00\u7684&#xff0c;\u8fd9\u662f\u591a\u5b9e\u4f8b\u89c2\u6d4b\u7684\u524d\u63d0\u3002<\/p>\n<h4>10.3 nginx \u914d\u7f6e&#xff1a;\u6bcf\u4e00\u884c\u4e3a\u4ec0\u4e48\u5fc5\u987b\u5199<\/h4>\n<p>\u8fd9\u4efd\u914d\u7f6e\u6211\u6539\u4e86\u5f88\u591a\u8f6e\u624d\u7a33\u5b9a&#xff0c;\u4e0b\u9762\u662f\u6700\u5173\u952e\u7684\u51e0\u6761&#xff08;\u5b8c\u6574\u6a21\u677f\u5728 nginx\/nginx.conf.tpl&#xff09;&#xff1a;<\/p>\n<p>upstream llm_gateway {<br \/>\n    zone llm_gateway 64k;      # \u591a worker \u5171\u4eab\u5065\u5eb7\u72b6\u6001\u4e0e\u8fde\u63a5\u6570&#xff08;least_conn \u4f9d\u8d56\u5b83&#xff09;<br \/>\n    least_conn;                # \u6309&#034;\u5f53\u524d\u8fde\u63a5\u6570\u6700\u5c11&#034;\u5206\u53d1&#xff1a;LLM \u5355\u8bf7\u6c42\u8017\u65f6\u957f&#xff0c;<br \/>\n                               # \u8f6e\u8be2\u5bb9\u6613\u628a\u65b0\u8bf7\u6c42\u4e22\u7ed9&#034;\u5df2\u7ecf\u538b\u7740\u51e0\u4e2a\u6162\u8bf7\u6c42&#034;\u7684\u5b9e\u4f8b<br \/>\n    server 127.0.0.1:8000 max_fails&#061;2 fail_timeout&#061;10s;<br \/>\n    server 127.0.0.1:8001 max_fails&#061;2 fail_timeout&#061;10s;<br \/>\n    server 127.0.0.1:8002 max_fails&#061;2 fail_timeout&#061;10s;<br \/>\n    keepalive 32;              # \u4e0e\u4e0a\u6e38\u4fdd\u6301 32 \u4e2a\u7a7a\u95f2\u957f\u8fde\u63a5&#xff0c;\u7701\u6389\u63e1\u624b\u5f00\u9500<br \/>\n}<\/p>\n<p>location \/ {<br \/>\n    proxy_pass http:\/\/llm_gateway;<\/p>\n<p>    proxy_http_version 1.1;    # \u2190 \u8fd9\u4e09\u884c\u9f50\u5199\u624d\u80fd\u4e0e\u4e0a\u6e38\u590d\u7528\u957f\u8fde\u63a5<br \/>\n    proxy_set_header Connection &#034;&#034;;<\/p>\n<p>    # \u2014\u2014 \u6d41\u5f0f&#xff08;SSE&#xff09;\u4e09\u4ef6\u5957&#xff1a;\u4e0d\u7f13\u51b2\u3001\u4e0d\u7f13\u5b58\u3001\u4e0d\u6512\u8bf7\u6c42\u4f53 \u2014\u2014<br \/>\n    proxy_buffering off;       # \u2605 \u6700\u5173\u952e\u7684\u4e00\u884c&#xff0c;\u5f00\u7740\u7684\u8bdd&#034;\u9010\u5b57\u8f93\u51fa&#034;\u4f1a\u53d8\u6210&#034;\u51e0\u79d2\u540e\u4e00\u6b21\u6027\u5410\u51fa&#034;<br \/>\n    proxy_cache off;<br \/>\n    proxy_request_buffering off;<\/p>\n<p>    proxy_connect_timeout 2s;  # \u6b7b\u5b9e\u4f8b\u5feb\u901f\u5931\u8d25&#xff0c;\u4ea4\u7ed9\u4e0b\u4e00\u4e2a\u4e0a\u6e38<br \/>\n    proxy_read_timeout   300s; # \u9ed8\u8ba4 60s \u4f1a\u628a\u957f\u56de\u7b54\u6390\u65ad<br \/>\n    proxy_send_timeout   300s;<br \/>\n    send_timeout         300s;<\/p>\n<p>    proxy_next_upstream error timeout;   # \u53ea\u5728&#034;\u8fde\u63a5\u9636\u6bb5&#034;\u5931\u8d25\u65f6\u6362\u673a<br \/>\n    proxy_next_upstream_tries 2;<br \/>\n}<\/p>\n<p>\u53e6\u5916\u4e24\u4e2a&#034;\u6559\u5b66\u5f0f&#034;\u7ec6\u8282&#xff1a;<\/p>\n<p># LB \u81ea\u8eab\u5065\u5eb7\u68c0\u67e5&#xff08;\u4e0d\u8f6c\u53d1\u7ed9\u7f51\u5173&#xff0c;nginx \u6d3b\u7740\u5c31\u80fd\u56de 200&#xff09;<br \/>\nlocation &#061; \/lb-health { access_log off; return 200 &#034;nginx lb ok\\\\n&#034;; }<\/p>\n<p># \/metrics \u5fc5\u987b\u76f4\u8fde\u6bcf\u4e2a\u5b9e\u4f8b\u6293&#xff1a;\u8d70 LB \u7684\u8bdd Prometheus \u5206\u4e0d\u6e05\u662f\u54ea\u4e2a\u5b9e\u4f8b\u7684\u6307\u6807&#xff0c;\u8fd9\u91cc\u663e\u5f0f\u62d2\u7edd<br \/>\nlocation &#061; \/metrics { return 404 &#034;metrics must be scraped per-instance directly\\\\n&#034;; }<\/p>\n<p>\u5173\u4e8e proxy_next_upstream \u6211\u8981\u7279\u522b\u8bf4\u4e00\u53e5&#xff1a;POST \u662f\u975e\u5e42\u7b49\u8bf7\u6c42&#xff0c; nginx \u9ed8\u8ba4\u4e0d\u4f1a\u628a&#034;\u5df2\u7ecf\u53d1\u51fa\u53bb\u7684\u8bf7\u6c42&#034;\u91cd\u8bd5\u5230\u4e0b\u4e00\u4e2a\u4e0a\u6e38\u2014\u2014\u8fd9\u6b63\u662f\u6211\u4eec\u60f3\u8981\u7684&#xff0c; \u5426\u5219\u7528\u6237\u63d0\u4e00\u4e2a\u95ee\u9898&#xff0c;\u53ef\u80fd\u4e24\u4e2a\u5b9e\u4f8b\u5404\u751f\u6210\u4e00\u904d&#xff08;\u70e7\u4e24\u500d\u7b97\u529b&#xff09;\u3002 \u8fd9\u91cc\u53ea\u5141\u8bb8&#034;\u8fde\u63a5\u9636\u6bb5\u5c31\u5931\u8d25&#034;\u7684\u6362\u673a&#xff0c;\u6211\u4e13\u95e8\u9a8c\u8bc1\u8fc7&#xff1a;\u6545\u969c\u8f6c\u79fb\u671f\u95f4\u6ca1\u6709\u51fa\u73b0\u91cd\u590d\u751f\u6210\u3002<\/p>\n<p>\u987a\u4fbf\u8bb0\u5f55\u4e00\u4e2a\u6211\u81ea\u5df1 review \u51fa\u6765\u7684\u4f4e\u7ea7\u9519\u8bef&#xff1a;\u6587\u6863\u91cc\u5199\u4e86&#034;\u7528 least_conn \u6309\u8fde\u63a5\u6570\u5206\u53d1&#034;&#xff0c; \u4f46\u6a21\u677f\u91cc\u5176\u5b9e\u6f0f\u4e86 least_conn; \u8fd9\u4e00\u884c&#xff0c;\u5b9e\u9645\u8dd1\u7684\u662f\u9ed8\u8ba4\u8f6e\u8be2\u3002 \u6211\u662f\u5199\u8fd9\u7bc7\u6587\u7ae0\u65f6\u9010\u884c\u5bf9\u7167\u914d\u7f6e\u624d\u53d1\u73b0\u7684&#xff0c;\u52a0\u4e0a\u5e76 reload \u4e4b\u540e\u4ecd\u7136\u662f 4\/4\/4 \u5747\u5206 &#xff08;\u8f7b\u8bf7\u6c42\u4e0b\u4e24\u79cd\u7b56\u7565\u5dee\u522b\u4e0d\u5927&#xff0c;\u4f46\u957f\u8bf7\u6c42\u573a\u666f least_conn \u624d\u66f4\u7a33&#xff09;\u3002 \u6559\u8bad&#xff1a;\u6587\u6863\u8bf4\u7684\u548c\u914d\u7f6e\u5199\u7684\u5fc5\u987b\u4e00\u81f4&#xff0c;\u800c\u552f\u4e00\u7684\u9a8c\u8bc1\u65b9\u5f0f\u662f&#034;\u53bb\u770b\u771f\u6b63\u751f\u6548\u7684\u914d\u7f6e&#034;\u3002<\/p>\n<h4>10.4 \u6d41\u5f0f&#xff08;SSE&#xff09;\u5230\u5e95\u6709\u6ca1\u6709\u88ab&#034;\u618b\u4f4f&#034;&#xff1f;\u2014\u2014\u672c\u7ae0\u6700\u91cd\u8981\u7684\u4e00\u6b21\u6d4b\u91cf<\/h4>\n<p>\u8d1f\u8f7d\u5747\u8861\u6700\u5bb9\u6613\u8e29\u7684\u5751\u5c31\u662f&#xff1a;\u672c\u5730\u76f4\u8fde\u597d\u597d\u7684\u9010\u5b57\u8f93\u51fa&#xff0c;\u63a5\u4e0a nginx \u4e4b\u540e\u53d8\u6210&#034;\u8f6c\u5708\u51e0\u79d2\u7136\u540e\u5237\u51fa\u5168\u6587&#034;\u3002 \u539f\u56e0\u5c31\u662f proxy_buffering on \u4f1a\u7b49\u7f13\u51b2\u6512\u6ee1\u624d\u4e0b\u53d1\u3002<\/p>\n<p>\u6211\u7528&#034;\u9996\u5b57\u8282\u65f6\u95f4 vs \u603b\u65f6\u957f&#034;\u505a\u5224\u636e&#xff1a;\u5982\u679c\u88ab\u7f13\u51b2&#xff0c;\u9996\u5b57\u8282\u4f1a\u63a5\u8fd1\u603b\u65f6\u957f&#xff08;\u6bd4\u4f8b\u2248100%&#xff09;\u3002<\/p>\n<p><span class=\"token comment\"># \u7ecf LB&#xff08;-N \u5173\u95ed curl \u81ea\u5df1\u7684\u7f13\u51b2&#xff09;<\/span><br \/>\n<span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-N<\/span> <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-w<\/span> <span class=\"token string\">&#039;[LB]   ttfb&#061;%{time_starttransfer}s total&#061;%{time_total}s\\\\n&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8080\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;\u6570\u5230\u4e8c\u5341&#034;}],&#034;stream&#034;:true,&#034;options&#034;:{&#034;num_predict&#034;:64}}&#039;<\/span><\/p>\n<p><span class=\"token comment\"># \u76f4\u8fde\u5b9e\u4f8b&#xff08;\u7ed5\u5f00 nginx&#xff09;<\/span><br \/>\n<span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-N<\/span> <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-w<\/span> <span class=\"token string\">&#039;[\u76f4\u8fde] ttfb&#061;%{time_starttransfer}s total&#061;%{time_total}s\\\\n&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8000\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;\u6570\u5230\u4e8c\u5341&#034;}],&#034;stream&#034;:true,&#034;options&#034;:{&#034;num_predict&#034;:64}}&#039;<\/span><\/p>\n<p>\u5b9e\u6d4b&#xff08;verify_cluster.py \u91cc\u540c\u4e00\u9879\u68c0\u67e5\u7684\u8f93\u51fa&#xff09;&#xff1a;<\/p>\n<p>\u7ecf LB : \u9996\u5b57\u8282 0.041s \/ \u603b\u65f6\u957f 0.710s     \u2190 \u9996\u5b57\u8282\u53ea\u5360 6%<br \/>\n\u76f4\u8fde\u5b9e\u4f8b: \u9996\u5b57\u8282 0.094s \/ \u603b\u65f6\u957f 0.346s<\/p>\n<p>\u9996\u5b57\u8282\u8fdc\u65e9\u4e8e\u603b\u65f6\u957f&#xff08;6%&#xff09;&#xff0c;\u8bf4\u660e nginx \u6ca1\u6709\u989d\u5916\u7f13\u51b2\u3002<\/p>\n<p>\u6211\u5728\u8fd9\u91cc\u88ab\u5751\u8fc7\u4e00\u6b21&#xff1a;\u6709\u4e00\u6b21\u5bf9\u6bd4\u6d4b\u51fa\u6765\u7ecf LB \u7684\u9996\u5b57\u8282\u8981 1.9 \u79d2&#xff0c; \u6211\u7b2c\u4e00\u53cd\u5e94\u662f&#034;nginx \u7f13\u51b2\u6ca1\u5173\u5e72\u51c0&#034;&#xff0c;\u67e5\u4e86\u534a\u5929\u914d\u7f6e\u90fd\u6ca1\u95ee\u9898\u3002 \u540e\u6765\u53d1\u73b0\u662f\u4e24\u6b21\u5bf9\u6bd4\u8bf7\u6c42\u7684 num_ctx \u4e0d\u4e00\u6837\u2014\u2014Ollama \u6362\u53c2\u4f1a\u91cd\u8f7d\u6a21\u578b&#xff08;\u51b7\u542f\u52a8 \u22482.1s&#xff09;&#xff0c; \u90a3 1.9 \u79d2\u662f\u6a21\u578b\u91cd\u8f7d\u65f6\u95f4&#xff0c;\u8ddf nginx \u4e00\u70b9\u5173\u7cfb\u90fd\u6ca1\u6709\u3002 \u6559\u8bad&#xff1a;\u5bf9\u6bd4\u5b9e\u9a8c\u7684\u53c2\u6570\u5fc5\u987b\u5b8c\u5168\u4e00\u81f4&#xff0c;\u5426\u5219\u4f60\u6d4b\u7684\u662f\u522b\u7684\u4e1c\u897f\u3002<\/p>\n<h4>10.5 \u6545\u969c\u8f6c\u79fb&#xff1a;\u6740\u6389\u4e00\u4e2a\u5b9e\u4f8b&#xff0c;\u5ba2\u6237\u7aef\u6beb\u65e0\u611f\u77e5<\/h4>\n<p>\u8fd9\u662f\u6700\u80fd\u4f53\u73b0&#034;\u8d1f\u8f7d\u5747\u8861\u503c\u4e0d\u503c&#034;\u7684\u4e00\u6b65\u3002\u6211\u76f4\u63a5 kill \u6389 8000 \u8fd9\u4e2a\u5b9e\u4f8b&#xff1a;<\/p>\n<p><span class=\"token function\">kill<\/span> <span class=\"token string\">&#034;<span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">cat<\/span> logs\/cluster\/run\/gateway-8000.pid<span class=\"token variable\">)<\/span><\/span>&#034;<\/span><br \/>\n<span class=\"token comment\"># \u7136\u540e\u7ecf LB \u8fde\u6253 12 \u6b21<\/span><br \/>\n<span class=\"token keyword\">for<\/span> <span class=\"token for-or-select variable\">i<\/span> <span class=\"token keyword\">in<\/span> <span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">seq<\/span> <span class=\"token number\">1<\/span> <span class=\"token number\">12<\/span><span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span> <span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-w<\/span> <span class=\"token string\">&#039;%{http_code} &#039;<\/span> http:\/\/localhost:8080\/health<span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">done<\/span><\/p>\n<p>\u5b9e\u6d4b\u7ed3\u679c&#xff1a;<\/p>\n<p>200 200 200 200 200 200 200 200 200 200 200 200      \u2190 12 \u6b21\u5168\u90e8\u6210\u529f<\/p>\n<p>\u5ba2\u6237\u7aef\u5b8c\u5168\u770b\u4e0d\u5230\u6545\u969c&#xff0c;\u4f46 nginx \u7684\u65e5\u5fd7\u91cc\u7559\u4e0b\u4e86\u5b8c\u6574\u8bc1\u636e&#xff1a;<\/p>\n<p><span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;uri&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;\/health&#034;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;status&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;request_time&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token number\">0.03<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;upstream_addr&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;127.0.0.1:8000, 127.0.0.1:8101&#034;<\/span><span class=\"token punctuation\">,<\/span>   \u2190 \u5148\u8bd5 <span class=\"token number\">8000<\/span>&#xff08;\u6b7b\u4e86&#xff09;&#xff0c;\u81ea\u52a8\u6362\u5230 <span class=\"token number\">8101<\/span><br \/>\n <span class=\"token string-property property\">&#034;upstream_status&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;502, 200&#034;<\/span><span class=\"token punctuation\">}<\/span>                        \u2190 \u7b2c\u4e00\u6b21 <span class=\"token number\">502<\/span>&#xff0c;\u6362\u673a\u540e <span class=\"token number\">200<\/span><\/p>\n<p>nginx-error.log:<br \/>\n[error] connect() failed (111: Connection refused) while connecting to upstream &#8230;<br \/>\n[error] upstream server temporarily disabled while connecting to upstream<\/p>\n<table>\n<tr>\u89c2\u5bdf\u70b9\u5b9e\u6d4b\u7ed3\u679c<\/tr>\n<tbody>\n<tr>\n<td>\u5ba2\u6237\u7aef\u770b\u5230\u7684 HTTP \u72b6\u6001<\/td>\n<td>12 \u6b21\u5168\u90e8 200&#xff08;\u6362\u673a\u5bf9\u5ba2\u6237\u7aef\u900f\u660e&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u54cd\u5e94\u91cc\u7684\u5b9e\u4f8b\u540d<\/td>\n<td>\u53ea\u5269\u5b58\u6d3b\u7684\u4e24\u4e2a\u5b9e\u4f8b<\/td>\n<\/tr>\n<tr>\n<td>\u6458\u9664\u662f\u5426\u751f\u6548<\/td>\n<td>\u540e\u7eed 8 \u6b21\u8bf7\u6c42\u7684\u8bbf\u95ee\u65e5\u5fd7\u91cc\u6ca1\u6709\u4efb\u4f55\u6362\u673a\u75d5\u8ff9&#xff08;\u4e0d\u518d\u53bb\u8fde\u6b7b\u5b9e\u4f8b&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u5b9e\u4f8b\u6062\u590d<\/td>\n<td>\u91cd\u65b0\u62c9\u8d77\u540e fail_timeout&#061;10s \u5185\u81ea\u52a8\u52a0\u56de\u4e0a\u6e38&#xff0c;\u65e0\u9700\u6539\u914d\u7f6e\u6216 reload<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u5c31\u662f max_fails&#061;2 \/ fail_timeout&#061;10s \u7684\u88ab\u52a8\u5065\u5eb7\u68c0\u67e5&#xff1a;\u4e0d\u9700\u8981\u989d\u5916\u90e8\u7f72\u4e00\u4e2a\u5065\u5eb7\u63a2\u6d4b\u670d\u52a1&#xff0c; \u4ee3\u4ef7\u662f&#034;\u6545\u969c\u521a\u5f00\u59cb\u7684 1~2 \u4e2a\u8bf7\u6c42\u8981\u9760\u6362\u673a\u91cd\u8bd5\u9876\u4f4f&#034;\u3002\u5bf9 LLM \u670d\u52a1\u6765\u8bf4\u8fd9\u4e2a\u4ee3\u4ef7\u53ef\u4ee5\u63a5\u53d7\u2014\u2014 \u4e00\u6b21\u63a8\u7406\u672c\u6765\u5c31\u8981\u597d\u51e0\u79d2&#xff0c;\u800c\u4e3b\u52a8\u5065\u5eb7\u68c0\u67e5\u8fd8\u5f97\u81ea\u5df1\u5199\u3002<\/p>\n<h4>10.6 \u8de8\u5b9e\u4f8b\u5171\u4eab\u9650\u6d41&#xff08;\u7ecf LB \u7684\u7248\u672c&#xff09;<\/h4>\n<p>\u540c\u6837\u7684&#034;\u62db\u724c\u5b9e\u9a8c&#034;&#xff0c;\u8fd9\u6b21\u7ecf LB \u6253&#xff08;3 \u4e2a\u5b9e\u4f8b\u5171\u4eab free \u6863 10 \u6b21\u989d\u5ea6&#xff09;&#xff1a;<\/p>\n<p>redis-cli <span class=\"token parameter variable\">-p<\/span> <span class=\"token number\">6379<\/span> FLUSHDB<br \/>\n<span class=\"token keyword\">for<\/span> <span class=\"token for-or-select variable\">i<\/span> <span class=\"token keyword\">in<\/span> <span class=\"token variable\"><span class=\"token variable\">$(<\/span><span class=\"token function\">seq<\/span> <span class=\"token number\">1<\/span> <span class=\"token number\">12<\/span><span class=\"token variable\">)<\/span><\/span><span class=\"token punctuation\">;<\/span> <span class=\"token keyword\">do<\/span><br \/>\n  <span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-o<\/span> \/dev\/null <span class=\"token parameter variable\">-w<\/span> <span class=\"token string\">&#039;%{http_code} &#039;<\/span> <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8080\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n    <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;x-api-key: sk-test-001&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n    <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;hi&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:2}}&#039;<\/span><br \/>\n<span class=\"token keyword\">done<\/span><\/p>\n<p>\u8fde\u8dd1\u4e24\u8f6e&#xff0c;\u7ed3\u679c\u5b8c\u5168\u4e00\u81f4&#xff1a;<\/p>\n<p>round 1 \u7ecf LB 12 \u6b21: 200 200 200 200 200 200 200 200 200 200 429 429<br \/>\nround 2 \u7ecf LB 12 \u6b21: 200 200 200 200 200 200 200 200 200 200 429 429<\/p>\n<p>3 \u4e2a\u5b9e\u4f8b\u300112 \u6b21\u8bf7\u6c42&#xff0c;\u53ea\u6709 10 \u6b21\u88ab\u653e\u884c\u2014\u2014\u989d\u5ea6\u786e\u5b9e\u662f\u5168\u5c40\u7684\u3002 \u518d\u52a0\u4e00\u4e2a\u53cd\u8bc1&#xff1a;\u4e0a\u9762\u8fd9 10 \u6b21\u989d\u5ea6\u7528\u5149\u540e&#xff0c;\u76f4\u8fde\u4efb\u610f\u4e00\u4e2a\u5b9e\u4f8b\u4e5f\u662f 12 \u4e2a 429&#xff1a;<\/p>\n<p>429 429 429 429 429 429 429 429 429 429 429 429      \u2190 \u76f4\u8fde :8000&#xff0c;\u989d\u5ea6\u5df2\u88ab\u5168\u5c40\u7528\u5b8c<\/p>\n<h4>10.7 \u6269\u7f29\u5bb9&#xff1a;\u987a\u5e8f\u4e0d\u4e00\u6837&#xff0c;\u7406\u7531\u5f88\u5b9e\u9645<\/h4>\n<p><span class=\"token function\">bash<\/span> scripts\/cluster.sh scale <span class=\"token number\">5<\/span>    <span class=\"token comment\"># \u6269\u5230 5 \u4e2a&#xff08;\u70ed\u52a0\u8f7d nginx&#xff0c;\u4e0d\u65ad\u8fde\u63a5&#xff09;<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/cluster.sh scale <span class=\"token number\">3<\/span>    <span class=\"token comment\"># \u7f29\u56de 3 \u4e2a<\/span><\/p>\n<table>\n<tr>\u64cd\u4f5c\u987a\u5e8f\u4e3a\u4ec0\u4e48<\/tr>\n<tbody>\n<tr>\n<td>\u6269\u5bb9<\/td>\n<td>\u5148\u8d77\u65b0\u5b9e\u4f8b \u2192 \u518d reload nginx<\/td>\n<td>\u65b0\u5b9e\u4f8b\u6ca1\u5c31\u7eea\u5c31\u88ab\u52a0\u8fdb\u4e0a\u6e38&#xff0c;\u4f1a\u767d\u6328\u51e0\u6b21\u5931\u8d25&#xff08;max_fails \u80fd\u515c\u4f4f\u4f46\u6ca1\u5fc5\u8981&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u7f29\u5bb9<\/td>\n<td>\u5148\u6539\u4e0a\u6e38\u5e76 reload \u2192 \u518d\u505c\u5b9e\u4f8b<\/td>\n<td>\u53cd\u8fc7\u6765\u7684\u8bdd&#xff0c;\u8bf7\u6c42\u8fd8\u5728\u5f80\u8981\u505c\u7684\u5b9e\u4f8b\u4e0a\u6253&#xff0c;\u4f1a\u51fa\u73b0 502<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u91cc\u8fd8\u8e29\u4e86\u4e2a\u811a\u672c\u5c42\u7684\u5751&#xff1a;\u6bcf\u6b21\u6267\u884c\u811a\u672c\u90fd\u662f\u65b0\u8fdb\u7a0b&#xff0c;\u73af\u5883\u53d8\u91cf\u8bb0\u4e0d\u4f4f&#034;\u73b0\u5728\u6709\u54ea\u51e0\u4e2a\u5b9e\u4f8b\u5728\u8dd1&#034;&#xff0c; \u6240\u4ee5 scale \u4e00\u5f00\u59cb\u6309\u73af\u5883\u53d8\u91cf\u7b97&#xff0c;\u7f29\u5bb9\u65f6\u63a8\u4e0d\u51fa\u5b9e\u4f8b\u5217\u8868&#xff0c;\u5e72\u8106\u4e0d\u505c\u5b9e\u4f8b\u3002 \u540e\u6765\u6539\u6210\u6309 pid \u6587\u4ef6\u53cd\u63a8\u5f53\u524d\u5728\u8dd1\u7684\u7aef\u53e3&#xff08;running_ports()&#xff09;&#xff0c;\u624d\u771f\u6b63\u505a\u5230\u6269\u5bb9\/\u7f29\u5bb9\u90fd\u5e42\u7b49\u3002<\/p>\n<hr \/>\n<h4>10.8 \u6211\u81ea\u5df1\u52a0\u7684\u4e24\u4e2a\u5b9e\u9a8c&#xff1a;\u8d1f\u8f7d\u5747\u8861\u5230\u5e95\u5e26\u6765\u4e86\u591a\u5c11\u6536\u76ca&#xff1f;<\/h4>\n<p>\u6587\u6863\u91cc\u7684\u9a8c\u6536\u9879\u53ea\u80fd\u8bc1\u660e&#034;\u5b83\u80fd\u7528&#034;&#xff0c;\u4f46\u6211\u66f4\u60f3\u77e5\u9053\u5b83\u503c\u591a\u5c11\u3002 \u4e8e\u662f\u6211\u53c8\u505a\u4e86\u4e24\u7ec4\u5b9e\u9a8c&#xff08;\u8fd9\u90e8\u5206\u662f\u6211\u81ea\u5df1\u52a0\u7684&#xff0c;\u4e0d\u5728\u6700\u521d\u7684\u5b66\u4e60\u8def\u7ebf\u91cc&#xff09;\u3002<\/p>\n<h5>\u5b9e\u9a8c A&#xff1a;\u5355\u5361\u5e76\u53d1\u4f1a\u4e0d\u4f1a\u628a TTFT \u62d6\u57ae&#xff1f;<\/h5>\n<p>\u7ecf LB \u53d1\u8d77 1\/2\/4\/8 \u5e76\u53d1&#xff08;stream&#061;true&#xff0c;num_predict&#061;128&#xff0c;\u5148\u505a\u4e00\u6b21 warmup \u628a\u6a21\u578b\u52a0\u8f7d\u8fdb\u663e\u5b58&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u5e76\u53d1\u6570TTFT \u5e73\u5747TTFT \u6700\u5927\u7aef\u5230\u7aef\u5e73\u5747\u7aef\u5230\u7aef\u6700\u5927\u843d\u70b9\u5206\u5e03<\/tr>\n<tbody>\n<tr>\n<td align=\"right\">1<\/td>\n<td align=\"right\">0.03s<\/td>\n<td align=\"right\">0.03s<\/td>\n<td align=\"right\">1.04s<\/td>\n<td align=\"right\">1.04s<\/td>\n<td>{8000:1}<\/td>\n<\/tr>\n<tr>\n<td align=\"right\">2<\/td>\n<td align=\"right\">0.57s<\/td>\n<td align=\"right\">1.09s<\/td>\n<td align=\"right\">1.59s<\/td>\n<td align=\"right\">2.12s<\/td>\n<td>{8000:1, 8002:1}<\/td>\n<\/tr>\n<tr>\n<td align=\"right\">4<\/td>\n<td align=\"right\">1.63s<\/td>\n<td align=\"right\">3.25s<\/td>\n<td align=\"right\">2.68s<\/td>\n<td align=\"right\">4.30s<\/td>\n<td>{8000:1, 8001:2, 8002:1}<\/td>\n<\/tr>\n<tr>\n<td align=\"right\">8<\/td>\n<td align=\"right\">3.62s<\/td>\n<td align=\"right\">7.26s<\/td>\n<td align=\"right\">4.62s<\/td>\n<td align=\"right\">8.26s<\/td>\n<td>{8000:3, 8001:2, 8002:3}<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8bfb\u6cd5\u5f88\u76f4\u63a5&#xff1a;<\/p>\n<ul>\n<li>nginx \u628a\u8bf7\u6c42\u5747\u5300\u5206\u7ed9\u4e86 3 \u4e2a\u5b9e\u4f8b&#xff08;8 \u5e76\u53d1\u65f6 3\/2\/3&#xff09;&#xff0c;\u8fd9\u4e00\u5c42\u6ca1\u95ee\u9898&#xff1b;<\/li>\n<li>\u4f46 TTFT \u968f\u5e76\u53d1\u7ebf\u6027\u4e0a\u6da8&#xff08;0.03 \u2192 0.57 \u2192 1.63 \u2192 3.62&#xff09;&#xff0c;\u56e0\u4e3a\u8bf7\u6c42\u6700\u7ec8\u90fd\u6392\u5728\u540c\u4e00\u4e2a Ollama \u961f\u5217\u91cc&#xff0c; \u5728\u5b83\u524d\u9762\u6709\u591a\u5c11\u4e2a\u8bf7\u6c42&#xff0c;\u5c31\u5f97\u7b49\u591a\u4e45\u3002<\/li>\n<\/ul>\n<p>\u8fd9\u5c31\u662f\u5355\u673a\u591a\u5b9e\u4f8b\u7684\u5929\u82b1\u677f&#xff1a;\u6269\u5bb9\u7684\u662f&#034;\u7f51\u5173\u5c42\u5e76\u53d1&#034;&#xff0c;\u4e0d\u662f&#034;\u7b97\u529b&#034;\u3002<\/p>\n<h5>\u5b9e\u9a8c B&#xff1a;\u8d1f\u8f7d\u5747\u8861\u7684\u541e\u5410\u6536\u76ca&#xff08;\u7528\u4e0d\u78b0 GPU \u7684\u8def\u5f84\u6d4b&#xff09;<\/h5>\n<p>\u4e3a\u4e86\u628a GPU \u8fd9\u4e2a\u53d8\u91cf\u6392\u9664\u6389&#xff0c;\u6211\u6311\u4e86\u4e24\u6761\u4e0d\u6d89\u53ca\u6a21\u578b\u63a8\u7406\u7684\u8def\u5f84\u6765\u538b&#xff1a;<\/p>\n<p><span class=\"token comment\"># \u6bcf\u6761\u8def\u5f84 300 \u6b21\u8bf7\u6c42&#xff1b;\u5e76\u53d1 8 \/ 16 \/ 32 \u5404\u8dd1\u4e00\u8f6e&#xff0c;\u6bcf\u8f6e\u91cd\u590d 3 \u6b21\u53d6\u5e73\u5747<\/span><br \/>\n<span class=\"token comment\"># A) GET  \/health                  \u2014\u2014 \u4f1a\u53bb\u95ee\u4e00\u6b21 Ollama&#xff08;\/api\/tags&#xff09;<\/span><br \/>\n<span class=\"token comment\"># B) POST \/v1\/chat\/completions     \u2014\u2014 \u6545\u610f\u7528\u9519 key&#xff0c;\u5728\u9274\u6743\u5c42\u5c31\u88ab 401 \u62e6\u4e0b&#xff08;\u5b8c\u5168\u4e0d\u78b0 GPU&#xff09;<\/span><\/p>\n<table>\n<tr>\u538b\u6d4b\u8def\u5f84\u5e76\u53d1\u76f4\u8fde\u5355\u5b9e\u4f8b\u7ecf nginx&#xff08;3 \u5b9e\u4f8b&#xff09;\u63d0\u5347<\/tr>\n<tbody>\n<tr>\n<td>GET \/health<\/td>\n<td align=\"right\">8<\/td>\n<td align=\"right\">85.2 req\/s<\/td>\n<td align=\"right\">196.8 req\/s<\/td>\n<td align=\"right\">2.31\u00d7<\/td>\n<\/tr>\n<tr>\n<td>GET \/health<\/td>\n<td align=\"right\">16<\/td>\n<td align=\"right\">96.3 req\/s<\/td>\n<td align=\"right\">208.8 req\/s<\/td>\n<td align=\"right\">2.17\u00d7<\/td>\n<\/tr>\n<tr>\n<td>GET \/health<\/td>\n<td align=\"right\">32<\/td>\n<td align=\"right\">97.4 req\/s<\/td>\n<td align=\"right\">208.4 req\/s<\/td>\n<td align=\"right\">2.14\u00d7<\/td>\n<\/tr>\n<tr>\n<td>POST 401&#xff08;\u9274\u6743\u62e6\u622a&#xff09;<\/td>\n<td align=\"right\">8<\/td>\n<td align=\"right\">924.4 req\/s<\/td>\n<td align=\"right\">759.9 req\/s<\/td>\n<td align=\"right\">0.82\u00d7<\/td>\n<\/tr>\n<tr>\n<td>POST 401&#xff08;\u9274\u6743\u62e6\u622a&#xff09;<\/td>\n<td align=\"right\">16<\/td>\n<td align=\"right\">979.6 req\/s<\/td>\n<td align=\"right\">759.8 req\/s<\/td>\n<td align=\"right\">0.78\u00d7<\/td>\n<\/tr>\n<tr>\n<td>POST 401&#xff08;\u9274\u6743\u62e6\u622a&#xff09;<\/td>\n<td align=\"right\">32<\/td>\n<td align=\"right\">889.6 req\/s<\/td>\n<td align=\"right\">694.5 req\/s<\/td>\n<td align=\"right\">0.78\u00d7<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u987a\u4fbf\u6d4b\u4e86 nginx \u81ea\u5df1\u7684\u88f8\u8f6c\u53d1\u4e0a\u9650&#xff08;\/lb-health&#xff0c;\u4e0d\u8f6c\u53d1\u4e5f\u4e0d\u8bb0\u65e5\u5fd7&#xff09;&#xff1a; 979.8 req\/s&#xff08;\u5e76\u53d1 16&#xff09;\u30011102.4 req\/s&#xff08;\u5e76\u53d1 32&#xff09;\u3002<\/p>\n<p>\u8fd9\u4e24\u884c\u6570\u636e\u80cc\u540e\u7684\u7ed3\u8bba\u5b8c\u5168\u76f8\u53cd&#xff0c;\u4f46\u90fd\u5f88\u6709\u4ef7\u503c&#xff1a;<\/p>\n<li>\/health \u8d70 LB \u63d0\u5347 2.1~2.3 \u500d&#xff08;\u4e0d\u662f 3 \u500d&#xff09;\u3002\u4e3a\u4ec0\u4e48\u4e0d\u662f 3 \u500d&#xff1f; \u56e0\u4e3a\u6bcf\u4e2a \/health \u90fd\u8981\u53bb\u95ee\u4e00\u6b21 Ollama&#xff0c;\u74f6\u9888\u5728\u5171\u4eab\u7684 Ollama \u4e0a&#xff0c; 3 \u4e2a\u5b9e\u4f8b\u53ea\u80fd\u4e00\u8d77\u6392\u961f\u7b49\u5b83\u3002\u8fd9\u5c31\u662f\u4e3a\u4ec0\u4e48&#034;\u6269\u5bb9\u524d\u5148\u627e\u74f6\u9888&#034;\u2014\u2014 \u74f6\u9888\u5728\u88ab\u5171\u4eab\u7684\u90a3\u4e00\u5c42&#xff0c;\u52a0\u673a\u5668\u662f\u767d\u52a0\u3002<\/li>\n<li>401 \u8def\u5f84\u8d70 LB \u53cd\u800c\u6162\u4e86 22%&#xff08;950 \u2192 760 req\/s&#xff09;\u3002 \u56e0\u4e3a\u8fd9\u6761\u8def\u5f84\u592a&#034;\u8f7b&#034;\u4e86&#xff1a;\u5355\u4e2a\u7f51\u5173\u8fdb\u7a0b\u672c\u6765\u5c31\u80fd\u625b\u4f4f ~950 req\/s&#xff0c; \u591a\u52a0\u4e00\u4e2a nginx \u53cd\u800c\u591a\u4e00\u8df3&#xff08;\u8fd8\u8981\u8bb0\u4e00\u884c JSON \u8bbf\u95ee\u65e5\u5fd7&#xff0c;\u800c nginx \u81ea\u5df1\u7684\u4e0a\u9650\u4e5f\u5c31 1100 req\/s&#xff09;\u3002 \u8d1f\u8f7d\u5747\u8861\u4e0d\u662f\u514d\u8d39\u7684\u2014\u2014\u5b83\u53ea\u6709\u5728&#034;\u5b9e\u4f8b\u771f\u7684\u88ab\u538b\u5230\u9971\u548c&#034;\u65f6\u624d\u5212\u7b97\u3002<\/li>\n<p>\u8fd9\u4e00\u7ec4\u5b9e\u9a8c\u662f\u6211\u8fd9\u6b21\u5b66\u4e60\u4e2d\u6700&#034;\u53cd\u9e21\u6c64&#034;\u7684\u6536\u83b7&#xff1a; \u6559\u79d1\u4e66\u90fd\u8bf4&#034;\u8d1f\u8f7d\u5747\u8861 \u2192 \u541e\u5410\u63d0\u5347&#034;&#xff0c;\u4f46\u6570\u5b57\u8981\u81ea\u5df1\u6d4b\u3002 \u6211\u539f\u672c\u4ee5\u4e3a 3 \u4e2a\u5b9e\u4f8b\u5c31\u662f 3 \u500d&#xff0c;\u5b9e\u6d4b\u662f 2.2 \u500d \/ 0.8 \u500d&#xff0c;\u53d6\u51b3\u4e8e\u74f6\u9888\u5728\u54ea\u4e00\u5c42\u3002<\/p>\n<h4>10.9 \u60f3\u6e05\u695a\u5929\u82b1\u677f&#xff1a;\u5355\u673a\u591a\u5b9e\u4f8b\u5230\u5e95\u89e3\u51b3\u4e86\u4ec0\u4e48<\/h4>\n<ul>\n<li>\u89e3\u51b3\u4e86&#xff1a;\u8fdb\u7a0b\u7ea7\u5e76\u53d1\u4e0e\u5355\u70b9\u6545\u969c\u3002\u4e00\u4e2a\u8fdb\u7a0b\u5d29\u4e86\u3001\u91cd\u542f&#xff0c;\u7528\u6237\u51e0\u4e4e\u65e0\u611f&#xff1b; Python \u5355\u8fdb\u7a0b\u5403\u6ee1\u4e00\u4e2a\u6838\u7684\u74f6\u9888\u4e5f\u88ab\u644a\u5f00\u4e86\u3002<\/li>\n<li>\u6ca1\u89e3\u51b3&#xff1a;GPU \u53ea\u6709\u4e00\u5757\u30023 \u4e2a\u5b9e\u4f8b\u6700\u7ec8\u8fd8\u662f\u628a\u8bf7\u6c42\u6392\u5230\u540c\u4e00\u4e2a Ollama \u961f\u5217\u91cc&#xff0c; \u7aef\u5230\u7aef\u5ef6\u8fdf\u4e0d\u4f1a\u56e0\u4e3a\u5b9e\u4f8b\u6570\u53d8\u591a\u800c\u4e0b\u964d&#xff08;\u5b9e\u9a8c A \u91cc\u5e76\u53d1\u8d8a\u9ad8\u5e73\u5747\u5ef6\u8fdf\u8d8a\u5927&#xff0c;\u5c31\u662f\u8fd9\u4e2a\u539f\u56e0&#xff09;\u3002<\/li>\n<li>\u771f\u6b63\u7684\u4e0b\u4e00\u6b65&#xff1a;\u628a OLLAMA_URL \u6307\u5411\u591a\u53f0 Ollama&#xff0c;\u5728\u7f51\u5173\u5185\u90e8\u6216\u518d\u5957\u4e00\u5c42\u8c03\u5ea6 &#xff08;\u6309\u6a21\u578b \/ \u6309\u961f\u5217\u957f\u5ea6\u9009\u540e\u7aef&#xff09;&#xff0c;\u624d\u80fd\u8ba9&#034;\u6c34\u5e73\u6269\u5bb9&#034;\u4e00\u8def\u6269\u5230\u63a8\u7406\u5c42\u3002<\/li>\n<\/ul>\n<p>\u4e00\u53e5\u503c\u5f97\u8bb0\u4f4f\u7684\u5de5\u7a0b\u5224\u65ad&#xff1a;\u8d1f\u8f7d\u5747\u8861\u80fd\u6d88\u9664\u74f6\u9888&#xff0c;\u4e0d\u80fd\u521b\u9020\u7b97\u529b\u3002 \u5148\u6d4b\u51fa\u74f6\u9888\u5728\u54ea&#xff08;Grafana &#043; \u538b\u6d4b&#xff09;&#xff0c;\u518d\u51b3\u5b9a\u5f80\u54ea\u4e00\u5c42\u52a0\u673a\u5668\u3002<\/p>\n<hr \/>\n<h3>11. \u8e29\u5751\u5408\u96c6&#xff1a;\u8fd9 10 \u4e2a\u5751&#xff0c;\u6211\u6bcf\u4e2a\u90fd\u82b1\u8fc7\u81f3\u5c11\u534a\u5c0f\u65f6<\/h3>\n<table>\n<tr>#\u73b0\u8c61\u771f\u56e0\u6b63\u89e3<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u63a5\u4e0a nginx \u540e&#034;\u9010\u5b57\u8f93\u51fa&#034;\u53d8\u6210\u4e00\u6b21\u6027\u5237\u51fa<\/td>\n<td>proxy_buffering \u9ed8\u8ba4 on&#xff0c;nginx \u6512\u591f\u7f13\u51b2\u624d\u4e0b\u53d1<\/td>\n<td>proxy_buffering off; &#043; \u5ba2\u6237\u7aef curl -N<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u5bf9\u6bd4\u6d4b\u51fa\u7ecf LB \u9996\u5b57\u8282 1.9s&#xff0c;\u4ee5\u4e3a\u88ab\u7f13\u51b2\u4e86<\/td>\n<td>\u4e24\u6b21\u8bf7\u6c42 num_ctx \u4e0d\u540c \u2192 Ollama \u91cd\u8f7d\u6a21\u578b&#xff08;\u51b7\u542f\u52a8 \u22482.1s&#xff09;<\/td>\n<td>\u5bf9\u6bd4\u5b9e\u9a8c\u53c2\u6570\u5fc5\u987b\u5b8c\u5168\u4e00\u81f4&#xff1b;\u6bcf\u7ec4\u5148 warmup<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>Prometheus \u91cc\u591a\u5b9e\u4f8b\u66f2\u7ebf\u7cca\u6210\u4e00\u6761<\/td>\n<td>\/metrics \u8d70\u4e86 LB&#xff0c;instance \u6807\u7b7e\u5168\u53d8\u6210 nginx \u5730\u5740<\/td>\n<td>\u76f4\u8fde\u6bcf\u4e2a\u5b9e\u4f8b\u6293&#xff1b;LB \u4e0a\u663e\u5f0f location &#061; \/metrics { return 404; }<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>\u957f\u56de\u7b54\u5199\u5230\u4e00\u534a\u8fde\u63a5\u65ad\u4e86<\/td>\n<td>nginx proxy_read_timeout \u9ed8\u8ba4 60s<\/td>\n<td>\u6539\u6210 300s&#xff08;proxy_send_timeout\/send_timeout \u4e00\u8d77&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u7f29\u5bb9\u540e\u5076\u53d1 502<\/td>\n<td>\u5148\u505c\u4e86\u5b9e\u4f8b&#xff0c;nginx \u8fd8\u5728\u5f80\u4e0a\u6253<\/td>\n<td>\u5148 reload \u4e0a\u6e38&#xff0c;\u518d\u505c\u5b9e\u4f8b&#xff08;\u6269\u5bb9\u987a\u5e8f\u76f8\u53cd&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u4e0a\u6e38 keepalive \u6ca1\u751f\u6548&#xff08;\u5168\u662f\u65b0\u8fde\u63a5&#xff09;<\/td>\n<td>\u5c11\u5199\u4e86\u4e00\u884c<\/td>\n<td>proxy_http_version 1.1 &#043; proxy_set_header Connection &#034;&#034; &#043; upstream { keepalive 32; } \u4e09\u8005\u7f3a\u4e00\u4e0d\u53ef<\/td>\n<\/tr>\n<tr>\n<td>7<\/td>\n<td>\u4e00\u8fb9\u538b\u6d4b\u4e00\u8fb9\u8dd1\u9a8c\u6536&#xff0c;\u4e24\u8fb9\u6570\u636e\u90fd\u4e0d\u5bf9<\/td>\n<td>\u62a2\u540c\u4e00\u4e2a\u9650\u6d41\u989d\u5ea6\u3001\u62a2\u540c\u4e00\u5757 GPU<\/td>\n<td>\u5b9e\u9a8c\u8981\u4e32\u884c\u3001\u8981\u9694\u79bb&#xff08;\u89c1 11.1&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>8<\/td>\n<td>cluster.sh scale \u7f29\u5bb9\u65f6\u4e0d\u505c\u5b9e\u4f8b<\/td>\n<td>\u65b0\u8fdb\u7a0b\u8bfb\u4e0d\u5230&#034;\u73b0\u5728\u8dd1\u7740\u54ea\u51e0\u4e2a\u5b9e\u4f8b&#034;\u7684\u73af\u5883\u53d8\u91cf<\/td>\n<td>\u6539\u6210\u6309 pid \u6587\u4ef6\u53cd\u63a8\u7aef\u53e3\u5217\u8868<\/td>\n<\/tr>\n<tr>\n<td>9<\/td>\n<td>\u8bbf\u95ee\u65e5\u5fd7\u91cc\u51fa\u73b0\u660e\u6587 API Key<\/td>\n<td>\u65e5\u5fd7\u683c\u5f0f\u91cc\u5199\u4e86 &#034;api_key&#034;:&#034;$http_x_api_key&#034;<\/td>\n<td>\u4e0a\u7ebf\u524d\u5220\u6389\u8be5\u5b57\u6bb5&#xff0c;\u6216\u53ea\u8bb0\u54c8\u5e0c&#xff08;\u89c1 11.2&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>10<\/td>\n<td>0.35 \u7248 Ollama \u4ece num_gpu&#061;12 \u5207\u56de -1 \u4e0d\u91cd\u8f7d<\/td>\n<td>Ollama \u5224\u5b9a&#034;\u914d\u7f6e\u672a\u53d8&#034;&#xff0c;\u4e0d\u6362 runner<\/td>\n<td>\u6362\u53c2\u524d\u5148 keep_alive&#061;0 \u5f3a\u5236\u5378\u8f7d<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>11.1 \u91cd\u70b9\u8bb2\u8bb2\u7b2c 7 \u4e2a\u5751&#xff1a;\u6211\u7684&#034;\u5b9e\u9a8c\u6c61\u67d3&#034;\u7ffb\u8f66\u73b0\u573a<\/h4>\n<p>\u8fd9\u4e2a\u5751\u5f88\u503c\u5f97\u5355\u72ec\u8bf4&#xff0c;\u56e0\u4e3a\u5b83\u662f\u65b9\u6cd5\u8bba\u7ea7\u522b\u7684\u5751&#xff0c;\u800c\u4e0d\u662f\u914d\u7f6e\u95ee\u9898\u3002<\/p>\n<p>\u90a3\u5929\u6211\u4e3a\u4e86\u7701\u65f6\u95f4&#xff0c;\u8ba9 verify_cluster.py&#xff08;34 \u9879\u81ea\u52a8\u9a8c\u6536&#xff09;\u5728\u540e\u53f0\u8dd1&#xff0c; \u540c\u65f6\u53e6\u5f00\u4e00\u4e2a\u7ec8\u7aef\u505a&#034;\u7ecf LB \u8fde\u53d1 12 \u6b21\u7684\u5171\u4eab\u9650\u6d41\u5b9e\u9a8c&#034;\u3002\u7ed3\u679c\u4e24\u8fb9\u90fd\u51fa\u95ee\u9898&#xff1a;<\/p>\n<p># \u9a8c\u6536\u811a\u672c\u6700\u540e\u7684\u8f93\u51fa&#xff08;\u7b2c\u4e00\u6b21&#xff09;<br \/>\n\u9a8c\u6536\u7ed3\u679c: 32\/34 \u9879\u901a\u8fc7<br \/>\n  FAIL  \u672a\u88ab nginx \u7f13\u51b2&#xff08;\u9996\u5b57\u8282\u8fdc\u65e9\u4e8e\u603b\u65f6\u957f&#xff0c;\u4e14\u4e0e\u76f4\u8fde\u76f8\u5f53&#xff09;<br \/>\n  FAIL  \u8de8 3 \u4e2a\u5b9e\u4f8b\u53ea\u653e\u884c 10 \u6b21&#xff08;\u989d\u5ea6\u771f\u5171\u4eab&#xff09;<\/p>\n<p># \u6211\u7684\u9650\u6d41\u5b9e\u9a8c&#xff08;\u7b2c\u4e00\u6b21&#xff09;<br \/>\n200 200 200 200 200 200 200 200 200 429 429 429      \u2190 \u53ea\u6709 9 \u4e2a 200&#xff1f;\u4e3a\u4ec0\u4e48\u4e0d\u662f 10&#xff1f;<\/p>\n<p>\u6211\u4e00\u5f00\u59cb\u4ee5\u4e3a\u662f\u4ee3\u7801\u6709 bug&#xff0c;\u5dee\u70b9\u53bb\u6539\u9650\u6d41\u903b\u8f91\u3002\u540e\u6765\u60f3\u660e\u767d\u4e86&#xff1a;<\/p>\n<li>\u9650\u6d41\u989d\u5ea6\u662f\u5168\u5c40\u7684&#xff1a;\u9a8c\u6536\u811a\u672c\u7528\u7684\u662f sk-test-001&#xff08;free \u6863 10 \u6b21\/\u5206\u949f&#xff09;&#xff0c; \u6211\u7684\u5b9e\u9a8c\u4e5f\u7528\u540c\u4e00\u4e2a key&#xff0c;\u4e24\u4e2a\u8fdb\u7a0b\u5728\u62a2\u540c\u4e00\u4e2a\u914d\u989d&#xff1b; \u800c\u4e14\u6211\u7684\u811a\u672c\u91cc\u6709 FLUSHDB&#xff0c;\u76f4\u63a5\u628a\u9a8c\u6536\u811a\u672c\u7684\u8ba1\u6570\u5668\u6e05\u7a7a\u4e86 \u2192 \u5b83\u5f53\u7136 PASS \u4e0d\u4e86\u3002<\/li>\n<li>GPU \u4e5f\u662f\u5168\u5c40\u7684&#xff1a;\u9a8c\u6536\u811a\u672c\u6d4b&#034;\u7ecf LB vs \u76f4\u8fde\u7684\u9996\u5b57\u8282\u65f6\u95f4&#034;\u65f6&#xff0c; \u6211\u7684\u538b\u6d4b\u6b63\u5360\u7740 1050Ti \u751f\u6210 token&#xff0c;\u4e24\u6b21\u6d4b\u91cf\u7684\u6392\u961f\u60c5\u51b5\u4e0d\u540c \u2192 \u8ba1\u65f6\u5bf9\u6bd4\u5931\u771f \u2192 FAIL\u3002<\/li>\n<p>\u628a\u4e24\u4e2a\u5b9e\u9a8c\u4e32\u884c\u8dd1\u4e4b\u540e&#xff1a;<\/p>\n<p>\u9a8c\u6536\u7ed3\u679c: 34\/34 \u9879\u901a\u8fc7<br \/>\nround 1 \u7ecf LB 12 \u6b21: 200 200 200 200 200 200 200 200 200 200 429 429<br \/>\nround 2 \u7ecf LB 12 \u6b21: 200 200 200 200 200 200 200 200 200 200 429 429<\/p>\n<p>\u5e72\u5e72\u51c0\u51c0&#xff0c;\u8fd8\u987a\u624b\u5f97\u5230\u4e86&#034;\u4e24\u6b21\u590d\u73b0\u4e00\u81f4&#034;\u8fd9\u4e2a\u989d\u5916\u8bc1\u636e\u3002<\/p>\n<p>\u8fd9\u6761\u7ecf\u9a8c\u6211\u89c9\u5f97\u6bd4\u4efb\u4f55\u914d\u7f6e\u90fd\u503c\u94b1&#xff1a; \u6027\u80fd\u4e0e\u884c\u4e3a\u5b9e\u9a8c\u5fc5\u987b\u72ec\u5360\u8d44\u6e90&#xff0c;\u5426\u5219\u4f60\u6d4b\u51fa\u6765\u7684&#034;\u6545\u969c&#034;\u53ef\u80fd\u53ea\u662f\u53e6\u4e00\u4e2a\u5b9e\u9a8c\u7684\u5f71\u5b50\u3002<\/p>\n<h4>11.2 \u91cd\u70b9\u8bb2\u8bb2\u7b2c 9 \u4e2a\u5751&#xff1a;\u4e00\u4e2a\u4e0d\u8be5\u51fa\u73b0\u5728\u65e5\u5fd7\u91cc\u7684\u5b57\u6bb5<\/h4>\n<p>\u8fd9\u662f\u6211\u5199\u5b8c nginx \u65e5\u5fd7\u683c\u5f0f\u4e4b\u540e\u81ea\u5df1 review \u51fa\u6765\u7684\u95ee\u9898\u3002\u4e3a\u4e86\u6392\u969c\u65b9\u4fbf&#xff0c; \u6211\u8ba9\u8bbf\u95ee\u65e5\u5fd7\u628a\u8bf7\u6c42\u5934\u4e5f\u8bb0\u4e86\u4e0b\u6765&#xff1a;<\/p>\n<p>log_format gw escape&#061;json &#039;{&#039;<br \/>\n    &#039;&#034;uri&#034;:&#034;$uri&#034;,&#039;<br \/>\n    &#039;&#034;upstream_addr&#034;:&#034;$upstream_addr&#034;,&#039;<br \/>\n    &#039;&#034;instance&#034;:&#034;$upstream_http_x_gateway_instance&#034;,&#039;<br \/>\n    &#039;&#034;api_key&#034;:&#034;$http_x_api_key&#034;&#039;      # \u2190 \u8fd9\u91cc<br \/>\n&#039;}&#039;;<\/p>\n<p>\u6253\u51fa\u6765\u7684\u65e5\u5fd7\u786e\u5b9e\u597d\u7528&#xff1a;<\/p>\n<p><span class=\"token punctuation\">{<\/span><span class=\"token string-property property\">&#034;time&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;2026-10-07T23:44:13&#043;08:00&#034;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;method&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;POST&#034;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;uri&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;\/v1\/chat\/completions&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;status&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token number\">200<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;request_time&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token number\">0.028<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;upstream_addr&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;127.0.0.1:8002&#034;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n <span class=\"token string-property property\">&#034;upstream_status&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;200&#034;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;instance&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;gateway-8002&#034;<\/span><span class=\"token punctuation\">,<\/span><span class=\"token string-property property\">&#034;api_key&#034;<\/span><span class=\"token operator\">:<\/span><span class=\"token string\">&#034;sk-prod-001&#034;<\/span><span class=\"token punctuation\">}<\/span><\/p>\n<p>\u4f46\u8fd9\u662f\u7edd\u5bf9\u4e0d\u80fd\u4e0a\u751f\u4ea7\u7684&#xff1a;\u8bbf\u95ee\u65e5\u5fd7\u4f1a\u88ab\u91c7\u96c6\u3001\u8f6c\u53d1\u3001\u843d\u76d8\u3001\u5f52\u6863&#xff0c; \u7b49\u4e8e\u628a API Key \u660e\u6587\u6492\u5f97\u6ee1\u5730\u90fd\u662f\u3002\u6b63\u786e\u505a\u6cd5\u662f\u53ea\u8bb0&#034;\u80fd\u5426\u8bc6\u522b\u7528\u6237&#034;&#xff0c;\u4e0d\u8bb0\u5bc6\u94a5\u672c\u4f53&#xff1a;<\/p>\n<p># \u65b9\u6848\u4e00&#xff1a;\u5e72\u8106\u4e0d\u8bb0&#xff08;\u6700\u7701\u4e8b&#xff0c;\u6392\u969c\u65f6\u7528\u7f51\u5173\u81ea\u5df1\u7684 JSONL \u65e5\u5fd7\u53bb\u5bf9&#xff09;<br \/>\n#   \u5220\u6389 &#039;&#034;api_key&#034;:&#034;$http_x_api_key&#034;&#039; \u8fd9\u4e00\u884c\u5373\u53ef<\/p>\n<p># \u65b9\u6848\u4e8c&#xff1a;\u8bb0\u4e00\u4e2a&#034;\u53ef\u8ffd\u6eaf\u4f46\u4e0d\u654f\u611f&#034;\u7684\u6807\u8bc6\u3002<br \/>\n#   \u5728\u7f51\u5173\u4fa7\u989d\u5916\u8fd4\u56de\u4e00\u4e2a\u63a9\u7801\u540e\u7684 key \u6807\u8bc6&#xff08;\u5982 sk-prod-***-1a2b&#xff09;&#xff0c;<br \/>\n#   nginx \u53ea\u8bb0\u8fd9\u4e2a\u5934&#xff0c;\u4e0d\u8bb0\u539f\u59cb\u5bc6\u94a5&#xff1a;<br \/>\n&#039;&#034;api_key_id&#034;:&#034;$upstream_http_x_api_key_id&#034;&#039;<\/p>\n<p>\u6559\u8bad&#xff1a;\u6392\u969c\u4fbf\u5229\u6027\u548c\u6570\u636e\u5b89\u5168\u7ecf\u5e38\u51b2\u7a81&#xff0c;\u9ed8\u8ba4\u5e94\u8be5\u9009\u5b89\u5168&#xff0c;\u9700\u8981\u65f6\u518d\u4e34\u65f6\u6253\u5f00\u3002 &#xff08;\u6211\u8fd9\u4efd\u6a21\u677f\u662f\u5b66\u4e60\u7528\u9014\u6240\u4ee5\u4fdd\u7559\u4e86\u8be5\u5b57\u6bb5&#xff0c;\u4f46\u5df2\u7ecf\u5728\u6587\u6863\u91cc\u6807\u6ce8&#034;\u4e0a\u7ebf\u5fc5\u987b\u5220&#034;\u3002&#xff09;<\/p>\n<hr \/>\n<h3>12. \u4e00\u5757 1050Ti \u6559\u4f1a\u6211\u7684 5 \u4ef6\u4e8b<\/h3>\n<p>\u5199\u5b8c 11 \u4e2a\u9636\u6bb5\u56de\u5934\u770b&#xff0c;\u771f\u6b63\u8ba9\u6211&#034;\u80fd\u529b\u5347\u7ea7&#034;\u7684\u4e0d\u662f\u4f1a\u7528\u591a\u5c11\u4e2a\u7ec4\u4ef6&#xff0c;\u800c\u662f\u4e0b\u9762\u8fd9 5 \u4e2a\u8ba4\u77e5&#xff1a;<\/p>\n<p>1. \u5148\u627e\u74f6\u9888&#xff0c;\u518d\u8c08\u4f18\u5316\u3002 \/health \u8d70 LB \u53ea\u6709 2.2 \u500d\u63d0\u5347&#xff08;\u800c\u4e0d\u662f 3 \u500d&#xff09;&#xff0c;\u56e0\u4e3a\u74f6\u9888\u5728\u5171\u4eab\u7684 Ollama \u4e0a&#xff1b; 401 \u8def\u5f84\u8d70 LB \u53cd\u800c\u6162 22%&#xff0c;\u56e0\u4e3a\u74f6\u9888\u53d8\u6210\u4e86&#034;\u591a\u4e00\u8df3&#034;\u3002 \u540c\u4e00\u4e2a&#034;\u52a0\u673a\u5668&#034;\u7684\u52a8\u4f5c&#xff0c;\u5728\u4e0d\u540c\u74f6\u9888\u4e0b\u6536\u76ca\u662f 2.3\u00d7 \u548c 0.8\u00d7 \u7684\u5dee\u522b\u3002 \u6570\u5b57\u4e0d\u6d4b\u51fa\u6765&#xff0c;\u4f60\u6839\u672c\u4e0d\u77e5\u9053\u8be5\u5f80\u54ea\u52a0\u3002<\/p>\n<p>2. \u89c2\u6d4b\u7ec4\u4ef6\u5fc5\u987b\u80fd\u5931\u8d25&#xff0c;\u4e14\u5931\u8d25\u4e0d\u80fd\u62d6\u6b7b\u4e3b\u94fe\u8def\u3002 Redis \u6302\u4e86\u9650\u6d41\u9000\u5316\u6210\u5355\u5b9e\u4f8b\u3001PG \u6302\u4e86\u53ea\u4e22\u5386\u53f2\u3001Kafka \u6302\u4e86\u5199\u672c\u5730\u6587\u4ef6\u2014\u2014 \u6bcf\u6b21\u6253\u6302\u4e00\u4e2a\u7ec4\u4ef6&#xff0c;\u63a8\u7406\u8bf7\u6c42\u90fd\u8fd8\u662f 200\u3002 \u8fd9\u662f&#034;\u80fd\u4e0a\u7ebf&#034;\u548c&#034;\u53ea\u80fd\u8dd1 demo&#034;\u4e4b\u95f4\u7684\u5206\u754c\u7ebf\u3002<\/p>\n<p>3. \u6240\u6709\u72b6\u6001\u90fd\u8981\u95ee\u4e00\u53e5&#034;\u5b83\u653e\u5728\u54ea&#034;\u3002 \u6211\u6700\u521d\u60f3\u4e0d\u901a&#034;\u4e3a\u4ec0\u4e48\u591a\u8d77\u4e00\u4e2a\u8fdb\u7a0b\u5c31\u7b97\u6269\u5bb9&#034;&#xff0c;\u540e\u6765\u53d1\u73b0\u7b54\u6848\u5f88\u7b80\u5355&#xff1a; \u56e0\u4e3a\u7f51\u5173\u6ca1\u6709\u72b6\u6001\u3002\u9650\u6d41\u5728 Redis\u3001\u5386\u53f2\u5728\u5e93\u3001\u4e8b\u4ef6\u5728\u961f\u5217\u3001\u914d\u7f6e\u5728\u73af\u5883\u53d8\u91cf\u2014\u2014 \u8fdb\u7a0b\u672c\u8eab\u53d8\u6210\u53ef\u968f\u610f\u66ff\u6362\u7684&#034;\u8ba1\u7b97\u5355\u5143&#034;&#xff0c;\u8fd9\u624d\u662f\u6c34\u5e73\u6269\u5bb9\u7684\u524d\u63d0\u3002<\/p>\n<p>4. \u4e00\u5207\u90fd\u8981\u80fd\u81ea\u52a8\u6062\u590d\u3002 Redis 5 \u79d2\u91cd\u8fde\u3001Kafka 10 \u79d2\u91cd\u8fde\u3001\u5386\u53f2\u5199\u5931\u8d25\u540e\u4e0b\u6279\u6362\u65b0\u8fde\u63a5\u3001 nginx \u5b9e\u4f8b 10 \u79d2\u540e\u81ea\u52a8\u52a0\u56de\u4e0a\u6e38\u3002 \u76ee\u6807\u662f\u4e0d\u9700\u8981&#034;\u534a\u591c\u8d77\u6765\u91cd\u542f\u7f51\u5173&#034;\u3002 \u8fd9\u4e00\u70b9\u5728\u5355\u673a\u73af\u5883\u91cc\u53cd\u800c\u66f4\u597d\u9a8c\u8bc1\u2014\u2014 kill \u4e00\u4e0b\u5c31\u77e5\u9053\u6062\u590d\u5f97\u5bf9\u4e0d\u5bf9\u3002<\/p>\n<p>5. \u6162\u4e0b\u6765&#xff0c;\u624d\u80fd\u770b\u6e05\u3002 1050Ti \u751f\u6210 0.5B \u6a21\u578b\u53ea\u6709 ~176 tok\/s&#xff0c;\u4e00\u6b21 128 token \u7684\u56de\u7b54\u8981 1 \u79d2\u591a\u3002 \u6b63\u56e0\u4e3a\u6162&#xff0c;\u5e76\u53d1\u4e00\u4e0a\u6765 TTFT \u4ece 0.03s \u6da8\u5230 3.62s \u7684\u66f2\u7ebf\u6e05\u6670\u53ef\u89c1&#xff0c; \u9650\u6d41\u3001\u964d\u7ea7\u3001\u6545\u969c\u8f6c\u79fb\u7684\u6bcf\u4e2a\u73b0\u8c61\u90fd\u80fd\u88ab\u8089\u773c\u89c2\u5bdf\u5230\u3002 \u5982\u679c\u4e00\u5f00\u59cb\u5c31\u7528 A100 \u8dd1 70B&#xff0c;\u6211\u5927\u6982\u4f1a\u628a\u6240\u6709\u73b0\u8c61\u90fd\u5f52\u56e0\u4e8e&#034;\u5b83\u592a\u5feb\u4e86&#xff0c;\u770b\u4e0d\u6e05&#034;\u3002<\/p>\n<p>\u6362\u53e5\u8bdd\u8bf4&#xff1a;4GB \u663e\u5b58\u7684\u9650\u5236&#xff0c;\u903c\u6211\u628a\u7cbe\u529b\u653e\u5728\u4e86\u771f\u6b63\u91cd\u8981\u3001\u4e5f\u771f\u6b63\u53ef\u8fc1\u79fb\u7684\u5730\u65b9\u2014\u2014 \u670d\u52a1\u5316\u3001\u53ef\u89c2\u6d4b\u3001\u964d\u7ea7\u3001\u6269\u5bb9\u3001\u9a8c\u6536\u3002<\/p>\n<hr \/>\n<h3>13. \u60f3\u590d\u73b0&#xff1f;\u6700\u77ed\u8def\u5f84&#xff08;\u7ea6\u4e00\u4e2a\u4e0b\u5348&#xff09;<\/h3>\n<p>\u5982\u679c\u4f60\u4e5f\u6709\u4e00\u5f20\u8001\u663e\u5361&#xff08;\u751a\u81f3\u6ca1\u6709\u663e\u5361&#xff0c;\u7eaf CPU \u4e5f\u80fd\u8dd1 0.5B&#xff09;&#xff0c;\u53ef\u4ee5\u8fd9\u6837\u5f00\u59cb&#xff1a;<\/p>\n<p><span class=\"token comment\"># 0) \u57fa\u7840&#xff1a;Ollama &#043; \u6a21\u578b<\/span><br \/>\nollama serve <span class=\"token operator\">&amp;<\/span><br \/>\nollama pull qwen2.5:0.5b<\/p>\n<p><span class=\"token comment\"># 1) \u8d77\u7f51\u5173&#xff08;\u96f6\u4f9d\u8d56\u4e5f\u80fd\u8dd1&#xff1a;\u9650\u6d41\u8d70\u5185\u5b58\u3001\u5386\u53f2\u8d70 SQLite\u3001\u4e8b\u4ef6\u5199\u672c\u5730\u6587\u4ef6&#xff09;<\/span><br \/>\npython main.py<br \/>\n<span class=\"token function\">curl<\/span> <span class=\"token parameter variable\">-s<\/span> <span class=\"token parameter variable\">-X<\/span> POST http:\/\/localhost:8000\/v1\/chat\/completions <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;X-API-Key: sk-prod-001&#039;<\/span> <span class=\"token parameter variable\">-H<\/span> <span class=\"token string\">&#039;Content-Type: application\/json&#039;<\/span> <span class=\"token punctuation\">\\\\<\/span><br \/>\n  <span class=\"token parameter variable\">-d<\/span> <span class=\"token string\">&#039;{&#034;model&#034;:&#034;qwen0.5b&#034;,&#034;messages&#034;:[{&#034;role&#034;:&#034;user&#034;,&#034;content&#034;:&#034;hi&#034;}],&#034;options&#034;:{&#034;num_predict&#034;:8}}&#039;<\/span><\/p>\n<p><span class=\"token comment\"># 2) \u63a5\u4e0a\u4e09\u4ef6\u57fa\u7840\u8bbe\u65bd&#xff08;\u514d root \u89e3\u5305 Redis \/ PostgreSQL \/ Kafka&#xff09;<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/infra.sh <span class=\"token function\">install<\/span> <span class=\"token operator\">&amp;&amp;<\/span> <span class=\"token function\">bash<\/span> scripts\/infra.sh start<\/p>\n<p><span class=\"token comment\"># 3) \u8d77\u76d1\u63a7<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/install_monitoring.sh <span class=\"token operator\">&amp;&amp;<\/span> <span class=\"token function\">bash<\/span> scripts\/monitoring.sh start<br \/>\n<span class=\"token comment\">#    Prometheus: http:\/\/localhost:9090     Grafana: http:\/\/localhost:3000\/d\/llm-gateway<\/span><\/p>\n<p><span class=\"token comment\"># 4) \u591a\u5b9e\u4f8b &#043; \u8d1f\u8f7d\u5747\u8861<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/nginx.sh <span class=\"token function\">install<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/cluster.sh start          <span class=\"token comment\"># 3 \u5b9e\u4f8b &#043; nginx :8080<\/span><br \/>\n<span class=\"token function\">bash<\/span> scripts\/cluster.sh scale <span class=\"token number\">5<\/span>        <span class=\"token comment\"># \u70ed\u6269\u5bb9<\/span><\/p>\n<p><span class=\"token comment\"># 5) \u9a8c\u6536&#xff08;\u8fd9\u4e24\u4e2a\u811a\u672c\u4f1a\u544a\u8bc9\u4f60&#034;\u4f60\u771f\u7684\u4f1a\u4e86\u6ca1\u6709&#034;&#xff09;<\/span><br \/>\npython verify_infra.py                 <span class=\"token comment\"># \u671f\u671b 21\/21 \u901a\u8fc7<\/span><br \/>\npython verify_cluster.py               <span class=\"token comment\"># \u671f\u671b 34\/34 \u901a\u8fc7<\/span><\/p>\n<p>\u5efa\u8bae\u7684\u5b66\u4e60\u987a\u5e8f&#xff08;\u4e5f\u662f\u6211\u5b9e\u9645\u8d70\u7684\u987a\u5e8f&#xff0c;\u8df3\u8fc7&#034;\u964d\u7ea7\u6f14\u7ec3&#034;\u4f1a\u5403\u4e8f&#xff09;&#xff1a;<\/p>\n<p>\u8dd1\u901a \u2192 \u8bfb\u61c2\u4e3b\u94fe\u8def \u2192 SSE \u2192 \u76d1\u63a7 \u2192 \u9650\u6d41 \u2192 \u5386\u53f2 \u2192 \u4e8b\u4ef6 \u2192 \u964d\u7ea7\u6f14\u7ec3 \u2192 \u8c03\u4f18 \u2192 \u591a\u5b9e\u4f8b<br \/>\n                                                 \u2191<br \/>\n                                     \u522b\u8df3\u8fc7\u8fd9\u4e00\u6b65&#xff0c;\u5b83\u662f\u5206\u6c34\u5cad<\/p>\n<p>\u96be\u5ea6\u4e0e\u65f6\u95f4\u53c2\u8003&#xff08;\u4e1a\u4f59\u65f6\u95f4\u505a&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u9636\u6bb5\u8017\u65f6\u96be\u5ea6<\/tr>\n<tbody>\n<tr>\n<td>\u8dd1\u901a &#043; \u4e3b\u94fe\u8def &#043; SSE<\/td>\n<td>1 \u5929<\/td>\n<td>\u2605<\/td>\n<\/tr>\n<tr>\n<td>\u76d1\u63a7&#xff08;Prometheus \/ Grafana&#xff09;<\/td>\n<td>1~2 \u5929<\/td>\n<td>\u2605\u2605<\/td>\n<\/tr>\n<tr>\n<td>\u5206\u5e03\u5f0f\u9650\u6d41<\/td>\n<td>1 \u5929<\/td>\n<td>\u2605\u2605<\/td>\n<\/tr>\n<tr>\n<td>\u5386\u53f2 &#043; \u4e8b\u4ef6&#xff08;PG \/ Kafka&#xff09;<\/td>\n<td>1~2 \u5929<\/td>\n<td>\u2605\u2605\u2605<\/td>\n<\/tr>\n<tr>\n<td>\u964d\u7ea7\u6f14\u7ec3<\/td>\n<td>\u534a\u5929<\/td>\n<td>\u2605\u2605&#xff08;\u6536\u83b7\u6700\u5927&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u538b\u6d4b\u4e0e\u53c2\u6570\u8c03\u4f18<\/td>\n<td>1 \u5929<\/td>\n<td>\u2605\u2605<\/td>\n<\/tr>\n<tr>\n<td>\u591a\u5b9e\u4f8b &#043; nginx<\/td>\n<td>1~2 \u5929<\/td>\n<td>\u2605\u2605\u2605<\/td>\n<\/tr>\n<tr>\n<td>12 \u9053\u8fdb\u9636\u52a8\u624b\u9898<\/td>\n<td>\u6309\u9700<\/td>\n<td>\u2605\u2605\u2605\u2605<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>14. \u5199\u5728\u6700\u540e<\/h3>\n<p>\u6709\u53e5\u8bdd\u6211\u7279\u522b\u60f3\u5bf9\u540c\u6837\u88ab\u786c\u4ef6\u5361\u4f4f\u7684\u540c\u5b66\u8bf4&#xff1a;<\/p>\n<p>AI Infra \u4e0d\u7b49\u4e8e&#034;\u8bad\u5927\u6a21\u578b&#034;\u3002<\/p>\n<p>\u5728\u4e00\u53f0 1050Ti \u7b14\u8bb0\u672c\u4e0a&#xff0c;\u6211\u7167\u6837\u628a\u5de5\u7a0b\u91cc\u6700\u96be\u4e5f\u6700\u503c\u94b1\u7684\u90a3\u90e8\u5206\u7ec3\u4e86\u4e00\u904d&#xff1a; \u9650\u6d41\u600e\u4e48\u505a\u624d\u4e0d\u4f1a\u8ba9\u4e00\u4e2a\u7528\u6237\u62d6\u6b7b\u6240\u6709\u4eba\u3001\u89c2\u6d4b\u7ec4\u4ef6\u6302\u4e86\u670d\u52a1\u8981\u4e0d\u8981\u8ddf\u7740\u6b7b\u3001 \u591a\u5b9e\u4f8b\u4e4b\u95f4\u72b6\u6001\u600e\u4e48\u5171\u4eab\u3001SSE \u600e\u4e48\u624d\u4e0d\u88ab\u4e2d\u95f4\u4ef6\u7f13\u51b2\u3001 \u4e00\u4e2a\u5b9e\u4f8b\u5d29\u4e86\u7528\u6237\u4e3a\u4ec0\u4e48\u6beb\u65e0\u611f\u77e5\u3001\u6269\u5bb9\u5230\u5e95\u8be5\u6269\u54ea\u4e00\u5c42\u3002<\/p>\n<p>\u8fd9\u4e9b\u80fd\u529b\u6362\u5230\u4efb\u4f55\u4e00\u5757 A100\/H100 \u4e0a\u90fd\u76f4\u63a5\u53ef\u7528\u2014\u2014\u53ea\u662f\u6570\u5b57\u4f1a\u4ece 176 tok\/s \u53d8\u6210 3000 tok\/s&#xff0c; 3 \u4e2a\u5b9e\u4f8b\u53d8\u6210 30 \u4e2a\u5b9e\u4f8b\u3002\u800c&#034;\u5148\u6d4b\u74f6\u9888\u518d\u4f18\u5316&#034;\u201c\u89c2\u6d4b\u662f\u589e\u5f3a\u4e0d\u662f\u4f9d\u8d56\u201d\u201c\u72b6\u6001\u8981\u5916\u7f6e\u201d\u201c\u4e00\u5207\u81ea\u52a8\u6062\u590d\u201d \u8fd9\u4e9b\u5224\u65ad&#xff0c;\u8ddf\u663e\u5361\u578b\u53f7\u6beb\u65e0\u5173\u7cfb\u3002<\/p>\n<p>\u771f\u6b63\u9650\u5236\u6211\u4eec\u7684&#xff0c;\u4ece\u6765\u4e0d\u662f 4GB \u663e\u5b58&#xff0c;\u800c\u662f**\u201c\u53ea\u628a\u670d\u52a1\u8dd1\u8d77\u6765&#034;\u548c&#034;\u8bf4\u6e05\u695a\u5b83\u5728\u4ec0\u4e48\u6761\u4ef6\u4e0b\u4f1a\u574f\u201d \u4e4b\u95f4\u7684\u90a3\u6bb5\u8ddd\u79bb**\u3002\u8fd9\u4e00\u6bb5&#xff0c;\u4e00\u5757 1050Ti \u5b8c\u5168\u591f\u7528\u3002<\/p>\n<hr \/>\n<p>\u672c\u6587\u6240\u6709\u6570\u636e\u5747\u4e3a\u672c\u673a\u5b9e\u6d4b&#xff08;GTX 1050 Ti 4GB \/ Ollama 0.35.0 \/ Prometheus 3.15.0 \/ nginx 1.28.3 \/ Redis 8.0.5 \/ PostgreSQL 18.6&#xff09;&#xff0c;\u9a8c\u6536\u7ed3\u679c 21\/21\u300134\/34&#xff1b; \u53c2\u6570\u626b\u63cf\u7684\u539f\u59cb\u660e\u7ec6\u5728 logs\/tuning_experiment.jsonl\u3002<\/p>\n<p>\u9879\u76ee\u6587\u4ef6\u901f\u67e5&#xff08;\u5982\u679c\u4f60\u4e5f\u60f3\u7167\u7740\u5199\u4e00\u4efd&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u6587\u4ef6\u4f5c\u7528<\/tr>\n<tbody>\n<tr>\n<td>main.py<\/td>\n<td>\u7f51\u5173\u4e3b\u94fe\u8def&#xff08;\u9274\u6743 \/ \u9650\u6d41 \/ \u8def\u7531 \/ \u8f6c\u53d1 \/ \u7559\u75d5&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>auth.py ratelimit.py router.py<\/td>\n<td>\u9274\u6743\u3001Redis&#043;Lua \u9650\u6d41\u3001\u6a21\u578b\u522b\u540d\u8def\u7531<\/td>\n<\/tr>\n<tr>\n<td>storage.py events.py logger.py<\/td>\n<td>\u5386\u53f2\u5e93&#xff08;SQLite\/PG&#xff09;\u3001Kafka \u4e8b\u4ef6\u3001JSONL \u65e5\u5fd7<\/td>\n<\/tr>\n<tr>\n<td>prometheus_metrics.py<\/td>\n<td>\u6307\u6807\u5b9a\u4e49&#xff08;Counter \/ Gauge \/ Histogram&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>scripts\/infra.sh scripts\/monitoring.sh<\/td>\n<td>Redis\/PG\/Kafka\u3001Prometheus\/Grafana \u7684\u4e00\u952e\u8d77\u505c&#xff08;\u514d root&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>scripts\/nginx.sh scripts\/cluster.sh<\/td>\n<td>nginx \u5b89\u88c5\/\u6e32\u67d3\/\u70ed\u52a0\u8f7d\u3001\u591a\u5b9e\u4f8b\u8d77\u505c\u4e0e\u6269\u7f29\u5bb9<\/td>\n<\/tr>\n<tr>\n<td>nginx\/nginx.conf.tpl<\/td>\n<td>\u8d1f\u8f7d\u5747\u8861\u914d\u7f6e\u6a21\u677f&#xff08;SSE\u3001\u5065\u5eb7\u68c0\u67e5\u3001\u65e5\u5fd7\u683c\u5f0f&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>verify_infra.py verify_cluster.py<\/td>\n<td>21 \u9879 \/ 34 \u9879\u81ea\u52a8\u9a8c\u6536<\/td>\n<\/tr>\n<tr>\n<td>tuning_experiment.py<\/td>\n<td>Ollama \u53c2\u6570\u53d7\u63a7\u626b\u63cf\u5b9e\u9a8c<\/td>\n<\/tr>\n<tr>\n<td>README.md study.md<\/td>\n<td>\u901f\u67e5\u624b\u518c &#043; 11 \u9636\u6bb5\u9010\u6b65\u6559\u5b66\u624b\u518c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n","protected":false},"excerpt":{"rendered":"<p>\u4e00\u5757 1050Ti \u5b66 AI Infra&#xff1a;\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173&#xff0c;\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 4GB \u663e\u5b58\u3001i5-7300HQ\u300111GB \u5185\u5b58\u7684\u8001\u7b14\u8bb0\u672c&#xff0c;\u8dd1\u4e0d\u52a8 7B \u6a21\u578b&#xff0c;\u66f4\u522b\u8bf4\u5206\u5e03\u5f0f\u63a8\u7406\u3002 \u4f46\u6211\u7528\u5b83\u628a AI Infra \u91cc\u6700\u5e38\u88ab\u95ee\u5230\u7684\u4e1c\u897f\u90fd\u505a\u4e86\u4e00\u904d&#xff1a; \u9274\u6743\u3001\u9650\u6d41\u3001\u6a21\u578b\u8def\u7531\u3001SSE \u6d41\u5f0f\u3001Prometheus\/Grafana\u3001Redis\u3001PostgreSQL\u3001Kafka\u3001\u964d\u7ea7\u6f14<\/p>\n","protected":false},"author":2,"featured_media":114264,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[1699,52,50,188,51],"topic":[],"class_list":["post-114265","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-agi","tag-aigc","tag-50","tag-188","tag-51"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/114265.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u4e00\u5757 1050Ti \u5b66 AI Infra&#xff1a;\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173&#xff0c;\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 4GB \u663e\u5b58\u3001i5-7300HQ\u300111GB \u5185\u5b58\u7684\u8001\u7b14\u8bb0\u672c&#xff0c;\u8dd1\u4e0d\u52a8 7B \u6a21\u578b&#xff0c;\u66f4\u522b\u8bf4\u5206\u5e03\u5f0f\u63a8\u7406\u3002 \u4f46\u6211\u7528\u5b83\u628a AI Infra \u91cc\u6700\u5e38\u88ab\u95ee\u5230\u7684\u4e1c\u897f\u90fd\u505a\u4e86\u4e00\u904d&#xff1a; \u9274\u6743\u3001\u9650\u6d41\u3001\u6a21\u578b\u8def\u7531\u3001SSE \u6d41\u5f0f\u3001Prometheus\/Grafana\u3001Redis\u3001PostgreSQL\u3001Kafka\u3001\u964d\u7ea7\u6f14\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/114265.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-08T10:00:44+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/10\/20261008100042-6ac769cabae90.png\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"17 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/114265.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/114265.html\",\"name\":\"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-10-08T10:00:44+00:00\",\"dateModified\":\"2026-10-08T10:00:44+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/114265.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/114265.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/114265.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/114265.html","og_locale":"zh_CN","og_type":"article","og_title":"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u4e00\u5757 1050Ti \u5b66 AI Infra&#xff1a;\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173&#xff0c;\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 4GB \u663e\u5b58\u3001i5-7300HQ\u300111GB \u5185\u5b58\u7684\u8001\u7b14\u8bb0\u672c&#xff0c;\u8dd1\u4e0d\u52a8 7B \u6a21\u578b&#xff0c;\u66f4\u522b\u8bf4\u5206\u5e03\u5f0f\u63a8\u7406\u3002 \u4f46\u6211\u7528\u5b83\u628a AI Infra \u91cc\u6700\u5e38\u88ab\u95ee\u5230\u7684\u4e1c\u897f\u90fd\u505a\u4e86\u4e00\u904d&#xff1a; \u9274\u6743\u3001\u9650\u6d41\u3001\u6a21\u578b\u8def\u7531\u3001SSE \u6d41\u5f0f\u3001Prometheus\/Grafana\u3001Redis\u3001PostgreSQL\u3001Kafka\u3001\u964d\u7ea7\u6f14","og_url":"https:\/\/www.wsisp.com\/helps\/114265.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-10-08T10:00:44+00:00","og_image":[{"url":"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/10\/20261008100042-6ac769cabae90.png"}],"author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"17 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/114265.html","url":"https:\/\/www.wsisp.com\/helps\/114265.html","name":"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-10-08T10:00:44+00:00","dateModified":"2026-10-08T10:00:44+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/114265.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/114265.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/114265.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u4e00\u5757 1050Ti \u5b66 AI Infra\uff1a\u6211\u624b\u6413\u4e86\u4e00\u4e2a LLM \u7f51\u5173\uff0c\u4ece Redis \u9650\u6d41\u4e00\u8def\u505a\u5230 nginx \u8d1f\u8f7d\u5747\u8861"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/114265","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=114265"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/114265\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media\/114264"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=114265"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=114265"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=114265"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=114265"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}