{"id":110862,"date":"2026-09-29T15:24:48","date_gmt":"2026-09-29T07:24:48","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/110862.html"},"modified":"2026-09-29T15:24:48","modified_gmt":"2026-09-29T07:24:48","slug":"%e6%a8%a1%e5%9e%8b%e7%9b%91%e6%8e%a7%e4%b8%8e%e5%91%8a%e8%ad%a6%ef%bc%9a%e6%80%a7%e8%83%bd%e6%8c%87%e6%a0%87%e3%80%81%e5%bc%82%e5%b8%b8%e6%a3%80%e6%b5%8b%e4%b8%8e%e6%97%a5%e5%bf%97","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/110862.html","title":{"rendered":"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7"},"content":{"rendered":"

\"\u5728\u8fd9\u91cc\u63d2\u5165\u56fe\u7247\u63cf\u8ff0\"<\/p>\n

\u76ee\u5f55<\/h3>\n
  • \u6a21\u578b\u76d1\u63a7\u6982\u8ff0<\/li>\n
  • \u6027\u80fd\u6307\u6807<\/li>\n
  • \u5f02\u5e38\u68c0\u6d4b<\/li>\n
  • \u65e5\u5fd7\u8bb0\u5f55<\/li>\n
  • \u544a\u8b66\u4f53\u7cfb<\/li>\n
  • \u76d1\u63a7\u5e73\u53f0<\/li>\n
  • \u5b9e\u8df5\u5efa\u8bae\u4e0e\u6700\u4f73\u5b9e\u8df5<\/li>\n

    \u6458\u8981<\/h3>\n

    \u672c\u6587\u4ee5 7B \u53c2\u6570\u5927\u8bed\u8a00\u6a21\u578b\u5728\u7ebf\u63a8\u7406\u670d\u52a1\u4e3a\u4f8b，\u6784\u5efa\u4e00\u5957\u8986\u76d6\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7\u8bb0\u5f55\u7684\u76d1\u63a7\u544a\u8b66\u4f53\u7cfb\u3002 \u751f\u4ea7\u6807\u51c6\u8bbe\u5b9a\u4e3a P99 \u9996 token \u5ef6\u8fdf\u4f4e\u4e8e 500ms\u3001\u5355\u5361\u751f\u6210\u541e\u5410\u9ad8\u4e8e 800 tokens\/s\u3001\u9519\u8bef\u7387\u4f4e\u4e8e 1%\u3002 \u5f02\u5e38\u68c0\u6d4b\u4ee5 3 \u4e2a\u6807\u51c6\u5dee\u4f5c\u4e3a\u7edf\u8ba1\u544a\u8b66\u9608\u503c，\u5c06\u544a\u8b66\u8bef\u62a5\u7387\u63a7\u5236\u5728 5% \u4ee5\u5185\u3002<\/p>\n

    1. \u6a21\u578b\u76d1\u63a7\u6982\u8ff0<\/h3>\n

    \u5927\u8bed\u8a00\u6a21\u578b\u63a8\u7406\u670d\u52a1\u4e0e\u4f20\u7edf Web \u670d\u52a1\u5728\u53ef\u89c2\u6d4b\u6027\u4e0a\u5b58\u5728\u672c\u8d28\u5dee\u5f02\u3002\u4f20\u7edf\u670d\u52a1\u7684\u8bf7\u6c42\u662f\u65e0\u72b6\u6001\u3001\u53ef\u91cd\u653e\u7684，\u54cd\u5e94\u8981\u4e48\u6210\u529f\u8981\u4e48\u5931\u8d25，\u9519\u8bef\u7801\u53ef\u4ee5\u76f4\u63a5\u66b4\u9732\u6545\u969c\u3002\u800c LLM \u63a8\u7406\u662f\u72b6\u6001\u6027\u7684：\u540c\u4e00\u8bf7\u6c42\u88ab\u62c6\u5206\u4e3a KV Cache \u5199\u5165\u663e\u5b58\u3001\u9010 token \u81ea\u56de\u5f52\u751f\u6210，\u671f\u95f4\u4efb\u610f\u4e00\u6b65\u7684\u663e\u5b58\u7d27\u5f20\u3001\u8c03\u5ea6\u6392\u961f\u6216\u6743\u91cd\u635f\u574f\u90fd\u4e0d\u4f1a\u4ea7\u751f\u6807\u51c6\u9519\u8bef\u7801，\u53ea\u8868\u73b0\u4e3a\u8f93\u51fa\u53d8\u6162\u3001\u622a\u65ad\u6216\u8bed\u4e49\u9000\u5316\u3002\u8fd9\u79cd"\u65e0\u58f0\u5931\u8d25"\u610f\u5473\u7740\u670d\u52a1\u53ef\u7528\u4e0d\u4ee3\u8868\u670d\u52a1\u6b63\u786e，\u76d1\u63a7\u5fc5\u987b\u540c\u65f6\u8986\u76d6\u8d44\u6e90\u3001\u6027\u80fd\u3001\u8d28\u91cf\u548c\u6210\u672c\u56db\u4e2a\u7ef4\u5ea6\u3002<\/p>\n

    \u76d1\u63a7\u7684\u843d\u5730\u5f62\u6001\u662f\u4e09\u5c42\u6570\u636e\u901a\u9053\u3002\u7b2c\u4e00\u5c42\u662f\u6307\u6807（Metrics）：\u4ee5\u6570\u503c\u5e8f\u5217\u63cf\u8ff0\u5ef6\u8fdf\u3001\u541e\u5410\u3001\u5229\u7528\u7387\u7b49\u53ef\u805a\u5408\u91cf，\u7531 Prometheus \u6bcf 15 \u79d2\u62c9\u53d6\u4e00\u6b21\u3002\u7b2c\u4e8c\u5c42\u662f\u65e5\u5fd7（Logs）：\u8bb0\u5f55\u6bcf\u4e2a\u8bf7\u6c42\u7684\u7ed3\u6784\u5316\u660e\u7ec6，\u4f9b\u5355\u8bf7\u6c42\u56de\u6eaf\u4e0e\u805a\u5408\u5206\u6790\u3002\u7b2c\u4e09\u5c42\u662f\u8c03\u7528\u94fe（Traces）：\u8ffd\u8e2a\u4e00\u6b21\u8bf7\u6c42\u5728\u7f51\u5173\u3001\u63a8\u7406\u5f15\u64ce\u3001\u5411\u91cf\u5e93\u4e4b\u95f4\u7684\u6d41\u8f6c\u8017\u65f6\u3002\u4e09\u5c42\u6570\u636e\u5728\u5f02\u5e38\u5224\u5b9a\u9636\u6bb5\u6c47\u5408，\u4e09\u5c42\u90fd\u6b63\u5e38\u7684\u8bf7\u6c42\u624d\u5224\u5b9a\u4e3a\u5065\u5eb7\u3002<\/p>\n

    #mermaid-svg-ThESkP0byS8E2ILJ{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-ThESkP0byS8E2ILJ .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-ThESkP0byS8E2ILJ .error-icon{fill:#552222;}#mermaid-svg-ThESkP0byS8E2ILJ .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-ThESkP0byS8E2ILJ .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-ThESkP0byS8E2ILJ .marker{fill:#333333;stroke:#333333;}#mermaid-svg-ThESkP0byS8E2ILJ .marker.cross{stroke:#333333;}#mermaid-svg-ThESkP0byS8E2ILJ svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-ThESkP0byS8E2ILJ p{margin:0;}#mermaid-svg-ThESkP0byS8E2ILJ .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-ThESkP0byS8E2ILJ .cluster-label text{fill:#333;}#mermaid-svg-ThESkP0byS8E2ILJ .cluster-label span{color:#333;}#mermaid-svg-ThESkP0byS8E2ILJ .cluster-label span p{background-color:transparent;}#mermaid-svg-ThESkP0byS8E2ILJ .label text,#mermaid-svg-ThESkP0byS8E2ILJ span{fill:#333;color:#333;}#mermaid-svg-ThESkP0byS8E2ILJ .node rect,#mermaid-svg-ThESkP0byS8E2ILJ .node circle,#mermaid-svg-ThESkP0byS8E2ILJ .node ellipse,#mermaid-svg-ThESkP0byS8E2ILJ .node polygon,#mermaid-svg-ThESkP0byS8E2ILJ .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-ThESkP0byS8E2ILJ .rough-node .label text,#mermaid-svg-ThESkP0byS8E2ILJ .node .label text,#mermaid-svg-ThESkP0byS8E2ILJ .image-shape .label,#mermaid-svg-ThESkP0byS8E2ILJ .icon-shape .label{text-anchor:middle;}#mermaid-svg-ThESkP0byS8E2ILJ .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-ThESkP0byS8E2ILJ .rough-node .label,#mermaid-svg-ThESkP0byS8E2ILJ .node .label,#mermaid-svg-ThESkP0byS8E2ILJ .image-shape .label,#mermaid-svg-ThESkP0byS8E2ILJ .icon-shape .label{text-align:center;}#mermaid-svg-ThESkP0byS8E2ILJ .node.clickable{cursor:pointer;}#mermaid-svg-ThESkP0byS8E2ILJ .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-ThESkP0byS8E2ILJ .arrowheadPath{fill:#333333;}#mermaid-svg-ThESkP0byS8E2ILJ .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-ThESkP0byS8E2ILJ .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-ThESkP0byS8E2ILJ .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-ThESkP0byS8E2ILJ .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-ThESkP0byS8E2ILJ .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-ThESkP0byS8E2ILJ .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-ThESkP0byS8E2ILJ .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-ThESkP0byS8E2ILJ .cluster text{fill:#333;}#mermaid-svg-ThESkP0byS8E2ILJ .cluster span{color:#333;}#mermaid-svg-ThESkP0byS8E2ILJ div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-ThESkP0byS8E2ILJ .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-ThESkP0byS8E2ILJ rect.text{fill:none;stroke-width:0;}#mermaid-svg-ThESkP0byS8E2ILJ .icon-shape,#mermaid-svg-ThESkP0byS8E2ILJ .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-ThESkP0byS8E2ILJ .icon-shape p,#mermaid-svg-ThESkP0byS8E2ILJ .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-ThESkP0byS8E2ILJ .icon-shape .label rect,#mermaid-svg-ThESkP0byS8E2ILJ .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-ThESkP0byS8E2ILJ .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-ThESkP0byS8E2ILJ .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-ThESkP0byS8E2ILJ :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-ThESkP0byS8E2ILJ .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-ThESkP0byS8E2ILJ .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-ThESkP0byS8E2ILJ .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u89c2\u6d4b\u91c7\u96c6\u5c42<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5728\u7ebf\u670d\u52a1\u5c42<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u52a8\u6001\u6279\u5904\u7406\u56de\u73af<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u672a\u8d8a\u754c<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8d8a\u754c<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5904\u7406\u7ed3\u8bba\u5199\u56de<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5ba2\u6237\u7aef\u8bf7\u6c42<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    API \u7f51\u5173<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u63a8\u7406\u5f15\u64ce<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8fd4\u56de\u54cd\u5e94<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6307\u6807\u63a2\u9488<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Prometheus<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u65e5\u5fd7\u91c7\u96c6\u5668<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Elasticsearch<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8c03\u7528\u94fe\u91c7\u6837<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8ffd\u8e2a\u5b58\u50a8<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7efc\u5408\u5224\u5b9a<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6307\u6807\u5f52\u6863<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u544a\u8b66\u7cfb\u7edf<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u503c\u73ed\u4eba\u5458<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    1.1 \u76d1\u63a7\u7684\u5fc5\u8981\u6027<\/h4>\n

    \u63a8\u7406\u670d\u52a1\u7684\u6545\u969c\u9762\u6bd4\u5e38\u89c4\u670d\u52a1\u5bbd\u5f97\u591a，\u4e14\u591a\u6570\u6545\u969c\u5728\u9519\u8bef\u7387\u6307\u6807\u4e0a\u770b\u4e0d\u51fa\u6765\u3002\u4ee5\u4e00\u4e2a 7B \u6a21\u578b\u5355\u5361\u90e8\u7f72\u4e3a\u4f8b：\u6743\u91cd FP16 \u7ea6 14GB，\u5355\u8bf7\u6c42\u4e0a\u4e0b\u6587 4096 token \u65f6 KV Cache \u7ea6\u5360\u7528 0.5GB，A100 80GB \u663e\u5b58\u4e0a\u5e76\u53d1 16 \u65f6\u663e\u5b58\u4f59\u91cf\u4e0d\u5230 30GB\u3002\u4e00\u65e6\u5e76\u53d1\u6a21\u578b\u6216\u63d0\u793a\u8bcd\u957f\u5ea6\u5206\u5e03\u53d8\u5316，\u663e\u5b58\u6c34\u4f4d\u4f1a\u5728\u51e0\u5206\u949f\u5185\u8d8a\u8fc7 95% \u7684\u544a\u8b66\u7ebf，\u89e6\u53d1 OOM \u6216\u62a2\u5360\u5f0f\u6279\u5904\u7406\u7684\u88ab\u52a8\u9010\u51fa，\u8868\u73b0\u4e3a P99 \u5ef6\u8fdf\u4ece 500ms \u6500\u5347\u5230 3s，\u800c\u5e73\u5747\u5ef6\u8fdf\u53ea\u4ece 380ms \u6da8\u5230 520ms\u3002\u53ea\u770b\u5e73\u5747\u503c\u4f1a\u6f0f\u6389\u8fd9\u7c7b\u6545\u969c\u3002<\/p>\n

    \u53e6\u4e00\u4e2a\u88ab\u4f4e\u4f30\u7684\u76d1\u63a7\u5bf9\u8c61\u662f\u6a21\u578b\u7248\u672c\u3002\u6a21\u578b\u6587\u4ef6\u635f\u574f\u3001\u91cf\u5316\u8bef\u5dee\u7d2f\u79ef\u3001\u6216\u52a0\u8f7d\u4e86\u9519\u8bef\u7684 checkpoint \u65f6，\u63a8\u7406\u670d\u52a1\u5b8c\u5168\u6b63\u5e38，\u4f46\u751f\u6210\u8d28\u91cf\u4e0b\u964d\u3002\u8fd9\u7c7b\u8d28\u91cf\u56de\u5f52\u53ea\u6709\u901a\u8fc7\u8f93\u51fa\u4fa7\u91c7\u6837\u8bc4\u4f30\u624d\u80fd\u53d1\u73b0：\u6bcf 1000 \u4e2a\u8bf7\u6c42\u62bd\u53d6 1 \u4e2a，\u7528\u56fa\u5b9a\u8bc4\u4f30\u96c6\u8ba1\u7b97\u56f0\u60d1\u5ea6\u53d8\u5316，\u504f\u5dee\u8d85\u8fc7 0.15 \u65f6\u56de\u6eda\u7248\u672c\u3002\u76d1\u63a7\u7684\u5fc5\u8981\u6027\u8fd8\u4f53\u73b0\u5728\u6210\u672c\u4e0a：\u4e00\u4e2a 7B \u6a21\u578b\u7684\u5355\u6b21\u751f\u6210\u7ea6\u6d88\u8017 7 GFLOPs \u91cf\u7ea7\u7684\u8ba1\u7b97，\u7a7a\u95f2 GPU \u6bcf\u5c0f\u65f6\u7684\u7535\u8d39\u4e0e\u644a\u9500\u6210\u672c\u6309 1.2 \u7f8e\u5143\u4f30\u7b97，\u76d1\u63a7\u662f\u53d1\u73b0\u95f2\u7f6e\u4e0e\u9519\u914d\u7684\u552f\u4e00\u9014\u5f84\u3002<\/p>\n

    \u76d1\u63a7\u7684\u6838\u5fc3\u4ef7\u503c\u662f\u7f29\u77ed\u5e73\u5747\u53d1\u73b0\u65f6\u95f4（MTTD）\u4e0e\u5e73\u5747\u6062\u590d\u65f6\u95f4（MTTR）\u3002\u63a5\u5165\u9010\u5c42\u57cb\u70b9\u540e，\u4e00\u6b21\u663e\u5b58\u544a\u8b66\u7684\u53d1\u73b0\u65f6\u95f4\u53ef\u4ee5\u4ece\u4eba\u5de5\u5de1\u68c0\u7684 30 \u5206\u949f\u538b\u7f29\u5230\u63a2\u9488\u4e0a\u62a5\u7684 15 \u79d2\u4ee5\u5185\u3002\u91cf\u5316\u6536\u76ca\u53ef\u4ee5\u8fd9\u6837\u4f30\u7b97：\u6bcf\u6708\u4e00\u6b21 P1 \u7ea7\u6545\u969c，\u6545\u969c\u6301\u7eed 2 \u5c0f\u65f6，\u6bcf 10 \u5206\u949f\u635f\u5931\u7ea6 40 \u6b21\u8bf7\u6c42\u7684 SLA \u8fdd\u7ea6，\u76d1\u63a7\u5230\u4f4d\u540e\u8fdd\u7ea6\u6b21\u6570\u53ef\u51cf\u5c11 80%\u3002<\/p>\n

    \u76d1\u63a7\u7f3a\u5931\u7684\u6210\u672c\u8fd8\u53ef\u4ee5\u4ece\u53e6\u4e00\u4e2a\u89d2\u5ea6\u91cf\u5316\u3002\u4e00\u6b21\u672a\u88ab\u53d1\u73b0\u7684\u6a21\u578b\u8d28\u91cf\u56de\u5f52，\u7528\u6237\u4fa7\u4f1a\u4ee5\u6295\u8bc9\u3001\u6d41\u5931\u548c\u5dee\u8bc4\u7684\u5f62\u5f0f\u4f53\u73b0，\u4e00\u6b21\u4e8b\u6545\u7684\u8d54\u507f\u4e0e\u53e3\u7891\u635f\u5931\u901a\u5e38\u8d85\u8fc7\u76d1\u63a7\u4f53\u7cfb\u4e00\u5e74\u7684\u8fd0\u884c\u6210\u672c\u3002\u4ee5\u4e00\u53f0 A100 \u6bcf\u5c0f\u65f6 3.5 \u7f8e\u5143\u7684\u79df\u8d41\u6210\u672c\u8ba1\u7b97，24 \u5c0f\u65f6\u65e0\u4eba\u503c\u5b88\u7684\u7a7a\u8f6c\u635f\u5931\u4e3a 84 \u7f8e\u5143；\u800c\u4e00\u5957\u8986\u76d6 10 \u53f0 GPU \u7684\u76d1\u63a7\u6808，Prometheus \u4e0e Grafana \u5747\u4e3a\u5f00\u6e90\u8f6f\u4ef6，\u6bcf\u6708\u7684\u4eba\u529b\u7ef4\u62a4\u6210\u672c\u6298\u5408\u7ea6 1 \u4eba\u5929\u3002\u6295\u5165\u4ea7\u51fa\u6bd4\u51b3\u5b9a\u4e86\u76d1\u63a7\u4e0d\u662f\u53ef\u9009\u4f18\u5316，\u800c\u662f\u63a8\u7406\u670d\u52a1\u4e0a\u7ebf\u7684\u5f3a\u5236\u524d\u7f6e\u6761\u4ef6\u3002<\/p>\n

    \u89c2\u6d4b\u6570\u636e\u7684\u4e09\u7c7b\u5f62\u6001\u5404\u6709\u8986\u76d6\u76f2\u533a，\u76d1\u63a7\u5fc5\u8981\u6027\u4e5f\u4f53\u73b0\u5728"\u76f2\u533a\u4e92\u8865"\u4e0a\u3002\u6307\u6807\u8986\u76d6\u8d44\u6e90\u4e0e\u6027\u80fd，\u65e5\u5fd7\u8986\u76d6\u8bf7\u6c42\u7ea7\u660e\u7ec6，\u8c03\u7528\u94fe\u8986\u76d6\u8de8\u670d\u52a1\u6d41\u8f6c，\u4e09\u8005\u5355\u72ec\u4f7f\u7528\u90fd\u4f1a\u6f0f\u6389\u6545\u969c：\u6307\u6807\u770b\u4e0d\u51fa\u5355\u4e2a\u8bf7\u6c42\u7684\u5931\u8d25\u539f\u56e0，\u65e5\u5fd7\u770b\u4e0d\u51fa\u5168\u5c40\u8d8b\u52bf，\u8c03\u7528\u94fe\u5728\u5355\u673a\u90e8\u7f72\u65f6\u6beb\u65e0\u4ef7\u503c\u3002\u53ea\u6709\u4e09\u7c7b\u6570\u636e\u5728\u540c\u4e00\u6545\u969c\u65f6\u95f4\u8f74\u4e0a\u5bf9\u9f50，\u624d\u80fd\u56de\u7b54"\u662f\u4ec0\u4e48\u3001\u4e3a\u4ec0\u4e48\u3001\u5f71\u54cd\u591a\u5927"\u4e09\u4e2a\u95ee\u9898，\u8fd9\u4e5f\u662f\u540e\u7eed\u7ae0\u8282\u5206\u5c42\u8bbe\u8ba1\u7684\u4f9d\u636e\u3002<\/p>\n

    \u76d1\u63a7\u7684\u5fc5\u8981\u6027\u6700\u7ec8\u843d\u5728\u7ec4\u7ec7\u80fd\u529b\u4e0a：\u6ca1\u6709\u76d1\u63a7\u5c31\u6ca1\u6709\u6307\u6807，\u6ca1\u6709\u6307\u6807\u5c31\u6ca1\u6709\u5bb9\u91cf\u51b3\u7b56\u4e0e\u6210\u672c\u6838\u7b97\u3002GPU \u96c6\u7fa4\u7684\u6269\u7f29\u5bb9\u3001\u6a21\u578b\u91cf\u5316\u65b9\u6848\u7684\u9009\u62e9\u3001\u5e76\u53d1\u53c2\u6570\u7684\u8c03\u4f18，\u6bcf\u4e00\u9879\u90fd\u4f9d\u8d56\u76d1\u63a7\u6570\u636e\u505a\u56de\u5f52\u9a8c\u8bc1\u3002\u63a8\u7406\u670d\u52a1\u4e0a\u7ebf\u76d1\u63a7，\u7b49\u4ef7\u4e8e\u4e3a\u6bcf\u4e00\u9879\u6280\u672f\u51b3\u7b56\u5b89\u88c5\u6d4b\u91cf\u4eea\u5668，\u8fd9\u5c42\u6d4b\u91cf\u80fd\u529b\u4e00\u65e6\u7f3a\u5931，\u540e\u7eed\u6240\u6709\u4f18\u5316\u90fd\u9000\u5316\u4e3a\u51ed\u611f\u89c9\u884c\u4e8b\u3002<\/p>\n

    # \u6765\u6e90：prometheus_client 0.20.0 \/ health_probe.py<\/span>
    \nfrom<\/span> prometheus_client import<\/span> Counter,<\/span> Gauge,<\/span> Histogram,<\/span> start_http_server
    \nimport<\/span> random
    \nimport<\/span> time<\/p>\n

    # \u4e09\u4e2a\u6838\u5fc3\u89c2\u6d4b\u91cf：\u8bf7\u6c42\u8ba1\u6570\u3001\u5ef6\u8fdf\u76f4\u65b9\u56fe\u3001\u6a21\u578b\u7248\u672c<\/span>
    \nREQUEST_TOTAL =<\/span> Counter(<\/span>"llm_requests_total"<\/span>,<\/span> "\u7d2f\u8ba1\u8bf7\u6c42\u6570"<\/span>,<\/span> [<\/span>"model"<\/span>,<\/span> "outcome"<\/span>]<\/span>)<\/span>
    \nLATENCY =<\/span> Histogram(<\/span>
    \n "llm_latency_seconds"<\/span>,<\/span>
    \n "\u8bf7\u6c42\u7aef\u5230\u7aef\u5ef6\u8fdf"<\/span>,<\/span>
    \n buckets=<\/span>(<\/span>0.1<\/span>,<\/span> 0.3<\/span>,<\/span> 0.5<\/span>,<\/span> 1.0<\/span>,<\/span> 2.0<\/span>,<\/span> 5.0<\/span>,<\/span> float<\/span>(<\/span>"inf"<\/span>)<\/span>)<\/span>,<\/span>
    \n)<\/span>
    \nVERSION =<\/span> Gauge(<\/span>"llm_model_version"<\/span>,<\/span> "\u5f53\u524d\u52a0\u8f7d\u7684\u6a21\u578b\u7248\u672c\u53f7"<\/span>)<\/span><\/p>\n

    def<\/span> main<\/span>(<\/span>)<\/span> –<\/span>><\/span> None<\/span>:<\/span>
    \n """\u66b4\u9732 \/metrics \u7aef\u53e3，\u6bcf 1 \u79d2\u4ea7\u751f\u4e00\u6761\u6a21\u62df\u89c2\u6d4b\u6837\u672c\u3002"""<\/span>
    \n start_http_server(<\/span>9101<\/span>)<\/span>
    \n VERSION.<\/span>set<\/span>(<\/span>37<\/span>)<\/span>
    \n while<\/span> True<\/span>:<\/span>
    \n outcome =<\/span> "ok"<\/span> if<\/span> random.<\/span>random(<\/span>)<\/span> <<\/span> 0.99<\/span> else<\/span> "error"<\/span>
    \n REQUEST_TOTAL.<\/span>labels(<\/span>model=<\/span>"qwen-7b"<\/span>,<\/span> outcome=<\/span>outcome)<\/span>.<\/span>inc(<\/span>)<\/span>
    \n # \u6a21\u62df\u5ef6\u8fdf\u5206\u5e03：\u5747\u503c 0.4s，P99 \u843d\u5728 0.5-1.0s \u6876\u5185<\/span>
    \n LATENCY.<\/span>observe(<\/span>max<\/span>(<\/span>0.0<\/span>,<\/span> random.<\/span>gauss(<\/span>0.4<\/span>,<\/span> 0.1<\/span>)<\/span>)<\/span>)<\/span>
    \n time.<\/span>sleep(<\/span>1<\/span>)<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n main(<\/span>)<\/span><\/p>\n

    \u4e0a\u8ff0\u63a2\u9488\u628a\u89c2\u6d4b\u4ece"\u4eba\u5de5\u5de1\u68c0"\u53d8\u4e3a"\u673a\u5668\u91c7\u96c6"\u3002\u6307\u6807\u5728\u8fdb\u7a0b\u5185\u5b8c\u6210\u5206\u6876\u805a\u5408，\u5355\u8fdb\u7a0b\u5f00\u9500\u5c0f\u4e8e 1% CPU，\u907f\u514d\u4e86\u4e3a\u6bcf\u4e2a\u8bf7\u6c42\u843d\u4e00\u6761\u65e5\u5fd7\u7684\u9ad8\u6602\u6210\u672c\u3002\u63a2\u9488\u7aef\u53e3\u7531 Prometheus \u6293\u53d6\u540e，\u5ef6\u8fdf\u5206\u4f4d\u6570\u3001\u9519\u8bef\u7387\u3001\u7248\u672c\u53f7\u7acb\u5373\u8fdb\u5165\u65f6\u5e8f\u5e93，\u540e\u7eed\u6240\u6709\u7ae0\u8282\u7684\u5f02\u5e38\u68c0\u6d4b\u4e0e\u544a\u8b66\u90fd\u5efa\u7acb\u5728\u8fd9\u4e00\u5c42\u89c2\u6d4b\u6570\u636e\u4e4b\u4e0a\u3002<\/p>\n

    1.2 \u76d1\u63a7\u6311\u6218<\/h4>\n

    LLM \u670d\u52a1\u7ed9\u76d1\u63a7\u7cfb\u7edf\u672c\u8eab\u5e26\u6765\u4e86\u56db\u4e2a\u53ef\u91cf\u5316\u7684\u6311\u6218\u3002\u7b2c\u4e00\u662f\u6807\u7b7e\u57fa\u6570\u7206\u70b8：\u8bf7\u6c42\u6309\u6a21\u578b\u7248\u672c\u3001\u91cf\u5316\u65b9\u5f0f\u3001GPU \u7f16\u53f7\u3001\u5730\u533a\u3001prompt \u524d\u7f00\u7b49\u591a\u7ef4\u6253\u6807，\u6807\u7b7e\u7b1b\u5361\u5c14\u79ef\u53ef\u8fbe 10 \u4e07\u7ea7\u3002Prometheus \u65f6\u5e8f\u5e93\u5bf9\u6bcf\u6761\u65f6\u95f4\u5e8f\u5217\u90fd\u8981\u4fdd\u7559\u5185\u5b58\u7d22\u5f15，\u5e8f\u5217\u6570\u4ece 1000 \u6da8\u5230 10 \u4e07\u65f6，\u5355\u8282\u70b9\u5185\u5b58\u5360\u7528\u4ece\u7ea6 50MB \u6da8\u5230 5GB，\u67e5\u8be2\u5ef6\u8fdf\u4ece 100ms \u7ea7\u9000\u5316\u5230\u79d2\u7ea7\u3002<\/p>\n

    \u7b2c\u4e8c\u662f\u9ad8\u541e\u5410\u4e0b\u7684\u5199\u5165\u538b\u529b\u3002100 QPS \u7684\u670d\u52a1\u82e5\u5bf9\u6bcf\u4e2a\u8bf7\u6c42\u8bb0\u5f55 6 \u4e2a\u6307\u6807\u6837\u672c，\u6bcf 15 \u79d2\u6293\u53d6\u5468\u671f\u5185\u8981\u5199\u5165\u7ea6 6 \u4e07\u6761\u6837\u672c；\u82e5\u5ef6\u8fdf\u76f4\u65b9\u56fe\u5206 7 \u4e2a\u6876，\u5b9e\u9645\u5199\u5165\u91cf\u4e58\u4ee5 7 \u500d，\u7ea6 42 \u4e07\u6837\u672c\u3002Prometheus TSDB \u5355\u8282\u70b9\u6bcf\u79d2\u53ef\u5438\u6536\u7ea6 50 \u4e07\u6837\u672c，\u4f46\u53e0\u52a0\u67e5\u8be2\u8d1f\u8f7d\u540e\u4f1a\u5f62\u6210\u5199\u653e\u5927，\u9700\u8981\u6309\u6307\u6807\u7c7b\u578b\u5206\u6d41\u5230\u4e0d\u540c job\u3002<\/p>\n

    \u7b2c\u4e09\u662f\u6307\u6807\u95f4\u76f8\u5173\u6027\u3002GPU \u5229\u7528\u7387\u3001KV Cache \u5360\u7528\u3001\u961f\u5217\u957f\u5ea6\u4e09\u4e2a\u6307\u6807\u5355\u72ec\u770b\u90fd\u53ef\u80fd\u5728\u6b63\u5e38\u533a\u95f4，\u8054\u5408\u8d77\u6765\u5374\u6307\u5411\u8c03\u5ea6\u5931\u8861：\u5229\u7528\u7387 60%\u3001\u7f13\u5b58\u5360\u7528 90%\u3001\u961f\u5217 40 \u4e2a，\u8bf4\u660e\u8bf7\u6c42\u5806\u79ef\u5728\u6392\u961f\u800c\u975e\u8ba1\u7b97\u3002\u5355\u6307\u6807\u9608\u503c\u5bf9\u6b64\u7c7b\u573a\u666f\u5b8c\u5168\u5931\u6548，\u9700\u8981\u591a\u5143\u5f02\u5e38\u68c0\u6d4b\u6216\u9884\u8bbe\u7684\u8054\u5408\u89c4\u5219\u3002<\/p>\n

    \u7b2c\u56db\u662f\u544a\u8b66\u75b2\u52b3\u3002\u89c4\u5219\u6570\u91cf\u4e0e\u8bef\u62a5\u7387\u6b63\u76f8\u5173，30 \u6761\u89c4\u5219\u3001\u6bcf\u6761\u8bef\u62a5\u7387 10% \u65f6，\u6bcf\u5929\u4f1a\u4ea7\u751f\u7ea6 72 \u6761\u65e0\u610f\u4e49\u544a\u8b66，\u503c\u73ed\u4eba\u5458\u5bf9\u544a\u8b66\u7684\u54cd\u5e94\u7387\u4f1a\u4ece 95% \u8dcc\u5230 40%\u3002\u6311\u6218\u7684\u8fb9\u754c\u5728\u4e8e：\u6307\u6807\u91c7\u96c6\u8d8a\u7ec6，\u8d8a\u80fd\u6355\u6349\u7ec6\u5fae\u9000\u5316，\u4f46\u5b58\u50a8\u4e0e\u544a\u8b66\u6210\u672c\u8d8a\u9ad8\u3002\u751f\u4ea7\u4e0a\u901a\u5e38\u5728 1% \u7684\u91c7\u6837\u7387\u4e0e\u5168\u91cf\u5173\u952e\u8def\u5f84\u6307\u6807\u4e4b\u95f4\u53d6\u5e73\u8861，\u975e\u5173\u952e\u6307\u6807\u964d\u91c7\u6837\u5230\u6bcf 60 \u79d2\u4e00\u4e2a\u70b9\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ cardinality_guard.py<\/span>
    \nimport<\/span> re<\/p>\n

    # \u6307\u6807\u540d\u53ea\u5141\u8bb8\u5c0f\u5199\u5b57\u6bcd\u3001\u6570\u5b57\u3001\u4e0b\u5212\u7ebf，\u6bb5\u4e4b\u95f4\u7528\u5355\u4e0b\u5212\u7ebf<\/span>
    \n_NAME_RE =<\/span> re.<\/span>compile<\/span>(<\/span>r"^[a-z][a-z0-9_]*(?:_[a-z0-9]+)*$"<\/span>)<\/span><\/p>\n

    class<\/span> LabelPolicy<\/span>:<\/span>
    \n """\u5728\u57cb\u70b9\u6ce8\u518c\u9636\u6bb5\u62e6\u622a\u9ad8\u57fa\u6570\u4e0e\u4e0d\u89c4\u8303\u6807\u7b7e，\u628a\u95ee\u9898\u6321\u5728\u5199\u5165\u4e4b\u524d\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> max_labels=<\/span>8<\/span>,<\/span> max_values=<\/span>20<\/span>)<\/span>:<\/span>
    \n self.<\/span>max_labels =<\/span> max_labels
    \n self.<\/span>max_values =<\/span> max_values
    \n self.<\/span>seen =<\/span> {<\/span>}<\/span><\/p>\n

    def<\/span> register<\/span>(<\/span>self,<\/span> metric,<\/span> labels)<\/span>:<\/span>
    \n if<\/span> not<\/span> _NAME_RE.<\/span>fullmatch(<\/span>metric)<\/span>:<\/span>
    \n raise<\/span> ValueError(<\/span>f"\u975e\u6cd5\u6307\u6807\u540d: <\/span>{<\/span>metric}<\/span><\/span>"<\/span><\/span>)<\/span>
    \n if<\/span> len<\/span>(<\/span>labels)<\/span> ><\/span> self.<\/span>max_labels:<\/span>
    \n raise<\/span> ValueError(<\/span>f"\u6807\u7b7e\u6570\u91cf\u8d85\u9650: <\/span>{<\/span>len<\/span>(<\/span>labels)<\/span>}<\/span><\/span> \u8d85\u8fc7 <\/span>{<\/span>self.<\/span>max_labels}<\/span><\/span>"<\/span><\/span>)<\/span>
    \n for<\/span> name,<\/span> value in<\/span> labels.<\/span>items(<\/span>)<\/span>:<\/span>
    \n if<\/span> name.<\/span>endswith(<\/span>"_id"<\/span>)<\/span> or<\/span> name ==<\/span> "request_id"<\/span>:<\/span>
    \n continue<\/span> # \u8bf7\u6c42\u7ea7 ID \u5c5e\u4e8e\u65e5\u5fd7\u57df，\u4e0d\u8fdb\u6307\u6807\u57df<\/span>
    \n bucket =<\/span> self.<\/span>seen.<\/span>setdefault(<\/span>(<\/span>metric,<\/span> name)<\/span>,<\/span> set<\/span>(<\/span>)<\/span>)<\/span>
    \n bucket.<\/span>add(<\/span>value)<\/span>
    \n if<\/span> len<\/span>(<\/span>bucket)<\/span> ><\/span> self.<\/span>max_values:<\/span>
    \n raise<\/span> RuntimeError(<\/span>
    \n f"<\/span>{<\/span>metric}<\/span><\/span>[<\/span>{<\/span>name}<\/span><\/span>] \u57fa\u6570\u8d85\u9650: <\/span>{<\/span>len<\/span>(<\/span>bucket)<\/span>}<\/span><\/span> \u4e2a\u552f\u4e00\u503c"<\/span><\/span>
    \n )<\/span>
    \n return<\/span> True<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n policy =<\/span> LabelPolicy(<\/span>max_values=<\/span>20<\/span>)<\/span>
    \n policy.<\/span>register(<\/span>"llm_latency_seconds"<\/span>,<\/span> {<\/span>"model"<\/span>:<\/span> "qwen-7b"<\/span>,<\/span> "gpu"<\/span>:<\/span> "gpu0"<\/span>}<\/span>)<\/span>
    \n policy.<\/span>register(<\/span>"llm_latency_seconds"<\/span>,<\/span> {<\/span>"model"<\/span>:<\/span> "qwen-13b"<\/span>,<\/span> "gpu"<\/span>:<\/span> "gpu0"<\/span>}<\/span>)<\/span>
    \n try<\/span>:<\/span>
    \n policy.<\/span>register(<\/span>"llm_requests_total"<\/span>,<\/span> {<\/span>"instance"<\/span>:<\/span> "host-%d"<\/span> %<\/span> 1<\/span>}<\/span>)<\/span>
    \n for<\/span> i in<\/span> range<\/span>(<\/span>25<\/span>)<\/span>:<\/span>
    \n policy.<\/span>register(<\/span>"llm_requests_total"<\/span>,<\/span> {<\/span>"instance"<\/span>:<\/span> "host-%d"<\/span> %<\/span> i}<\/span>)<\/span>
    \n except<\/span> RuntimeError as<\/span> exc:<\/span>
    \n print<\/span>(<\/span>"\u62e6\u622a\u5230\u9ad8\u57fa\u6570:"<\/span>,<\/span> exc)<\/span><\/p>\n

    \u6807\u7b7e\u7b56\u7565\u4e0e\u5199\u653e\u5927\u662f\u540c\u4e00\u679a\u786c\u5e01\u7684\u4e24\u9762\u3002\u628a\u8bf7\u6c42 ID\u3001\u65f6\u95f4\u6233\u7b49\u9ad8\u57fa\u6570\u5b57\u6bb5\u6321\u5728\u6307\u6807\u57df\u4e4b\u5916\u3001\u53ea\u8fdb\u65e5\u5fd7\u57df，\u65e2\u63a7\u5236\u4e86\u65f6\u5e8f\u5e93\u7684\u5e8f\u5217\u6570\u91cf，\u53c8\u4fdd\u7559\u4e86\u9010\u8bf7\u6c42\u7684\u53ef\u56de\u6eaf\u80fd\u529b\u3002\u8fd0\u884c\u65f6\u518d\u53e0\u52a0\u57fa\u6570\u5b88\u536b，\u57cb\u70b9\u4ee3\u7801\u5373\u4f7f\u5199\u9519，\u4e5f\u4f1a\u5728\u6ce8\u518c\u9636\u6bb5\u88ab\u62d2\u7edd，\u800c\u4e0d\u662f\u6c61\u67d3\u7ebf\u4e0a\u65f6\u5e8f\u5e93\u3002<\/p>\n

    1.3 \u76d1\u63a7\u76ee\u6807<\/h4>\n

    \u76d1\u63a7\u76ee\u6807\u5fc5\u987b\u5148\u7528\u670d\u52a1\u7b49\u7ea7\u76ee\u6807（SLO）\u951a\u5b9a，\u518d\u53cd\u5411\u63a8\u5bfc\u91c7\u96c6\u4e0e\u544a\u8b66\u9700\u6c42\u3002\u4e09\u8005\u7684\u8fb9\u754c\u9700\u8981\u660e\u786e：SLI \u662f\u53ef\u6d4b\u91cf\u7684\u6307\u6807\u503c，SLO \u662f\u4e3a\u8be5\u6307\u6807\u8bbe\u5b9a\u7684\u76ee\u6807\u533a\u95f4，SLA \u662f\u4e0e\u5ba2\u6237\u7ea6\u5b9a\u7684\u8fdd\u7ea6\u6761\u6b3e\u3002\u4ee5 99.9% \u53ef\u7528\u6027\u76ee\u6807\u4e3a\u4f8b，30 \u5929\u9519\u8bef\u9884\u7b97\u4e3a 43.2 \u5206\u949f，\u4e00\u65e6\u7d2f\u8ba1\u4e0d\u53ef\u7528\u65f6\u95f4\u8d85\u8fc7\u9884\u7b97\u7684\u4e00\u534a，\u5c31\u5fc5\u987b\u8fdb\u5165\u6b62\u635f\u6d41\u7a0b，\u800c\u4e0d\u662f\u7b49\u5230\u6708\u5e95\u624d\u53d1\u73b0\u8fdd\u7ea6\u3002<\/p>\n

    \u751f\u4ea7\u73af\u5883\u7684\u76d1\u63a7\u76ee\u6807\u6309\u56db\u5c42\u5206\u89e3\u3002\u7b2c\u4e00\u5c42\u662f\u53ef\u7528\u6027：99.9% \u7684\u8bf7\u6c42\u6210\u529f\u8fd4\u56de，\u9519\u8bef\u9884\u7b97 43.2 \u5206\u949f\/\u6708\u3002\u7b2c\u4e8c\u5c42\u662f\u6027\u80fd：P99 \u9996 token \u5ef6\u8fdf\u4f4e\u4e8e 500ms，P99 \u7aef\u5230\u7aef\u5ef6\u8fdf\u4f4e\u4e8e 2s，P99 \u6bcf token \u751f\u6210\u5ef6\u8fdf\u4f4e\u4e8e 60ms\u3002\u7b2c\u4e09\u5c42\u662f\u6548\u7387：\u5355\u5361\u751f\u6210\u541e\u5410\u4e0d\u4f4e\u4e8e 800 tokens\/s，GPU \u8ba1\u7b97\u5229\u7528\u7387\u4e0d\u4f4e\u4e8e 70%，KV Cache \u6c34\u4f4d\u4e0d\u8d85\u8fc7 90%\u3002\u7b2c\u56db\u5c42\u662f\u8d28\u91cf：\u9ec4\u91d1\u8bc4\u4f30\u96c6\u51c6\u786e\u7387\u5468\u73af\u6bd4\u4e0b\u964d\u4e0d\u8d85\u8fc7 2%，\u8f93\u5165\u5206\u5e03 PSI \u504f\u79fb\u4e0d\u8d85\u8fc7 0.2\u3002<\/p>\n

    \u76ee\u6807\u4e0e\u91c7\u96c6\u4e4b\u95f4\u5b58\u5728\u6570\u91cf\u5173\u7cfb\u3002P99 \u5ef6\u8fdf\u8981\u53ef\u4fe1，\u81f3\u5c11\u9700\u8981 500 \u4e2a\u6837\u672c\u624d\u80fd\u8ba9 99% \u5206\u4f4d\u6570\u7684\u7f6e\u4fe1\u533a\u95f4\u6536\u7a84\u5230 10% \u4ee5\u5185；\u6309 100 QPS \u8ba1\u7b97，500 \u4e2a\u6837\u672c\u53ea\u9700 5 \u79d2\u3002\u800c P99.99 \u5206\u4f4d\u6570\u9700\u8981 1 \u4e07\u4e2a\u6837\u672c，\u540c\u4e00\u6d41\u91cf\u4e0b\u9700\u8981 100 \u79d2，\u8fd9\u610f\u5473\u7740\u6781\u957f\u5c3e\u6307\u6807\u8981\u4e48\u7528\u66f4\u5927\u7684\u65f6\u95f4\u7a97\u53e3，\u8981\u4e48\u63a5\u53d7\u66f4\u5927\u7684\u4e0d\u786e\u5b9a\u6027\u3002\u76ee\u6807\u7684\u8bbe\u5b9a\u8fd8\u8981\u4e0e\u5bb9\u91cf\u89c4\u5212\u8054\u52a8：GPU \u5229\u7528\u7387\u76ee\u6807 70% \u4e3a\u5cf0\u503c\u7559\u51fa 30% \u7684\u4f59\u91cf，\u5426\u5219\u4e00\u6b21\u6d41\u91cf\u6bdb\u523a\u5c31\u4f1a\u628a\u5229\u7528\u7387\u9876\u5230 95% \u5e76\u89e6\u53d1\u544a\u8b66\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ error_budget.py<\/span>
    \nclass<\/span> ErrorBudget<\/span>:<\/span>
    \n """\u57fa\u4e8e 99.9% SLO \u7684\u6708\u5ea6\u9519\u8bef\u9884\u7b97\u8ba1\u7b97\u5668\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> slo_nines=<\/span>3<\/span>,<\/span> period_days=<\/span>30<\/span>)<\/span>:<\/span>
    \n # \u4e00\u4e2a\u6708\u6309 30 \u5929\u6362\u7b97\u6210\u5206\u949f，\u8fdd\u7ea6\u5206\u949f\u6570\u7531 1 \u51cf\u53ef\u7528\u6027\u5f97\u51fa<\/span>
    \n # 99.9% \u53ef\u7528\u6027\u5bf9\u5e94 0.1% \u8fdd\u7ea6\u65f6\u95f4，30 \u5929\u5171 43.2 \u5206\u949f<\/span>
    \n self.<\/span>limit =<\/span> period_days *<\/span> 24<\/span> *<\/span> 60<\/span> *<\/span> (<\/span>10<\/span> **<\/span> (<\/span>–<\/span>slo_nines)<\/span>)<\/span>
    \n self.<\/span>burned =<\/span> 0.0<\/span><\/p>\n

    def<\/span> record<\/span>(<\/span>self,<\/span> minutes)<\/span>:<\/span>
    \n self.<\/span>burned +=<\/span> minutes
    \n return<\/span> self.<\/span>remaining(<\/span>)<\/span><\/p>\n

    def<\/span> remaining<\/span>(<\/span>self)<\/span>:<\/span>
    \n return<\/span> self.<\/span>limit –<\/span> self.<\/span>burned<\/p>\n

    def<\/span> status<\/span>(<\/span>self)<\/span>:<\/span>
    \n ratio =<\/span> self.<\/span>burned \/<\/span> self.<\/span>limit
    \n if<\/span> ratio ><\/span> 0.5<\/span>:<\/span>
    \n return<\/span> "\u5df2\u6d88\u8017\u8d85\u8fc7\u4e00\u534a\u9884\u7b97，\u8fdb\u5165\u6b62\u635f\u6d41\u7a0b"<\/span>
    \n if<\/span> ratio ><\/span> 0.2<\/span>:<\/span>
    \n return<\/span> "\u9884\u7b97\u6d88\u8017\u504f\u5feb，\u68c0\u67e5\u8fd1\u671f\u53d8\u66f4"<\/span>
    \n return<\/span> "\u9884\u7b97\u5145\u8db3"<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n budget =<\/span> ErrorBudget(<\/span>slo_nines=<\/span>3<\/span>,<\/span> period_days=<\/span>30<\/span>)<\/span>
    \n # \u6a21\u62df\u4e00\u6b21 25 \u5206\u949f\u6545\u969c\u4e0e\u4e24\u6b21 8 \u5206\u949f\u6296\u52a8<\/span>
    \n for<\/span> minutes in<\/span> (<\/span>25<\/span>,<\/span> 8<\/span>,<\/span> 8<\/span>)<\/span>:<\/span>
    \n budget.<\/span>record(<\/span>minutes)<\/span>
    \n print<\/span>(<\/span>f"\u9884\u7b97\u4e0a\u9650 <\/span>{<\/span>budget.<\/span>limit:<\/span>.1f<\/span>}<\/span><\/span> \u5206\u949f，\u5269\u4f59 <\/span>{<\/span>budget.<\/span>remaining(<\/span>)<\/span>:<\/span>.1f<\/span>}<\/span><\/span> \u5206\u949f"<\/span><\/span>)<\/span>
    \n print<\/span>(<\/span>budget.<\/span>status(<\/span>)<\/span>)<\/span><\/p>\n

    \u9519\u8bef\u9884\u7b97\u628a\u76d1\u63a7\u4ece"\u4e8b\u540e\u67e5\u770b"\u53d8\u6210"\u4e8b\u524d\u9608\u503c"\u3002\u5f53\u9884\u7b97\u6d88\u8017\u6bd4\u4f8b\u8d85\u8fc7 50% \u65f6\u81ea\u52a8\u8fdb\u5165\u6b62\u635f，\u8fd9\u4e0e\u540e\u7eed\u7ae0\u8282\u7684\u544a\u8b66\u9608\u503c\u4e92\u4e3a\u8865\u5145：\u524d\u8005\u7ba1\u4e1a\u52a1\u53ef\u7528\u6027，\u540e\u8005\u7ba1\u6280\u672f\u6307\u6807\u3002\u76ee\u6807\u8bbe\u5b9a\u5b8c\u6bd5\u540e，\u6240\u6709\u91c7\u96c6\u9879\u90fd\u5fc5\u987b\u80fd\u5bf9\u63a5\u5230\u67d0\u4e00\u5c42\u76ee\u6807，\u65e0\u6cd5\u5bf9\u63a5\u5230\u76ee\u6807\u7684\u57cb\u70b9\u5c31\u662f\u65e0\u6548\u57cb\u70b9，\u5e94\u5f53\u5728\u8bc4\u5ba1\u65f6\u5220\u9664\u3002<\/p>\n

    2. \u6027\u80fd\u6307\u6807<\/h3>\n

    \u6027\u80fd\u6307\u6807\u5206\u5728\u7ebf\u4e0e\u79bb\u7ebf\u4e24\u7c7b，\u4e8c\u8005\u56de\u7b54\u7684\u95ee\u9898\u4e0d\u540c\u3002\u5728\u7ebf\u6307\u6807\u56de\u7b54"\u670d\u52a1\u73b0\u5728\u8dd1\u5f97\u5982\u4f55"，\u91c7\u96c6\u81ea\u8fd0\u884c\u65f6\u63a2\u9488，\u7c92\u5ea6\u662f\u79d2\u7ea7，\u8986\u76d6\u5ef6\u8fdf\u3001\u541e\u5410\u3001\u5e76\u53d1\u3001\u8d44\u6e90\u5229\u7528\u7387；\u79bb\u7ebf\u6307\u6807\u56de\u7b54"\u6a21\u578b\u8d28\u91cf\u662f\u5426\u9000\u5316"，\u91c7\u96c6\u81ea\u6279\u91cf\u8bc4\u4f30\u4efb\u52a1，\u7c92\u5ea6\u662f\u5929\u7ea7，\u8986\u76d6\u56f0\u60d1\u5ea6\u3001\u5206\u6b65\u6307\u6807\u3001\u5206\u5e03\u504f\u79fb\u3002\u5728\u7ebf\u6307\u6807\u65e0\u6cd5\u53d1\u73b0\u8bed\u4e49\u8d28\u91cf\u4e0b\u964d，\u79bb\u7ebf\u6307\u6807\u65e0\u6cd5\u53d1\u73b0\u5ef6\u8fdf\u6bdb\u523a，\u4e00\u5957\u5b8c\u6574\u7684\u76d1\u63a7\u4f53\u7cfb\u5fc5\u987b\u4e24\u8005\u5e76\u884c\u3002<\/p>\n

    \u5728\u7ebf\u6307\u6807\u5185\u90e8\u8fd8\u8981\u533a\u5206"\u670d\u52a1\u4e8e\u7528\u6237"\u7684\u6307\u6807\u4e0e"\u63cf\u8ff0\u8d44\u6e90"\u7684\u6307\u6807\u3002\u524d\u8005\u5305\u62ec\u9996 token \u5ef6\u8fdf\u3001\u751f\u6210\u5ef6\u8fdf\u3001\u7aef\u5230\u7aef\u5ef6\u8fdf，\u76f4\u63a5\u51b3\u5b9a\u7528\u6237\u4f53\u9a8c；\u540e\u8005\u5305\u62ec GPU \u5229\u7528\u7387\u3001\u663e\u5b58\u5360\u7528\u3001KV Cache \u5927\u5c0f\u3001\u6279\u5904\u7406\u89c4\u6a21，\u95f4\u63a5\u5f71\u54cd\u524d\u8005\u7684\u7a33\u5b9a\u6027\u3002\u8d44\u6e90\u6307\u6807\u662f\u5ef6\u8fdf\u6307\u6807\u7684\u6839\u56e0\u5c42：80% \u7684\u5ef6\u8fdf\u52a3\u5316\u6848\u4f8b\u6700\u7ec8\u53ef\u4ee5\u5f52\u56e0\u5230\u663e\u5b58\u3001\u6279\u5904\u7406\u6216\u961f\u5217\u7684\u53d8\u5316，\u56e0\u6b64\u91c7\u96c6\u65f6\u8981\u628a\u4e24\u8005\u5728\u65f6\u95f4\u8f74\u4e0a\u5bf9\u9f50，\u624d\u80fd\u505a\u6839\u56e0\u5173\u8054\u3002<\/p>\n

    #mermaid-svg-2YsEF2z7x8Qe234f{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-2YsEF2z7x8Qe234f .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-2YsEF2z7x8Qe234f .error-icon{fill:#552222;}#mermaid-svg-2YsEF2z7x8Qe234f .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-2YsEF2z7x8Qe234f .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-2YsEF2z7x8Qe234f .marker{fill:#333333;stroke:#333333;}#mermaid-svg-2YsEF2z7x8Qe234f .marker.cross{stroke:#333333;}#mermaid-svg-2YsEF2z7x8Qe234f svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-2YsEF2z7x8Qe234f p{margin:0;}#mermaid-svg-2YsEF2z7x8Qe234f .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-2YsEF2z7x8Qe234f .cluster-label text{fill:#333;}#mermaid-svg-2YsEF2z7x8Qe234f .cluster-label span{color:#333;}#mermaid-svg-2YsEF2z7x8Qe234f .cluster-label span p{background-color:transparent;}#mermaid-svg-2YsEF2z7x8Qe234f .label text,#mermaid-svg-2YsEF2z7x8Qe234f span{fill:#333;color:#333;}#mermaid-svg-2YsEF2z7x8Qe234f .node rect,#mermaid-svg-2YsEF2z7x8Qe234f .node circle,#mermaid-svg-2YsEF2z7x8Qe234f .node ellipse,#mermaid-svg-2YsEF2z7x8Qe234f .node polygon,#mermaid-svg-2YsEF2z7x8Qe234f .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-2YsEF2z7x8Qe234f .rough-node .label text,#mermaid-svg-2YsEF2z7x8Qe234f .node .label text,#mermaid-svg-2YsEF2z7x8Qe234f .image-shape .label,#mermaid-svg-2YsEF2z7x8Qe234f .icon-shape .label{text-anchor:middle;}#mermaid-svg-2YsEF2z7x8Qe234f .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-2YsEF2z7x8Qe234f .rough-node .label,#mermaid-svg-2YsEF2z7x8Qe234f .node .label,#mermaid-svg-2YsEF2z7x8Qe234f .image-shape .label,#mermaid-svg-2YsEF2z7x8Qe234f .icon-shape .label{text-align:center;}#mermaid-svg-2YsEF2z7x8Qe234f .node.clickable{cursor:pointer;}#mermaid-svg-2YsEF2z7x8Qe234f .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-2YsEF2z7x8Qe234f .arrowheadPath{fill:#333333;}#mermaid-svg-2YsEF2z7x8Qe234f .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-2YsEF2z7x8Qe234f .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-2YsEF2z7x8Qe234f .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-2YsEF2z7x8Qe234f .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-2YsEF2z7x8Qe234f .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-2YsEF2z7x8Qe234f .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-2YsEF2z7x8Qe234f .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-2YsEF2z7x8Qe234f .cluster text{fill:#333;}#mermaid-svg-2YsEF2z7x8Qe234f .cluster span{color:#333;}#mermaid-svg-2YsEF2z7x8Qe234f div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-2YsEF2z7x8Qe234f .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-2YsEF2z7x8Qe234f rect.text{fill:none;stroke-width:0;}#mermaid-svg-2YsEF2z7x8Qe234f .icon-shape,#mermaid-svg-2YsEF2z7x8Qe234f .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-2YsEF2z7x8Qe234f .icon-shape p,#mermaid-svg-2YsEF2z7x8Qe234f .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-2YsEF2z7x8Qe234f .icon-shape .label rect,#mermaid-svg-2YsEF2z7x8Qe234f .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-2YsEF2z7x8Qe234f .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-2YsEF2z7x8Qe234f .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-2YsEF2z7x8Qe234f :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-2YsEF2z7x8Qe234f .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-2YsEF2z7x8Qe234f .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-2YsEF2z7x8Qe234f .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u79bb\u7ebf\u6307\u6807<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5728\u7ebf\u6307\u6807<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5426<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u662f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5426<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u662f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6b21\u65e5\u5bf9\u6bd4<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u89e6\u53d1\u544a\u8b66<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    TTFT \u9996 token \u5ef6\u8fdf<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    TPOT \u6bcf token \u5ef6\u8fdf<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P99 \u7aef\u5230\u7aef\u5ef6\u8fdf<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u541e\u5410 tokens\/s<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u56f0\u60d1\u5ea6<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    PSI \u5206\u5e03\u504f\u79fb<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u9ec4\u91d1\u96c6\u51c6\u786e\u7387<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5ef6\u8fdf\u76f4\u65b9\u56fe\u5206\u6876<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u901f\u7387\u4e0e\u9971\u548c\u5ea6\u5206\u6790<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6bcf\u65e5\u6279\u91cf\u8bc4\u4f30<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5ef6\u8fdf\u662f\u5426\u8d8a\u754c<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8d28\u91cf\u662f\u5426\u56de\u5f52<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u57fa\u7ebf\u5b58\u50a8<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8fdb\u5165\u5f02\u5e38\u68c0\u6d4b<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u544a\u8b66\u7cfb\u7edf<\/p>\n

    <\/span><\/p>\n

    2.1 \u5728\u7ebf\u6307\u6807<\/h4>\n

    \u5728\u7ebf\u6307\u6807\u7684\u6838\u5fc3\u662f\u5ef6\u8fdf\u5206\u89e3\u3002\u4e00\u6b21 LLM \u8bf7\u6c42\u7684\u7aef\u5230\u7aef\u5ef6\u8fdf\u7531\u56db\u6bb5\u76f8\u52a0：\u7f51\u7edc\u4f20\u8f93\u3001\u8c03\u5ea6\u6392\u961f\u3001\u9996 token \u8ba1\u7b97\u3001\u9010 token \u751f\u6210\u3002\u9996 token \u5ef6\u8fdf（TTFT）\u662f\u8c03\u5ea6\u4e0e\u9884\u586b\u5145\u9636\u6bb5\u7684\u603b\u548c，\u751f\u6210\u5ef6\u8fdf（TPOT）\u662f\u5355\u4e2a token \u7684\u81ea\u56de\u5f52\u8ba1\u7b97\u65f6\u95f4，\u4e24\u8005\u91cf\u7ea7\u5dee\u5f02\u60ac\u6b8a：TTFT \u901a\u5e38\u5728 200-600ms，\u800c TPOT \u5728 40ms \u5de6\u53f3\u3002\u5982\u679c\u7aef\u5230\u7aef\u5ef6\u8fdf\u52a3\u5316，\u5fc5\u987b\u5148\u5224\u65ad\u52a3\u5316\u6bb5\u843d\u5728\u54ea\u4e00\u6bb5，\u624d\u80fd\u51b3\u5b9a\u662f\u6269\u5bb9\u3001\u8c03\u6279\u5904\u7406\u8fd8\u662f\u6362\u91cf\u5316\u3002<\/p>\n

    \u5ef6\u8fdf\u91c7\u96c6\u5fc5\u987b\u7528\u76f4\u65b9\u56fe\u800c\u4e0d\u662f\u5747\u503c\u3002\u5747\u503c\u5bf9\u957f\u5c3e\u5206\u5e03\u6ca1\u6709\u610f\u4e49：\u4e00\u4e2a 7B \u6a21\u578b\u5728\u52a8\u6001\u6279\u5904\u7406\u4e0b\u7684\u5ef6\u8fdf\u5448\u660e\u663e\u957f\u5c3e，P50 \u53ef\u80fd\u53ea\u6709 300ms，P99 \u5374\u5230 800ms，P99.9 \u5230 2s\u3002\u7528\u5747\u503c\u505a\u544a\u8b66，\u957f\u5c3e\u52a3\u5316\u4f1a\u88ab\u9ad8\u5e76\u53d1\u8bf7\u6c42\u7684\u5e73\u5747\u6548\u5e94\u6df9\u6ca1\u3002\u76f4\u65b9\u56fe\u5728\u5ba2\u6237\u7aef\u6309\u6876\u805a\u5408，\u518d\u5728\u67e5\u8be2\u7aef\u7528 histogram_quantile \u8fd8\u539f\u5206\u4f4d\u6570，\u65e2\u964d\u4f4e\u4f20\u8f93\u91cf，\u53c8\u4fdd\u7559\u5206\u5e03\u4fe1\u606f\u3002<\/p>\n

    \u751f\u4ea7\u76ee\u6807\u7684\u91cf\u5316\u57fa\u7ebf\u5982\u4e0b\u8868，\u5747\u4ee5\u5355\u5361 A100\u30017B \u6a21\u578b\u3001\u8f93\u5165\u8f93\u51fa\u5404 512 token \u4e3a\u524d\u63d0\u6d4b\u5f97\u3002\u8868\u4e2d P99 \u662f\u544a\u8b66\u7684\u4e3b\u6307\u6807，P50 \u7528\u4e8e\u65e5\u5e38\u8d8b\u52bf\u89c2\u5bdf，P99.9 \u7528\u4e8e\u957f\u5c3e\u6cbb\u7406\u3002<\/p>\n\n\u6307\u6807\u5b9a\u4e49P50P99\u544a\u8b66\u9608\u503c<\/tr>\n\n\n\n\n\n\n
    TTFT \u9996 token \u5ef6\u8fdf<\/td>\n\u8bf7\u6c42\u5230\u8fbe\u81f3\u9996\u4e2a token \u8f93\u51fa<\/td>\n300ms<\/td>\n500ms<\/td>\nP99 \u5927\u4e8e 500ms \u6301\u7eed 5 \u5206\u949f<\/td>\n<\/tr>\n
    TPOT \u751f\u6210\u5ef6\u8fdf<\/td>\n\u5355\u4e2a token \u7684\u81ea\u56de\u5f52\u8017\u65f6<\/td>\n40ms<\/td>\n60ms<\/td>\nP99 \u5927\u4e8e 80ms \u6301\u7eed 10 \u5206\u949f<\/td>\n<\/tr>\n
    \u7aef\u5230\u7aef\u5ef6\u8fdf<\/td>\n\u5b8c\u6574\u8bf7\u6c42\u8017\u65f6<\/td>\n1.2s<\/td>\n2.0s<\/td>\nP99 \u5927\u4e8e 2s \u6301\u7eed 5 \u5206\u949f<\/td>\n<\/tr>\n
    \u751f\u6210\u541e\u5410<\/td>\n\u6bcf\u5361\u6bcf\u79d2\u751f\u6210 token \u6570<\/td>\n800<\/td>\n1200<\/td>\n\u4f4e\u4e8e 600 \u6301\u7eed 10 \u5206\u949f<\/td>\n<\/tr>\n
    \u9519\u8bef\u7387<\/td>\n5xx \u4e0e\u975e 200 \u8bf7\u6c42\u5360\u6bd4<\/td>\n0.1%<\/td>\n0.5%<\/td>\n\u5927\u4e8e 1% \u6301\u7eed 10 \u5206\u949f<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    \u957f\u5c3e\u5ef6\u8fdf\u7684\u6839\u56e0\u901a\u5e38\u662f\u6392\u961f\u800c\u975e\u8ba1\u7b97\u3002\u5f53\u5e76\u53d1\u8bf7\u6c42\u6570\u8d85\u8fc7\u52a8\u6001\u6279\u5904\u7406\u5bb9\u91cf，\u8bf7\u6c42\u8fdb\u5165\u8c03\u5ea6\u961f\u5217，\u7b49\u5f85\u65f6\u95f4\u968f\u961f\u5217\u957f\u5ea6\u7ebf\u6027\u589e\u957f：\u961f\u5217 20 \u4e2a\u8bf7\u6c42\u3001\u5355\u8bf7\u6c42\u5e73\u5747\u670d\u52a1\u65f6\u95f4 1.5s \u65f6，\u961f\u5c3e\u8bf7\u6c42\u7684\u6392\u961f\u5ef6\u8fdf\u7ea6 30s\u3002\u56e0\u6b64\u5728\u7ebf\u6307\u6807\u5fc5\u987b\u540c\u65f6\u91c7\u96c6\u961f\u5217\u957f\u5ea6\u4e0e\u5e76\u53d1\u6570，\u961f\u5217\u957f\u5ea6\u8d85\u8fc7 10 \u4e14\u6301\u7eed\u4e0a\u5347\u65f6，\u5373\u4f7f\u5ef6\u8fdf\u5c1a\u672a\u8d8a\u754c，\u4e5f\u5e94\u89e6\u53d1\u6269\u5bb9\u544a\u8b66\u3002<\/p>\n

    # \u6765\u6e90：prometheus_client 0.20.0 \/ latency_tracker.py<\/span>
    \nfrom<\/span> prometheus_client import<\/span> Histogram,<\/span> generate_latest
    \nimport<\/span> random<\/p>\n

    # \u5ef6\u8fdf\u76f4\u65b9\u56fe\u6309\u4e1a\u52a1\u9608\u503c\u5207\u6876，\u4fdd\u8bc1\u5206\u4f4d\u6570\u8fd8\u539f\u7cbe\u5ea6<\/span>
    \nTTFT =<\/span> Histogram(<\/span>
    \n "llm_ttft_seconds"<\/span>,<\/span> "\u9996 token \u5ef6\u8fdf"<\/span>,<\/span> buckets=<\/span>(<\/span>0.1<\/span>,<\/span> 0.2<\/span>,<\/span> 0.3<\/span>,<\/span> 0.5<\/span>,<\/span> 0.8<\/span>,<\/span> 1.0<\/span>,<\/span> 2.0<\/span>)<\/span>
    \n)<\/span>
    \nTPOT =<\/span> Histogram(<\/span>
    \n "llm_tpot_seconds"<\/span>,<\/span> "\u6bcf token \u751f\u6210\u5ef6\u8fdf"<\/span>,<\/span> buckets=<\/span>(<\/span>0.01<\/span>,<\/span> 0.02<\/span>,<\/span> 0.04<\/span>,<\/span> 0.08<\/span>,<\/span> 0.16<\/span>)<\/span>
    \n)<\/span><\/p>\n

    def<\/span> serve<\/span>(<\/span>prompt_tokens,<\/span> output_tokens)<\/span>:<\/span>
    \n """\u8bb0\u5f55\u4e00\u6b21\u63a8\u7406\u7684\u8017\u65f6\u89c2\u6d4b：\u9884\u586b\u5145\u4e0e\u81ea\u56de\u5f52\u4e24\u6bb5\u3002
    \n \u751f\u4ea7\u73af\u5883\u5728\u771f\u5b9e\u8bf7\u6c42\u8def\u5f84\u4e0a\u8c03\u7528，\u8fd9\u91cc\u7528\u968f\u673a\u6837\u672c\u6a21\u62df\u6d4b\u91cf\u3002"""<\/span>
    \n ttft =<\/span> max<\/span>(<\/span>0.0<\/span>,<\/span> random.<\/span>gauss(<\/span>0.35<\/span>,<\/span> 0.05<\/span>)<\/span>)<\/span> # \u9884\u586b\u5145\u4e0e\u8c03\u5ea6\u8017\u65f6<\/span>
    \n tpot =<\/span> max<\/span>(<\/span>0.0<\/span>,<\/span> random.<\/span>gauss(<\/span>0.04<\/span>,<\/span> 0.008<\/span>)<\/span>)<\/span> # \u5355 token \u751f\u6210\u8017\u65f6<\/span>
    \n TTFT.<\/span>observe(<\/span>ttft)<\/span>
    \n TPOT.<\/span>observe(<\/span>tpot)<\/span>
    \n return<\/span> ttft +<\/span> tpot *<\/span> output_tokens<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n for<\/span> _ in<\/span> range<\/span>(<\/span>2000<\/span>)<\/span>:<\/span>
    \n serve(<\/span>random.<\/span>randint(<\/span>32<\/span>,<\/span> 512<\/span>)<\/span>,<\/span> random.<\/span>randint(<\/span>16<\/span>,<\/span> 256<\/span>)<\/span>)<\/span>
    \n for<\/span> line in<\/span> generate_latest(<\/span>)<\/span>.<\/span>splitlines(<\/span>)<\/span>:<\/span>
    \n text =<\/span> line.<\/span>decode(<\/span>)<\/span>
    \n if<\/span> text.<\/span>startswith(<\/span>"llm_ttft_seconds_bucket"<\/span>)<\/span> or<\/span> text.<\/span>startswith(<\/span>
    \n "llm_ttft_seconds_count"<\/span>
    \n )<\/span>:<\/span>
    \n print<\/span>(<\/span>text)<\/span><\/p>\n

    2.2 \u79bb\u7ebf\u6307\u6807<\/h4>\n

    \u79bb\u7ebf\u6307\u6807\u56de\u7b54\u5728\u7ebf\u6307\u6807\u56de\u7b54\u4e0d\u4e86\u7684\u95ee\u9898：\u6a21\u578b\u8f93\u51fa\u8d28\u91cf\u662f\u5426\u9000\u5316\u3002\u4e00\u4e2a\u6587\u672c\u751f\u6210\u6a21\u578b\u53ef\u80fd\u5ef6\u8fdf\u3001\u541e\u5410\u3001\u9519\u8bef\u7387\u5168\u90e8\u6b63\u5e38，\u4f46\u751f\u6210\u7ed3\u679c\u4ece"\u6b63\u786e\u6458\u8981"\u9000\u5316\u4e3a"\u5e7b\u89c9\u5f0f\u6458\u8981"，\u8fd9\u79cd\u9000\u5316\u53ea\u80fd\u901a\u8fc7\u79bb\u7ebf\u6279\u91cf\u8bc4\u4f30\u6355\u6349\u3002\u79bb\u7ebf\u8bc4\u4f30\u7684\u57fa\u672c\u5355\u5143\u662f\u9ec4\u91d1\u8bc4\u4f30\u96c6：2000 \u6761\u5e26\u6807\u51c6\u7b54\u6848\u7684\u63d0\u793a\u8bcd，\u6bcf\u5929\u6216\u6bcf\u7248\u672c\u8dd1\u4e00\u8f6e，\u5bf9\u6bd4\u56f0\u60d1\u5ea6\u3001ROUGE\u3001\u51c6\u786e\u7387\u7b49\u6307\u6807\u7684\u53d8\u5316\u3002<\/p>\n

    \u5206\u5e03\u504f\u79fb\u662f\u79bb\u7ebf\u76d1\u63a7\u7684\u53e6\u4e00\u7c7b\u6838\u5fc3\u5bf9\u8c61\u3002\u7ebf\u4e0a\u8f93\u5165\u5206\u5e03\u4f1a\u968f\u4e1a\u52a1\u6f14\u8fdb\u6f02\u79fb：\u65b0\u7684 prompt \u6a21\u677f\u3001\u65b0\u7684\u8bed\u8a00\u6df7\u5408\u3001\u7528\u6237\u884c\u4e3a\u7684\u5b63\u8282\u6027\u53d8\u5316，\u90fd\u4f1a\u8ba9\u8f93\u5165\u5206\u5e03\u504f\u79bb\u8bad\u7ec3\u5206\u5e03\u3002\u91cf\u5316\u504f\u79fb\u7684\u5e38\u7528\u6307\u6807\u662f\u7fa4\u4f53\u7a33\u5b9a\u6027\u6307\u6570（PSI），\u4ee5 10 \u4e2a\u5206\u7bb1\u8ba1\u7b97\u65b0\u65e7\u5206\u5e03\u7684\u5dee\u5f02，PSI \u8d85\u8fc7 0.2 \u5373\u5224\u5b9a\u4e3a\u660e\u663e\u504f\u79fb，\u9700\u8981\u89e6\u53d1\u91cd\u8bad\u7ec3\u6216\u7279\u5f81\u5de5\u7a0b\u8c03\u6574\u3002\u5d4c\u5165\u5411\u91cf\u7684\u4f59\u5f26\u8ddd\u79bb\u5206\u5e03\u4e5f\u53ef\u4f5c\u4e3a\u5206\u5e03\u76d1\u63a7\u7684\u8865\u5145，\u91c7\u96c6\u6210\u672c\u9ad8\u4e8e PSI，\u4f46\u80fd\u6355\u6349\u8bed\u4e49\u7ea7\u6f02\u79fb\u3002<\/p>\n

    \u79bb\u7ebf\u8bc4\u4f30\u6709\u660e\u786e\u7684\u8c03\u5ea6\u4e0e\u6210\u672c\u7ea6\u675f\u30022000 \u6761\u63d0\u793a\u8bcd\u3001\u5355\u6761\u6700\u957f 2048 token，\u5728\u4e00\u5f20 A100 \u4e0a\u5b8c\u6574\u8bc4\u4f30\u7ea6\u8017\u65f6 40 \u5206\u949f，\u4ea7\u751f\u7ea6 12 \u4e07\u6761\u63a8\u7406\u65e5\u5fd7\u3002\u8bc4\u4f30\u4efb\u52a1\u901a\u5e38\u5b89\u6392\u5728\u6d41\u91cf\u4f4e\u8c37\u7684\u51cc\u6668 2 \u70b9\u6267\u884c，\u7ed3\u679c\u5199\u5165\u72ec\u7acb\u7684\u8bc4\u4f30\u65f6\u5e8f\u5e93，\u4e0e\u5728\u7ebf\u6307\u6807\u5206\u5f00\u5b58\u50a8，\u907f\u514d\u8bc4\u4f30\u8d1f\u8f7d\u6c61\u67d3\u5728\u7ebf\u76d1\u63a7\u6570\u636e\u3002\u8bc4\u4f30\u7ed3\u679c\u7684\u6bd4\u5bf9\u5bf9\u8c61\u662f\u6700\u8fd1 7 \u5929\u7684\u57fa\u7ebf：\u5355\u65e5\u4e0b\u964d\u8d85\u8fc7 5% \u9700\u8981\u4eba\u5de5\u590d\u6838，\u8d85\u8fc7 10% \u81ea\u52a8\u51bb\u7ed3\u6a21\u578b\u53d1\u5e03\u3002<\/p>\n

    \u79bb\u7ebf\u4e0e\u5728\u7ebf\u7684\u8fb9\u754c\u5728\u4e8e\u65f6\u95f4\u5c3a\u5ea6：\u79bb\u7ebf\u6307\u6807\u662f\u5206\u949f\u5230\u5c0f\u65f6\u7ea7\u5ef6\u8fdf\u7684"\u6162\u4fe1\u53f7"，\u65e0\u6cd5\u7528\u4e8e\u5b9e\u65f6\u544a\u8b66，\u4f46\u80fd\u53d1\u73b0\u5728\u7ebf\u6307\u6807\u6c38\u8fdc\u770b\u4e0d\u89c1\u7684\u8d28\u91cf\u9000\u5316\u3002\u4e24\u8005\u914d\u5408\u7684\u5178\u578b\u52a8\u4f5c\u94fe\u662f：PSI \u8d85\u8fc7 0.2 \u89e6\u53d1\u79bb\u7ebf\u590d\u8bc4，\u590d\u8bc4\u786e\u8ba4\u51c6\u786e\u7387\u4e0b\u964d 8%，\u968f\u540e\u89e6\u53d1\u6a21\u578b\u56de\u6eda\u4e0e\u91cd\u8bad\u7ec3\u6392\u671f\u3002\u8fd9\u4e00\u94fe\u8def\u628a\u76d1\u63a7\u4ece"\u6551\u706b"\u5347\u7ea7\u4e3a"\u9884\u9632"\u3002<\/p>\n

    \u79bb\u7ebf\u8bc4\u4f30\u7684\u8986\u76d6\u8303\u56f4\u8981\u4e0e\u5728\u7ebf\u6d41\u91cf\u7ed3\u6784\u5bf9\u9f50\u3002\u82e5\u7ebf\u4e0a 70% \u8bf7\u6c42\u6765\u81ea\u5ba2\u670d\u6458\u8981\u573a\u666f\u300120% \u6765\u81ea\u6587\u6863\u95ee\u7b54，\u9ec4\u91d1\u8bc4\u4f30\u96c6\u5c31\u5e94\u8be5\u6309 7:2:1 \u7684\u6bd4\u4f8b\u5206\u914d\u573a\u666f，\u5426\u5219\u8bc4\u4f30\u7ed3\u679c\u4f1a\u88ab\u9ad8\u9891\u573a\u666f\u7a00\u91ca，\u4f4e\u9891\u573a\u666f\u7684\u8d28\u91cf\u56de\u5f52\u88ab\u5e73\u5747\u6570\u63a9\u76d6\u3002\u8bc4\u4f30\u96c6\u573a\u666f\u6bd4\u4f8b\u6bcf\u5b63\u5ea6\u6838\u5bf9\u4e00\u6b21，\u4e0e\u7ebf\u4e0a\u6d41\u91cf\u91c7\u6837\u5bf9\u6bd4，\u504f\u5dee\u8d85\u8fc7 10% \u5c31\u91cd\u65b0\u914d\u6bd4\u3002\u8bc4\u4f30\u8f93\u51fa\u8fd8\u8981\u5206\u5c42：\u56f0\u60d1\u5ea6\u8fd9\u7c7b\u5168\u5c40\u6307\u6807\u53ea\u80fd\u7ed9\u51fa"\u662f\u5426\u6709\u95ee\u9898"\u7684\u7ed3\u8bba，\u5206\u573a\u666f\u7684 ROUGE \u4e0e\u51c6\u786e\u7387\u624d\u80fd\u5b9a\u4f4d"\u95ee\u9898\u5728\u54ea\u7c7b\u8f93\u5165\u4e0a"，\u540e\u8005\u624d\u662f\u56de\u6eda\u51b3\u7b56\u7684\u4f9d\u636e\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ psi_drift.py<\/span>
    \nimport<\/span> math
    \nimport<\/span> random<\/p>\n

    def<\/span> psi<\/span>(<\/span>expected,<\/span> actual,<\/span> eps=<\/span>1e-6<\/span>)<\/span>:<\/span>
    \n """\u7fa4\u4f53\u7a33\u5b9a\u6027\u6307\u6570，\u5927\u4e8e 0.2 \u5224\u5b9a\u4e3a\u660e\u663e\u5206\u5e03\u504f\u79fb\u3002"""<\/span>
    \n if<\/span> len<\/span>(<\/span>expected)<\/span> !=<\/span> len<\/span>(<\/span>actual)<\/span>:<\/span>
    \n raise<\/span> ValueError(<\/span>"\u4e24\u7ec4\u5206\u5e03\u7684\u5206\u7bb1\u6570\u4e0d\u4e00\u81f4"<\/span>)<\/span>
    \n score =<\/span> 0.0<\/span>
    \n for<\/span> e,<\/span> a in<\/span> zip<\/span>(<\/span>expected,<\/span> actual)<\/span>:<\/span>
    \n e =<\/span> max<\/span>(<\/span>e,<\/span> eps)<\/span>
    \n a =<\/span> max<\/span>(<\/span>a,<\/span> eps)<\/span>
    \n score +=<\/span> (<\/span>a –<\/span> e)<\/span> *<\/span> math.<\/span>log(<\/span>a \/<\/span> e)<\/span>
    \n return<\/span> score<\/p>\n

    def<\/span> quantize<\/span>(<\/span>values,<\/span> low,<\/span> high,<\/span> bins=<\/span>10<\/span>)<\/span>:<\/span>
    \n """\u628a\u8fde\u7eed\u503c\u6620\u5c04\u5230 [low, high] \u533a\u95f4\u7684 10 \u4e2a\u7b49\u5bbd\u5206\u7bb1\u3002"""<\/span>
    \n width =<\/span> (<\/span>high –<\/span> low)<\/span> \/<\/span> bins
    \n counts =<\/span> [<\/span>0<\/span>]<\/span> *<\/span> bins
    \n for<\/span> v in<\/span> values:<\/span>
    \n idx =<\/span> min<\/span>(<\/span>int<\/span>(<\/span>(<\/span>v –<\/span> low)<\/span> \/<\/span> width)<\/span>,<\/span> bins –<\/span> 1<\/span>)<\/span>
    \n counts[<\/span>idx]<\/span> +=<\/span> 1<\/span>
    \n total =<\/span> sum<\/span>(<\/span>counts)<\/span> or<\/span> 1<\/span>
    \n return<\/span> [<\/span>c \/<\/span> total for<\/span> c in<\/span> counts]<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n # \u57fa\u7ebf\u8f93\u5165\u957f\u5ea6\u4e0e\u504f\u79fb\u540e\u7684\u8f93\u5165\u957f\u5ea6<\/span>
    \n baseline =<\/span> [<\/span>random.<\/span>uniform(<\/span>32<\/span>,<\/span> 512<\/span>)<\/span> for<\/span> _ in<\/span> range<\/span>(<\/span>5000<\/span>)<\/span>]<\/span>
    \n drifted =<\/span> [<\/span>random.<\/span>uniform(<\/span>32<\/span>,<\/span> 768<\/span>)<\/span> for<\/span> _ in<\/span> range<\/span>(<\/span>5000<\/span>)<\/span>]<\/span>
    \n p =<\/span> psi(<\/span>quantize(<\/span>baseline,<\/span> 32<\/span>,<\/span> 768<\/span>)<\/span>,<\/span> quantize(<\/span>drifted,<\/span> 32<\/span>,<\/span> 768<\/span>)<\/span>)<\/span>
    \n verdict =<\/span> "\u89e6\u53d1\u91cd\u8bad\u7ec3"<\/span> if<\/span> p ><\/span> 0.2<\/span> else<\/span> "\u5206\u5e03\u7a33\u5b9a"<\/span>
    \n print<\/span>(<\/span>f"PSI=<\/span>{<\/span>p:<\/span>.3f<\/span>}<\/span><\/span>，<\/span>{<\/span>verdict}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    2.3 \u6307\u6807\u91c7\u96c6<\/h4>\n

    \u6307\u6807\u91c7\u96c6\u7684\u901a\u9053\u9009\u62e9\u51b3\u5b9a\u6570\u636e\u5b8c\u6574\u6027\u4e0e\u7cfb\u7edf\u590d\u6742\u5ea6\u3002Pull \u6a21\u578b（Prometheus \u6bcf 15 \u79d2\u4e3b\u52a8\u6293\u53d6）\u9002\u5408\u57fa\u7840\u8bbe\u65bd\u4e0e\u957f\u751f\u547d\u5468\u671f\u8fdb\u7a0b，\u670d\u52a1\u65b9\u53ea\u9700\u66b4\u9732 \/metrics \u7aef\u70b9，\u6293\u53d6\u8282\u594f\u7531\u76d1\u63a7\u7aef\u7edf\u4e00\u63a7\u5236，\u5929\u7136\u652f\u6301\u670d\u52a1\u53d1\u73b0\u3002Push \u6a21\u578b（StatsD \u5ba2\u6237\u7aef\u4e3b\u52a8\u4e0a\u62a5）\u9002\u5408\u77ed\u751f\u547d\u5468\u671f\u4efb\u52a1\u548c\u9700\u8981\u9010\u8bf7\u6c42\u8ba1\u6570\u7684\u573a\u666f，\u5982\u6279\u91cf\u8bc4\u4f30\u4f5c\u4e1a\u3001\u6279\u5904\u7406\u5bb9\u5668，\u4efb\u52a1\u7ed3\u675f\u5373\u4e0a\u62a5\u5b8c\u6bd5\u3002<\/p>\n

    \u4e24\u7c7b\u901a\u9053\u5728 LLM \u573a\u666f\u7684\u9009\u578b\u8fb9\u754c\u5f88\u6e05\u6670：\u5e38\u9a7b\u63a8\u7406\u8fdb\u7a0b\u7528 Pull，\u63a2\u9488\u4ee5 15 \u79d2\u95f4\u9694\u6293\u53d6，\u6bcf\u4e2a\u8fdb\u7a0b\u66b4\u9732\u7ea6 50 \u6761\u65f6\u95f4\u5e8f\u5217；\u79bb\u7ebf\u8bc4\u4f30\u3001\u6570\u636e\u7ba1\u7ebf\u7b49\u6279\u4efb\u52a1\u7528 Push，\u4efb\u52a1\u7ed3\u675f\u65f6\u4e0a\u62a5\u6c47\u603b\u503c\u3002\u6df7\u5408\u90e8\u7f72\u65f6\u5fc5\u987b\u5728\u6307\u6807\u547d\u540d\u4e0a\u533a\u5206\u6765\u6e90，\u4f8b\u5982 llm_infra_ \u524d\u7f00\u6765\u81ea Pull\u3001llm_batch_ \u524d\u7f00\u6765\u81ea Push，\u5426\u5219\u805a\u5408\u65f6\u4f1a\u628a\u4e24\u4e2a\u8bed\u4e49\u4e0d\u540c\u7684\u5e8f\u5217\u6df7\u5728\u4e00\u8d77\u3002<\/p>\n

    \u91c7\u96c6\u6210\u672c\u9700\u8981\u91cf\u5316\u3002\u6bcf\u6761\u65f6\u95f4\u5e8f\u5217\u4ee5 15 \u79d2\u95f4\u9694\u91c7\u6837，\u6bcf\u5929\u4ea7\u751f 5760 \u4e2a\u6837\u672c，TSDB \u538b\u7f29\u540e\u7ea6 1.5 \u5b57\u8282\u6bcf\u6837\u672c，\u5373\u6bcf\u5929\u7ea6 8.6KB\u30023000 \u6761\u6d3b\u8dc3\u5e8f\u5217\u5bf9\u5e94\u6bcf\u5929\u7ea6 26MB\u3001\u4fdd\u7559 15 \u5929\u7ea6 390MB，Prometheus \u5355\u8282\u70b9\u5b8c\u5168\u627f\u53d7\u5f97\u8d77\u3002\u771f\u6b63\u7684\u5927\u5934\u662f\u76f4\u65b9\u56fe：7 \u4e2a\u6876\u7684\u76f4\u65b9\u56fe\u4f1a\u4ea7\u751f 8 \u6761\u5e8f\u5217（7 \u6876\u52a0 count \u4e0e sum \u5171 9 \u6761），3000 \u4e2a\u76f4\u65b9\u56fe\u5e8f\u5217\u5bf9\u5e94\u7684\u5b9e\u9645\u78c1\u76d8\u5360\u7528\u662f\u666e\u901a\u5e8f\u5217\u7684 4 \u500d\u4ee5\u4e0a，\u56e0\u6b64\u53ea\u5bf9\u5ef6\u8fdf\u8fd9\u7c7b\u5173\u952e\u6307\u6807\u542f\u7528\u76f4\u65b9\u56fe，\u5176\u4f59\u6307\u6807\u7528 Gauge\u3002<\/p>\n

    \u91c7\u96c6\u914d\u7f6e\u7684\u53e6\u4e00\u4e2a\u5173\u952e\u53c2\u6570\u662f\u6293\u53d6\u95f4\u9694\u4e0e\u8bc4\u4f30\u95f4\u9694\u300215 \u79d2\u7684\u9ed8\u8ba4\u503c\u5bf9\u5206\u949f\u7ea7\u544a\u8b66\u8db3\u591f，\u4f46\u5bf9\u79d2\u7ea7\u6bdb\u523a\u65e0\u80fd\u4e3a\u529b：\u4e00\u6b21\u6301\u7eed 20 \u79d2\u7684\u5ef6\u8fdf\u6bdb\u523a\u4f1a\u88ab 15 \u79d2\u91c7\u6837\u5b8c\u5168\u9519\u8fc7\u3002\u9700\u8981\u6355\u83b7\u79d2\u7ea7\u6bdb\u523a\u7684\u6307\u6807（\u5982 OOM \u524d\u7684\u663e\u5b58\u6c34\u4f4d）\u5e94\u7f29\u77ed\u5230 5 \u79d2\u6293\u53d6，\u4ee3\u4ef7\u662f\u5199\u5165\u91cf\u4e58\u4ee5 3\u3002\u5b9e\u8df5\u4e2d\u6309\u6307\u6807\u91cd\u8981\u6027\u5206\u4e24\u6863：\u5173\u952e\u6307\u6807 5 \u79d2\u3001\u5e38\u89c4\u6307\u6807 15 \u79d2\u3002<\/p>\n

    \u91c7\u96c6\u5931\u8d25\u7684\u5904\u7406\u673a\u5236\u540c\u6837\u8981\u5b9a\u4e49\u6e05\u695a\u3002\u6293\u53d6\u8d85\u65f6\u6216\u8fd4\u56de\u975e 200 \u65f6\u5e94\u91cd\u8bd5\u5e76\u8bb0\u5f55\u5931\u8d25\u8ba1\u6570，\u8fde\u7eed 3 \u6b21\u5931\u8d25\u89e6\u53d1\u91c7\u96c6\u6e90\u544a\u8b66，\u9632\u6b62"\u76d1\u63a7\u9759\u9ed8\u5931\u6548"\u3002\u91c7\u96c6\u7aef\u4e0e TSDB \u4e4b\u95f4\u7684\u80cc\u538b\u7b56\u7565\u5e94\u660e\u786e：TSDB \u5199\u5165\u62e5\u585e\u65f6\u4f18\u5148\u4e22\u5f03\u4f4e\u4f18\u5148\u7ea7\u5e8f\u5217\u6837\u672c，\u4fdd\u7559\u5173\u952e\u6307\u6807，\u907f\u514d\u6574\u6761\u5199\u5165\u94fe\u8def\u88ab\u62d6\u57ae\u3002\u91c7\u96c6\u914d\u7f6e\u7eb3\u5165\u7248\u672c\u7ba1\u7406，\u53d8\u66f4\u8d70\u8bc4\u5ba1，\u914d\u7f6e\u9519\u8bef\u5bfc\u81f4\u7684\u6307\u6807\u7f3a\u53e3\u5e94\u53ef\u56de\u6eaf\u5230\u5177\u4f53\u53d8\u66f4\u3002<\/p>\n

    \u91c7\u96c6\u7684\u5408\u89c4\u4e0e\u5ba1\u8ba1\u8981\u6c42：\u6d89\u53ca\u7528\u6237\u6570\u636e\u7684\u6307\u6807（\u5982\u8bf7\u6c42\u5185\u5bb9\u957f\u5ea6\u3001\u8131\u654f\u540e\u7684 token \u6570）\u5e94\u8bb0\u5f55\u91c7\u96c6\u5b57\u6bb5\u6e05\u5355\u4e0e\u7559\u5b58\u5468\u671f，\u8d85\u8fc7\u4fdd\u7559\u671f\u7684\u6570\u636e\u5b9a\u671f\u6e05\u7406\u3002\u6307\u6807\u91c7\u96c6\u5668\u7684\u8bbf\u95ee\u6743\u9650\u6700\u5c0f\u5316，\u5199\u6743\u9650\u53ea\u6388\u4e88\u76d1\u63a7\u7ec4\u4ef6\u8d26\u53f7，\u4eba\u5de5\u67e5\u8be2\u8d70\u53ea\u8bfb\u901a\u9053，\u91c7\u96c6\u94fe\u8def\u53d8\u66f4\u8bb0\u5f55\u5ba1\u8ba1\u65e5\u5fd7\u3002<\/p>\n

    \u91c7\u96c6\u4e0e\u544a\u8b66\u7684\u8054\u52a8\u8fb9\u754c：\u91c7\u96c6\u53ea\u8d1f\u8d23\u628a\u6570\u636e\u9001\u8fdb TSDB，\u5224\u5b9a\u4e0e\u544a\u8b66\u5728\u89c4\u5219\u5c42\u5b8c\u6210，\u4e24\u8005\u89e3\u8026\u540e\u91c7\u96c6\u5347\u7ea7\u6216\u89c4\u5219\u8c03\u6574\u4e92\u4e0d\u5f71\u54cd\u3002\u91c7\u96c6\u914d\u7f6e\u53d8\u66f4\u524d\u7528\u5f71\u5b50\u91c7\u96c6\u9a8c\u8bc1\u517c\u5bb9\u6027，\u907f\u514d\u5347\u7ea7\u5bfc\u81f4\u6307\u6807\u65ad\u6d41\u3002<\/p>\n

    # \u6765\u6e90：statsd 4.0.1 \/ \u5ba2\u6237\u7aef\u7528\u6cd5，main \u4e3a\u81ea\u5b9e\u73b0<\/span>
    \nimport<\/span> random
    \nimport<\/span> time<\/p>\n

    import<\/span> statsd
    \nfrom<\/span> prometheus_client import<\/span> Counter,<\/span> Gauge,<\/span> start_http_server<\/p>\n

    # Push \u901a\u9053：StatsD \u5ba2\u6237\u7aef，\u524d\u7f00 llm，\u4e0a\u62a5\u5230\u672c\u673a 8125 \u7aef\u53e3<\/span>
    \nstats =<\/span> statsd.<\/span>StatsClient(<\/span>"127.0.0.1"<\/span>,<\/span> 8125<\/span>,<\/span> prefix=<\/span>"llm"<\/span>)<\/span><\/p>\n

    # Pull \u901a\u9053：Prometheus \u63a2\u9488，\u4f9b\u6293\u53d6<\/span>
    \nTOKENS =<\/span> Counter(<\/span>"llm_tokens_generated_total"<\/span>,<\/span> "\u7d2f\u8ba1\u751f\u6210 token \u6570"<\/span>)<\/span>
    \nBATCH =<\/span> Gauge(<\/span>"llm_current_batch_size"<\/span>,<\/span> "\u5f53\u524d\u52a8\u6001\u6279\u5927\u5c0f"<\/span>)<\/span><\/p>\n

    def<\/span> emit<\/span>(<\/span>batch_size,<\/span> gen_tokens,<\/span> latency_ms)<\/span>:<\/span>
    \n """\u540c\u4e00\u4efd\u89c2\u6d4b\u540c\u65f6\u5199\u4e24\u6761\u901a\u9053，\u6f14\u793a\u53cc\u901a\u9053\u534f\u540c\u3002"""<\/span>
    \n BATCH.<\/span>set<\/span>(<\/span>batch_size)<\/span>
    \n TOKENS.<\/span>inc(<\/span>gen_tokens)<\/span>
    \n stats.<\/span>gauge(<\/span>"batch_size"<\/span>,<\/span> batch_size)<\/span>
    \n stats.<\/span>timing(<\/span>"tpot_ms"<\/span>,<\/span> latency_ms)<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n start_http_server(<\/span>9102<\/span>)<\/span>
    \n for<\/span> _ in<\/span> range<\/span>(<\/span>30<\/span>)<\/span>:<\/span>
    \n emit(<\/span>random.<\/span>randint(<\/span>1<\/span>,<\/span> 64<\/span>)<\/span>,<\/span> random.<\/span>randint(<\/span>10<\/span>,<\/span> 256<\/span>)<\/span>,<\/span> random.<\/span>randint(<\/span>20<\/span>,<\/span> 120<\/span>)<\/span>)<\/span>
    \n time.<\/span>sleep(<\/span>1<\/span>)<\/span>
    \n print<\/span>(<\/span>"\u5df2\u5411 StatsD \u4e0a\u62a5 30 \u7ec4\u6837\u672c，\u5e76\u5411 Prometheus \u66b4\u9732 \/metrics"<\/span>)<\/span><\/p>\n

    \u53cc\u901a\u9053\u534f\u540c\u7684\u4ef7\u503c\u5728\u6545\u969c\u573a\u666f\u4e2d\u6700\u660e\u663e：Pull \u4fa7\u6301\u7eed\u8bb0\u5f55\u7a33\u6001\u6307\u6807，Push \u4fa7\u8bb0\u5f55\u6279\u4efb\u52a1\u7684\u4e00\u6b21\u6027\u7ed3\u679c\u3002\u5f53\u4e00\u6b21\u6279\u91cf\u8bc4\u4f30\u7684 PSI \u503c\u901a\u8fc7 Push \u4e0a\u62a5\u540e\u8d85\u8fc7 0.2，\u91c7\u96c6\u5c42\u518d\u7ed3\u5408 Pull \u4fa7\u7684\u5728\u7ebf\u5ef6\u8fdf\u5e8f\u5217\u5224\u65ad\u5f71\u54cd\u9762，\u4e24\u79cd\u901a\u9053\u7684\u6570\u636e\u5728\u540c\u4e00\u65f6\u5e8f\u5e93\u4e2d\u5bf9\u9f50，\u5f02\u5e38\u68c0\u6d4b\u624d\u80fd\u540c\u65f6\u62ff\u5230\u8d44\u6e90\u4fa7\u4e0e\u8d28\u91cf\u4fa7\u7684\u8bc1\u636e\u3002<\/p>\n

    3. \u5f02\u5e38\u68c0\u6d4b<\/h3>\n

    \u5f02\u5e38\u68c0\u6d4b\u7684\u76ee\u6807\u662f\u628a"\u6307\u6807\u504f\u79bb"\u7ffb\u8bd1\u6210"\u6545\u969c\u4fe1\u53f7"，\u518d\u51b3\u5b9a\u662f\u5426\u544a\u8b66\u3002\u5b83\u4f4d\u4e8e\u6307\u6807\u91c7\u96c6\u4e0e\u544a\u8b66\u4f53\u7cfb\u4e4b\u95f4，\u627f\u62c5\u4e24\u5c42\u8fc7\u6ee4：\u7b2c\u4e00\u5c42\u662f\u628a\u6b63\u5e38\u6ce2\u52a8\u4e0e\u771f\u5b9e\u5f02\u5e38\u5206\u5f00，\u7b2c\u4e8c\u5c42\u662f\u628a\u771f\u5b9e\u5f02\u5e38\u4ece\u5076\u53d1\u566a\u58f0\u4e2d\u7a33\u5b9a\u5730\u786e\u8ba4\u4e0b\u6765\u3002\u5f02\u5e38\u68c0\u6d4b\u65b9\u6cd5\u5206\u7edf\u8ba1\u65b9\u6cd5\u4e0e\u673a\u5668\u5b66\u4e60\u65b9\u6cd5\u4e24\u5927\u9635\u8425，\u524d\u8005\u53ef\u89e3\u91ca\u3001\u5f00\u9500\u5c0f，\u9002\u5408\u5355\u6307\u6807；\u540e\u8005\u80fd\u6355\u6349\u591a\u6307\u6807\u8054\u5408\u5f02\u5e38，\u4f46\u9700\u8981\u8bad\u7ec3\u6570\u636e\u4e0e\u518d\u6821\u51c6\u5468\u671f\u3002<\/p>\n

    \u5f02\u5e38\u68c0\u6d4b\u5fc5\u987b\u660e\u786e\u81ea\u5df1\u7684\u5931\u8d25\u6a21\u5f0f\u3002\u7edf\u8ba1\u65b9\u6cd5\u5bf9\u5e73\u7a33\u5206\u5e03\u7684\u5047\u8bbe\u5728\u6d41\u91cf\u5468\u671f\u6027\u6ce2\u52a8\u4e0b\u7ecf\u5e38\u5931\u6548\u2014\u2014\u767d\u5929 800 QPS\u3001\u591c\u95f4 80 QPS \u7684\u8d1f\u8f7d\u5dee\u5f02\u4f1a\u8ba9\u56fa\u5b9a\u9608\u503c\u5728\u591c\u95f4\u628a\u6b63\u5e38\u6d41\u91cf\u8bef\u5224\u4e3a\u5f02\u5e38\u3002\u673a\u5668\u5b66\u4e60\u65b9\u6cd5\u5219\u53d7\u8bad\u7ec3\u6570\u636e\u5206\u5e03\u7ea6\u675f，\u4e0a\u7ebf\u540e\u9047\u5230\u8bad\u7ec3\u671f\u6ca1\u89c1\u8fc7\u7684\u8d1f\u8f7d\u5f62\u6001\u4f1a\u4ea7\u751f\u5931\u63a7\u7684\u865a\u8b66\u3002\u56e0\u6b64\u751f\u4ea7\u73af\u5883\u51e0\u4e4e\u90fd\u91c7\u7528"\u591a\u65b9\u6cd5\u6295\u7968"\u7ed3\u6784：\u7edf\u8ba1\u65b9\u6cd5\u4e0e\u673a\u5668\u5b66\u4e60\u65b9\u6cd5\u5e76\u884c\u6253\u5206，\u81f3\u5c11\u4e24\u7968\u4e00\u81f4\u624d\u5224\u5b9a\u5f02\u5e38，\u628a\u5355\u4e00\u65b9\u6cd5\u7684\u504f\u5dee\u7a00\u91ca\u6389\u3002<\/p>\n

    #mermaid-svg-FXarhr3ErTLVhrWn{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-FXarhr3ErTLVhrWn .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-FXarhr3ErTLVhrWn .error-icon{fill:#552222;}#mermaid-svg-FXarhr3ErTLVhrWn .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-FXarhr3ErTLVhrWn .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-FXarhr3ErTLVhrWn .marker{fill:#333333;stroke:#333333;}#mermaid-svg-FXarhr3ErTLVhrWn .marker.cross{stroke:#333333;}#mermaid-svg-FXarhr3ErTLVhrWn svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-FXarhr3ErTLVhrWn p{margin:0;}#mermaid-svg-FXarhr3ErTLVhrWn .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-FXarhr3ErTLVhrWn .cluster-label text{fill:#333;}#mermaid-svg-FXarhr3ErTLVhrWn .cluster-label span{color:#333;}#mermaid-svg-FXarhr3ErTLVhrWn .cluster-label span p{background-color:transparent;}#mermaid-svg-FXarhr3ErTLVhrWn .label text,#mermaid-svg-FXarhr3ErTLVhrWn span{fill:#333;color:#333;}#mermaid-svg-FXarhr3ErTLVhrWn .node rect,#mermaid-svg-FXarhr3ErTLVhrWn .node circle,#mermaid-svg-FXarhr3ErTLVhrWn .node ellipse,#mermaid-svg-FXarhr3ErTLVhrWn .node polygon,#mermaid-svg-FXarhr3ErTLVhrWn .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-FXarhr3ErTLVhrWn .rough-node .label text,#mermaid-svg-FXarhr3ErTLVhrWn .node .label text,#mermaid-svg-FXarhr3ErTLVhrWn .image-shape .label,#mermaid-svg-FXarhr3ErTLVhrWn .icon-shape .label{text-anchor:middle;}#mermaid-svg-FXarhr3ErTLVhrWn .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-FXarhr3ErTLVhrWn .rough-node .label,#mermaid-svg-FXarhr3ErTLVhrWn .node .label,#mermaid-svg-FXarhr3ErTLVhrWn .image-shape .label,#mermaid-svg-FXarhr3ErTLVhrWn .icon-shape .label{text-align:center;}#mermaid-svg-FXarhr3ErTLVhrWn .node.clickable{cursor:pointer;}#mermaid-svg-FXarhr3ErTLVhrWn .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-FXarhr3ErTLVhrWn .arrowheadPath{fill:#333333;}#mermaid-svg-FXarhr3ErTLVhrWn .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-FXarhr3ErTLVhrWn .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-FXarhr3ErTLVhrWn .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-FXarhr3ErTLVhrWn .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-FXarhr3ErTLVhrWn .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-FXarhr3ErTLVhrWn .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-FXarhr3ErTLVhrWn .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-FXarhr3ErTLVhrWn .cluster text{fill:#333;}#mermaid-svg-FXarhr3ErTLVhrWn .cluster span{color:#333;}#mermaid-svg-FXarhr3ErTLVhrWn div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-FXarhr3ErTLVhrWn .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-FXarhr3ErTLVhrWn rect.text{fill:none;stroke-width:0;}#mermaid-svg-FXarhr3ErTLVhrWn .icon-shape,#mermaid-svg-FXarhr3ErTLVhrWn .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-FXarhr3ErTLVhrWn .icon-shape p,#mermaid-svg-FXarhr3ErTLVhrWn .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-FXarhr3ErTLVhrWn .icon-shape .label rect,#mermaid-svg-FXarhr3ErTLVhrWn .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-FXarhr3ErTLVhrWn .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-FXarhr3ErTLVhrWn .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-FXarhr3ErTLVhrWn :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-FXarhr3ErTLVhrWn .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-FXarhr3ErTLVhrWn .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-FXarhr3ErTLVhrWn .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u673a\u5668\u5b66\u4e60\u65b9\u6cd5<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7edf\u8ba1\u65b9\u6cd5<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u81f3\u5c11\u4e24\u7968\u4e00\u81f4<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5355\u7968\u547d\u4e2d<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6301\u7eed 3 \u4e2a\u7a97\u53e3<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u503c\u73ed\u786e\u8ba4\u540e\u53cd\u9988<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Z \u5206\u6570 3 \u4e2a\u6807\u51c6\u5dee<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    IQR 1.5 \u500d\u56db\u5206\u4f4d\u8ddd<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    CUSUM \u7d2f\u79ef\u548c<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5b64\u7acb\u68ee\u6797<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u81ea\u7f16\u7801\u5668\u91cd\u6784\u8bef\u5dee<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u65f6\u5e8f\u9884\u6d4b\u6b8b\u5dee<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6307\u6807\u5e8f\u5217<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6ed1\u52a8\u7a97\u53e3 30 \u5206\u949f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u591a\u65b9\u6cd5\u6295\u7968<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5224\u5b9a\u5f02\u5e38<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8fdb\u5165\u89c2\u5bdf\u5217\u8868<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5199\u5165\u544a\u8b66<\/p>\n

    <\/span><\/p>\n

    3.1 \u7edf\u8ba1\u65b9\u6cd5<\/h4>\n

    \u7edf\u8ba1\u65b9\u6cd5\u4ee5\u5206\u5e03\u5047\u8bbe\u4e3a\u57fa\u7840，\u628a\u504f\u79bb\u4e2d\u5fc3\u8d8b\u52bf\u5230\u7279\u5b9a\u8ddd\u79bb\u7684\u89c2\u6d4b\u5224\u4e3a\u5f02\u5e38\u3002Z \u5206\u6570\u662f\u6700\u76f4\u63a5\u7684\u5ea6\u91cf：\u7528\u6700\u8fd1 30 \u5206\u949f\u7a97\u53e3\u7684\u5747\u503c\u4e0e\u6807\u51c6\u5dee\u5f52\u4e00\u5316\u5f53\u524d\u503c，|Z| \u5927\u4e8e 3 \u5373\u8d85\u8fc7 3 \u4e2a\u6807\u51c6\u5dee，\u5224\u4e3a\u5f02\u5e38\u3002\u5bf9\u6b63\u6001\u5206\u5e03\u800c\u8a00，3 \u4e2a\u6807\u51c6\u5dee\u4e4b\u5916\u7684\u6837\u672c\u5360\u6bd4\u7ea6 0.27%，\u5373\u6bcf 1000 \u4e2a\u6837\u672c\u6700\u591a 3 \u4e2a，\u8bef\u62a5\u7387\u5929\u7136\u88ab\u538b\u5728 1% \u4ee5\u5185\u3002\u4ee3\u4ef7\u662f\u5bf9\u975e\u6b63\u6001\u5206\u5e03（\u5ef6\u8fdf\u3001\u8bf7\u6c42\u91cf\u8fd9\u7c7b\u53f3\u504f\u5206\u5e03）\u4f1a\u4ea7\u751f\u7cfb\u7edf\u6027\u8bef\u5224，\u9700\u8981\u5148\u505a\u5bf9\u6570\u53d8\u6362\u6216\u5206\u4f4d\u6570\u53d8\u6362\u3002<\/p>\n

    IQR \u65b9\u6cd5\u5bf9\u5206\u5e03\u5f62\u72b6\u66f4\u7a33\u5065\u3002\u4ee5 P25 \u4e0e P75 \u4e4b\u5dee\u4f5c\u4e3a\u79bb\u6563\u5ea6\u57fa\u51c6，\u8d85\u8fc7 Q3 + 1.5 \u500d IQR \u6216\u4f4e\u4e8e Q1 – 1.5 \u500d IQR \u5224\u4e3a\u5f02\u5e38\u3002\u5bf9\u5ef6\u8fdf\u8fd9\u7c7b\u91cd\u5c3e\u6570\u636e，IQR \u65b9\u6cd5\u6bd4 Z \u5206\u6570\u66f4\u6297\u6781\u7aef\u503c\u6c61\u67d3，\u56e0\u4e3a\u5b83\u7528\u5206\u4f4d\u6570\u800c\u975e\u5747\u503c\u4e0e\u65b9\u5dee\u3002\u5b9e\u9645\u90e8\u7f72\u4e2d\u4e24\u7c7b\u65b9\u6cd5\u540c\u65f6\u8fd0\u884c：Z \u5206\u6570\u6355\u83b7\u5747\u503c\u6f02\u79fb，IQR \u6355\u83b7\u5c3e\u90e8\u7a81\u53d8，\u4efb\u4e00\u547d\u4e2d\u90fd\u8bb0\u4e3a\u7edf\u8ba1\u65b9\u6cd5\u4e00\u7968\u3002<\/p>\n

    \u7d2f\u79ef\u548c（CUSUM）\u89e3\u51b3\u7684\u662f"\u6301\u7eed\u5c0f\u5e45\u504f\u79fb"\u95ee\u9898\u30023 \u4e2a\u6807\u51c6\u5dee\u7684\u9608\u503c\u5728\u5747\u503c\u6f02\u79fb 0.3 \u4e2a\u6807\u51c6\u5dee\u65f6，\u9700\u8981\u6570\u5c0f\u65f6\u624d\u80fd\u7d2f\u79ef\u51fa\u8db3\u591f\u8bc1\u636e，CUSUM \u901a\u8fc7\u7d2f\u79ef\u6b63\u504f\u5dee\u4e0e\u8d1f\u504f\u5dee，\u628a 0.3 \u4e2a\u6807\u51c6\u5dee\u7684\u6301\u7eed\u504f\u79fb\u5728 20-30 \u4e2a\u6837\u672c\u5185（\u7ea6 5-8 \u5206\u949f）\u68c0\u6d4b\u51fa\u6765\u3002\u6307\u6570\u52a0\u6743\u79fb\u52a8\u5e73\u5747（EWMA）\u4e0e\u6b64\u4e92\u8865：\u4ee5 0.3 \u7684\u8870\u51cf\u7cfb\u6570\u5e73\u6ed1\u5e8f\u5217，\u5bf9\u8fd1\u671f\u6837\u672c\u52a0\u6743\u66f4\u9ad8，\u9002\u5408\u6355\u6349\u8d8b\u52bf\u53d8\u5316\u3002\u56db\u7c7b\u65b9\u6cd5\u7684\u68c0\u6d4b\u5ef6\u8fdf\u4e0e\u8bef\u62a5\u7387\u5bf9\u6bd4\u5982\u4e0b\u3002<\/p>\n\n\u65b9\u6cd5\u9608\u503c\/\u53c2\u6570\u68c0\u6d4b\u5ef6\u8fdf\u8bef\u62a5\u7387\u9002\u7528\u573a\u666f<\/tr>\n\n\n\n\n\n
    Z \u5206\u6570<\/td>\n3 \u4e2a\u6807\u51c6\u5dee<\/td>\n1 \u4e2a\u7a97\u53e3<\/td>\n\u7ea6 0.3%<\/td>\n\u5e73\u7a33\u6307\u6807\u5747\u503c\u6f02\u79fb<\/td>\n<\/tr>\n
    IQR<\/td>\n1.5 \u500d\u56db\u5206\u4f4d\u8ddd<\/td>\n1 \u4e2a\u7a97\u53e3<\/td>\n\u5bf9\u91cd\u5c3e\u66f4\u7a33<\/td>\n\u5ef6\u8fdf\u3001\u65f6\u5ef6\u7c7b\u6307\u6807<\/td>\n<\/tr>\n
    EWMA<\/td>\n\u8870\u51cf\u7cfb\u6570 0.3<\/td>\n2-3 \u4e2a\u7a97\u53e3<\/td>\n\u7ea6 1%<\/td>\n\u8d8b\u52bf\u5e73\u6ed1\u4e0e\u8d8b\u52bf\u53d8\u5316<\/td>\n<\/tr>\n
    CUSUM<\/td>\n\u6f02\u79fb 0.5 \u500d\u6807\u51c6\u5dee<\/td>\n20-30 \u4e2a\u6837\u672c<\/td>\n\u7ea6 2%<\/td>\n\u5c0f\u5e45\u6301\u7eed\u504f\u79fb<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    \u7edf\u8ba1\u65b9\u6cd5\u7684\u8fb9\u754c\u5728\u975e\u5e73\u7a33\u5e8f\u5217\u9762\u524d\u66b4\u9732\u5f97\u6700\u5f7b\u5e95\u3002\u4e1a\u52a1\u6d41\u91cf\u6709\u660e\u663e\u7684\u65e5\u5185\u5468\u671f\u65f6，\u56fa\u5b9a\u7a97\u53e3\u7684\u5747\u503c\u672c\u8eab\u5c31\u5728\u53d8\u5316，Z \u5206\u6570\u4f1a\u8bef\u62a5\u3002\u89e3\u6cd5\u662f\u5468\u671f\u5316\u7a97\u53e3：\u6309"\u8fc7\u53bb 7 \u5929\u540c\u4e00\u5c0f\u65f6"\u7684\u6837\u672c\u6784\u5efa\u53c2\u8003\u5206\u5e03，\u518d\u8ba1\u7b97\u5f53\u524d\u503c\u7684\u504f\u79bb\u3002\u8fd9\u7c7b\u5468\u671f\u5bf9\u9f50\u628a 30 \u5206\u949f\u7a97\u53e3\u6269\u5c55\u5230 7 \u5929\u91c7\u6837，\u8ba1\u7b97\u91cf\u589e\u957f\u7ea6 336 \u500d，\u4f46\u8bef\u62a5\u7387\u53ef\u4ee5\u964d\u5230 0.1% \u4ee5\u4e0b\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ ewma_cusum.py<\/span>
    \nimport<\/span> random<\/p>\n

    class<\/span> CusumDetector<\/span>:<\/span>
    \n """\u8868\u683c\u6cd5 CUSUM：\u7d2f\u79ef\u6b63\u8d1f\u504f\u5dee，\u8d85\u8fc7\u9608\u503c\u5224\u4e3a\u6f02\u79fb，\u5168\u7a0b\u4f7f\u7528\u539f\u59cb\u91cf\u7eb2\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> mean,<\/span> std,<\/span> drift=<\/span>0.5<\/span>,<\/span> threshold=<\/span>5.0<\/span>)<\/span>:<\/span>
    \n self.<\/span>base =<\/span> mean
    \n self.<\/span>drift =<\/span> drift *<\/span> std # \u5141\u8bb8\u7684\u65e0\u6f02\u79fb\u533a\u95f4 k，\u53d6 0.5 \u500d\u6807\u51c6\u5dee<\/span>
    \n self.<\/span>h =<\/span> threshold *<\/span> std # \u5224\u5b9a\u9608\u503c h，\u53d6 5 \u500d\u6807\u51c6\u5dee<\/span>
    \n self.<\/span>cum_hi =<\/span> 0.0<\/span>
    \n self.<\/span>cum_lo =<\/span> 0.0<\/span><\/p>\n

    def<\/span> update<\/span>(<\/span>self,<\/span> x)<\/span>:<\/span>
    \n self.<\/span>cum_hi =<\/span> max<\/span>(<\/span>0.0<\/span>,<\/span> self.<\/span>cum_hi +<\/span> (<\/span>x –<\/span> self.<\/span>base –<\/span> self.<\/span>drift)<\/span>)<\/span>
    \n self.<\/span>cum_lo =<\/span> max<\/span>(<\/span>0.0<\/span>,<\/span> self.<\/span>cum_lo +<\/span> (<\/span>self.<\/span>base –<\/span> x –<\/span> self.<\/span>drift)<\/span>)<\/span>
    \n return<\/span> self.<\/span>cum_hi ><\/span> self.<\/span>h or<\/span> self.<\/span>cum_lo ><\/span> self.<\/span>h<\/p>\n

    class<\/span> Ewma<\/span>:<\/span>
    \n """\u6307\u6570\u52a0\u6743\u79fb\u52a8\u5e73\u5747，alpha \u8d8a\u5927\u5bf9\u8fd1\u671f\u6837\u672c\u8d8a\u654f\u611f\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> alpha=<\/span>0.3<\/span>)<\/span>:<\/span>
    \n self.<\/span>alpha =<\/span> alpha
    \n self.<\/span>value =<\/span> None<\/span><\/p>\n

    def<\/span> update<\/span>(<\/span>self,<\/span> x)<\/span>:<\/span>
    \n self.<\/span>value =<\/span> x if<\/span> self.<\/span>value is<\/span> None<\/span> else<\/span> self.<\/span>alpha *<\/span> x +<\/span> (<\/span>1<\/span> –<\/span> self.<\/span>alpha)<\/span> *<\/span> self.<\/span>value
    \n return<\/span> self.<\/span>value<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n detector =<\/span> CusumDetector(<\/span>mean=<\/span>0.5<\/span>,<\/span> std=<\/span>0.1<\/span>,<\/span> drift=<\/span>0.5<\/span>,<\/span> threshold=<\/span>5.0<\/span>)<\/span>
    \n hits =<\/span> 0<\/span>
    \n for<\/span> i in<\/span> range<\/span>(<\/span>200<\/span>)<\/span>:<\/span>
    \n # \u524d 150 \u70b9\u670d\u4ece\u539f\u5206\u5e03，\u4e4b\u540e\u53e0\u52a0 0.12 \u7684\u5747\u503c\u504f\u79fb<\/span>
    \n x =<\/span> random.<\/span>gauss(<\/span>0.5<\/span>,<\/span> 0.1<\/span>)<\/span> +<\/span> (<\/span>0.12<\/span> if<\/span> i >=<\/span> 150<\/span> else<\/span> 0<\/span>)<\/span>
    \n if<\/span> detector.<\/span>update(<\/span>x)<\/span>:<\/span>
    \n hits +=<\/span> 1<\/span>
    \n print<\/span>(<\/span>f"CUSUM \u5728 200 \u4e2a\u6837\u672c\u4e2d\u89e6\u53d1 <\/span>{<\/span>hits}<\/span><\/span> \u6b21，\u504f\u79fb\u91cf 0.12 \u7ea6\u7b49\u4e8e 1.2 \u4e2a\u6807\u51c6\u5dee"<\/span><\/span>)<\/span><\/p>\n

    3.2 \u673a\u5668\u5b66\u4e60\u65b9\u6cd5<\/h4>\n

    \u673a\u5668\u5b66\u4e60\u65b9\u6cd5\u5904\u7406\u7edf\u8ba1\u65b9\u6cd5\u65e0\u6cd5\u8986\u76d6\u7684\u591a\u6307\u6807\u8054\u5408\u5f02\u5e38\u3002\u5355\u4e2a\u6307\u6807\u6b63\u5e38\u3001\u7ec4\u5408\u5f02\u5e38\u7684\u573a\u666f\u5728 LLM \u670d\u52a1\u4e2d\u5f88\u5e38\u89c1：GPU \u5229\u7528\u7387 60%\u3001KV Cache \u5360\u7528 92%\u3001\u961f\u5217\u957f\u5ea6 40，\u5355\u72ec\u770b\u90fd\u4e0d\u8d8a\u754c，\u8054\u5408\u770b\u5374\u6307\u5411"\u8bf7\u6c42\u5806\u5728\u6392\u961f\u3001\u663e\u5b58\u88ab\u7f13\u5b58\u5360\u6ee1\u3001\u8ba1\u7b97\u7a7a\u95f2"\u7684\u8c03\u5ea6\u5931\u8861\u3002\u591a\u6307\u6807\u65b9\u6cd5\u628a 8 \u7ef4\u7279\u5f81\u5411\u91cf\u5f53\u4f5c\u6574\u4f53\u5224\u65ad，\u8fd9\u7c7b\u8054\u5408\u5f02\u5e38\u53ea\u6709\u591a\u7ef4\u5efa\u6a21\u624d\u80fd\u53d1\u73b0\u3002<\/p>\n

    \u5b64\u7acb\u68ee\u6797（Isolation Forest）\u662f\u751f\u4ea7\u73af\u5883\u6700\u5e38\u7528\u7684\u9009\u62e9，\u56e0\u4e3a\u5b83\u4e0d\u5047\u8bbe\u5206\u5e03\u5f62\u72b6\u3001\u8bad\u7ec3\u5f00\u9500\u5c0f\u3002\u4ee5 8 \u7ef4\u7279\u5f81（GPU \u5229\u7528\u7387\u3001\u663e\u5b58\u3001\u961f\u5217\u3001P99 \u5ef6\u8fdf\u3001\u9519\u8bef\u7387\u3001\u6279\u5927\u5c0f\u3001\u541e\u5410\u3001KV Cache）\u8bad\u7ec3 200 \u68f5\u6811，3000 \u6761\u6b63\u5e38\u6837\u672c\u8bad\u7ec3\u8017\u65f6\u7ea6 2 \u79d2，\u6253\u5206\u5355\u6837\u672c\u4e9a\u6beb\u79d2\u7ea7\u3002\u5f02\u5e38\u5206\u6570\u6765\u81ea\u8def\u5f84\u957f\u5ea6：\u6837\u672c\u88ab\u968f\u673a\u5207\u5206\u5f97\u8d8a\u65e9，\u8bf4\u660e\u5b83\u8d8a\u5bb9\u6613\u88ab"\u5b64\u7acb"，\u8d8a\u53ef\u80fd\u662f\u5f02\u5e38\u3002contamination \u53c2\u6570\u8bbe\u4e3a 0.01，\u5373\u9884\u8bbe 1% \u7684\u5f02\u5e38\u7387，\u5bf9\u5e94\u9694\u79bb\u6811\u8def\u5f84\u957f\u5ea6\u7684\u5224\u5b9a\u9608\u503c\u3002<\/p>\n

    \u81ea\u7f16\u7801\u5668\u8d70\u53e6\u4e00\u6761\u8def\u7ebf：\u7528\u6b63\u5e38\u6570\u636e\u8bad\u7ec3\u91cd\u6784\u7f51\u7edc，\u6b63\u5e38\u6837\u672c\u91cd\u6784\u8bef\u5dee\u5c0f\u3001\u5f02\u5e38\u6837\u672c\u91cd\u6784\u8bef\u5dee\u5927，\u4ee5\u91cd\u6784\u8bef\u5dee\u7684 P99 \u4f5c\u4e3a\u9608\u503c\u3002\u5b83\u7684\u4f18\u52bf\u662f\u80fd\u5b66\u5230\u6b63\u5e38\u6a21\u5f0f\u7684\u975e\u7ebf\u6027\u7ed3\u6784，\u52a3\u52bf\u662f\u9700\u8981\u8c03\u7f51\u7edc\u7ed3\u6784\u3001\u8bad\u7ec3\u6210\u672c\u9ad8（30 \u5929\u6570\u636e\u7ea6 10 \u5206\u949f\u8bad\u7ec3），\u4e14\u5bf9\u8bad\u7ec3\u6570\u636e\u7684\u4ee3\u8868\u6027\u654f\u611f\u3002\u65f6\u5e8f\u9884\u6d4b\u6b8b\u5dee\u6cd5\u5219\u7528 Prophet \u6216 ARIMA \u9884\u6d4b\u4e0b\u4e00\u65f6\u523b\u503c，\u5b9e\u9645\u503c\u4e0e\u9884\u6d4b\u503c\u7684\u6b8b\u5dee\u8d85\u8fc7 3 \u500d\u5386\u53f2\u6b8b\u5dee\u6807\u51c6\u5dee\u5373\u544a\u8b66，\u64c5\u957f\u6355\u6349\u8d8b\u52bf\u7a81\u53d8\u66f4\u80fd\u5bb9\u5fcd\u5b63\u8282\u6ce2\u52a8\u3002<\/p>\n

    \u4e09\u7c7b\u65b9\u6cd5\u7684\u6210\u672c\u4e0e\u9002\u7528\u8fb9\u754c\u9700\u8981\u91cf\u5316\u6743\u8861，\u5982\u4e0b\u8868\u3002\u751f\u4ea7\u5b9e\u8df5\u4e2d\u4ee5\u9694\u79bb\u68ee\u6797\u4e3a\u4e3b\u68c0\u6d4b\u5668\u3001\u4ee5\u65f6\u5e8f\u6b8b\u5dee\u6cd5\u4e3a\u6162\u68c0\u6d4b\u5668，\u81ea\u7f16\u7801\u5668\u4ec5\u5728\u8054\u5408\u5f02\u5e38\u9891\u53d1\u7684\u573a\u666f\u542f\u7528\u3002<\/p>\n\n\u65b9\u6cd5\u8bad\u7ec3\u6210\u672c\u5355\u6837\u672c\u63a8\u7406\u4f18\u70b9\u5c40\u9650<\/tr>\n\n\n\n\n
    \u5b64\u7acb\u68ee\u6797<\/td>\n3000 \u6837\u672c\u7ea6 2s<\/td>\n\u4e9a\u6beb\u79d2<\/td>\n\u65e0\u9700\u5206\u5e03\u5047\u8bbe\u3001\u5feb<\/td>\n\u5bf9\u70b9\u5f02\u5e38\u654f\u611f，\u5bf9\u5e8f\u5217\u4e0a\u4e0b\u6587\u5f31<\/td>\n<\/tr>\n
    \u81ea\u7f16\u7801\u5668<\/td>\n30 \u5929\u6570\u636e\u7ea6 10 \u5206\u949f<\/td>\n\u6beb\u79d2\u7ea7<\/td>\n\u6355\u6349\u975e\u7ebf\u6027\u6b63\u5e38\u6a21\u5f0f<\/td>\n\u8c03\u53c2\u590d\u6742\u3001\u4f9d\u8d56\u6570\u636e\u4ee3\u8868\u6027<\/td>\n<\/tr>\n
    \u65f6\u5e8f\u6b8b\u5dee\u6cd5<\/td>\n\u9700 30 \u5929\u4ee5\u4e0a\u5386\u53f2<\/td>\n\u6beb\u79d2\u7ea7<\/td>\n\u517c\u987e\u5b63\u8282\u4e0e\u8d8b\u52bf<\/td>\n\u5bf9\u7a81\u53d8\u54cd\u5e94\u6162\u4e00\u4e2a\u9884\u6d4b\u5468\u671f<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    \u673a\u5668\u5b66\u4e60\u65b9\u6cd5\u7684\u8fb9\u754c\u662f\u6982\u5ff5\u6f02\u79fb：\u8bad\u7ec3\u6570\u636e\u6765\u81ea\u67d0\u4e2a\u8d1f\u8f7d\u5f62\u6001，\u8d1f\u8f7d\u7ed3\u6784\u53d8\u5316\u540e\u6a21\u578b\u8f93\u51fa\u4f1a\u5931\u771f\u3002\u9694\u79bb\u68ee\u6797\u8bad\u7ec3 30 \u5929\u3001\u4e4b\u540e\u6bcf\u6708\u91cd\u8bad\u662f\u5e38\u89c1\u8282\u594f；\u4e0a\u7ebf\u521d\u671f\u6ca1\u6709\u5386\u53f2\u6570\u636e\u65f6，\u5148\u7528\u7edf\u8ba1\u65b9\u6cd5\u9876\u4f4f，\u7d2f\u79ef\u4e24\u5468\u6570\u636e\u540e\u518d\u5207\u6362\u673a\u5668\u5b66\u4e60\u65b9\u6cd5，\u907f\u514d\u51b7\u542f\u52a8\u671f\u7684\u9ad8\u8bef\u62a5\u3002<\/p>\n

    # \u6765\u6e90：scikit-learn 1.3.2 \/ sklearn.ensemble.IsolationForest<\/span>
    \nimport<\/span> numpy as<\/span> np
    \nfrom<\/span> sklearn.<\/span>ensemble import<\/span> IsolationForest<\/p>\n

    FEATURES =<\/span> [<\/span>
    \n "gpu_util"<\/span>,<\/span> "vram_gb"<\/span>,<\/span> "queue_len"<\/span>,<\/span> "p99_latency_ms"<\/span>,<\/span>
    \n "error_rate"<\/span>,<\/span> "batch_size"<\/span>,<\/span> "tokens_per_s"<\/span>,<\/span> "kv_cache_gb"<\/span>,<\/span>
    \n]<\/span><\/p>\n

    def<\/span> train<\/span>(<\/span>history)<\/span>:<\/span>
    \n """\u7528\u6b63\u5e38\u5386\u53f2\u6570\u636e\u8bad\u7ec3\u9694\u79bb\u68ee\u6797，\u9884\u8bbe 1% \u5f02\u5e38\u7387\u3002"""<\/span>
    \n model =<\/span> IsolationForest(<\/span>n_estimators=<\/span>200<\/span>,<\/span> contamination=<\/span>0.01<\/span>,<\/span> random_state=<\/span>42<\/span>)<\/span>
    \n model.<\/span>fit(<\/span>history)<\/span>
    \n return<\/span> model<\/p>\n

    def<\/span> score<\/span>(<\/span>model,<\/span> sample)<\/span>:<\/span>
    \n """\u8fd4\u56de\u5f02\u5e38\u5206\u6570\u4e0e\u5224\u5b9a\u7ed3\u679c，\u5f97\u5206\u4f4e\u4e8e -0.5 \u5224\u4e3a\u5f02\u5e38\u3002"""<\/span>
    \n raw =<\/span> model.<\/span>score_samples(<\/span>[<\/span>sample]<\/span>)<\/span>[<\/span>0<\/span>]<\/span>
    \n return<\/span> raw,<\/span> raw <<\/span> –<\/span>0.5<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n rng =<\/span> np.<\/span>random.<\/span>default_rng(<\/span>7<\/span>)<\/span>
    \n # \u6784\u9020 3000 \u6761\u6b63\u5e38\u6837\u672c：8 \u4e2a\u7279\u5f81\u5404\u81ea\u56f4\u7ed5\u5178\u578b\u503c\u5206\u5e03<\/span>
    \n normal =<\/span> rng.<\/span>normal(<\/span>
    \n [<\/span>80<\/span>,<\/span> 20<\/span>,<\/span> 5<\/span>,<\/span> 400<\/span>,<\/span> 0.5<\/span>,<\/span> 32<\/span>,<\/span> 800<\/span>,<\/span> 12<\/span>]<\/span>,<\/span>
    \n [<\/span>5<\/span>,<\/span> 2<\/span>,<\/span> 1<\/span>,<\/span> 40<\/span>,<\/span> 0.1<\/span>,<\/span> 8<\/span>,<\/span> 60<\/span>,<\/span> 1<\/span>]<\/span>,<\/span>
    \n size=<\/span>(<\/span>3000<\/span>,<\/span> 8<\/span>)<\/span>,<\/span>
    \n )<\/span>
    \n model =<\/span> train(<\/span>normal)<\/span>
    \n # \u6784\u9020\u8054\u5408\u5f02\u5e38：\u663e\u5b58\u4e0e\u961f\u5217\u540c\u65f6\u98d9\u9ad8\u3001\u541e\u5410\u9aa4\u964d<\/span>
    \n joint_anomaly =<\/span> [<\/span>95<\/span>,<\/span> 40<\/span>,<\/span> 60<\/span>,<\/span> 1500<\/span>,<\/span> 4.0<\/span>,<\/span> 4<\/span>,<\/span> 120<\/span>,<\/span> 30<\/span>]<\/span>
    \n raw,<\/span> is_anomaly =<\/span> score(<\/span>model,<\/span> joint_anomaly)<\/span>
    \n print<\/span>(<\/span>f"\u5f02\u5e38\u5206\u6570 <\/span>{<\/span>raw:<\/span>.3f<\/span>}<\/span><\/span>，\u5224\u5b9a <\/span>{<\/span>'\u5f02\u5e38'<\/span> if<\/span> is_anomaly else<\/span> '\u6b63\u5e38'<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    3.3 \u544a\u8b66\u9608\u503c<\/h4>\n

    \u544a\u8b66\u9608\u503c\u662f\u5f02\u5e38\u68c0\u6d4b\u4e0e\u544a\u8b66\u4f53\u7cfb\u7684\u63a5\u53e3，\u8bbe\u5b9a\u8d28\u91cf\u76f4\u63a5\u51b3\u5b9a\u544a\u8b66\u75b2\u52b3\u4e0e\u6f0f\u62a5\u7684\u5e73\u8861\u3002\u9608\u503c\u6765\u6e90\u5206\u4e24\u7c7b：\u7edf\u8ba1\u9608\u503c\u6765\u81ea\u5206\u5e03\u8ba1\u7b97，\u5982 3 \u4e2a\u6807\u51c6\u5dee\u30011.5 \u500d IQR，\u4f18\u70b9\u662f\u81ea\u52a8\u9002\u914d\u6570\u636e；\u4e1a\u52a1\u9608\u503c\u6765\u81ea SLO \u4e0e\u5bb9\u91cf\u89c4\u5212，\u5982 P99 \u5ef6\u8fdf 500ms\u3001\u9519\u8bef\u7387 1%，\u4f18\u70b9\u662f\u53ef\u76f4\u63a5\u5bf9\u8d26\u5230\u4e1a\u52a1\u76ee\u6807\u3002\u751f\u4ea7\u5b9e\u8df5\u4e2d\u4e24\u8005\u7ed3\u5408：\u7edf\u8ba1\u9608\u503c\u505a\u9884\u7b5b，\u4e1a\u52a1\u9608\u503c\u505a\u6700\u7ec8\u88c1\u5b9a\u3002<\/p>\n

    \u9608\u503c\u5fc5\u987b\u914d\u5957\u6301\u7eed\u6761\u4ef6，\u5426\u5219\u5355\u70b9\u6bdb\u523a\u5c31\u4f1a\u5237\u5c4f\u3002\u5e38\u89c1\u7684\u6301\u7eed\u6761\u4ef6\u662f"N \u5206\u949f\u5185 M \u5206\u949f\u8d8a\u754c"：P99 \u5ef6\u8fdf\u5927\u4e8e 500ms \u4e14\u6301\u7eed 5 \u5206\u949f\u624d\u544a\u8b66，\u4e00\u6b21\u6301\u7eed 20 \u79d2\u7684\u6bdb\u523a（\u901a\u5e38\u7531\u6279\u5904\u7406\u6ce2\u52a8\u5f15\u8d77）\u88ab\u6291\u5236\u3002\u7a97\u53e3\u8d8a\u957f\u8bef\u62a5\u8d8a\u4f4e\u4f46\u53d1\u73b0\u8d8a\u6162：5 \u5206\u949f\u7a97\u53e3\u7684\u6f0f\u62a5\u98ce\u9669\u7ea6 3%，1 \u5206\u949f\u7a97\u53e3\u7684\u8bef\u62a5\u7387\u4f1a\u9ad8\u5230 30%\u3002\u5bf9\u8d44\u6e90\u7c7b\u6307\u6807\u8fd8\u53ef\u4ee5\u52a0\u8fdf\u6ede（hysteresis）：\u8fdb\u5165\u544a\u8b66\u9700\u8d85\u8fc7 95%，\u6062\u590d\u9700\u56de\u843d\u5230 90% \u4ee5\u4e0b，\u907f\u514d\u5728\u9608\u503c\u8fb9\u7f18\u53cd\u590d\u6a2a\u8df3\u3002<\/p>\n

    \u5177\u4f53\u5230 LLM \u670d\u52a1\u7684\u5178\u578b\u89c4\u5219\u96c6\u5982\u4e0b\u8868\u3002\u6ce8\u610f\u6bcf\u6761\u89c4\u5219\u90fd\u6807\u6ce8\u4e86\u4e25\u91cd\u7ea7\u522b\u4e0e\u9884\u671f\u7684\u8bef\u62a5\u7387，\u89c4\u5219\u8bc4\u5ba1\u65f6\u4ee5\u8bef\u62a5\u7387\u4e3a\u95e8\u69db，\u8d85\u8fc7 5% \u7684\u89c4\u5219\u5fc5\u987b\u91cd\u8c03\u3002<\/p>\n\n\u89c4\u5219\u89e6\u53d1\u6761\u4ef6\u4e25\u91cd\u7ea7\u522b\u9884\u671f\u8bef\u62a5\u7387<\/tr>\n\n\n\n\n\n\n
    \u5ef6\u8fdf\u52a3\u5316<\/td>\nP99 \u5927\u4e8e 500ms \u6301\u7eed 5 \u5206\u949f<\/td>\nP1<\/td>\n2%<\/td>\n<\/tr>\n
    \u9519\u8bef\u7387\u4e0a\u5347<\/td>\n\u9519\u8bef\u7387\u5927\u4e8e 1% \u6301\u7eed 10 \u5206\u949f<\/td>\nP1<\/td>\n1%<\/td>\n<\/tr>\n
    \u663e\u5b58\u6c34\u4f4d<\/td>\n\u663e\u5b58\u5360\u7528\u5927\u4e8e 95% \u6301\u7eed 3 \u5206\u949f<\/td>\nP2<\/td>\n3%<\/td>\n<\/tr>\n
    \u541e\u5410\u9aa4\u964d<\/td>\n\u541e\u5410\u4f4e\u4e8e 600 tokens\/s \u6301\u7eed 10 \u5206\u949f<\/td>\nP2<\/td>\n4%<\/td>\n<\/tr>\n
    \u8d28\u91cf\u56de\u5f52<\/td>\nPSI \u5927\u4e8e 0.2 \u6216\u51c6\u786e\u7387\u4e0b\u964d 8%<\/td>\nP1<\/td>\n5%<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    \u9608\u503c\u8bbe\u5b9a\u7684\u8fb9\u754c\u6848\u4f8b\u96c6\u4e2d\u5728\u6d41\u91cf\u6ce2\u52a8\u671f\u3002\u53d1\u5e03\u65b0\u7248\u672c\u540e\u7684 1 \u5c0f\u65f6\u3001\u4fc3\u9500\u6d3b\u52a8\u7684\u9ad8\u5cf0\u671f，\u6307\u6807\u6574\u4f53\u4e0a\u79fb，\u56fa\u5b9a\u9608\u503c\u4f1a\u7cfb\u7edf\u6027\u8bef\u62a5\u3002\u89e3\u6cd5\u662f\u9608\u503c\u5206\u7ea7：\u628a\u4e00\u5929\u5206\u6210\u9ad8\u5cf0（10-22 \u70b9）\u4e0e\u4f4e\u8c37\u4e24\u6863，\u5404\u81ea\u7ef4\u62a4\u4e00\u5957\u7edf\u8ba1\u9608\u503c；\u6216\u8005\u5728\u53d1\u5e03\u6d41\u7a0b\u4e2d\u81ea\u52a8\u6302\u9759\u9ed8，\u53d1\u5e03\u7a97\u53e3\u5185\u53ea\u8bb0\u5f55\u4e0d\u544a\u8b66\u3002\u7edf\u8ba1\u9608\u503c\u672c\u8eab\u4e5f\u6709\u81ea\u6821\u51c6\u5468\u671f：\u6bcf\u5468\u91cd\u7b97\u4e00\u6b21\u53c2\u8003\u5206\u5e03，\u907f\u514d\u9608\u503c\u968f\u6570\u636e\u6f02\u79fb\u5931\u6548\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ rule_engine.py<\/span>
    \nfrom<\/span> collections import<\/span> deque<\/p>\n

    class<\/span> SlidingRule<\/span>:<\/span>
    \n """\u6ed1\u52a8\u7a97\u53e3\u89c4\u5219：\u7a97\u53e3\u5185\u8d8a\u754c\u6bd4\u4f8b\u8fbe\u5230\u9608\u503c\u624d\u89e6\u53d1，\u6291\u5236\u5355\u70b9\u6bdb\u523a\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> rule_id,<\/span> kind,<\/span> threshold,<\/span> window=<\/span>5<\/span>,<\/span> hysteresis=<\/span>0.1<\/span>,<\/span> min_ratio=<\/span>0.6<\/span>)<\/span>:<\/span>
    \n self.<\/span>rule_id =<\/span> rule_id
    \n self.<\/span>kind =<\/span> kind
    \n self.<\/span>threshold =<\/span> threshold
    \n self.<\/span>hysteresis =<\/span> hysteresis
    \n self.<\/span>min_ratio =<\/span> min_ratio
    \n self.<\/span>history =<\/span> deque(<\/span>maxlen=<\/span>window)<\/span><\/p>\n

    def<\/span> feed<\/span>(<\/span>self,<\/span> value)<\/span>:<\/span>
    \n self.<\/span>history.<\/span>append(<\/span>value)<\/span>
    \n if<\/span> len<\/span>(<\/span>self.<\/span>history)<\/span> <<\/span> self.<\/span>history.<\/span>maxlen:<\/span>
    \n return<\/span> False<\/span>,<\/span> 0.0<\/span>
    \n over =<\/span> sum<\/span>(<\/span>1<\/span> for<\/span> v in<\/span> self.<\/span>history if<\/span> self.<\/span>_breach(<\/span>v)<\/span>)<\/span>
    \n ratio =<\/span> over \/<\/span> self.<\/span>history.<\/span>maxlen
    \n return<\/span> ratio >=<\/span> self.<\/span>min_ratio,<\/span> ratio<\/p>\n

    def<\/span> _breach<\/span>(<\/span>self,<\/span> v)<\/span>:<\/span>
    \n # \u8fdf\u6ede：\u8d8a\u754c\u65f6\u9700\u8d85\u8fc7 threshold + hysteresis<\/span>
    \n if<\/span> self.<\/span>kind ==<\/span> "high"<\/span>:<\/span>
    \n return<\/span> v ><\/span> self.<\/span>threshold +<\/span> self.<\/span>hysteresis
    \n return<\/span> v <<\/span> self.<\/span>threshold –<\/span> self.<\/span>hysteresis<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n rule =<\/span> SlidingRule(<\/span>"p99_latency"<\/span>,<\/span> "high"<\/span>,<\/span> threshold=<\/span>500<\/span>,<\/span> window=<\/span>5<\/span>,<\/span> hysteresis=<\/span>20<\/span>,<\/span> min_ratio=<\/span>0.6<\/span>)<\/span>
    \n samples =<\/span> [<\/span>400<\/span>,<\/span> 420<\/span>,<\/span> 560<\/span>,<\/span> 620<\/span>,<\/span> 590<\/span>,<\/span> 610<\/span>,<\/span> 540<\/span>,<\/span> 480<\/span>]<\/span>
    \n for<\/span> i,<\/span> v in<\/span> enumerate<\/span>(<\/span>samples)<\/span>:<\/span>
    \n fired,<\/span> ratio =<\/span> rule.<\/span>feed(<\/span>v)<\/span>
    \n mark =<\/span> "\u89e6\u53d1\u544a\u8b66"<\/span> if<\/span> fired else<\/span> ""<\/span>
    \n print<\/span>(<\/span>f"\u7b2c <\/span>{<\/span>i +<\/span> 1<\/span>}<\/span><\/span> \u4e2a\u6837\u672c: \u5ef6\u8fdf <\/span>{<\/span>v}<\/span><\/span>ms，\u8d8a\u754c\u6bd4\u4f8b <\/span>{<\/span>ratio:<\/span>.0%<\/span>}<\/span><\/span> <\/span>{<\/span>mark}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    4. \u65e5\u5fd7\u8bb0\u5f55<\/h3>\n

    \u65e5\u5fd7\u8bb0\u5f55\u8865\u5168\u6307\u6807\u4e0e\u8c03\u7528\u7684\u76f2\u533a：\u6307\u6807\u56de\u7b54"\u53d1\u751f\u4e86\u4ec0\u4e48"，\u65e5\u5fd7\u56de\u7b54"\u5177\u4f53\u662f\u54ea\u4e2a\u8bf7\u6c42\u3001\u54ea\u4e2a\u6a21\u578b\u7248\u672c\u3001\u54ea\u5757 GPU"\u3002\u4e00\u6b21 P1 \u5ef6\u8fdf\u544a\u8b66\u89e6\u53d1\u540e，\u5b9a\u4f4d\u6839\u56e0\u5fc5\u987b\u56de\u5230\u65e5\u5fd7：\u662f\u67d0\u4e2a\u7279\u5b9a prompt \u89e6\u53d1\u4e86\u8d85\u957f\u9884\u586b\u5145，\u8fd8\u662f\u67d0\u4e2a GPU \u7684 KV Cache \u788e\u7247\u5316，\u53ea\u6709\u9010\u8bf7\u6c42\u7684\u7ed3\u6784\u5316\u65e5\u5fd7\u80fd\u7ed9\u51fa\u7b54\u6848\u3002\u65e5\u5fd7\u56e0\u6b64\u627f\u62c5\u7740\u53ef\u8ffd\u6eaf\u6027\u4e0e\u53d6\u8bc1\u6027，\u5176\u4ef7\u503c\u5728\u6545\u969c\u590d\u76d8\u65f6\u624d\u4f1a\u5b8c\u5168\u663e\u73b0\u3002<\/p>\n

    \u65e5\u5fd7\u4f53\u7cfb\u7684\u4e09\u5c42\u8bbe\u8ba1\u5bf9\u5e94\u91c7\u96c6\u3001\u5b58\u50a8\u4e0e\u6d88\u8d39\u3002\u91c7\u96c6\u5c42\u628a\u6563\u843d\u5728\u5404\u8fdb\u7a0b\u7684\u6807\u51c6\u8f93\u51fa\u6c47\u603b\u4e3a\u7edf\u4e00\u4e8b\u4ef6\u6d41；\u5b58\u50a8\u5c42\u4ee5\u7d22\u5f15\u5316\u65b9\u5f0f\u4fdd\u5b58，\u652f\u6491\u79d2\u7ea7\u68c0\u7d22；\u6d88\u8d39\u5c42\u5b8c\u6210\u4e24\u7c7b\u5206\u6790\u2014\u2014\u6309\u8bf7\u6c42 ID \u7684\u5168\u94fe\u8def\u8fd8\u539f，\u4e0e\u6309\u7ef4\u5ea6\u7684\u805a\u5408\u7edf\u8ba1\u3002\u4e09\u5c42\u4e4b\u95f4\u7528\u7f13\u51b2\u89e3\u8026：\u91c7\u96c6\u5c42\u7a81\u53d1\u5199\u5165\u4e0d\u76f4\u63a5\u538b\u57ae\u5b58\u50a8\u5c42，\u5b58\u50a8\u5c42\u6545\u969c\u65f6\u65e5\u5fd7\u5728\u7f13\u51b2\u4e2d\u7b49\u5f85\u91cd\u8bd5，\u800c\u4e0d\u662f\u76f4\u63a5\u4e22\u5f03\u3002<\/p>\n

    #mermaid-svg-o8CPnjrP3HNAjhzY{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-o8CPnjrP3HNAjhzY .error-icon{fill:#552222;}#mermaid-svg-o8CPnjrP3HNAjhzY .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-o8CPnjrP3HNAjhzY .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-o8CPnjrP3HNAjhzY .marker{fill:#333333;stroke:#333333;}#mermaid-svg-o8CPnjrP3HNAjhzY .marker.cross{stroke:#333333;}#mermaid-svg-o8CPnjrP3HNAjhzY svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-o8CPnjrP3HNAjhzY p{margin:0;}#mermaid-svg-o8CPnjrP3HNAjhzY .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster-label text{fill:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster-label span{color:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster-label span p{background-color:transparent;}#mermaid-svg-o8CPnjrP3HNAjhzY .label text,#mermaid-svg-o8CPnjrP3HNAjhzY span{fill:#333;color:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY .node rect,#mermaid-svg-o8CPnjrP3HNAjhzY .node circle,#mermaid-svg-o8CPnjrP3HNAjhzY .node ellipse,#mermaid-svg-o8CPnjrP3HNAjhzY .node polygon,#mermaid-svg-o8CPnjrP3HNAjhzY .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-o8CPnjrP3HNAjhzY .rough-node .label text,#mermaid-svg-o8CPnjrP3HNAjhzY .node .label text,#mermaid-svg-o8CPnjrP3HNAjhzY .image-shape .label,#mermaid-svg-o8CPnjrP3HNAjhzY .icon-shape .label{text-anchor:middle;}#mermaid-svg-o8CPnjrP3HNAjhzY .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-o8CPnjrP3HNAjhzY .rough-node .label,#mermaid-svg-o8CPnjrP3HNAjhzY .node .label,#mermaid-svg-o8CPnjrP3HNAjhzY .image-shape .label,#mermaid-svg-o8CPnjrP3HNAjhzY .icon-shape .label{text-align:center;}#mermaid-svg-o8CPnjrP3HNAjhzY .node.clickable{cursor:pointer;}#mermaid-svg-o8CPnjrP3HNAjhzY .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-o8CPnjrP3HNAjhzY .arrowheadPath{fill:#333333;}#mermaid-svg-o8CPnjrP3HNAjhzY .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-o8CPnjrP3HNAjhzY .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-o8CPnjrP3HNAjhzY .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-o8CPnjrP3HNAjhzY .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-o8CPnjrP3HNAjhzY .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-o8CPnjrP3HNAjhzY .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster text{fill:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY .cluster span{color:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-o8CPnjrP3HNAjhzY .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-o8CPnjrP3HNAjhzY rect.text{fill:none;stroke-width:0;}#mermaid-svg-o8CPnjrP3HNAjhzY .icon-shape,#mermaid-svg-o8CPnjrP3HNAjhzY .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-o8CPnjrP3HNAjhzY .icon-shape p,#mermaid-svg-o8CPnjrP3HNAjhzY .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-o8CPnjrP3HNAjhzY .icon-shape .label rect,#mermaid-svg-o8CPnjrP3HNAjhzY .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-o8CPnjrP3HNAjhzY .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-o8CPnjrP3HNAjhzY .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-o8CPnjrP3HNAjhzY :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-o8CPnjrP3HNAjhzY .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-o8CPnjrP3HNAjhzY .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-o8CPnjrP3HNAjhzY .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u91c7\u96c6\u7ba1\u9053<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u65e5\u5fd7\u4ea7\u751f<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6309 request_id \u68c0\u7d22<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6309\u6a21\u578b\u7248\u672c\u805a\u5408<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7ed3\u8bba\u56de\u5199<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6d88\u8d39\u79ef\u538b\u8d85\u8fc7\u9608\u503c<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u63a8\u7406\u670d\u52a1<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7ed3\u6784\u5316 JSON \u65e5\u5fd7<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    PII \u8131\u654f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Filebeat<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Logstash \u89e3\u6790<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Kafka \u7f13\u51b2<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Elasticsearch \u7d22\u5f15<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u662f\u5426\u9700\u8981\u6392\u67e5<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5355\u8bf7\u6c42\u5168\u94fe\u8def\u8fd8\u539f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u9519\u8bef\u7387\u4e0e\u5ef6\u8fdf\u805a\u5408<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6839\u56e0\u5206\u6790<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u4e22\u5f03\u4f4e\u4f18\u5148\u7ea7\u65e5\u5fd7<\/p>\n

    <\/span><\/p>\n

    4.1 \u65e5\u5fd7\u89c4\u8303<\/h4>\n

    \u65e5\u5fd7\u89c4\u8303\u7684\u6838\u5fc3\u662f\u7ed3\u6784\u7edf\u4e00\u4e0e\u5b57\u6bb5\u9f50\u5168\u3002\u6bcf\u6761\u8bf7\u6c42\u65e5\u5fd7\u5fc5\u987b\u662f\u5355\u884c JSON，\u5b57\u6bb5\u5206\u4e3a\u6807\u8bc6\u57df\u3001\u4e0a\u4e0b\u6587\u57df\u4e0e\u5ea6\u91cf\u57df\u3002\u6807\u8bc6\u57df\u5305\u542b request_id\u3001trace_id\u3001model_version，\u662f\u8de8\u7cfb\u7edf\u5173\u8054\u7684\u4e3b\u952e\u3002\u4e0a\u4e0b\u6587\u57df\u5305\u542b prompt_hash\u3001num_input_tokens\u3001num_output_tokens\u3001gpu_id\u3001region，\u662f\u805a\u5408\u5206\u6790\u7684\u7ef4\u5ea6\u3002\u5ea6\u91cf\u57df\u5305\u542b latency_ms\u3001ttft_ms\u3001tpot_ms\u3001cache_hit\u3001error_code，\u662f\u6027\u80fd\u8fd8\u539f\u7684\u6570\u636e\u6e90\u3002\u5b57\u6bb5\u5728\u57cb\u70b9\u89c4\u8303\u6587\u6863\u4e2d\u7edf\u4e00\u5b9a\u4e49，\u65b0\u589e\u5b57\u6bb5\u5fc5\u987b\u8d70\u8bc4\u5ba1\u3002<\/p>\n

    \u65e5\u5fd7\u7b49\u7ea7\u5212\u5206\u5fc5\u987b\u4e0e\u7ecf\u6d4e\u6027\u6302\u94a9\u3002INFO \u8bb0\u5f55\u6bcf\u6b21\u8bf7\u6c42\u7684\u5b8c\u6210\u4fe1\u606f，\u7528\u4e8e\u6027\u80fd\u4e0e\u4e1a\u52a1\u5206\u6790；WARN \u8bb0\u5f55\u53ef\u6062\u590d\u7684\u964d\u7ea7（\u91cd\u8bd5\u6210\u529f\u3001\u6392\u961f\u8d85\u65f6），\u6570\u91cf\u7ea7\u7ea6\u4e3a INFO \u7684 5%；ERROR \u8bb0\u5f55\u5931\u8d25\u8bf7\u6c42\u4e0e\u5185\u90e8\u5f02\u5e38，\u5fc5\u987b\u5e26 error_code \u4e0e\u5806\u6808\u6458\u8981\u3002DEBUG \u9ed8\u8ba4\u5173\u95ed，\u4ec5\u5728\u6309 request_id \u5b9a\u4f4d\u95ee\u9898\u65f6\u4e34\u65f6\u5f00\u542f，\u907f\u514d\u5199\u5165\u91cf\u7206\u70b8\u3002\u65e5\u5fd7\u91cf\u4e0e\u6210\u672c\u7684\u5173\u7cfb\u662f\u786c\u7ea6\u675f：\u5355\u6761\u8bf7\u6c42\u65e5\u5fd7\u7ea6 600 \u5b57\u8282，100 QPS \u65f6\u6bcf\u5929\u4ea7\u751f\u7ea6 5.2GB，\u4fdd\u7559 90 \u5929\u7ea6 470GB，\u8fd9\u662f\u9009\u578b Elasticsearch \u96c6\u7fa4\u89c4\u6a21\u7684\u76f4\u63a5\u8f93\u5165\u3002<\/p>\n

    \u654f\u611f\u6570\u636e\u6cbb\u7406\u662f\u65e5\u5fd7\u89c4\u8303\u4e0d\u53ef\u8df3\u8fc7\u7684\u4e00\u73af\u3002prompt \u4e0e\u54cd\u5e94\u53ef\u80fd\u5305\u542b\u7528\u6237\u9690\u79c1，\u8131\u654f\u5728\u65e5\u5fd7\u4ea7\u751f\u8fdb\u7a0b\u5185\u5b8c\u6210：\u90ae\u7bb1\u3001\u624b\u673a\u53f7\u7528\u6b63\u5219\u63a9\u7801，token \u5e8f\u5217\u53ea\u4fdd\u7559\u524d 64 \u4e0e\u540e 64 \u4e2a，\u7528\u6237 ID \u4ee5\u76d0\u503c\u54c8\u5e0c\u5b58\u50a8\u3002\u8131\u654f\u5fc5\u987b\u53d1\u751f\u5728\u5e8f\u5217\u5316\u4e4b\u524d，\u800c\u4e0d\u662f\u5b58\u50a8\u4e4b\u540e，\u5426\u5219\u660e\u6587\u4f1a\u5148\u843d\u5230\u78c1\u76d8\u518d\u8fdb\u5165\u5907\u4efd，\u8865\u6551\u6210\u672c\u6210\u500d\u4e0a\u5347\u3002\u7070\u5ea6\u671f\u7684\u7ecf\u9a8c\u6570\u636e\u662f：\u672a\u8131\u654f\u65e5\u5fd7\u6cc4\u9732\u4e00\u6b21，\u5408\u89c4\u6574\u6539\u7684\u4eba\u529b\u6210\u672c\u7ea6 2 \u4eba\u5468\u3002<\/p>\n

    # \u6765\u6e90：Python 3.11 logging + \u81ea\u5b9e\u73b0 JSONFormatter<\/span>
    \nimport<\/span> hashlib
    \nimport<\/span> json
    \nimport<\/span> logging
    \nimport<\/span> re
    \nimport<\/span> sys
    \nimport<\/span> time<\/p>\n

    _PII =<\/span> re.<\/span>compile<\/span>(<\/span>r"[\\\\w.+-]+@[\\\\w-]+\\\\.[\\\\w.]+|\\\\b1[3-9]\\\\d{9}\\\\b"<\/span>)<\/span><\/p>\n

    def<\/span> mask<\/span>(<\/span>text)<\/span>:<\/span>
    \n """\u628a\u90ae\u7bb1\u4e0e\u624b\u673a\u53f7\u7edf\u4e00\u63a9\u7801\u4e3a\u7b49\u957f\u661f\u53f7\u3002"""<\/span>
    \n return<\/span> _PII.<\/span>sub(<\/span>lambda<\/span> m:<\/span> "*"<\/span> *<\/span> len<\/span>(<\/span>m.<\/span>group(<\/span>)<\/span>)<\/span>,<\/span> text or<\/span> ""<\/span>)<\/span><\/p>\n

    class<\/span> JsonFormatter<\/span>(<\/span>logging.<\/span>Formatter)<\/span>:<\/span>
    \n """\u628a logging \u8bb0\u5f55\u5e8f\u5217\u5316\u4e3a\u5355\u884c JSON，\u5e76\u9644\u52a0\u7edf\u4e00\u65f6\u95f4\u6233\u3002"""<\/span><\/p>\n

    def<\/span> format<\/span>(<\/span>self,<\/span> record)<\/span>:<\/span>
    \n payload =<\/span> {<\/span>
    \n "ts"<\/span>:<\/span> round<\/span>(<\/span>time.<\/span>time(<\/span>)<\/span> *<\/span> 1000<\/span>)<\/span>,<\/span>
    \n "level"<\/span>:<\/span> record.<\/span>levelname,<\/span>
    \n "logger"<\/span>:<\/span> record.<\/span>name,<\/span>
    \n "msg"<\/span>:<\/span> mask(<\/span>record.<\/span>getMessage(<\/span>)<\/span>)<\/span>,<\/span>
    \n }<\/span>
    \n for<\/span> key in<\/span> (<\/span>"request_id"<\/span>,<\/span> "model_version"<\/span>,<\/span> "prompt_hash"<\/span>,<\/span> "latency_ms"<\/span>,<\/span> "tokens"<\/span>)<\/span>:<\/span>
    \n if<\/span> hasattr<\/span>(<\/span>record,<\/span> key)<\/span>:<\/span>
    \n payload[<\/span>key]<\/span> =<\/span> getattr<\/span>(<\/span>record,<\/span> key)<\/span>
    \n return<\/span> json.<\/span>dumps(<\/span>payload,<\/span> ensure_ascii=<\/span>False<\/span>)<\/span><\/p>\n

    logger =<\/span> logging.<\/span>getLogger(<\/span>"llm.access"<\/span>)<\/span>
    \nlogger.<\/span>setLevel(<\/span>logging.<\/span>INFO)<\/span>
    \nhandler =<\/span> logging.<\/span>StreamHandler(<\/span>sys.<\/span>stdout)<\/span> # \u8f93\u51fa\u5230\u6807\u51c6\u8f93\u51fa，\u4fbf\u4e8e\u7ba1\u9053\u91c7\u96c6<\/span>
    \nhandler.<\/span>setFormatter(<\/span>JsonFormatter(<\/span>)<\/span>)<\/span>
    \nlogger.<\/span>addHandler(<\/span>handler)<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n logger.<\/span>info(<\/span>"request finished"<\/span>,<\/span> extra=<\/span>{<\/span>
    \n "request_id"<\/span>:<\/span> "req-0001"<\/span>,<\/span>
    \n "model_version"<\/span>:<\/span> "qwen-7b-v37"<\/span>,<\/span>
    \n "prompt_hash"<\/span>:<\/span> hashlib.<\/span>sha256(<\/span>b"user prompt"<\/span>)<\/span>.<\/span>hexdigest(<\/span>)<\/span>[<\/span>:<\/span>12<\/span>]<\/span>,<\/span>
    \n "latency_ms"<\/span>:<\/span> 486<\/span>,<\/span>
    \n "tokens"<\/span>:<\/span> 96<\/span>,<\/span>
    \n }<\/span>)<\/span><\/p>\n

    4.2 \u65e5\u5fd7\u91c7\u96c6<\/h4>\n

    \u65e5\u5fd7\u91c7\u96c6\u7684\u7ba1\u9053\u8bbe\u8ba1\u8981\u89e3\u51b3\u4e09\u4e2a\u95ee\u9898：\u4e0d\u4e22\u3001\u4e0d\u5835\u3001\u53ef\u56de\u6eaf\u3002Filebeat \u4f5c\u4e3a\u91c7\u96c6\u5668\u5e38\u9a7b\u5404\u8282\u70b9，\u4ee5 tail \u6a21\u5f0f\u8bfb\u53d6 JSON \u65e5\u5fd7\u6587\u4ef6，\u6309 4096 \u6761\u4e00\u6279\u6216 5MB \u89e6\u53d1\u4e00\u6b21\u6279\u91cf\u63a8\u9001，\u907f\u514d\u6bcf\u6761\u65e5\u5fd7\u4e00\u6b21\u7f51\u7edc\u8bf7\u6c42\u3002\u91c7\u96c6\u5668\u672c\u5730\u7ef4\u62a4\u8bfb\u53d6\u504f\u79fb，\u8fdb\u7a0b\u91cd\u542f\u540e\u4ece\u65ad\u70b9\u7ee7\u7eed，\u4fdd\u8bc1\u81f3\u5c11\u4e00\u6b21\u8bed\u4e49\u3002Logstash \u6216 Vector \u5728\u4e2d\u95f4\u5c42\u5b8c\u6210\u5b57\u6bb5\u89e3\u6790\u4e0e\u6807\u51c6\u5316，\u628a\u81ea\u7531\u6587\u672c\u8f6c\u6210\u7edf\u4e00 schema\u3002<\/p>\n

    \u7f13\u51b2\u5c42\u89e3\u51b3\u751f\u4ea7\u4e0e\u6d88\u8d39\u7684\u901f\u5ea6\u5dee\u3002\u63a8\u7406\u670d\u52a1\u5728\u6d41\u91cf\u6bdb\u523a\u65f6\u65e5\u5fd7\u4ea7\u751f\u901f\u7387\u53ef\u8fbe\u7a33\u6001\u7684 5 \u500d，\u76f4\u63a5\u5199\u5165 Elasticsearch \u4f1a\u9020\u6210\u5199\u961f\u5217\u5806\u79ef\u4e0e\u62d2\u7edd\u3002Kafka \u4ee5 3 \u526f\u672c\u3001\u4fdd\u7559 24 \u5c0f\u65f6\u4f5c\u4e3a\u7f13\u51b2，\u91c7\u96c6\u5668\u53ea\u9762\u5bf9 Kafka，\u6d88\u8d39\u7aef\u6309\u81ea\u8eab\u80fd\u529b\u62c9\u53d6\u3002\u79ef\u538b\u76d1\u63a7\u662f\u7ba1\u9053\u5065\u5eb7\u7684\u5173\u952e：Kafka \u6d88\u8d39 lag \u8d85\u8fc7 500 \u4e07\u6761\u65f6，\u6309\u4f18\u5148\u7ea7\u4e22\u5f03\u2014\u2014\u4f4e\u4f18\u5148\u7ea7\u65e5\u5fd7（INFO \u4ee5\u4e0b）\u5148\u4e22，ERROR \u65e5\u5fd7\u6700\u540e\u4e22，\u4fdd\u8bc1\u6545\u969c\u73b0\u573a\u7684\u9ad8\u4ef7\u503c\u65e5\u5fd7\u4e0d\u4e22\u5931\u3002<\/p>\n

    \u7ba1\u9053\u6545\u969c\u7684\u8fb9\u754c\u884c\u4e3a\u5fc5\u987b\u9884\u6f14\u3002Elasticsearch \u5199\u5165\u5931\u8d25\u65f6，\u6d88\u8d39\u7aef\u9000\u907f\u91cd\u8bd5，\u91cd\u8bd5 3 \u6b21\u4ecd\u5931\u8d25\u5219\u628a\u65e5\u5fd7\u8f6c\u5b58\u51b7\u8def\u5f84（\u5bf9\u8c61\u5b58\u50a8），\u800c\u4e0d\u662f\u65e0\u9650\u91cd\u8bd5\u62d6\u57ae\u6d88\u8d39\u8fdb\u7a0b\u3002Filebeat \u81ea\u8eab\u7684\u65e5\u5fd7\u91cf\u4e0e\u78c1\u76d8\u914d\u989d\u4e5f\u8981\u76d1\u63a7：\u5355\u8282\u70b9\u65e5\u5fd7\u76ee\u5f55 20GB \u4e0a\u9650，\u8d85\u8fc7\u4e0a\u9650\u89e6\u53d1\u8f6e\u8f6c，\u8f6e\u8f6c\u6587\u4ef6\u4fdd\u7559 7 \u5929\u3002\u91c7\u96c6\u7ba1\u9053\u7684\u76d1\u63a7\u6307\u6807（\u91c7\u96c6\u5ef6\u8fdf\u3001\u6d88\u8d39 lag\u3001\u4e22\u5f03\u91cf）\u672c\u8eab\u4e5f\u8981\u63a5\u5165\u76d1\u63a7\u4f53\u7cfb，\u5426\u5219\u65e5\u5fd7\u7ba1\u9053\u6545\u969c\u4f1a\u6210\u4e3a\u76f2\u533a\u3002<\/p>\n

    # \u6765\u6e90：Elastic Beats Filebeat 8.11 \/ filebeat.yml<\/span>
    \nfilebeat.inputs<\/span>:<\/span>
    \n –<\/span> type<\/span>:<\/span> filestream
    \n id<\/span>:<\/span> llm–<\/span>access–<\/span>logs
    \n paths<\/span>:<\/span>
    \n –<\/span> \/var\/log\/llm\/access.jsonl
    \n parsers<\/span>:<\/span>
    \n –<\/span> ndjson<\/span>:<\/span>
    \n target<\/span>:<\/span> ""<\/span><\/p>\n

    filebeat.autodiscover<\/span>:<\/span>
    \n providers<\/span>:<\/span>
    \n –<\/span> type<\/span>:<\/span> kubernetes<\/p>\n

    output.elasticsearch<\/span>:<\/span>
    \n hosts<\/span>:<\/span> [<\/span>"es-master:9200"<\/span>]<\/span>
    \n bulk_max_size<\/span>:<\/span> 4096<\/span>
    \n flush_bytes<\/span>:<\/span> 5242880<\/span>
    \n backoff.init<\/span>:<\/span> 1s
    \n backoff.max<\/span>:<\/span> 60s<\/p>\n

    # \u6765\u6e90：confluent-kafka 2.3.0 \/ \u6d88\u8d39\u7aef\u68c0\u67e5\u811a\u672c，main \u4e3a\u81ea\u5b9e\u73b0<\/span>
    \nfrom<\/span> confluent_kafka import<\/span> Consumer<\/p>\n

    def<\/span> lag_check<\/span>(<\/span>broker,<\/span> topic,<\/span> group)<\/span>:<\/span>
    \n """\u8fd4\u56de\u6307\u5b9a\u6d88\u8d39\u7ec4\u7684\u79ef\u538b\u6761\u6570，\u8d85\u8fc7 500 \u4e07\u89c6\u4e3a\u7ba1\u9053\u544a\u8b66\u3002"""<\/span>
    \n c =<\/span> Consumer(<\/span>{<\/span>"bootstrap.servers"<\/span>:<\/span> broker,<\/span> "group.id"<\/span>:<\/span> group}<\/span>)<\/span>
    \n try<\/span>:<\/span>
    \n high =<\/span> c.<\/span>get_watermark_offsets(<\/span>topic,<\/span> 0<\/span>)<\/span>[<\/span>1<\/span>]<\/span>
    \n committed =<\/span> c.<\/span>committed(<\/span>[<\/span>c.<\/span>list_topics(<\/span>topic)<\/span>.<\/span>topics[<\/span>topic]<\/span>.<\/span>partitions[<\/span>0<\/span>]<\/span>]<\/span>)<\/span>
    \n pos =<\/span> committed[<\/span>0<\/span>]<\/span>.<\/span>offset if<\/span> committed and<\/span> committed[<\/span>0<\/span>]<\/span>.<\/span>offset else<\/span> 0<\/span>
    \n return<\/span> max<\/span>(<\/span>0<\/span>,<\/span> high –<\/span> pos)<\/span>
    \n finally<\/span>:<\/span>
    \n c.<\/span>close(<\/span>)<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n lag =<\/span> lag_check(<\/span>"kafka-1:9092"<\/span>,<\/span> "llm-access-logs"<\/span>,<\/span> "es-indexer"<\/span>)<\/span>
    \n level =<\/span> "ERROR"<\/span> if<\/span> lag ><\/span> 5_000_000<\/span> else<\/span> "INFO"<\/span>
    \n print<\/span>(<\/span>f"<\/span>{<\/span>level}<\/span><\/span> \u6d88\u8d39\u79ef\u538b <\/span>{<\/span>lag}<\/span><\/span> \u6761"<\/span><\/span>)<\/span><\/p>\n

    4.3 \u65e5\u5fd7\u5206\u6790<\/h4>\n

    \u65e5\u5fd7\u5206\u6790\u7684\u4e24\u79cd\u4e3b\u8def\u5f84\u8986\u76d6\u6545\u969c\u5904\u7406\u4e0e\u8d8b\u52bf\u6d1e\u5bdf\u3002\u6545\u969c\u5904\u7406\u8def\u5f84\u6309 request_id \u5168\u94fe\u8def\u8fd8\u539f：\u8f93\u5165\u65e5\u5fd7\u3001\u8f93\u51fa\u65e5\u5fd7\u3001\u8c03\u5ea6\u65e5\u5fd7\u3001GPU \u65e5\u5fd7\u4ee5 request_id \u4e3a join \u952e\u5408\u5e76，\u8fd8\u539f\u51fa\u8be5\u8bf7\u6c42\u7684\u5b8c\u6574\u751f\u547d\u5468\u671f\u3002\u4e00\u6b21\u5ef6\u8fdf\u544a\u8b66\u7684\u5b9a\u4f4d\u65f6\u95f4\u53ef\u4ee5\u4ece 30 \u5206\u949f\u538b\u5230 2 \u5206\u949f：\u5148\u770b ttft_ms \u4e0e tpot_ms \u7684\u62c6\u5206，\u82e5 ttft \u5f02\u5e38\u5219\u68c0\u67e5\u9884\u586b\u5145\u957f\u5ea6\u4e0e\u961f\u5217，\u82e5 tpot \u5f02\u5e38\u5219\u68c0\u67e5 GPU \u90bb\u5c45\u8bf7\u6c42\u7684\u5e76\u53d1\u5f71\u54cd\u3002<\/p>\n

    \u8d8b\u52bf\u6d1e\u5bdf\u8def\u5f84\u6309\u7ef4\u5ea6\u805a\u5408\u3002\u6a21\u578b\u7248\u672c\u662f\u6700\u91cd\u8981\u7684\u805a\u5408\u7ef4\u5ea6：\u65b0\u7248\u672c\u4e0a\u7ebf\u540e，\u5bf9\u6bd4\u65b0\u65e7\u7248\u672c\u7684\u9519\u8bef\u7387\u4e0e\u5ef6\u8fdf\u5206\u5e03，\u5dee\u5f02\u8d85\u8fc7 2% \u5373\u89e6\u53d1\u7248\u672c\u56de\u6eda\u8bc4\u4f30\u3002GPU \u7ef4\u5ea6\u805a\u5408\u80fd\u53d1\u73b0\u5355\u5361\u52a3\u5316：\u540c\u4e00\u6279\u6b21 8 \u5361\u4e2d\u67d0\u4e00\u5361\u7684\u9519\u8bef\u7387\u663e\u8457\u9ad8\u4e8e\u5176\u4f59 7 \u5361，\u6307\u5411\u663e\u5b58 ECC \u7ea0\u9519\u6216\u6563\u70ed\u95ee\u9898\u3002\u8fd9\u7c7b\u805a\u5408\u5206\u6790\u7528 Elasticsearch \u7684 terms \u805a\u5408\u4e0e\u65e5\u671f\u76f4\u65b9\u56fe\u5b9e\u73b0，\u67e5\u8be2\u8017\u65f6\u5728 3 \u8282\u70b9\u30012 \u4ebf\u6761\u6587\u6863\u89c4\u6a21\u4e0b\u53ef\u4ee5\u63a7\u5236\u5728 1 \u79d2\u4ee5\u5185\u3002<\/p>\n

    \u65e5\u5fd7\u68c0\u7d22\u7684\u5b58\u50a8\u7b56\u7565\u7531\u7d22\u5f15\u751f\u547d\u5468\u671f\u7ba1\u7406（ILM）\u6267\u884c：\u70ed\u9636\u6bb5 7 \u5929，\u4fdd\u5b58\u5728 SSD，\u652f\u6301\u6beb\u79d2\u7ea7\u68c0\u7d22；\u6696\u9636\u6bb5 7-30 \u5929，\u8fc1\u79fb\u5230 HDD；\u51b7\u9636\u6bb5 30-90 \u5929，\u4ee5 searchable snapshot \u5b58\u5728\u5bf9\u8c61\u5b58\u50a8，\u68c0\u7d22\u53d8\u6162\u4f46\u6210\u672c\u964d\u4f4e\u7ea6 80%；\u8d85\u8fc7 90 \u5929\u5220\u9664\u3002\u5bb9\u91cf\u89c4\u5212\u6309 4.1 \u8282\u7684 5.2GB\/\u5929\u63a8\u7b97：\u4e09\u526f\u672c\u5b58\u50a8 90 \u5929\u7ea6 1.4TB，\u5bf9\u8c61\u5b58\u50a8\u51b7\u6570\u636e\u6309 0.02 \u7f8e\u5143\u6bcf GB \u6bcf\u6708\u8ba1\u7b97，\u6708\u5b58\u50a8\u6210\u672c\u7ea6 9 \u7f8e\u5143，\u8fd9\u662f\u53ef\u4ee5\u63a5\u53d7\u7684\u65e5\u5fd7\u6cbb\u7406\u6210\u672c\u3002<\/p>\n

    \u65e5\u5fd7\u5206\u6790\u7684\u8fb9\u754c\u5728\u4e8e\u566a\u97f3\u4e0e\u8986\u76d6\u7387\u3002\u62bd\u6837\u7387\u4e0d\u8db3\u65f6\u805a\u5408\u7ed3\u8bba\u5931\u771f：\u53ea\u91c7\u6837 1% \u8bf7\u6c42\u8ba1\u7b97\u9519\u8bef\u7387，\u6837\u672c\u91cf 100 \u65f6 95% \u7f6e\u4fe1\u533a\u95f4\u5bbd\u8fbe\u6b63\u8d1f 2%，\u65e0\u6cd5\u533a\u5206 1% \u4e0e 3% \u7684\u5dee\u5f02\u3002\u56e0\u6b64\u9519\u8bef\u7387\u8fd9\u7c7b\u5173\u952e\u805a\u5408\u5fc5\u987b\u5168\u91cf\u91c7\u96c6，\u800c\u6027\u80fd\u5206\u5e03\u5206\u6790\u53ef\u4ee5\u62bd\u6837\u3002\u65e5\u5fd7\u4e0e\u6307\u6807\u7684\u6821\u51c6\u4e5f\u503c\u5f97\u6ce8\u610f：\u65e5\u5fd7\u7edf\u8ba1\u7684\u5ef6\u8fdf\u901a\u5e38\u6bd4\u6307\u6807\u7565\u9ad8，\u56e0\u4e3a\u6307\u6807\u5728\u8fdb\u7a0b\u5185\u805a\u5408\u3001\u65e5\u5fd7\u5728\u5e8f\u5217\u5316\u540e\u7edf\u8ba1，\u4e24\u8005\u504f\u5dee\u7ea6 2-5% \u5c5e\u4e8e\u6b63\u5e38\u8303\u56f4\u3002<\/p>\n

    # \u6765\u6e90：elasticsearch-py 8.12 \/ \u81ea\u5b9e\u73b0\u5206\u6790\u811a\u672c<\/span>
    \nfrom<\/span> elasticsearch import<\/span> Elasticsearch<\/p>\n

    INDEX =<\/span> "llm-access-*"<\/span><\/p>\n

    def<\/span> error_rate_by_version<\/span>(<\/span>es,<\/span> minutes=<\/span>60<\/span>)<\/span>:<\/span>
    \n """\u6309\u6a21\u578b\u7248\u672c\u805a\u5408\u8fc7\u53bb\u4e00\u5c0f\u65f6\u5404\u7248\u672c\u7684\u9519\u8bef\u7387\u3002"""<\/span>
    \n body =<\/span> {<\/span>
    \n "size"<\/span>:<\/span> 0<\/span>,<\/span>
    \n "query"<\/span>:<\/span> {<\/span>"range"<\/span>:<\/span> {<\/span>"ts"<\/span>:<\/span> {<\/span>"gte"<\/span>:<\/span> f"now-<\/span>{<\/span>minutes}<\/span><\/span>m"<\/span><\/span>}<\/span>}<\/span>}<\/span>,<\/span>
    \n "aggs"<\/span>:<\/span> {<\/span>
    \n "by_version"<\/span>:<\/span> {<\/span>
    \n "terms"<\/span>:<\/span> {<\/span>"field"<\/span>:<\/span> "model_version"<\/span>,<\/span> "size"<\/span>:<\/span> 10<\/span>}<\/span>,<\/span>
    \n "aggs"<\/span>:<\/span> {<\/span>
    \n "total"<\/span>:<\/span> {<\/span>"value_count"<\/span>:<\/span> {<\/span>"field"<\/span>:<\/span> "ts"<\/span>}<\/span>}<\/span>,<\/span>
    \n "errors"<\/span>:<\/span> {<\/span>"filter"<\/span>:<\/span> {<\/span>"term"<\/span>:<\/span> {<\/span>"level"<\/span>:<\/span> "ERROR"<\/span>}<\/span>}<\/span>}<\/span>,<\/span>
    \n }<\/span>,<\/span>
    \n }<\/span>
    \n }<\/span>,<\/span>
    \n }<\/span>
    \n resp =<\/span> es.<\/span>search(<\/span>index=<\/span>INDEX,<\/span> body=<\/span>body)<\/span>
    \n out =<\/span> [<\/span>]<\/span>
    \n for<\/span> bucket in<\/span> resp[<\/span>"aggregations"<\/span>]<\/span>[<\/span>"by_version"<\/span>]<\/span>[<\/span>"buckets"<\/span>]<\/span>:<\/span>
    \n total =<\/span> bucket[<\/span>"total"<\/span>]<\/span>[<\/span>"value"<\/span>]<\/span>
    \n errors =<\/span> bucket[<\/span>"errors"<\/span>]<\/span>[<\/span>"doc_count"<\/span>]<\/span>
    \n out.<\/span>append(<\/span>(<\/span>bucket[<\/span>"key"<\/span>]<\/span>,<\/span> errors \/<\/span> total if<\/span> total else<\/span> 0.0<\/span>)<\/span>)<\/span>
    \n return<\/span> out<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n es =<\/span> Elasticsearch(<\/span>"http:\/\/127.0.0.1:9200"<\/span>,<\/span> request_timeout=<\/span>10<\/span>)<\/span>
    \n for<\/span> version,<\/span> rate in<\/span> error_rate_by_version(<\/span>es,<\/span> minutes=<\/span>60<\/span>)<\/span>:<\/span>
    \n flag =<\/span> "\u9700\u56de\u6eda\u8bc4\u4f30"<\/span> if<\/span> rate ><\/span> 0.02<\/span> else<\/span> ""<\/span>
    \n print<\/span>(<\/span>f"<\/span>{<\/span>version}<\/span><\/span>: \u9519\u8bef\u7387 <\/span>{<\/span>rate:<\/span>.2%<\/span>}<\/span><\/span> <\/span>{<\/span>flag}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    5. \u544a\u8b66\u4f53\u7cfb<\/h3>\n

    \u544a\u8b66\u4f53\u7cfb\u662f\u628a\u5f02\u5e38\u68c0\u6d4b\u7ed3\u8bba\u8f6c\u5316\u4e3a\u503c\u73ed\u884c\u52a8\u7684\u6700\u540e\u4e00\u9053\u5de5\u5e8f，\u5176\u8bbe\u8ba1\u76ee\u6807\u4e0d\u662f"\u53d1\u51fa\u544a\u8b66"，\u800c\u662f"\u8ba9\u503c\u73ed\u4eba\u5458\u5728\u6700\u77ed\u65f6\u95f4\u5185\u91c7\u53d6\u6b63\u786e\u52a8\u4f5c"\u3002\u544a\u8b66\u4f53\u7cfb\u7531\u4e09\u5757\u7ec4\u6210：\u5206\u7ea7\u5b9a\u4e49\u544a\u8b66\u7684\u4e25\u91cd\u6027\u4e0e\u54cd\u5e94\u65f6\u9650，\u8def\u7531\u628a\u544a\u8b66\u9001\u5230\u6b63\u786e\u7684\u56e2\u961f\u4e0e\u901a\u9053，\u53bb\u91cd\u628a\u76f8\u540c\u6216\u76f8\u5173\u7684\u544a\u8b66\u6536\u655b\u6210\u4e00\u6761\u53ef\u6267\u884c\u7684\u4efb\u52a1\u3002\u4e09\u8005\u5171\u540c\u51b3\u5b9a\u544a\u8b66\u7684\u7cbe\u51c6\u5ea6：\u7406\u60f3\u72b6\u6001\u4e0b\u6bcf\u6761\u544a\u8b66\u90fd\u5bf9\u5e94\u4e00\u6b21\u771f\u5b9e\u6545\u969c，\u4e14\u9644\u5e26\u5b9a\u4f4d\u4fe1\u606f\u3002<\/p>\n

    \u544a\u8b66\u4f53\u7cfb\u7684\u5931\u6548\u6a21\u5f0f\u8868\u73b0\u4e3a\u4e24\u7ea7：\u6f0f\u62a5（\u8be5\u544a\u6ca1\u544a）\u4e0e\u566a\u97f3（\u4e0d\u8be5\u544a\u7684\u5237\u5c4f）\u3002\u6f0f\u62a5\u7531\u9608\u503c\u4e0e\u68c0\u6d4b\u65b9\u6cd5\u51b3\u5b9a，\u5c5e\u4e8e\u5f02\u5e38\u68c0\u6d4b\u7684\u8303\u7574；\u566a\u97f3\u7531\u5206\u7ea7\u3001\u8def\u7531\u3001\u53bb\u91cd\u4e0d\u5f53\u5bfc\u81f4，\u4e00\u6761\u6545\u969c\u4ea7\u751f 20 \u6761\u91cd\u590d\u544a\u8b66\u5c31\u662f\u5178\u578b\u7684\u8def\u7531\u6536\u655b\u5931\u8d25\u3002\u751f\u4ea7\u5b9e\u8df5\u7684\u7ecf\u9a8c\u6570\u636e\u662f：\u53bb\u91cd\u4e0e\u5206\u7ec4\u53ef\u4ee5\u628a\u544a\u8b66\u6570\u91cf\u538b\u7f29 70%，\u544a\u8b66\u7cbe\u786e\u7387（\u544a\u8b66\u5bf9\u5e94\u771f\u5b9e\u6545\u969c\u7684\u6bd4\u4f8b）\u4ece 30% \u63d0\u5347\u5230 85% \u4ee5\u4e0a\u3002<\/p>\n

    #mermaid-svg-XJ8iOnbxiHyDeIZK{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-XJ8iOnbxiHyDeIZK .error-icon{fill:#552222;}#mermaid-svg-XJ8iOnbxiHyDeIZK .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-XJ8iOnbxiHyDeIZK .marker{fill:#333333;stroke:#333333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .marker.cross{stroke:#333333;}#mermaid-svg-XJ8iOnbxiHyDeIZK svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-XJ8iOnbxiHyDeIZK p{margin:0;}#mermaid-svg-XJ8iOnbxiHyDeIZK .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster-label text{fill:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster-label span{color:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster-label span p{background-color:transparent;}#mermaid-svg-XJ8iOnbxiHyDeIZK .label text,#mermaid-svg-XJ8iOnbxiHyDeIZK span{fill:#333;color:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .node rect,#mermaid-svg-XJ8iOnbxiHyDeIZK .node circle,#mermaid-svg-XJ8iOnbxiHyDeIZK .node ellipse,#mermaid-svg-XJ8iOnbxiHyDeIZK .node polygon,#mermaid-svg-XJ8iOnbxiHyDeIZK .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .rough-node .label text,#mermaid-svg-XJ8iOnbxiHyDeIZK .node .label text,#mermaid-svg-XJ8iOnbxiHyDeIZK .image-shape .label,#mermaid-svg-XJ8iOnbxiHyDeIZK .icon-shape .label{text-anchor:middle;}#mermaid-svg-XJ8iOnbxiHyDeIZK .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .rough-node .label,#mermaid-svg-XJ8iOnbxiHyDeIZK .node .label,#mermaid-svg-XJ8iOnbxiHyDeIZK .image-shape .label,#mermaid-svg-XJ8iOnbxiHyDeIZK .icon-shape .label{text-align:center;}#mermaid-svg-XJ8iOnbxiHyDeIZK .node.clickable{cursor:pointer;}#mermaid-svg-XJ8iOnbxiHyDeIZK .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .arrowheadPath{fill:#333333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-XJ8iOnbxiHyDeIZK .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-XJ8iOnbxiHyDeIZK .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-XJ8iOnbxiHyDeIZK .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster text{fill:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK .cluster span{color:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-XJ8iOnbxiHyDeIZK .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-XJ8iOnbxiHyDeIZK rect.text{fill:none;stroke-width:0;}#mermaid-svg-XJ8iOnbxiHyDeIZK .icon-shape,#mermaid-svg-XJ8iOnbxiHyDeIZK .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-XJ8iOnbxiHyDeIZK .icon-shape p,#mermaid-svg-XJ8iOnbxiHyDeIZK .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-XJ8iOnbxiHyDeIZK .icon-shape .label rect,#mermaid-svg-XJ8iOnbxiHyDeIZK .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-XJ8iOnbxiHyDeIZK .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-XJ8iOnbxiHyDeIZK .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-XJ8iOnbxiHyDeIZK :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-XJ8iOnbxiHyDeIZK .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-XJ8iOnbxiHyDeIZK .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-XJ8iOnbxiHyDeIZK .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u8def\u7531\u4e0e\u6536\u655b<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u544a\u8b66\u5206\u7ea7<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P0<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P1 \u4ee5\u4e0b<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u672a\u786e\u8ba4<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u91cd\u8bd5<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P0 \u670d\u52a1\u4e0d\u53ef\u7528<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    15 \u5206\u949f\u54cd\u5e94<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P1 \u5ef6\u8fdf\u8d8a\u754c<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    30 \u5206\u949f\u54cd\u5e94<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    P2 \u8d44\u6e90\u6c34\u4f4d<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    4 \u5c0f\u65f6\u54cd\u5e94<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u544a\u8b66\u5b9e\u4f8b<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6309\u544a\u8b66\u540d\u4e0e\u96c6\u7fa4\u5206\u7ec4<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u53bb\u91cd\u6307\u7eb9<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u4e25\u91cd\u7ea7\u522b<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7535\u8bdd\u4e0e\u503c\u73ed<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5373\u65f6\u6d88\u606f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6291\u5236\u4f4e\u7ea7\u522b\u544a\u8b66<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u9759\u9ed8\u7ef4\u62a4\u7a97\u53e3<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u901a\u77e5\u901a\u9053<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5347\u7ea7\u5230\u4e0b\u4e00\u7ea7\u522b<\/p>\n

    <\/span><\/p>\n

    5.1 \u544a\u8b66\u5206\u7ea7<\/h4>\n

    \u544a\u8b66\u5206\u7ea7\u628a\u6545\u969c\u6309\u5f71\u54cd\u9762\u4e0e\u7d27\u8feb\u5ea6\u6392\u5e8f，\u6838\u5fc3\u4ea7\u51fa\u662f\u54cd\u5e94\u65f6\u9650\u4e0e\u670d\u52a1\u627f\u8bfa\u3002\u5206\u7ea7\u4f9d\u636e\u4e09\u4e2a\u7ef4\u5ea6：\u5f71\u54cd\u8303\u56f4（\u5168\u5c40\u6216\u5c40\u90e8）\u3001\u7528\u6237\u53ef\u611f\u77e5\u7a0b\u5ea6（\u53ef\u7528\u6216\u964d\u7ea7）\u3001\u6b62\u635f\u52a8\u4f5c\u590d\u6742\u5ea6（\u56de\u6eda\u6216\u6269\u5bb9）\u3002LLM \u670d\u52a1\u7684\u5178\u578b\u5206\u7ea7\u5982\u4e0b\u8868，\u6bcf\u4e00\u7ea7\u90fd\u7ed1\u5b9a\u660e\u786e\u7684\u54cd\u5e94\u65f6\u9650\u4e0e\u5347\u7ea7\u8def\u5f84，\u503c\u73ed\u804c\u8d23\u8868\u636e\u6b64\u6392\u73ed\u3002<\/p>\n\n\u7ea7\u522b\u5178\u578b\u573a\u666f\u54cd\u5e94\u65f6\u9650\u901a\u77e5\u901a\u9053\u6b62\u635f\u52a8\u4f5c<\/tr>\n\n\n\n\n\n
    P0<\/td>\n\u670d\u52a1\u4e0d\u53ef\u7528\u3001\u63a8\u7406\u7ed3\u679c\u635f\u574f<\/td>\n15 \u5206\u949f<\/td>\n\u7535\u8bdd + \u503c\u73ed<\/td>\n\u7acb\u5373\u56de\u6eda\u6216\u5207\u6362\u5bb9\u707e<\/td>\n<\/tr>\n
    P1<\/td>\nP99 \u5ef6\u8fdf\u5927\u4e8e 2s\u3001\u9519\u8bef\u7387\u5927\u4e8e 5%<\/td>\n30 \u5206\u949f<\/td>\n\u5373\u65f6\u6d88\u606f + \u7535\u8bdd<\/td>\n\u56de\u6eda\u6a21\u578b\u7248\u672c<\/td>\n<\/tr>\n
    P2<\/td>\n\u663e\u5b58\u5927\u4e8e 90%\u3001\u541e\u5410\u4e0b\u964d<\/td>\n4 \u5c0f\u65f6<\/td>\n\u5373\u65f6\u6d88\u606f<\/td>\n\u89c2\u5bdf\u5e76\u51c6\u5907\u6269\u5bb9<\/td>\n<\/tr>\n
    P3<\/td>\n\u7f13\u5b58\u547d\u4e2d\u7387\u4e0b\u964d\u3001\u65e5\u5fd7\u79ef\u538b<\/td>\n\u6b21\u65e5<\/td>\n\u5de5\u5355<\/td>\n\u6392\u671f\u5904\u7406<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

    \u5206\u7ea7\u4e0d\u80fd\u62cd\u8111\u888b\u5b9a\u7ea7，\u5fc5\u987b\u4e0e SLO \u5bf9\u8d26\u3002P0 \u5bf9\u5e94\u9519\u8bef\u9884\u7b97\u4e2d\u7684\u707e\u96be\u6027\u4e8b\u4ef6，\u5355\u6b21\u6545\u969c\u6d88\u8017\u9884\u7b97\u8d85\u8fc7 10% \u5373\u5347\u7ea7\u4e3a P0；P1 \u5bf9\u5e94\u9884\u7b97\u6d88\u8017\u7387\u8d85\u8fc7\u544a\u8b66\u7ebf\u7684\u6301\u7eed\u52a3\u5316；P2\u3001P3 \u662f\u6f5c\u5728\u98ce\u9669\u4e0e\u4f18\u5316\u9879\u3002\u5b9a\u7ea7\u7684\u53e6\u4e00\u6761\u51c6\u5219\u662f"\u80fd\u5426\u88ab\u6b62\u635f\u52a8\u4f5c\u6d88\u89e3"：\u53ef\u56de\u6eda\u7684\u6545\u969c\u5b9a P1，\u4e0d\u53ef\u56de\u6eda\u7684\u5b9a P0，\u53ea\u6709\u8f6f\u6062\u590d\u624b\u6bb5\u7684\u5b9a P2\u3002\u8fd9\u4e00\u51c6\u5219\u8ba9\u5206\u7ea7\u5929\u7136\u4e0e\u8fd0\u7ef4\u80fd\u529b\u5bf9\u9f50，\u907f\u514d\u5168\u5458 P0 \u7684\u6ee5\u7528\u3002<\/p>\n

    \u5206\u7ea7\u8fd8\u51b3\u5b9a\u503c\u73ed\u4eba\u529b\u4e0e\u6392\u73ed\u6210\u672c\u3002P0 \u9700\u8981 7 \u5929 24 \u5c0f\u65f6\u7535\u8bdd\u503c\u73ed，\u4e00\u4e2a 6 \u4eba\u56e2\u961f\u6392 3 \u73ed\u3001\u6bcf\u73ed 8 \u5c0f\u65f6，\u6bcf\u4eba\u6bcf\u5468\u7ea6 28 \u5c0f\u65f6\u503c\u73ed\u8d1f\u62c5；P1 \u53ea\u9700\u5de5\u4f5c\u65e5 9-21 \u70b9\u5373\u65f6\u6d88\u606f\u503c\u73ed\u3002\u5206\u7ea7\u8d8a\u4e25\u683c，\u503c\u73ed\u6210\u672c\u8d8a\u9ad8，\u56e0\u6b64\u5206\u7ea7\u8bc4\u5ba1\u6bcf\u6708\u4e00\u6b21，\u628a\u957f\u671f\u65e0 P0 \u7684\u89c4\u5219\u964d\u7ea7\u5230 P1，\u628a\u9891\u7e41 P2 \u7684\u89c4\u5219\u5347\u7ea7\u5230 P1，\u4fdd\u6301\u5206\u7ea7\u4e0e\u771f\u5b9e\u98ce\u9669\u5339\u914d\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ severity_policy.py<\/span>
    \nimport<\/span> time
    \nfrom<\/span> dataclasses import<\/span> dataclass<\/p>\n

    # \u5404\u7ea7\u522b\u54cd\u5e94\u65f6\u9650（\u5206\u949f），\u8d85\u8fc7\u5373\u5347\u7ea7\u901a\u77e5<\/span>
    \nSEVERITY =<\/span> {<\/span>"P0"<\/span>:<\/span> 15<\/span>,<\/span> "P1"<\/span>:<\/span> 30<\/span>,<\/span> "P2"<\/span>:<\/span> 240<\/span>,<\/span> "P3"<\/span>:<\/span> 1440<\/span>}<\/span><\/p>\n

    @dataclass<\/span>
    \nclass<\/span> Alert<\/span>:<\/span>
    \n name:<\/span> str<\/span>
    \n severity:<\/span> str<\/span>
    \n started:<\/span> float<\/span><\/p>\n

    class<\/span> Escalator<\/span>:<\/span>
    \n """\u8ddf\u8e2a\u672a\u786e\u8ba4\u544a\u8b66，\u8d85\u65f6\u672a\u54cd\u5e94\u5219\u89e6\u53d1\u5347\u7ea7\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self)<\/span>:<\/span>
    \n self.<\/span>active =<\/span> {<\/span>}<\/span><\/p>\n

    def<\/span> fire<\/span>(<\/span>self,<\/span> name,<\/span> severity)<\/span>:<\/span>
    \n self.<\/span>active[<\/span>name]<\/span> =<\/span> Alert(<\/span>name,<\/span> severity,<\/span> time.<\/span>time(<\/span>)<\/span>)<\/span><\/p>\n

    def<\/span> overdue<\/span>(<\/span>self,<\/span> now=<\/span>None<\/span>)<\/span>:<\/span>
    \n now =<\/span> now or<\/span> time.<\/span>time(<\/span>)<\/span>
    \n result =<\/span> [<\/span>]<\/span>
    \n for<\/span> alert in<\/span> self.<\/span>active.<\/span>values(<\/span>)<\/span>:<\/span>
    \n limit =<\/span> SEVERITY[<\/span>alert.<\/span>severity]<\/span> *<\/span> 60<\/span>
    \n if<\/span> now –<\/span> alert.<\/span>started ><\/span> limit:<\/span>
    \n result.<\/span>append(<\/span>alert)<\/span>
    \n return<\/span> result<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n esc =<\/span> Escalator(<\/span>)<\/span>
    \n esc.<\/span>fire(<\/span>"p99_latency_2s"<\/span>,<\/span> "P1"<\/span>)<\/span>
    \n esc.<\/span>fire(<\/span>"service_down"<\/span>,<\/span> "P0"<\/span>)<\/span>
    \n # \u6a21\u62df 31 \u5206\u949f\u540e\u4ecd\u672a\u786e\u8ba4，\u68c0\u67e5\u5347\u7ea7\u6e05\u5355<\/span>
    \n fake_now =<\/span> time.<\/span>time(<\/span>)<\/span> +<\/span> 31<\/span> *<\/span> 60<\/span>
    \n for<\/span> alert in<\/span> esc.<\/span>overdue(<\/span>fake_now)<\/span>:<\/span>
    \n print<\/span>(<\/span>f"<\/span>{<\/span>alert.<\/span>name}<\/span><\/span> \u8d85\u8fc7 <\/span>{<\/span>SEVERITY[<\/span>alert.<\/span>severity]<\/span>}<\/span><\/span> \u5206\u949f\u65f6\u9650，\u5347\u7ea7\u901a\u77e5"<\/span><\/span>)<\/span><\/p>\n

    5.2 \u544a\u8b66\u8def\u7531<\/h4>\n

    \u544a\u8b66\u8def\u7531\u51b3\u5b9a"\u544a\u8b66\u53d1\u7ed9\u8c01"，\u662f\u544a\u8b66\u4f53\u7cfb\u907f\u514d\u5168\u5458\u8f70\u70b8\u7684\u5173\u952e\u3002\u8def\u7531\u89c4\u5219\u4ee5\u544a\u8b66\u6807\u7b7e\u4e3a\u8f93\u5165：\u670d\u52a1\u3001\u56e2\u961f\u3001\u4e25\u91cd\u7ea7\u522b\u3001\u96c6\u7fa4，\u901a\u8fc7\u8def\u7531\u6811\u9010\u5c42\u5339\u914d\u3002Prometheus Alertmanager \u662f\u4e8b\u5b9e\u4e0a\u7684\u8def\u7531\u6807\u51c6，\u5176\u8def\u7531\u6811\u652f\u6301 matcher \u5339\u914d\u3001\u5d4c\u5957\u8def\u7531\u4e0e receiver \u7ed1\u5b9a，\u4e00\u6761\u544a\u8b66\u4ece\u6839\u8def\u7531\u51fa\u53d1，\u6309\u5339\u914d\u89c4\u5219\u4e0b\u6c89\u5230\u53f6\u5b50\u8282\u70b9\u5bf9\u5e94\u7684\u63a5\u6536\u65b9\u3002<\/p>\n

    \u8def\u7531\u914d\u7f6e\u7684\u4e09\u4e2a\u6838\u5fc3\u53c2\u6570\u662f\u5206\u7ec4\u3001\u7b49\u5f85\u4e0e\u91cd\u590d\u3002\u5206\u7ec4\u628a\u76f8\u540c\u544a\u8b66\u540d\u4e0e\u96c6\u7fa4\u7684\u544a\u8b66\u5408\u5e76\u4e3a\u4e00\u6761\u901a\u77e5，\u907f\u514d 50 \u53f0 GPU \u540c\u65f6\u544a\u8b66\u5237 50 \u6761\u6d88\u606f\u3002group_wait \u63a7\u5236\u5728\u5206\u7ec4\u5efa\u7acb\u540e\u7b49\u5f85\u591a\u4e45\u518d\u53d1\u9001，\u9ed8\u8ba4 30 \u79d2，\u7528\u4e8e\u7b49\u5f85\u540c\u7ec4\u544a\u8b66\u805a\u9f50\u3002group_interval \u63a7\u5236\u540c\u4e00\u5206\u7ec4\u4e24\u6b21\u901a\u77e5\u7684\u6700\u5c0f\u95f4\u9694，\u9ed8\u8ba4 5 \u5206\u949f\u3002repeat_interval \u63a7\u5236\u672a\u6062\u590d\u544a\u8b66\u7684\u91cd\u590d\u901a\u77e5\u5468\u671f，\u9ed8\u8ba4 4 \u5c0f\u65f6\u3002\u8fd9\u4e09\u4e2a\u53c2\u6570\u51b3\u5b9a\u901a\u77e5\u91cf：50 \u6761 GPU \u544a\u8b66\u7ecf\u5206\u7ec4\u6536\u655b\u4e3a 1 \u6761，\u518d\u7ecf repeat_interval \u628a 4 \u5c0f\u65f6\u5185\u7684\u91cd\u590d\u901a\u77e5\u4ece 200 \u6b21\u538b\u5230 1 \u6b21\u3002<\/p>\n

    \u8def\u7531\u6811\u7684\u8fb9\u754c\u5728\u4e8e\u8de8\u56e2\u961f\u4e0e\u8de8\u7cfb\u7edf\u7684\u544a\u8b66\u5f52\u5c5e\u3002\u6a21\u578b\u8d28\u91cf\u544a\u8b66\u5f52\u7b97\u6cd5\u56e2\u961f\u3001\u57fa\u7840\u8bbe\u65bd\u544a\u8b66\u5f52\u5e73\u53f0\u56e2\u961f\u3001\u4e1a\u52a1\u53ef\u7528\u6027\u544a\u8b66\u5f52 SRE，\u4e00\u65e6\u4e00\u6761\u544a\u8b66\u540c\u65f6\u6d89\u53ca\u591a\u4e2a\u56e2\u961f，\u5fc5\u987b\u6709\u660e\u786e\u7684"\u4e3b\u63a5\u6536\u65b9 + \u6284\u9001"\u7ea6\u5b9a，\u5426\u5219\u4f1a\u51fa\u73b0\u4e09\u5934\u4e0d\u63a5\u7684\u771f\u7a7a\u3002\u8def\u7531\u914d\u7f6e\u9700\u8981\u7248\u672c\u5316\u7ba1\u7406\u5e76\u7eb3\u5165\u8bc4\u5ba1：\u8def\u7531\u9519\u8bef\u662f\u544a\u8b66\u6f0f\u53d1\u7684\u9996\u8981\u539f\u56e0，\u5176\u5f71\u54cd\u6bd4\u9608\u503c\u9519\u8bef\u66f4\u9690\u853d\u2014\u2014\u9608\u503c\u9519\u8bef\u4ea7\u751f\u8bef\u62a5，\u8def\u7531\u9519\u8bef\u76f4\u63a5\u541e\u6389\u544a\u8b66\u3002<\/p>\n

    # \u6765\u6e90：Prometheus Alertmanager 0.26.0 \/ alertmanager.yml<\/span>
    \nroute<\/span>:<\/span>
    \n group_by<\/span>:<\/span> [<\/span>"alertname"<\/span>,<\/span> "cluster"<\/span>]<\/span>
    \n group_wait<\/span>:<\/span> 30s
    \n group_interval<\/span>:<\/span> 5m
    \n repeat_interval<\/span>:<\/span> 4h
    \n receiver<\/span>:<\/span> platform–<\/span>default
    \n routes<\/span>:<\/span>
    \n –<\/span> matchers<\/span>:<\/span> [<\/span>"severity=P0"<\/span>]<\/span>
    \n receiver<\/span>:<\/span> oncall–<\/span>phone
    \n continue<\/span>:<\/span> false<\/span>
    \n –<\/span> matchers<\/span>:<\/span> [<\/span>"team=ml"<\/span>]<\/span>
    \n receiver<\/span>:<\/span> ml–<\/span>team–<\/span>im<\/p>\n

    inhibit_rules<\/span>:<\/span>
    \n –<\/span> source_matchers<\/span>:<\/span> [<\/span>"severity=P0"<\/span>]<\/span>
    \n target_matchers<\/span>:<\/span> [<\/span>"severity=^(P1|P2)$"<\/span>]<\/span>
    \n equal<\/span>:<\/span> [<\/span>"alertname"<\/span>,<\/span> "cluster"<\/span>]<\/span><\/p>\n

    receivers<\/span>:<\/span>
    \n –<\/span> name<\/span>:<\/span> oncall–<\/span>phone
    \n webhook_configs<\/span>:<\/span>
    \n –<\/span> url<\/span>:<\/span> http:<\/span>\/\/pagerduty.example.com\/events\/v2\/enqueue
    \n –<\/span> name<\/span>:<\/span> ml–<\/span>team–<\/span>im
    \n slack_configs<\/span>:<\/span>
    \n –<\/span> channel<\/span>:<\/span> "#ml-ops"<\/span>
    \n title<\/span>:<\/span> "{{ .CommonAnnotations.summary }}"<\/span><\/p>\n

    # \u6765\u6e90：PyYAML 6.0 \/ alert_routes_checker.py<\/span>
    \nimport<\/span> yaml<\/p>\n

    # \u6f14\u793a\u7528\u6700\u5c0f\u914d\u7f6e；\u751f\u4ea7\u73af\u5883\u8bfb\u53d6\u7248\u672c\u5316\u7684 alertmanager.yml<\/span>
    \n_EXAMPLE =<\/span> """
    \nroute:
    \n group_by: ["alertname", "cluster"]
    \n group_wait: 30s
    \n receiver: platform-default
    \n routes:
    \n – matchers: ["severity=P0"]
    \n receiver: oncall-phone
    \n"""<\/span><\/p>\n

    def<\/span> walk<\/span>(<\/span>route,<\/span> depth=<\/span>0<\/span>,<\/span> parents=<\/span>(<\/span>)<\/span>)<\/span>:<\/span>
    \n """\u904d\u5386\u8def\u7531\u6811，\u8fd4\u56de\u6bcf\u6761\u53f6\u5b50\u8def\u7531\u7684\u63a5\u6536\u65b9\u4e0e\u5c42\u7ea7\u3002"""<\/span>
    \n routes =<\/span> route.<\/span>get(<\/span>"routes"<\/span>)<\/span> or<\/span> [<\/span>]<\/span>
    \n if<\/span> routes:<\/span>
    \n for<\/span> r in<\/span> routes:<\/span>
    \n yield<\/span> from<\/span> walk(<\/span>r,<\/span> depth +<\/span> 1<\/span>,<\/span> parents +<\/span> (<\/span>route.<\/span>get(<\/span>"receiver"<\/span>)<\/span>,<\/span>)<\/span>)<\/span>
    \n else<\/span>:<\/span>
    \n yield<\/span> route.<\/span>get(<\/span>"receiver"<\/span>)<\/span>,<\/span> depth,<\/span> parents<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n try<\/span>:<\/span>
    \n with<\/span> open<\/span>(<\/span>"alertmanager.yml"<\/span>,<\/span> encoding=<\/span>"utf-8"<\/span>)<\/span> as<\/span> f:<\/span>
    \n cfg =<\/span> yaml.<\/span>safe_load(<\/span>f)<\/span>
    \n except<\/span> FileNotFoundError:<\/span>
    \n cfg =<\/span> yaml.<\/span>safe_load(<\/span>_EXAMPLE)<\/span>
    \n root =<\/span> cfg[<\/span>"route"<\/span>]<\/span>
    \n print<\/span>(<\/span>"\u6839\u8def\u7531\u5206\u7ec4\u5b57\u6bb5:"<\/span>,<\/span> root.<\/span>get(<\/span>"group_by"<\/span>)<\/span>)<\/span>
    \n for<\/span> receiver,<\/span> depth,<\/span> parents in<\/span> walk(<\/span>root)<\/span>:<\/span>
    \n print<\/span>(<\/span>f"\u53f6\u5b50\u8def\u7531 \u63a5\u6536\u65b9=<\/span>{<\/span>receiver}<\/span><\/span> \u5c42\u7ea7=<\/span>{<\/span>depth}<\/span><\/span> \u7236\u63a5\u6536\u65b9=<\/span>{<\/span>parents}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    5.3 \u544a\u8b66\u53bb\u91cd<\/h4>\n

    \u544a\u8b66\u53bb\u91cd\u89e3\u51b3"\u4e00\u6761\u6545\u969c\u591a\u6761\u544a\u8b66"\u7684\u901a\u77e5\u653e\u5927\u95ee\u9898，\u673a\u5236\u5206\u4e09\u5c42\u3002\u7b2c\u4e00\u5c42\u662f\u5206\u7ec4：\u540c\u544a\u8b66\u540d\u3001\u540c\u96c6\u7fa4\u3001\u540c\u5b9e\u4f8b\u7684\u544a\u8b66\u5728 Alertmanager \u5185\u5408\u5e76，\u53ea\u53d1\u4e00\u6761\u901a\u77e5\u3002\u7b2c\u4e8c\u5c42\u662f\u6307\u7eb9\u53bb\u91cd：\u6bcf\u6761\u544a\u8b66\u6309\u544a\u8b66\u540d\u3001\u5b9e\u4f8b\u3001\u4e25\u91cd\u7ea7\u522b\u751f\u6210\u6307\u7eb9，\u540c\u4e00\u6307\u7eb9\u5728\u53bb\u91cd\u7a97\u53e3\u5185\u53ea\u901a\u77e5\u4e00\u6b21，\u7a97\u53e3\u9ed8\u8ba4 1 \u5c0f\u65f6\u3002\u7b2c\u4e09\u5c42\u662f\u6291\u5236：\u9ad8\u4f18\u5148\u7ea7\u544a\u8b66\u6291\u5236\u540c\u6e90\u7684\u4f4e\u4f18\u5148\u7ea7\u544a\u8b66，P0 \u89e6\u53d1\u65f6\u540c\u96c6\u7fa4\u7684 P1\u3001P2 \u4e00\u5f8b\u9759\u9ed8，\u56e0\u4e3a\u6839\u56e0\u5df2\u88ab P0 \u8986\u76d6\u3002<\/p>\n

    \u6291\u5236\u89c4\u5219\u8981\u5c0f\u5fc3\u7684\u8fb9\u754c\u662f"\u8bef\u6291\u5236"\u3002P0 \u6291\u5236 P1 \u7684\u524d\u63d0\u662f\u4e8c\u8005\u540c\u6e90：inhibit_rules \u7528 equal \u5b57\u6bb5\u7ea6\u675f\u544a\u8b66\u540d\u4e0e\u96c6\u7fa4\u76f8\u540c，\u8de8\u96c6\u7fa4\u6216\u8de8\u544a\u8b66\u540d\u7684 P0 \u4e0d\u6291\u5236 P1\u3002\u53e6\u4e00\u4e2a\u53bb\u91cd\u7ef4\u5ea6\u7684\u8865\u5145\u662f\u6296\u52a8\u68c0\u6d4b：\u544a\u8b66\u5728\u89e6\u53d1\u4e0e\u6062\u590d\u95f4\u53cd\u590d\u6a2a\u8df3（flapping），1 \u5c0f\u65f6\u5185\u72b6\u6001\u7ffb\u8f6c 4 \u6b21\u4ee5\u4e0a\u5373\u5224\u5b9a\u6296\u52a8，\u6296\u52a8\u544a\u8b66\u8fdb\u5165\u89c2\u5bdf\u5217\u8868\u800c\u4e0d\u662f\u91cd\u590d\u901a\u77e5，\u5f85\u7a33\u5b9a\u540e\u518d\u51b3\u5b9a\u662f\u5426\u5347\u7ea7\u3002<\/p>\n

    \u53bb\u91cd\u7684\u91cf\u5316\u6548\u679c\u53ef\u4ee5\u9a8c\u8bc1\u3002\u4e00\u6b21 GPU \u663e\u5b58\u544a\u8b66，8 \u5361\u96c6\u7fa4\u4f1a\u4ea7\u751f 8 \u6761\u540c\u544a\u8b66\u540d\u544a\u8b66，\u7ecf\u5206\u7ec4\u6536\u655b\u4e3a 1 \u6761；\u82e5\u540c\u65f6\u89e6\u53d1\u5ef6\u8fdf\u3001\u9519\u8bef\u7387\u3001\u541e\u5410 3 \u6761\u5173\u8054\u544a\u8b66，\u7ecf\u6291\u5236\u53ea\u4fdd\u7559\u5ef6\u8fdf\u4e00\u6761\u3002\u5b9e\u6d4b\u4e00\u4e2a 30 \u6761\u89c4\u5219\u7684\u76d1\u63a7\u4f53\u7cfb，\u53bb\u91cd\u524d\u65e5\u5747\u901a\u77e5 40 \u6761，\u53bb\u91cd\u540e 12 \u6761，\u538b\u7f29\u7387 70%，\u503c\u73ed\u4eba\u5458\u5bf9\u544a\u8b66\u7684\u54cd\u5e94\u7387\u4ece 40% \u56de\u5347\u5230 90%\u3002\u53bb\u91cd\u5931\u6548\u7684\u6700\u5e38\u89c1\u539f\u56e0\u662f\u6807\u7b7e\u4e0d\u4e00\u81f4：\u540c\u4e00\u6307\u6807\u5728\u4e0d\u540c exporter \u6253\u4e86\u4e0d\u540c\u5b9e\u4f8b\u6807\u7b7e，\u6307\u7eb9\u6c38\u8fdc\u4e0d\u540c，\u53bb\u91cd\u5b8c\u5168\u5931\u6548，\u8fd9\u53c8\u56de\u5230\u6807\u7b7e\u89c4\u8303\u7684\u6cbb\u7406\u95ee\u9898\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ dedup.py<\/span>
    \nimport<\/span> hashlib
    \nimport<\/span> time
    \nfrom<\/span> collections import<\/span> defaultdict,<\/span> deque<\/p>\n

    class<\/span> Deduper<\/span>:<\/span>
    \n """\u57fa\u4e8e\u544a\u8b66\u6307\u7eb9\u7684\u53bb\u91cd\u5668，\u7a97\u53e3\u5185\u540c\u6307\u7eb9\u53ea\u901a\u77e5\u4e00\u6b21\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> window=<\/span>3600<\/span>)<\/span>:<\/span>
    \n self.<\/span>window =<\/span> window
    \n self.<\/span>fired =<\/span> {<\/span>}<\/span><\/p>\n

    def<\/span> fingerprint<\/span>(<\/span>self,<\/span> alert)<\/span>:<\/span>
    \n raw =<\/span> f"<\/span>{<\/span>alert[<\/span>'name'<\/span>]<\/span>}<\/span><\/span>|<\/span>{<\/span>alert[<\/span>'instance'<\/span>]<\/span>}<\/span><\/span>|<\/span>{<\/span>alert[<\/span>'severity'<\/span>]<\/span>}<\/span><\/span>"<\/span><\/span>
    \n return<\/span> hashlib.<\/span>sha1(<\/span>raw.<\/span>encode(<\/span>)<\/span>)<\/span>.<\/span>hexdigest(<\/span>)<\/span><\/p>\n

    def<\/span> should_send<\/span>(<\/span>self,<\/span> alert)<\/span>:<\/span>
    \n key =<\/span> self.<\/span>fingerprint(<\/span>alert)<\/span>
    \n now =<\/span> time.<\/span>time(<\/span>)<\/span>
    \n if<\/span> key in<\/span> self.<\/span>fired and<\/span> now –<\/span> self.<\/span>fired[<\/span>key]<\/span> <<\/span> self.<\/span>window:<\/span>
    \n return<\/span> False<\/span>
    \n self.<\/span>fired[<\/span>key]<\/span> =<\/span> now
    \n return<\/span> True<\/span><\/p>\n

    class<\/span> FlapDetector<\/span>:<\/span>
    \n """\u7edf\u8ba1 1 \u5c0f\u65f6\u5185\u72b6\u6001\u7ffb\u8f6c\u6b21\u6570，\u8d85\u8fc7\u9608\u503c\u5224\u5b9a\u4e3a\u6296\u52a8\u3002"""<\/span><\/p>\n

    def<\/span> __init__<\/span>(<\/span>self,<\/span> limit=<\/span>4<\/span>,<\/span> span=<\/span>3600<\/span>)<\/span>:<\/span>
    \n self.<\/span>limit =<\/span> limit
    \n self.<\/span>span =<\/span> span
    \n self.<\/span>spans =<\/span> defaultdict(<\/span>lambda<\/span>:<\/span> deque(<\/span>)<\/span>)<\/span><\/p>\n

    def<\/span> observe<\/span>(<\/span>self,<\/span> key,<\/span> state)<\/span>:<\/span>
    \n queue =<\/span> self.<\/span>spans[<\/span>key]<\/span>
    \n now =<\/span> time.<\/span>time(<\/span>)<\/span>
    \n queue.<\/span>append(<\/span>(<\/span>now,<\/span> state)<\/span>)<\/span>
    \n while<\/span> queue and<\/span> now –<\/span> queue[<\/span>0<\/span>]<\/span>[<\/span>0<\/span>]<\/span> ><\/span> self.<\/span>span:<\/span>
    \n queue.<\/span>popleft(<\/span>)<\/span>
    \n flips =<\/span> sum<\/span>(<\/span>1<\/span> for<\/span> i in<\/span> range<\/span>(<\/span>1<\/span>,<\/span> len<\/span>(<\/span>queue)<\/span>)<\/span> if<\/span> queue[<\/span>i]<\/span>[<\/span>1<\/span>]<\/span> !=<\/span> queue[<\/span>i –<\/span> 1<\/span>]<\/span>[<\/span>1<\/span>]<\/span>)<\/span>
    \n return<\/span> flips >=<\/span> self.<\/span>limit,<\/span> flips<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n d =<\/span> Deduper(<\/span>window=<\/span>3600<\/span>)<\/span>
    \n sample =<\/span> {<\/span>"name"<\/span>:<\/span> "p99_latency_2s"<\/span>,<\/span> "instance"<\/span>:<\/span> "serving-1:9101"<\/span>,<\/span> "severity"<\/span>:<\/span> "P1"<\/span>}<\/span>
    \n print<\/span>(<\/span>"\u9996\u6b21\u53d1\u9001:"<\/span>,<\/span> d.<\/span>should_send(<\/span>sample)<\/span>)<\/span>
    \n print<\/span>(<\/span>"\u7a97\u53e3\u5185\u91cd\u590d\u88ab\u62e6\u622a:"<\/span>,<\/span> not<\/span> d.<\/span>should_send(<\/span>sample)<\/span>)<\/span>
    \n fd =<\/span> FlapDetector(<\/span>limit=<\/span>3<\/span>,<\/span> span=<\/span>60<\/span>)<\/span>
    \n flips =<\/span> 0<\/span>
    \n for<\/span> state in<\/span> (<\/span>True<\/span>,<\/span> False<\/span>,<\/span> True<\/span>,<\/span> False<\/span>)<\/span>:<\/span>
    \n _,<\/span> flips =<\/span> fd.<\/span>observe(<\/span>"node-x"<\/span>,<\/span> state)<\/span>
    \n print<\/span>(<\/span>f"1 \u5206\u949f\u5185\u7ffb\u8f6c <\/span>{<\/span>flips}<\/span><\/span> \u6b21，\u5224\u5b9a\u4e3a <\/span>{<\/span>'\u6296\u52a8'<\/span> if<\/span> flips >=<\/span> 3<\/span> else<\/span> '\u7a33\u5b9a'<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    6. \u76d1\u63a7\u5e73\u53f0<\/h3>\n

    \u76d1\u63a7\u5e73\u53f0\u662f\u524d\u4e09\u7ae0\u7684\u627f\u8f7d\u5b9e\u4f53：\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u3001\u544a\u8b66\u3001\u65e5\u5fd7\u90fd\u8981\u5728\u7edf\u4e00\u7684\u5e73\u53f0\u4e0a\u843d\u5730\u8fd0\u884c，\u5e73\u53f0\u67b6\u6784\u51b3\u5b9a\u4e86\u6574\u5957\u76d1\u63a7\u4f53\u7cfb\u7684\u53ef\u6269\u5c55\u6027\u4e0e\u53ef\u7528\u6027\u3002\u5e73\u53f0\u9009\u578b\u7684\u5206\u6c34\u5cad\u662f\u56e2\u961f\u89c4\u6a21\u4e0e\u6570\u636e\u89c4\u6a21：3 \u4eba\u4ee5\u5185\u7684\u5c0f\u56e2\u961f\u7528 Prometheus \u52a0 Grafana \u52a0 Elasticsearch \u4e09\u4ef6\u5957\u5373\u53ef\u8986\u76d6\u5168\u90e8\u9700\u6c42；\u6570\u636e\u91cf\u8d85\u8fc7\u5343\u4e07\u6837\u672c\u6bcf\u5929\u3001\u6216\u9700\u8981\u8de8\u56e2\u961f\u5171\u4eab\u65f6，\u624d\u9700\u8981\u8003\u8651 Thanos\u3001VictoriaMetrics \u4e0e\u96c6\u4e2d\u5f0f\u65e5\u5fd7\u5e73\u53f0\u7684\u5f15\u5165\u3002<\/p>\n

    \u5e73\u53f0\u7684\u90e8\u7f72\u5f62\u6001\u8981\u907f\u514d\u4e24\u4e2a\u6781\u7aef\u3002\u8fc7\u5ea6\u5806\u53e0\u7684\u6781\u7aef\u662f"\u76d1\u63a7\u7cfb\u7edf\u7684\u76d1\u63a7"：\u6bcf\u4e00\u5c42\u90fd\u5f15\u5165\u9ad8\u53ef\u7528\u7ec4\u4ef6，Prometheus \u4e3b\u5907\u3001Alertmanager \u4e09\u526f\u672c\u3001ES \u4e09\u8282\u70b9，\u603b\u8d44\u6e90\u5f00\u9500\u53ef\u8fbe\u76d1\u63a7\u6d41\u91cf\u7684 3 \u500d，\u590d\u6742\u5ea6\u53cd\u8fc7\u6765\u541e\u566c\u4e86\u76d1\u63a7\u7684\u4ef7\u503c\u3002\u8fc7\u5ea6\u7cbe\u7b80\u7684\u6781\u7aef\u662f\u5355\u70b9：Prometheus \u5355\u8282\u70b9\u3001\u65e0\u6301\u4e45\u5316，\u4e00\u6b21\u91cd\u542f\u4e22\u5149 15 \u5929\u5386\u53f2\u3002\u751f\u4ea7\u4e0a\u91c7\u7528"\u4e24\u8282\u70b9 Prometheus \u52a0\u8fdc\u7aef\u5bf9\u8c61\u5b58\u50a8\u5907\u4efd\u3001Alertmanager \u53cc\u526f\u672c\u3001ES \u4e09\u8282\u70b9"\u7684\u6298\u4e2d，\u65e2\u625b\u5f97\u4f4f\u5355\u8282\u70b9\u6545\u969c，\u53c8\u4e0d\u81f3\u4e8e\u4e3a\u76d1\u63a7\u517b\u4e00\u652f\u8fd0\u7ef4\u56e2\u961f\u3002<\/p>\n

    #mermaid-svg-o93WxEckQEVtfPdy{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-o93WxEckQEVtfPdy .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-o93WxEckQEVtfPdy .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-o93WxEckQEVtfPdy .error-icon{fill:#552222;}#mermaid-svg-o93WxEckQEVtfPdy .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-o93WxEckQEVtfPdy .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-o93WxEckQEVtfPdy .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-o93WxEckQEVtfPdy .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-o93WxEckQEVtfPdy .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-o93WxEckQEVtfPdy .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-o93WxEckQEVtfPdy .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-o93WxEckQEVtfPdy .marker{fill:#333333;stroke:#333333;}#mermaid-svg-o93WxEckQEVtfPdy .marker.cross{stroke:#333333;}#mermaid-svg-o93WxEckQEVtfPdy svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-o93WxEckQEVtfPdy p{margin:0;}#mermaid-svg-o93WxEckQEVtfPdy .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-o93WxEckQEVtfPdy .cluster-label text{fill:#333;}#mermaid-svg-o93WxEckQEVtfPdy .cluster-label span{color:#333;}#mermaid-svg-o93WxEckQEVtfPdy .cluster-label span p{background-color:transparent;}#mermaid-svg-o93WxEckQEVtfPdy .label text,#mermaid-svg-o93WxEckQEVtfPdy span{fill:#333;color:#333;}#mermaid-svg-o93WxEckQEVtfPdy .node rect,#mermaid-svg-o93WxEckQEVtfPdy .node circle,#mermaid-svg-o93WxEckQEVtfPdy .node ellipse,#mermaid-svg-o93WxEckQEVtfPdy .node polygon,#mermaid-svg-o93WxEckQEVtfPdy .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-o93WxEckQEVtfPdy .rough-node .label text,#mermaid-svg-o93WxEckQEVtfPdy .node .label text,#mermaid-svg-o93WxEckQEVtfPdy .image-shape .label,#mermaid-svg-o93WxEckQEVtfPdy .icon-shape .label{text-anchor:middle;}#mermaid-svg-o93WxEckQEVtfPdy .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-o93WxEckQEVtfPdy .rough-node .label,#mermaid-svg-o93WxEckQEVtfPdy .node .label,#mermaid-svg-o93WxEckQEVtfPdy .image-shape .label,#mermaid-svg-o93WxEckQEVtfPdy .icon-shape .label{text-align:center;}#mermaid-svg-o93WxEckQEVtfPdy .node.clickable{cursor:pointer;}#mermaid-svg-o93WxEckQEVtfPdy .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-o93WxEckQEVtfPdy .arrowheadPath{fill:#333333;}#mermaid-svg-o93WxEckQEVtfPdy .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-o93WxEckQEVtfPdy .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-o93WxEckQEVtfPdy .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-o93WxEckQEVtfPdy .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-o93WxEckQEVtfPdy .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-o93WxEckQEVtfPdy .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-o93WxEckQEVtfPdy .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-o93WxEckQEVtfPdy .cluster text{fill:#333;}#mermaid-svg-o93WxEckQEVtfPdy .cluster span{color:#333;}#mermaid-svg-o93WxEckQEVtfPdy div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-o93WxEckQEVtfPdy .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-o93WxEckQEVtfPdy rect.text{fill:none;stroke-width:0;}#mermaid-svg-o93WxEckQEVtfPdy .icon-shape,#mermaid-svg-o93WxEckQEVtfPdy .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-o93WxEckQEVtfPdy .icon-shape p,#mermaid-svg-o93WxEckQEVtfPdy .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-o93WxEckQEVtfPdy .icon-shape .label rect,#mermaid-svg-o93WxEckQEVtfPdy .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-o93WxEckQEVtfPdy .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-o93WxEckQEVtfPdy .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-o93WxEckQEVtfPdy :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-o93WxEckQEVtfPdy .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-o93WxEckQEVtfPdy .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-o93WxEckQEVtfPdy .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u5e94\u7528\u5c42<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5b58\u50a8\u5c42<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u91c7\u96c6\u5c42<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u662f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5426<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5468\u671f\u5de1\u68c0<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8fdb\u7a0b Exporter<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8282\u70b9 Exporter<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    GPU Exporter<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u81ea\u5b9a\u4e49\u6307\u6807<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Prometheus \u4e3b<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Prometheus \u5907<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Elasticsearch \u4e09\u8282\u70b9<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Alertmanager<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    Grafana<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u56de\u8c03\u901a\u9053<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    SLO \u662f\u5426\u5065\u5eb7<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u7ee7\u7eed\u89c2\u5bdf<\/p>\n

    <\/span><\/p>\n

    6.1 \u5e73\u53f0\u67b6\u6784<\/h4>\n

    \u5e73\u53f0\u7684\u91c7\u96c6\u5c42\u6309\u6307\u6807\u6765\u6e90\u5212\u5206 exporter\u3002\u8fdb\u7a0b exporter \u66b4\u9732\u63a8\u7406\u5f15\u64ce\u81ea\u8eab\u7684\u6307\u6807（\u961f\u5217\u957f\u5ea6\u3001\u6279\u5927\u5c0f\u3001KV Cache \u4f7f\u7528），\u8282\u70b9 exporter \u66b4\u9732\u4e3b\u673a\u6307\u6807（CPU\u3001\u5185\u5b58\u3001\u7f51\u7edc），GPU exporter \u66b4\u9732\u663e\u5b58\u4e0e\u8ba1\u7b97\u5229\u7528\u7387，\u81ea\u5b9a\u4e49 exporter \u66b4\u9732\u4e1a\u52a1\u6307\u6807（SLO \u8fbe\u6210\u7387\u3001PSI \u504f\u79fb）\u3002exporter \u9075\u5faa\u4e00\u4e2a\u51fa\u53e3\u539f\u5219：\u6bcf\u4e2a\u8fdb\u7a0b\u53ea\u66b4\u9732\u4e00\u4e2a \/metrics \u7aef\u70b9，\u5168\u90e8\u6307\u6807\u5728\u540c\u4e00\u7aef\u53e3\u805a\u5408，Prometheus \u7684 scrape \u914d\u7f6e\u968f\u4e4b\u7b80\u5316\u3002<\/p>\n

    \u5b58\u50a8\u5c42\u7684\u53ef\u7528\u6027\u8bbe\u8ba1\u51b3\u5b9a\u6545\u969c\u6062\u590d\u80fd\u529b\u3002Prometheus \u4ee5\u4e24\u8282\u70b9\u5e76\u884c\u6293\u53d6\u76f8\u540c\u76ee\u6807，\u4e3b\u8282\u70b9\u6545\u969c\u65f6\u5907\u8282\u70b9\u63a5\u7ba1\u67e5\u8be2，\u4e24\u8282\u70b9\u95f4\u901a\u8fc7\u8fdc\u7aef\u5199\u5165\u4e0e\u5bf9\u8c61\u5b58\u50a8\u5f52\u6863\u5171\u4eab\u5386\u53f2\u3002Prometheus \u65f6\u5e8f\u5e93\u7684\u9ed8\u8ba4\u4fdd\u7559\u671f\u8bbe\u4e3a 15 \u5929，\u8d85\u8fc7 15 \u5929\u7684\u6570\u636e\u901a\u8fc7 thanos \u6216 victoriametrics \u4fa7\u5199\u5230\u5bf9\u8c61\u5b58\u50a8，\u4fdd\u7559 180 \u5929\u7528\u4e8e\u6708\u5ea6\u8d8b\u52bf\u5206\u6790\u3002ES \u4e09\u8282\u70b9\u6309\u4e3b\u4ece\u5206\u7247\u627f\u8f7d\u65e5\u5fd7，\u4e3b\u8282\u70b9\u6545\u969c\u65f6\u4ece\u5206\u7247\u63d0\u5347，\u5199\u5165\u4e0d\u505c\u3002<\/p>\n

    \u544a\u8b66\u94fe\u8def\u7684\u9ad8\u53ef\u7528\u662f\u5e73\u53f0\u67b6\u6784\u91cc\u6700\u5bb9\u6613\u88ab\u5ffd\u89c6\u7684\u4e00\u73af\u3002Alertmanager \u53cc\u526f\u672c\u4ee5 gossip \u534f\u8bae\u5171\u4eab\u544a\u8b66\u72b6\u6001，\u5355\u526f\u672c\u6545\u969c\u4e0d\u4e22\u544a\u8b66；Prometheus \u7684 rule_files \u91cc\u52a0\u8f7d\u544a\u8b66\u89c4\u5219，\u8bc4\u4f30\u95f4\u9694\u4e0e\u6293\u53d6\u95f4\u9694\u5bf9\u9f50\u4e3a 15 \u79d2\u3002\u94fe\u8def\u7684\u6bcf\u4e00\u8df3\u90fd\u8981\u6709\u81ea\u8eab\u7684\u5b58\u6d3b\u6307\u6807：Prometheus \u7684\u4e0a\u6e38\u6293\u53d6\u5931\u8d25\u7387\u3001Alertmanager \u7684\u901a\u77e5\u5931\u8d25\u7387\u3001ES \u7684\u5199\u5165\u5ef6\u8fdf，\u8fd9\u4e9b\u5143\u6307\u6807\u5728\u72ec\u7acb\u7684 dashboards \u4e0a\u5448\u73b0，\u5426\u5219\u76d1\u63a7\u5e73\u53f0\u81ea\u8eab\u4f1a\u6210\u4e3a\u6700\u5927\u7684\u76f2\u533a\u3002<\/p>\n

    \u5e73\u53f0\u67b6\u6784\u7684\u53e6\u4e00\u4e2a\u7ef4\u5ea6\u662f\u5b89\u5168\u4e0e\u8bbf\u95ee\u63a7\u5236\u3002Grafana \u4e0e Kibana \u7684\u67e5\u770b\u6743\u9650\u6309\u56e2\u961f\u5212\u5206：SRE \u62e5\u6709\u5168\u90e8\u770b\u677f\u7f16\u8f91\u6743，\u7b97\u6cd5\u56e2\u961f\u53ea\u6709\u672c\u6a21\u578b\u76f8\u5173\u770b\u677f\u7684\u53ea\u8bfb\u6743\u9650，\u503c\u73ed\u4eba\u5458\u6709\u544a\u8b66\u786e\u8ba4\u6743\u4f46\u65e0\u9608\u503c\u4fee\u6539\u6743\u3002Prometheus \u7684\u67e5\u8be2 API \u4e0e ES \u7684\u8bfb\u53d6\u90fd\u8981\u8d70\u7edf\u4e00\u7f51\u5173，\u5ba1\u8ba1\u65e5\u5fd7\u8bb0\u5f55\u6bcf\u4e00\u6b21\u6570\u636e\u8bbf\u95ee，\u914d\u5408 4.1 \u8282\u7684 PII \u8131\u654f\u7b56\u7565，\u6ee1\u8db3\u5408\u89c4\u5ba1\u8ba1\u8981\u6c42\u3002\u6743\u9650\u7684\u6700\u5c0f\u5316\u4e0d\u662f\u6d41\u7a0b\u5197\u4f59，\u800c\u662f\u9632\u6b62\u503c\u73ed\u4eba\u5458\u5728\u6545\u969c\u73b0\u573a\u8bef\u6539\u9608\u503c\u5bfc\u81f4\u6f0f\u62a5\u7684\u515c\u5e95\u624b\u6bb5\u3002<\/p>\n

    # \u6765\u6e90：Prometheus 2.51 \/ prometheus.yml<\/span>
    \nglobal<\/span>:<\/span>
    \n scrape_interval<\/span>:<\/span> 15s
    \n evaluation_interval<\/span>:<\/span> 15s<\/p>\n

    rule_files<\/span>:<\/span>
    \n –<\/span> \/etc\/prometheus\/rules\/*.yml<\/span><\/p>\n

    scrape_configs<\/span>:<\/span>
    \n –<\/span> job_name<\/span>:<\/span> "llm-serving"<\/span>
    \n static_configs<\/span>:<\/span>
    \n –<\/span> targets<\/span>:<\/span> [<\/span>"serving-1:9101"<\/span>,<\/span> "serving-2:9101"<\/span>]<\/span>
    \n –<\/span> job_name<\/span>:<\/span> "gpu-exporter"<\/span>
    \n static_configs<\/span>:<\/span>
    \n –<\/span> targets<\/span>:<\/span> [<\/span>"gpu-1:9400"<\/span>,<\/span> "gpu-2:9400"<\/span>,<\/span> "gpu-3:9400"<\/span>]<\/span>
    \n –<\/span> job_name<\/span>:<\/span> "alertmanager"<\/span>
    \n static_configs<\/span>:<\/span>
    \n –<\/span> targets<\/span>:<\/span> [<\/span>"alertmanager-1:9093"<\/span>,<\/span> "alertmanager-2:9093"<\/span>]<\/span><\/p>\n

    # \u6765\u6e90：PyYAML 6.0 \/ scrape_validator.py<\/span>
    \nimport<\/span> yaml<\/p>\n

    # \u6f14\u793a\u7528\u6700\u5c0f\u914d\u7f6e；\u751f\u4ea7\u73af\u5883\u8bfb\u53d6\u7248\u672c\u5316\u7684 prometheus.yml<\/span>
    \n_EXAMPLE =<\/span> """
    \nglobal:
    \n scrape_interval: 15s
    \nscrape_configs:
    \n – job_name: llm-serving
    \n static_configs:
    \n – targets: ["serving-1:9101"]
    \n"""<\/span><\/p>\n

    def<\/span> validate<\/span>(<\/span>path)<\/span>:<\/span>
    \n """\u6821\u9a8c\u6293\u53d6\u914d\u7f6e：\u68c0\u67e5\u5168\u5c40\u95f4\u9694\u4e0e\u5404 job \u662f\u5426\u6709 target\u3002"""<\/span>
    \n try<\/span>:<\/span>
    \n with<\/span> open<\/span>(<\/span>path,<\/span> encoding=<\/span>"utf-8"<\/span>)<\/span> as<\/span> f:<\/span>
    \n cfg =<\/span> yaml.<\/span>safe_load(<\/span>f)<\/span>
    \n except<\/span> FileNotFoundError:<\/span>
    \n cfg =<\/span> yaml.<\/span>safe_load(<\/span>_EXAMPLE)<\/span>
    \n interval =<\/span> cfg[<\/span>"global"<\/span>]<\/span>[<\/span>"scrape_interval"<\/span>]<\/span>
    \n missing =<\/span> [<\/span>]<\/span>
    \n for<\/span> job in<\/span> cfg[<\/span>"scrape_configs"<\/span>]<\/span>:<\/span>
    \n if<\/span> not<\/span> job.<\/span>get(<\/span>"static_configs"<\/span>)<\/span>:<\/span>
    \n missing.<\/span>append(<\/span>job[<\/span>"job_name"<\/span>]<\/span>)<\/span>
    \n return<\/span> interval,<\/span> missing<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n interval,<\/span> missing =<\/span> validate(<\/span>"prometheus.yml"<\/span>)<\/span>
    \n print<\/span>(<\/span>f"\u5168\u5c40\u6293\u53d6\u95f4\u9694 <\/span>{<\/span>interval}<\/span><\/span>，\u7f3a\u5c11 target \u7684 job: <\/span>{<\/span>missing or<\/span> '\u65e0'<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

    6.2 \u770b\u677f\u8bbe\u8ba1<\/h4>\n

    \u770b\u677f\u8bbe\u8ba1\u7684\u539f\u5219\u662f"\u4e00\u5c4f\u56de\u7b54\u4e00\u4e2a\u95ee\u9898"\u3002\u9876\u5c42\u770b\u677f\u56de\u7b54"SLO \u662f\u5426\u5065\u5eb7"：\u8bf7\u6c42\u6210\u529f\u7387\u3001\u9519\u8bef\u9884\u7b97\u6d88\u8017\u3001P99 \u5ef6\u8fdf\u4e09\u5757\u5927\u56fe\u5e76\u6392，\u7ea2\u7eff\u9608\u503c\u4e00\u76ee\u4e86\u7136\u3002\u7b2c\u4e8c\u5c42\u770b\u677f\u56de\u7b54"\u670d\u52a1\u8dd1\u5f97\u5982\u4f55"：\u5ef6\u8fdf\u70ed\u529b\u56fe\u3001\u541e\u5410\u3001\u5e76\u53d1\u3001\u961f\u5217\u957f\u5ea6\u56db\u5757，\u7528\u4e8e\u503c\u73ed\u5de1\u68c0\u3002\u7b2c\u4e09\u5c42\u770b\u677f\u56de\u7b54"\u8d44\u6e90\u662f\u5426\u5145\u8db3"：GPU \u5229\u7528\u7387\u3001\u663e\u5b58\u3001KV Cache\u3001\u6279\u5927\u5c0f\u56db\u5757，\u7528\u4e8e\u5bb9\u91cf\u89c4\u5212\u3002\u4e09\u5c42\u770b\u677f\u4e4b\u95f4\u7684\u8df3\u8f6c\u9760 URL \u53d8\u91cf\u8d2f\u901a：\u9876\u5c42\u70b9\u51fb\u5b9e\u4f8b\u8fdb\u5165\u7b2c\u4e8c\u5c42，\u7b2c\u4e8c\u5c42\u70b9\u51fb\u5b9e\u4f8b\u8fdb\u5165\u7b2c\u4e09\u5c42\u3002<\/p>\n

    \u5ef6\u8fdf\u770b\u677f\u5fc5\u987b\u7528\u76f4\u65b9\u56fe\u70ed\u529b\u56fe\u800c\u975e\u5355\u6761\u6298\u7ebf\u3002\u5355\u6761 P99 \u6298\u7ebf\u65e0\u6cd5\u56de\u7b54"\u957f\u5c3e\u5206\u5e03\u7684\u5f62\u72b6\u5982\u4f55"，\u70ed\u529b\u56fe\u6309\u65f6\u95f4\u4e0e\u5ef6\u8fdf\u6876\u4e24\u4e2a\u7ef4\u5ea6\u7740\u8272，\u80fd\u76f4\u63a5\u770b\u51fa\u6bdb\u523a\u662f\u96c6\u4e2d\u5728\u67d0\u4e2a\u6876\u8fd8\u662f\u6574\u4f53\u53f3\u79fb\u3002PromQL \u7684\u5b9e\u73b0\u662f histogram_quantile \u914d\u5408 sum(rate(bucket)) \u805a\u5408，\u67e5\u8be2\u95f4\u9694\u7528 Grafana \u7684 $__rate_interval \u53d8\u91cf\u5bf9\u9f50\u6293\u53d6\u5468\u671f，\u907f\u514d\u901f\u7387\u7a97\u53e3\u4e0e\u6293\u53d6\u95f4\u9694\u4e0d\u5339\u914d\u5bfc\u81f4\u7684\u952f\u9f7f\u3002<\/p>\n

    \u770b\u677f\u8981\u5904\u7406\u591a\u6a21\u578b\u7248\u672c\u5e76\u5b58\u7684\u53ef\u6bd4\u6027\u95ee\u9898\u3002\u7ebf\u4e0a\u540c\u65f6\u8fd0\u884c v36 \u4e0e v37 \u4e24\u4e2a\u7248\u672c\u505a\u7070\u5ea6\u65f6，\u770b\u677f\u5fc5\u987b\u6309\u7248\u672c\u5206\u9762\u677f，\u624d\u80fd\u5bf9\u6bd4\u4e24\u7248\u672c\u7684\u5ef6\u8fdf\u4e0e\u9519\u8bef\u7387\u5dee\u5f02\u3002PromQL \u7684 by (model_version) \u805a\u5408\u52a0\u4e0a Grafana \u7684 legend \u5206\u7ec4\u5373\u53ef\u5b9e\u73b0\u3002\u770b\u677f\u81ea\u8eab\u7684\u6cbb\u7406\u4e5f\u91cd\u8981：\u9762\u677f\u6570\u91cf\u8d85\u8fc7 100 \u65f6\u67e5\u8be2\u5e76\u53d1\u4f1a\u62d6\u6162\u52a0\u8f7d，\u8d85\u671f\u672a\u4f7f\u7528\u7684\u9762\u677f\u5f52\u6863，\u547d\u540d\u5e26\u8d1f\u8d23\u4eba\u6807\u7b7e，\u770b\u677f\u53d8\u66f4\u8d70 Git \u8bc4\u5ba1\u800c\u4e0d\u662f\u5728\u754c\u9762\u4e0a\u76f4\u63a5\u6539\u3002<\/p>\n

    # \u6765\u6e90：Grafana 10.2 \u9762\u677f JSON \/ latency_panel.py<\/span>
    \nPANEL =<\/span> {<\/span>
    \n "title"<\/span>:<\/span> "P99 \u5ef6\u8fdf"<\/span>,<\/span>
    \n "type"<\/span>:<\/span> "timeseries"<\/span>,<\/span>
    \n "targets"<\/span>:<\/span> [<\/span>
    \n {<\/span>
    \n "expr"<\/span>:<\/span> (<\/span>
    \n "histogram_quantile(0.99, "<\/span>
    \n "sum(rate(llm_ttft_seconds_bucket[$__rate_interval])) by (le))"<\/span>
    \n )<\/span>
    \n }<\/span>
    \n ]<\/span>,<\/span>
    \n "fieldConfig"<\/span>:<\/span> {<\/span>
    \n "defaults"<\/span>:<\/span> {<\/span>
    \n "unit"<\/span>:<\/span> "s"<\/span>,<\/span>
    \n "thresholds"<\/span>:<\/span> {<\/span>
    \n "steps"<\/span>:<\/span> [<\/span>
    \n {<\/span>"value"<\/span>:<\/span> 0<\/span>,<\/span> "color"<\/span>:<\/span> "green"<\/span>}<\/span>,<\/span>
    \n {<\/span>"value"<\/span>:<\/span> 0.5<\/span>,<\/span> "color"<\/span>:<\/span> "red"<\/span>}<\/span>,<\/span>
    \n ]<\/span>
    \n }<\/span>,<\/span>
    \n }<\/span>
    \n }<\/span>,<\/span>
    \n}<\/span><\/p>\n

    def<\/span> build_promql<\/span>(<\/span>metric,<\/span> quantiles=<\/span>(<\/span>0.5<\/span>,<\/span> 0.9<\/span>,<\/span> 0.99<\/span>)<\/span>)<\/span>:<\/span>
    \n """\u4e3a\u7ed9\u5b9a\u76f4\u65b9\u56fe\u6307\u6807\u751f\u6210\u591a\u6761\u5206\u4f4d\u7ebf\u67e5\u8be2\u3002"""<\/span>
    \n exprs =<\/span> [<\/span>]<\/span>
    \n for<\/span> q in<\/span> quantiles:<\/span>
    \n exprs.<\/span>append(<\/span>
    \n f"histogram_quantile(<\/span>{<\/span>q}<\/span><\/span>, "<\/span><\/span>
    \n f"sum(rate(<\/span>{<\/span>metric}<\/span><\/span>_bucket[$__rate_interval])) by (le))"<\/span><\/span>
    \n )<\/span>
    \n return<\/span> exprs<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n print<\/span>(<\/span>"\u9ed8\u8ba4\u9762\u677f\u67e5\u8be2:"<\/span>,<\/span> PANEL[<\/span>"targets"<\/span>]<\/span>[<\/span>0<\/span>]<\/span>[<\/span>"expr"<\/span>]<\/span>)<\/span>
    \n print<\/span>(<\/span>"\u6269\u5c55\u5206\u4f4d\u7ebf:"<\/span>)<\/span>
    \n for<\/span> expr in<\/span> build_promql(<\/span>"llm_ttft_seconds"<\/span>)<\/span>:<\/span>
    \n print<\/span>(<\/span>" "<\/span>,<\/span> expr)<\/span><\/p>\n

    6.3 \u6570\u636e\u5b58\u50a8<\/h4>\n

    \u6570\u636e\u5b58\u50a8\u7684\u5206\u5de5\u662f：\u65f6\u5e8f\u6570\u636e\u8fdb\u65f6\u5e8f\u5e93，\u65e5\u5fd7\u6570\u636e\u8fdb\u5168\u6587\u7d22\u5f15\u5e93，\u4e24\u79cd\u5f15\u64ce\u7684\u9009\u578b\u4f9d\u636e\u662f\u8bbf\u95ee\u6a21\u5f0f\u3002\u65f6\u5e8f\u5e93\u7684\u8bbf\u95ee\u6a21\u5f0f\u662f"\u6309\u65f6\u95f4\u8303\u56f4\u805a\u5408\u5206\u4f4d\u6570"，Prometheus TSDB \u7684\u5206\u5757\u538b\u7f29\u4e0e\u5012\u6392\u7d22\u5f15\u4e3a\u6b64\u4f18\u5316；\u65e5\u5fd7\u5e93\u7684\u8bbf\u95ee\u6a21\u5f0f\u662f"\u6309\u5b57\u6bb5\u8fc7\u6ee4\u68c0\u7d22\u539f\u6587"，Elasticsearch \u7684\u5012\u6392\u7d22\u5f15\u4e3a\u6b64\u4f18\u5316\u3002\u628a\u65e5\u5fd7\u585e\u8fdb\u65f6\u5e8f\u5e93\u6216\u628a\u6307\u6807\u585e\u8fdb\u65e5\u5fd7\u5e93\u90fd\u4f1a\u5728\u67e5\u8be2\u6027\u80fd\u4e0e\u5b58\u50a8\u6210\u672c\u4e0a\u540c\u65f6\u5403\u4e8f\u3002<\/p>\n

    \u65f6\u5e8f\u5b58\u50a8\u7684\u6210\u672c\u6a21\u578b\u53ef\u4ee5\u7cbe\u786e\u4f30\u7b97\u3002\u5355\u6761\u5e8f\u5217\u4ee5 15 \u79d2\u91c7\u6837，TSDB \u538b\u7f29\u540e\u7ea6 1.5 \u5b57\u8282\u6bcf\u6837\u672c，\u6bcf\u5929 8.6KB；3000 \u6761\u6d3b\u8dc3\u5e8f\u5217\u6bcf\u5929\u7ea6 26MB，15 \u5929\u4fdd\u7559\u671f\u7ea6 390MB，Prometheus \u5355\u8282\u70b9\u7ef0\u7ef0\u6709\u4f59\u3002\u4f46\u76f4\u65b9\u56fe\u6307\u6807\u653e\u5927 9 \u500d（7 \u6876\u52a0 count \u52a0 sum），\u4e14\u591a\u526f\u672c\u544a\u8b66\u7b49\u90fd\u4f1a\u8ba9\u5e8f\u5217\u6570\u81a8\u80c0\u3002\u5e8f\u5217\u6570\u8d85\u8fc7 10 \u4e07\u65f6，\u5185\u5b58\u7d22\u5f15\u4ece 100MB \u91cf\u7ea7\u6da8\u5230 5GB，\u67e5\u8be2\u4ece\u6beb\u79d2\u7ea7\u9000\u5316\u5230\u79d2\u7ea7，\u6b64\u65f6\u5fc5\u987b\u5f15\u5165 VictoriaMetrics \u7684\u4e0b\u91c7\u6837\u6216 Thanos \u7684\u5782\u76f4\u5206\u533a\u3002<\/p>\n

    \u65e5\u5fd7\u5b58\u50a8\u7684\u5bb9\u91cf\u6309 4.1 \u8282\u7684\u63a8\u7b97\u6267\u884c：100 QPS\u3001\u5355\u6761 600 \u5b57\u8282，\u6bcf\u5929 5.2GB，90 \u5929\u4e09\u526f\u672c\u7ea6 1.4TB\u3002ES \u7d22\u5f15\u751f\u547d\u5468\u671f\u7ba1\u7406\u5206\u56db\u9636\u6bb5：\u70ed 7 \u5929 SSD\u3001\u6696 8-30 \u5929 HDD\u3001\u51b7 31-90 \u5929\u5feb\u7167\u5230\u5bf9\u8c61\u5b58\u50a8\u300190 \u5929\u5220\u9664\u3002\u51b7\u9636\u6bb5\u7684\u68c0\u7d22\u4ece\u6beb\u79d2\u9000\u5316\u5230\u79d2\u7ea7，\u4f46\u6210\u672c\u964d\u5230\u539f\u6765\u7684 20%\u3002\u65f6\u5e8f\u4e0e\u65e5\u5fd7\u7684\u4fdd\u7559\u671f\u5dee\u5f02\u8981\u5199\u8fdb\u8fd0\u7ef4\u624b\u518c：\u5206\u6790\u5e08\u67e5 60 \u5929\u524d\u7684\u5ef6\u8fdf\u8981\u7528\u65e5\u5fd7\u51b7\u6570\u636e，\u67e5 60 \u5929\u524d\u7684\u65f6\u5e8f\u5206\u4f4d\u6570\u5219\u76f4\u63a5\u5931\u6548，\u56e0\u4e3a\u65f6\u5e8f\u53ea\u4fdd\u7559 15 \u5929\u3002<\/p>\n

    {<\/span>
    \n "policy"<\/span>:<\/span> "llm-log-policy"<\/span>,<\/span>
    \n "phases"<\/span>:<\/span> {<\/span>
    \n "hot"<\/span>:<\/span> {<\/span>
    \n "min_age"<\/span>:<\/span> "0ms"<\/span>,<\/span>
    \n "actions"<\/span>:<\/span> {<\/span>
    \n "rollover"<\/span>:<\/span> {<\/span> "max_size"<\/span>:<\/span> "50gb"<\/span>,<\/span> "max_age"<\/span>:<\/span> "1d"<\/span> }<\/span>
    \n }<\/span>
    \n }<\/span>,<\/span>
    \n "warm"<\/span>:<\/span> {<\/span>
    \n "min_age"<\/span>:<\/span> "7d"<\/span>,<\/span>
    \n "actions"<\/span>:<\/span> {<\/span>
    \n "forcemerge"<\/span>:<\/span> {<\/span> "max_num_segments"<\/span>:<\/span> 1<\/span> }<\/span>
    \n }<\/span>
    \n }<\/span>,<\/span>
    \n "cold"<\/span>:<\/span> {<\/span>
    \n "min_age"<\/span>:<\/span> "30d"<\/span>,<\/span>
    \n "actions"<\/span>:<\/span> {<\/span>
    \n "searchable_snapshot"<\/span>:<\/span> {<\/span> "snapshot_repository"<\/span>:<\/span> "backup"<\/span> }<\/span>
    \n }<\/span>
    \n }<\/span>,<\/span>
    \n "delete"<\/span>:<\/span> {<\/span>
    \n "min_age"<\/span>:<\/span> "90d"<\/span>,<\/span>
    \n "actions"<\/span>:<\/span> {<\/span> "delete"<\/span>:<\/span> {<\/span>}<\/span> }<\/span>
    \n }<\/span>
    \n }<\/span>
    \n}<\/span><\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ storage_budget.py<\/span>
    \ndef<\/span> storage_budget<\/span>(<\/span>req_per_s,<\/span> bytes_per_log,<\/span> retention_days=<\/span>90<\/span>,<\/span> redundancy=<\/span>1.1<\/span>)<\/span>:<\/span>
    \n """\u6309\u8bf7\u6c42\u91cf\u4e0e\u5355\u6761\u65e5\u5fd7\u5927\u5c0f\u63a8\u7b97\u5b58\u50a8\u6210\u672c\u3002"""<\/span>
    \n per_day =<\/span> req_per_s *<\/span> bytes_per_log *<\/span> 86400<\/span> \/<\/span> (<\/span>1024<\/span> **<\/span> 3<\/span>)<\/span>
    \n total =<\/span> per_day *<\/span> retention_days *<\/span> redundancy
    \n return<\/span> per_day,<\/span> total<\/p>\n

    def<\/span> series_budget<\/span>(<\/span>series_count,<\/span> scrape_interval_s=<\/span>15<\/span>,<\/span> bytes_per_sample=<\/span>1.5<\/span>,<\/span> retention_days=<\/span>15<\/span>)<\/span>:<\/span>
    \n """\u6309\u65f6\u5e8f\u5e8f\u5217\u6570\u4e0e\u91c7\u6837\u95f4\u9694\u63a8\u7b97\u78c1\u76d8\u5360\u7528\u3002"""<\/span>
    \n samples =<\/span> series_count *<\/span> (<\/span>86400<\/span> \/<\/span> scrape_interval_s)<\/span>
    \n per_day =<\/span> samples *<\/span> bytes_per_sample \/<\/span> (<\/span>1024<\/span> **<\/span> 2<\/span>)<\/span>
    \n return<\/span> per_day,<\/span> per_day *<\/span> retention_days<\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n log_day,<\/span> log_total =<\/span> storage_budget(<\/span>req_per_s=<\/span>100<\/span>,<\/span> bytes_per_log=<\/span>600<\/span>)<\/span>
    \n metric_day,<\/span> metric_total =<\/span> series_budget(<\/span>series_count=<\/span>3000<\/span>)<\/span>
    \n print<\/span>(<\/span>f"\u65e5\u5fd7: \u6bcf\u5929 <\/span>{<\/span>log_day:<\/span>.1f<\/span>}<\/span><\/span> GB，90 \u5929\u4e09\u526f\u672c\u7ea6 <\/span>{<\/span>log_total:<\/span>.1f<\/span>}<\/span><\/span> GB"<\/span><\/span>)<\/span>
    \n print<\/span>(<\/span>f"\u65f6\u5e8f: \u6bcf\u5929 <\/span>{<\/span>metric_day:<\/span>.1f<\/span>}<\/span><\/span> MB，15 \u5929\u7ea6 <\/span>{<\/span>metric_total:<\/span>.1f<\/span>}<\/span><\/span> MB"<\/span><\/span>)<\/span><\/p>\n

    7. \u5b9e\u8df5\u5efa\u8bae\u4e0e\u6700\u4f73\u5b9e\u8df5<\/h3>\n

    \u76d1\u63a7\u4f53\u7cfb\u7684\u6210\u8d25\u4e0d\u5728\u5de5\u5177\u9009\u578b，\u800c\u5728\u6d41\u7a0b\u4e0e\u6cbb\u7406\u3002\u5de5\u5177\u53ea\u89e3\u51b3"\u80fd\u4e0d\u80fd\u91c7\u5230\u3001\u80fd\u4e0d\u80fd\u770b\u5230"，\u6d41\u7a0b\u89e3\u51b3"\u91c7\u5230\u4ec0\u4e48\u7b97\u6570\u3001\u544a\u8b66\u7531\u8c01\u54cd\u5e94\u3001\u574f\u4e86\u8c01\u4fee"，\u6cbb\u7406\u89e3\u51b3"\u6307\u6807\u4e0d\u81a8\u80c0\u3001\u544a\u8b66\u4e0d\u5931\u63a7\u3001\u6570\u636e\u4e0d\u51fa\u683c"\u3002\u672c\u7ae0\u628a\u524d\u516d\u7ae0\u6c89\u6dc0\u4e3a\u4e00\u5957\u53ef\u6267\u884c\u7684\u843d\u5730\u65b9\u6cd5：\u5148\u5b9a\u4e49\u4e0a\u7ebf\u6d41\u7a0b，\u518d\u5efa\u7acb\u6cbb\u7406\u89c4\u5219，\u6700\u540e\u7528\u6301\u7eed\u6539\u8fdb\u95ed\u73af\u8ba9\u76d1\u63a7\u4f53\u7cfb\u81ea\u8eab\u4fdd\u6301\u5065\u5eb7\u3002<\/p>\n

    \u843d\u5730\u7684\u987a\u5e8f\u4e5f\u6709\u8bb2\u7a76\u3002\u6b63\u786e\u7684\u987a\u5e8f\u662f\u5148\u5b9a SLO \u518d\u57cb\u70b9，\u5148\u5efa\u770b\u677f\u518d\u914d\u544a\u8b66，\u5148\u914d\u8def\u7531\u518d\u8c03\u9608\u503c\u3002\u53cd\u8fc7\u6765\u7684\u987a\u5e8f（\u5148\u914d\u544a\u8b66\u518d\u8865 SLO）\u4f1a\u5728\u4e00\u4e2a\u6708\u540e\u4ea7\u751f\u5927\u91cf\u65e0\u6cd5\u5bf9\u8d26\u5230\u76ee\u6807\u7684\u544a\u8b66\u89c4\u5219，\u5c4a\u65f6\u5220\u9664\u89c4\u5219\u6bd4\u65b0\u589e\u89c4\u5219\u96be\u5f97\u591a\u3002\u6700\u4f73\u5b9e\u8df5\u7684\u68c0\u9a8c\u6807\u51c6\u662f：\u4efb\u610f\u4e00\u6761\u544a\u8b66，\u503c\u73ed\u4eba\u5458\u90fd\u80fd\u5728 5 \u5206\u949f\u5185\u56de\u7b54\u4e09\u4e2a\u95ee\u9898\u2014\u2014\u89e6\u53d1\u6761\u4ef6\u662f\u4ec0\u4e48\u3001\u5f71\u54cd\u8303\u56f4\u591a\u5927\u3001\u6b62\u635f\u52a8\u4f5c\u5728\u54ea\u4efd\u6587\u6863\u91cc\u3002<\/p>\n

    #mermaid-svg-6JKV9qv5LG0fQbjY{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-6JKV9qv5LG0fQbjY .error-icon{fill:#552222;}#mermaid-svg-6JKV9qv5LG0fQbjY .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-6JKV9qv5LG0fQbjY .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-6JKV9qv5LG0fQbjY .marker{fill:#333333;stroke:#333333;}#mermaid-svg-6JKV9qv5LG0fQbjY .marker.cross{stroke:#333333;}#mermaid-svg-6JKV9qv5LG0fQbjY svg{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-6JKV9qv5LG0fQbjY p{margin:0;}#mermaid-svg-6JKV9qv5LG0fQbjY .label{font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;color:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster-label text{fill:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster-label span{color:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster-label span p{background-color:transparent;}#mermaid-svg-6JKV9qv5LG0fQbjY .label text,#mermaid-svg-6JKV9qv5LG0fQbjY span{fill:#333;color:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY .node rect,#mermaid-svg-6JKV9qv5LG0fQbjY .node circle,#mermaid-svg-6JKV9qv5LG0fQbjY .node ellipse,#mermaid-svg-6JKV9qv5LG0fQbjY .node polygon,#mermaid-svg-6JKV9qv5LG0fQbjY .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-6JKV9qv5LG0fQbjY .rough-node .label text,#mermaid-svg-6JKV9qv5LG0fQbjY .node .label text,#mermaid-svg-6JKV9qv5LG0fQbjY .image-shape .label,#mermaid-svg-6JKV9qv5LG0fQbjY .icon-shape .label{text-anchor:middle;}#mermaid-svg-6JKV9qv5LG0fQbjY .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-6JKV9qv5LG0fQbjY .rough-node .label,#mermaid-svg-6JKV9qv5LG0fQbjY .node .label,#mermaid-svg-6JKV9qv5LG0fQbjY .image-shape .label,#mermaid-svg-6JKV9qv5LG0fQbjY .icon-shape .label{text-align:center;}#mermaid-svg-6JKV9qv5LG0fQbjY .node.clickable{cursor:pointer;}#mermaid-svg-6JKV9qv5LG0fQbjY .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-6JKV9qv5LG0fQbjY .arrowheadPath{fill:#333333;}#mermaid-svg-6JKV9qv5LG0fQbjY .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-6JKV9qv5LG0fQbjY .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-6JKV9qv5LG0fQbjY .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-6JKV9qv5LG0fQbjY .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-6JKV9qv5LG0fQbjY .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-6JKV9qv5LG0fQbjY .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster text{fill:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY .cluster span{color:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-6JKV9qv5LG0fQbjY .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-6JKV9qv5LG0fQbjY rect.text{fill:none;stroke-width:0;}#mermaid-svg-6JKV9qv5LG0fQbjY .icon-shape,#mermaid-svg-6JKV9qv5LG0fQbjY .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-6JKV9qv5LG0fQbjY .icon-shape p,#mermaid-svg-6JKV9qv5LG0fQbjY .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-6JKV9qv5LG0fQbjY .icon-shape .label rect,#mermaid-svg-6JKV9qv5LG0fQbjY .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-6JKV9qv5LG0fQbjY .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-6JKV9qv5LG0fQbjY .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-6JKV9qv5LG0fQbjY :root{–mermaid-font-family:\\”trebuchet ms\\”,verdana,arial,sans-serif;}#mermaid-svg-6JKV9qv5LG0fQbjY .default>*{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-6JKV9qv5LG0fQbjY .default span{fill:#faf9f5!important;stroke:#ffffff!important;color:#000000!important;stroke-width:0px!important;}#mermaid-svg-6JKV9qv5LG0fQbjY .default tspan{fill:#000000!important;}<\/p>\n

    <\/p>\n

    \u6301\u7eed\u6539\u8fdb<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8fd0\u884c\u4e2d<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u4e0a\u7ebf\u524d<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u662f<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5426<\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u5b9a\u4e49 SLO<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u57cb\u70b9\u4e0e\u6307\u6807\u547d\u540d<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u642d\u5efa\u770b\u677f\u4e0e\u544a\u8b66\u89c4\u5219<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u4f8b\u884c\u5de1\u68c0<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u544a\u8b66\u662f\u5426\u547d\u4e2d<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6267\u884c Runbook<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8bb0\u5f55\u89c2\u5bdf<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u590d\u76d8\u4e0e\u4f18\u5316<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u6708\u5ea6\u544a\u8b66\u8d28\u91cf\u8bc4\u5ba1<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u9608\u503c\u91cd\u6821\u51c6<\/p>\n

    <\/span><\/p>\n

    <\/p>\n

    \u8bc4\u4f30\u96c6\u66f4\u65b0<\/p>\n

    <\/span><\/p>\n

    7.1 \u76d1\u63a7\u6d41\u7a0b<\/h4>\n

    \u76d1\u63a7\u6d41\u7a0b\u4ee5\u670d\u52a1\u4e0a\u7ebf\u4e3a\u8d77\u70b9，\u5206\u56db\u6b65\u9a8c\u6536\u3002\u7b2c\u4e00\u6b65\u5b9a\u4e49 SLO：\u4e0e\u4e1a\u52a1\u65b9\u786e\u8ba4\u53ef\u7528\u6027\u4e0e\u5ef6\u8fdf\u627f\u8bfa，\u4ea7\u51fa SLI \u6e05\u5355\u4e0e\u9519\u8bef\u9884\u7b97\u3002\u7b2c\u4e8c\u6b65\u57cb\u70b9：\u6309\u6307\u6807\u547d\u540d\u89c4\u8303\u5b8c\u6210\u63a2\u9488\u63a5\u5165，\u9a8c\u8bc1 \/metrics \u7aef\u70b9\u8f93\u51fa\u4e0e\u76f4\u65b9\u56fe\u6876\u4f4d\u3002\u7b2c\u4e09\u6b65\u770b\u677f\u4e0e\u544a\u8b66：\u4e09\u5c42\u770b\u677f\u5c31\u4f4d，\u544a\u8b66\u89c4\u5219\u6309\u4e25\u91cd\u7ea7\u522b\u5206\u6863，\u8def\u7531\u914d\u5230\u63a5\u6536\u65b9\u3002\u7b2c\u56db\u6b65\u6f14\u7ec3：\u7528\u6df7\u6c8c\u624b\u6bb5\u6ce8\u5165\u4e00\u6b21 GPU OOM\u3001\u4e00\u6b21\u5ef6\u8fdf\u6bdb\u523a\u3001\u4e00\u6b21\u6a21\u578b\u7248\u672c\u56de\u6eda，\u9a8c\u8bc1\u544a\u8b66\u80fd\u5728 5 \u5206\u949f\u5185\u5230\u8fbe\u503c\u73ed\u4eba\u5458\u3001Runbook \u80fd\u5728 30 \u5206\u949f\u5185\u6267\u884c\u5b8c\u6bd5\u3002<\/p>\n

    Runbook \u662f\u6d41\u7a0b\u4e2d\u6bd4\u544a\u8b66\u66f4\u91cd\u8981\u7684\u8d44\u4ea7\u3002\u6bcf\u6761\u544a\u8b66\u89c4\u5219\u5fc5\u987b\u5173\u8054\u4e00\u4efd Runbook，\u5199\u660e\u89e6\u53d1\u6761\u4ef6\u3001\u5f71\u54cd\u9762\u5224\u65ad\u3001\u6b62\u635f\u6b65\u9aa4\u3001\u5347\u7ea7\u8054\u7cfb\u4eba\u3002\u4ee5 P99 \u5ef6\u8fdf\u544a\u8b66\u4e3a\u4f8b，Runbook \u7684\u6392\u67e5\u987a\u5e8f\u56fa\u5b9a\u4e3a：\u786e\u8ba4\u544a\u8b66\u5b9e\u4f8b\u4e0e\u6a21\u578b\u7248\u672c\u3001\u68c0\u67e5 GPU \u5229\u7528\u7387\u4e0e\u961f\u5217\u957f\u5ea6\u3001\u56de\u6eda\u5230\u4e0a\u4e00\u7a33\u5b9a\u7248\u672c\u3001\u9a8c\u8bc1 P99 \u56de\u843d\u5230 500ms \u4ee5\u4e0b\u3002Runbook \u7684\u7ef4\u62a4\u4e0e\u4ee3\u7801\u540c\u7b49\u5bf9\u5f85：\u6bcf\u6b21\u6545\u969c\u590d\u76d8\u53d1\u73b0 Runbook \u7f3a\u6b65\u9aa4\u5c31\u8865，\u6539\u5b8c\u8d70\u8bc4\u5ba1\u5408\u5e76，\u800c\u4e0d\u662f\u53e3\u5934\u4f20\u8fbe\u3002<\/p>\n

    \u6d41\u7a0b\u7684\u6709\u6548\u6027\u9760\u6f14\u7ec3\u4e0e\u590d\u76d8\u68c0\u9a8c\u3002\u6bcf\u4e24\u5468\u4e00\u6b21\u6545\u969c\u6f14\u7ec3，\u4ea4\u66ff\u6ce8\u5165\u57fa\u7840\u8bbe\u65bd\u6545\u969c\u4e0e\u6a21\u578b\u5c42\u6545\u969c，\u6f14\u7ec3\u901a\u8fc7\u7387\u4f4e\u4e8e 80% \u5c31\u8981\u56de\u7089\u57f9\u8bad；\u6bcf\u6b21\u771f\u5b9e\u6545\u969c\u540e 48 \u5c0f\u65f6\u5185\u5b8c\u6210\u590d\u76d8，\u8f93\u51fa MTTD\u3001MTTR\u3001\u8bef\u5224\u73af\u8282\u4e09\u9879\u6307\u6807\u3002\u6301\u7eed\u6539\u8fdb\u7684\u76ee\u6807\u662f\u53ef\u91cf\u5316\u7684：MTTD \u4ece 30 \u5206\u949f\u964d\u5230 5 \u5206\u949f\u4ee5\u5185，MTTR \u4ece 2 \u5c0f\u65f6\u964d\u5230 30 \u5206\u949f\u4ee5\u5185，\u8fd9\u662f\u6d41\u7a0b\u8bbe\u8ba1\u800c\u975e\u5de5\u5177\u9009\u578b\u5e26\u6765\u7684\u6536\u76ca\u3002<\/p>\n

    \u6d41\u7a0b\u4e0e\u53d8\u66f4\u7ba1\u7406\u662f\u8026\u5408\u7684\u3002\u4e00\u6b21\u6a21\u578b\u7248\u672c\u53d1\u5e03\u65e2\u662f\u53d8\u66f4\u4e5f\u662f\u76d1\u63a7\u7684\u8f93\u5165：\u53d1\u5e03\u524d\u786e\u8ba4\u65b0\u7248\u672c\u7684\u6307\u6807\u57fa\u7ebf\u3001\u53d1\u5e03\u4e2d\u5f00\u542f\u53d1\u5e03\u7a97\u53e3\u7684\u544a\u8b66\u9759\u9ed8（\u53ea\u8bb0\u5f55\u4e0d\u6253\u6270）\u3001\u53d1\u5e03\u540e 30 \u5206\u949f\u5bf9\u6bd4\u65b0\u65e7\u7248\u672c\u7684\u5ef6\u8fdf\u4e0e\u9519\u8bef\u7387\u66f2\u7ebf\u3002\u82e5\u65b0\u7248\u672c P99 \u5ef6\u8fdf\u9ad8\u51fa\u57fa\u7ebf 10% \u4ee5\u4e0a，\u53d1\u5e03\u6d41\u7a0b\u81ea\u52a8\u89e6\u53d1\u56de\u6eda，\u800c\u4e0d\u662f\u7b49\u76d1\u63a7\u4f53\u7cfb\u5728\u4e8b\u540e\u53d1\u73b0\u3002\u53d8\u66f4\u95e8\u7981\u628a\u76d1\u63a7\u6307\u6807\u4ece"\u88ab\u52a8\u5c55\u793a"\u63d0\u5347\u4e3a"\u4e3b\u52a8\u51b3\u7b56\u8f93\u5165"，\u8fd9\u662f\u76d1\u63a7\u6d41\u7a0b\u4e0e CI\/CD \u6d41\u6c34\u7ebf\u8854\u63a5\u7684\u5173\u952e\u4e00\u73af\u3002<\/p>\n

    # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ runbook.py<\/span>
    \nfrom<\/span> dataclasses import<\/span> dataclass<\/p>\n

    @dataclass<\/span>
    \nclass<\/span> Step<\/span>:<\/span>
    \n order:<\/span> int<\/span>
    \n action:<\/span> str<\/span>
    \n owner:<\/span> str<\/span>
    \n timeout_min:<\/span> int<\/span><\/p>\n

    RUNBOOKS =<\/span> {<\/span>
    \n "p99_latency_2s"<\/span>:<\/span> [<\/span>
    \n Step(<\/span>1<\/span>,<\/span> "\u786e\u8ba4\u544a\u8b66\u6765\u6e90\u5b9e\u4f8b\u4e0e\u6a21\u578b\u7248\u672c"<\/span>,<\/span> "oncall"<\/span>,<\/span> 5<\/span>)<\/span>,<\/span>
    \n Step(<\/span>2<\/span>,<\/span> "\u68c0\u67e5 GPU \u5229\u7528\u7387\u548c\u961f\u5217\u957f\u5ea6"<\/span>,<\/span> "oncall"<\/span>,<\/span> 5<\/span>)<\/span>,<\/span>
    \n Step(<\/span>3<\/span>,<\/span> "\u56de\u6eda\u6a21\u578b\u7248\u672c\u5230\u4e0a\u4e00\u7a33\u5b9a\u7248\u672c"<\/span>,<\/span> "ml-engineer"<\/span>,<\/span> 15<\/span>)<\/span>,<\/span>
    \n Step(<\/span>4<\/span>,<\/span> "\u9a8c\u8bc1 P99 \u5ef6\u8fdf\u56de\u843d\u81f3 500ms \u4ee5\u4e0b"<\/span>,<\/span> "oncall"<\/span>,<\/span> 10<\/span>)<\/span>,<\/span>
    \n ]<\/span>,<\/span>
    \n "vram_90pct"<\/span>:<\/span> [<\/span>
    \n Step(<\/span>1<\/span>,<\/span> "\u786e\u8ba4\u663e\u5b58\u5360\u7528\u6700\u9ad8\u7684\u5b9e\u4f8b"<\/span>,<\/span> "oncall"<\/span>,<\/span> 5<\/span>)<\/span>,<\/span>
    \n Step(<\/span>2<\/span>,<\/span> "\u964d\u4f4e\u5e76\u53d1\u6216\u8fc1\u79fb\u90e8\u5206\u6d41\u91cf"<\/span>,<\/span> "oncall"<\/span>,<\/span> 10<\/span>)<\/span>,<\/span>
    \n Step(<\/span>3<\/span>,<\/span> "\u68c0\u67e5 KV Cache \u662f\u5426\u53ef\u56de\u6536"<\/span>,<\/span> "ml-engineer"<\/span>,<\/span> 15<\/span>)<\/span>,<\/span>
    \n ]<\/span>,<\/span>
    \n}<\/span><\/p>\n

    def<\/span> run<\/span>(<\/span>runbook_name)<\/span>:<\/span>
    \n steps =<\/span> RUNBOOKS.<\/span>get(<\/span>runbook_name)<\/span>
    \n if<\/span> not<\/span> steps:<\/span>
    \n return<\/span> "\u672a\u627e\u5230\u5bf9\u5e94 Runbook，\u8f6c\u4eba\u5de5"<\/span>
    \n return<\/span> " -> "<\/span>.<\/span>join(<\/span>
    \n f"<\/span>{<\/span>s.<\/span>order}<\/span><\/span>.<\/span>{<\/span>s.<\/span>action}<\/span><\/span>(<\/span>{<\/span>s.<\/span>owner}<\/span><\/span>,<\/span>{<\/span>s.<\/span>timeout_min}<\/span><\/span>\u5206\u949f)"<\/span><\/span> for<\/span> s in<\/span> steps
    \n )<\/span><\/p>\n

    if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
    \n print<\/span>(<\/span>run(<\/span>"p99_latency_2s"<\/span>)<\/span>)<\/span><\/p>\n

    7.2 \u76d1\u63a7\u6cbb\u7406<\/h4>\n

    \u76d1\u63a7\u6cbb\u7406\u7ba1\u4e09\u4ef6\u4e8b：\u547d\u540d\u3001\u57fa\u6570\u4e0e\u8d23\u4efb\u3002\u547d\u540d\u89c4\u8303\u662f\u6cbb\u7406\u7684\u7b2c\u4e00\u9053\u95f8\u95e8，\u6307\u6807\u540d\u9075\u5faa namespace_subsystem_name \u4e09\u6bb5\u5f0f，\u4f8b\u5982 llm_latency_seconds \u4e2d llm \u662f\u547d\u540d\u7a7a\u95f4\u3001latency \u662f\u5b50\u7cfb\u7edf\u3001seconds \u662f\u5355\u4f4d\u3002\u5355\u4f4d\u540e\u7f00\u5fc5\u987b\u51fa\u73b0\u5728\u6307\u6807\u540d\u91cc，seconds\u3001bytes\u3001total\u3001ratio \u56db\u7c7b\u540e\u7f00\u662f\u786c\u6027\u8981\u6c42，\u5426\u5219\u770b\u677f\u4e0e\u544a\u8b66\u8868\u8fbe\u5f0f\u91cc\u65e0\u6cd5\u51ed\u540d\u5224\u65ad\u91cf\u7eb2，\u8de8\u56e2\u961f\u534f\u4f5c\u65f6\u5fc5\u7136\u51fa\u9519\u3002<\/p>\n

    \u6807\u7b7e\u57fa\u6570\u662f\u6cbb\u7406\u7684\u7b2c\u4e8c\u9053\u95f8\u95e8\u3002\u6bcf\u4e2a\u6807\u7b7e\u7684\u53d6\u503c\u6570\u91cf\u4e0a\u9650\u8bbe\u4e3a 20，\u8bf7\u6c42\u7ea7 ID \u4e0d\u8fdb\u6307\u6807\u57df，\u9ad8\u57fa\u6570\u7ef4\u5ea6\u4e00\u5f8b\u8d70\u65e5\u5fd7\u57df\u3002\u57fa\u6570\u7684\u63a7\u5236\u5206\u4e24\u5c42：\u57cb\u70b9\u89c4\u8303\u5728\u4ee3\u7801\u8bc4\u5ba1\u65f6\u68c0\u67e5，\u8fd0\u884c\u65f6\u7528\u57fa\u6570\u5b88\u536b\u62e6\u622a\u8d8a\u754c\u5199\u5165\u3002\u8d85\u8fc7\u57fa\u6570\u7684\u6307\u6807\u4e00\u65e6\u8fdb\u5165\u65f6\u5e8f\u5e93，\u5220\u9664\u6210\u672c\u8fdc\u9ad8\u4e8e\u963b\u6b62\u6210\u672c，\u56e0\u6b64\u5b88\u536b\u5fc5\u987b\u914d\u7f6e\u5728\u5199\u5165\u8def\u5f84\u4e0a\u800c\u4e0d\u662f\u4e8b\u540e\u6e05\u7406\u811a\u672c\u4e0a\u3002<\/p>\n

    \u8d23\u4efb\u5f52\u5c5e\u662f\u6cbb\u7406\u7684\u7b2c\u4e09\u9053\u95f8\u95e8\u3002\u6bcf\u6761\u544a\u8b66\u89c4\u5219\u5fc5\u987b\u7ed1\u5b9a\u8d1f\u8d23\u4eba\u4e0e\u56e2\u961f\u6807\u7b7e，\u8d1f\u8d23\u4eba\u627f\u62c5\u89c4\u5219\u7684\u51c6\u786e\u6027\u8d23\u4efb：\u9608\u503c\u8bef\u62a5\u7387\u8d85\u8fc7 5% \u7531\u8d1f\u8d23\u4eba\u8c03\u4f18，\u8fde\u7eed 3 \u4e2a\u6708\u96f6\u89e6\u53d1\u7684\u89c4\u5219\u7531\u8d1f\u8d23\u4eba\u7ed9\u51fa\u4fdd\u7559\u6216\u5220\u9664\u7ed3\u8bba\u3002\u6570\u636e\u6cbb\u7406\u4e0e\u544a\u8b66\u6cbb\u7406\u5e76\u5217：\u65e5\u5fd7\u4fdd\u7559\u671f\u3001PII \u8131\u654f\u7b56\u7565\u3001\u5ba1\u8ba1\u8bbf\u95ee\u8bb0\u5f55，\u7531\u5408\u89c4\u89d2\u8272\u6bcf\u5b63\u5ea6\u590d\u6838\u4e00\u6b21\u3002\u6cbb\u7406\u7684\u76ee\u6807\u662f\u8ba9\u76d1\u63a7\u4f53\u7cfb\u5728\u4eba\u5458\u6d41\u52a8\u7684\u60c5\u51b5\u4e0b\u4f9d\u7136\u53ef\u7ef4\u62a4，\u8d1f\u8d23\u4eba\u79bb\u804c\u65f6\u89c4\u5219\u4e0d\u53d8\u6210\u5b64\u513f\u3002<\/p>\n

    \u6cbb\u7406\u7684\u843d\u5730\u8981\u4f9d\u6258\u53ef\u6267\u884c\u7684\u5236\u5ea6\u800c\u975e\u5021\u8bae\u3002\u6307\u6807\u8bc4\u5ba1\u968f\u4ee3\u7801\u8bc4\u5ba1\u4e00\u8d77\u6267\u884c，\u65b0\u589e\u6307\u6807\u672a\u8fc7\u547d\u540d\u4e0e\u57fa\u6570\u6821\u9a8c\u4e0d\u5408\u5e76；\u544a\u8b66\u89c4\u5219\u4ee5\u4ee3\u7801\u4ed3\u5e93\u7ba1\u7406，\u6539\u9608\u503c\u5fc5\u987b\u63d0\u4ea4\u53d8\u66f4\u8bb0\u5f55，\u6ce8\u660e\u539f\u56e0\u4e0e\u9884\u671f\u6548\u679c\u3002\u503c\u5b88\u6392\u73ed\u4e0e\u89c4\u5219\u8d1f\u8d23\u4eba\u5bf9\u9f50：\u540c\u4e00\u56e2\u961f\u7684\u89c4\u5219\u7531\u672c\u56e2\u961f\u503c\u73ed\u4eba\u5458\u5904\u7406，\u8de8\u56e2\u961f\u7684\u89c4\u5219\u4ee5\u5bf9\u63a5\u4eba\u6e05\u5355\u4e3a\u51c6，\u907f\u514d\u544a\u8b66\u53d1\u51fa\u540e\u65e0\u4eba\u8ba4\u9886\u7684\u771f\u7a7a\u671f\u3002<\/p>\n

    \u6cbb\u7406\u7684\u8fb9\u754c\u8fd8\u6d89\u53ca\u76d1\u63a7\u8d44\u6e90\u672c\u8eab\u7684\u9884\u7b97\u3002\u6bcf\u4e2a\u670d\u52a1\u5206\u914d\u7684\u76d1\u63a7\u8d44\u6e90（\u65f6\u5e8f\u5e8f\u5217\u6570\u3001\u65e5\u5fd7\u914d\u989d）\u7eb3\u5165\u5bb9\u91cf\u7ba1\u7406，\u8d85\u9884\u7b97\u7684\u76d1\u63a7\u9700\u6c42\u8d70\u7533\u8bf7\u8bc4\u5ba1，\u800c\u4e0d\u662f\u65e0\u9650\u589e\u957f\u3002\u4e00\u4e2a 3000 \u6761\u5e8f\u5217\u3001\u6bcf\u5929 5.2GB \u65e5\u5fd7\u7684\u56e2\u961f\u9884\u7b97\u4e0a\u9650\u5199\u8fdb\u5b63\u5ea6\u89c4\u5212，\u8d85\u8fc7\u4e0a\u9650\u7531\u5e73\u53f0\u56e2\u961f\u88c1\u5b9a\u53d6\u820d：\u4f18\u5148\u4fdd\u7559 SLO \u76f8\u5173\u6307\u6807，\u5220\u9664\u957f\u671f\u65e0\u4eba\u67e5\u770b\u7684\u770b\u677f\u4e0e\u89c4\u5219\u3002\u8d44\u6e90\u9884\u7b97\u628a\u76d1\u63a7\u6cbb\u7406\u4ece\u53e3\u53f7\u53d8\u6210\u53ef\u5ba1\u8ba1\u7684\u5de5\u7a0b\u7ea6\u675f\u3002<\/p>\n

    \u6cbb\u7406\u68c0\u67e5\u6e05\u5355\u5728\u670d\u52a1\u4e0a\u7ebf\u8bc4\u5ba1\u65f6\u9010\u9879\u6253\u52fe，\u7f3a\u5c11\u4efb\u4f55\u4e00\u9879\u90fd\u4e0d\u5f97\u8fdb\u5165\u751f\u4ea7：<\/p>\n

      \n
    • \u6307\u6807\u547d\u540d\u901a\u8fc7 lint \u6821\u9a8c，\u5355\u4f4d\u540e\u7f00\u9f50\u5168，\u65e0\u8bf7\u6c42\u7ea7 ID \u8fdb\u5165\u6307\u6807\u57df\u3002<\/li>\n
    • \u6bcf\u4e2a\u6807\u7b7e\u53d6\u503c\u6570\u4e0d\u8d85\u8fc7 20，\u8fd0\u884c\u65f6\u57fa\u6570\u5b88\u536b\u5df2\u90e8\u7f72\u5728\u5199\u5165\u8def\u5f84\u3002<\/li>\n
    • \u6bcf\u6761\u544a\u8b66\u89c4\u5219\u7ed1\u5b9a\u8d1f\u8d23\u4eba\u4e0e\u56e2\u961f\u6807\u7b7e，\u5173\u8054\u7684 Runbook \u5df2\u8bc4\u5ba1\u5408\u5e76\u3002<\/li>\n
    • \u65e5\u5fd7\u4fdd\u7559\u671f\u4e0e PII \u8131\u654f\u7b56\u7565\u5df2\u786e\u8ba4，\u654f\u611f\u5b57\u6bb5\u5728\u5e8f\u5217\u5316\u524d\u5b8c\u6210\u63a9\u7801\u3002<\/li>\n
    • \u770b\u677f\u6309\u4e09\u5c42\u5212\u5206\u4e14\u8d1f\u8d23\u4eba\u660e\u786e，\u65e0\u4eba\u7ef4\u62a4\u7684\u770b\u677f\u6807\u8bb0\u4e3a\u5f85\u5f52\u6863\u3002<\/li>\n
    • \u76d1\u63a7\u8d44\u6e90\u9884\u7b97\u5728\u914d\u989d\u5185，\u65b0\u589e\u5e8f\u5217\u6570\u4e0e\u65e5\u5fd7\u91cf\u5df2\u8ba1\u5165\u5bb9\u91cf\u89c4\u5212\u3002<\/li>\n<\/ul>\n

      \u4ee5\u4e0a\u68c0\u67e5\u9879\u5bf9\u5e94\u672c\u7ae0\u6b63\u6587\u7684\u547d\u540d\u3001\u57fa\u6570\u3001\u8d23\u4efb\u4e0e\u8d44\u6e90\u56db\u4e2a\u6cbb\u7406\u95f8\u95e8，\u4efb\u4f55\u4e00\u9879\u672a\u843d\u5b9e\u90fd\u4f1a\u5728\u6545\u969c\u65f6\u4ee5"\u544a\u8b66\u6ca1\u4eba\u770b\u3001\u6307\u6807\u770b\u4e0d\u61c2\u3001\u6570\u636e\u6ca1\u7559\u591f"\u7684\u5f62\u5f0f\u53cd\u566c\u3002<\/p>\n

      # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ metric_linter.py<\/span>
      \nimport<\/span> re<\/p>\n

      NAME_RE =<\/span> re.<\/span>compile<\/span>(<\/span>r"^[a-z][a-z0-9_]*(_[a-z0-9]+)*$"<\/span>)<\/span>
      \nUNIT_SUFFIX =<\/span> {<\/span>"seconds"<\/span>,<\/span> "bytes"<\/span>,<\/span> "total"<\/span>,<\/span> "ratio"<\/span>,<\/span> "tokens"<\/span>}<\/span><\/p>\n

      def<\/span> lint<\/span>(<\/span>metric)<\/span>:<\/span>
      \n """\u6309\u547d\u540d\u89c4\u8303\u6821\u9a8c\u6307\u6807\u540d，\u8fd4\u56de\u8fdd\u89c4\u6e05\u5355\u3002"""<\/span>
      \n problems =<\/span> [<\/span>]<\/span>
      \n if<\/span> not<\/span> NAME_RE.<\/span>fullmatch(<\/span>metric)<\/span>:<\/span>
      \n problems.<\/span>append(<\/span>"\u547d\u540d\u4e0d\u89c4\u8303，\u5e94\u4e3a\u5c0f\u5199\u5b57\u6bcd\u6570\u5b57\u4e0b\u5212\u7ebf"<\/span>)<\/span>
      \n parts =<\/span> metric.<\/span>split(<\/span>"_"<\/span>)<\/span>
      \n if<\/span> len<\/span>(<\/span>parts)<\/span> <<\/span> 3<\/span>:<\/span>
      \n problems.<\/span>append(<\/span>"\u7f3a\u5c11 namespace \u6216 subsystem \u6bb5"<\/span>)<\/span>
      \n if<\/span> parts[<\/span>–<\/span>1<\/span>]<\/span> not<\/span> in<\/span> UNIT_SUFFIX:<\/span>
      \n problems.<\/span>append(<\/span>f"\u7f3a\u5c11\u5355\u4f4d\u540e\u7f00，\u5408\u6cd5\u540e\u7f00: <\/span>{<\/span>sorted<\/span>(<\/span>UNIT_SUFFIX)<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span>
      \n return<\/span> problems<\/p>\n

      if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
      \n for<\/span> name in<\/span> (<\/span>"llm_latency_seconds"<\/span>,<\/span> "p99_latency"<\/span>,<\/span> "GPU_util"<\/span>)<\/span>:<\/span>
      \n issues =<\/span> lint(<\/span>name)<\/span>
      \n print<\/span>(<\/span>f"<\/span>{<\/span>name}<\/span><\/span>: <\/span>{<\/span>'\u5408\u89c4'<\/span> if<\/span> not<\/span> issues else<\/span> '；'<\/span>.<\/span>join(<\/span>issues)<\/span>}<\/span><\/span>"<\/span><\/span>)<\/span><\/p>\n

      7.3 \u6301\u7eed\u6539\u8fdb<\/h4>\n

      \u6301\u7eed\u6539\u8fdb\u662f\u76d1\u63a7\u4f53\u7cfb\u7684\u5143\u5faa\u73af：\u76d1\u63a7\u81ea\u8eab\u7684\u5065\u5eb7\u5ea6\u4e5f\u8981\u88ab\u76d1\u63a7\u3002\u6708\u5ea6\u8bc4\u5ba1\u7684\u8f93\u5165\u662f\u544a\u8b66\u8d28\u91cf\u6570\u636e：\u672c\u6708\u544a\u8b66\u603b\u6570\u3001\u8bef\u62a5\u6570\u3001\u6f0f\u62a5\u6570\u3001MTTD\u3001MTTR，\u8f93\u51fa\u662f\u4e09\u7c7b\u8c03\u6574\u2014\u2014\u9608\u503c\u91cd\u6821\u51c6\u3001\u89c4\u5219\u589e\u5220\u3001\u8bc4\u4f30\u96c6\u66f4\u65b0\u3002\u8bc4\u5ba1\u7684\u91cf\u5316\u76ee\u6807\u8bbe\u5b9a\u4e3a：\u544a\u8b66\u7cbe\u786e\u7387\u4e0d\u4f4e\u4e8e 90%（\u6bcf\u6761\u544a\u8b66\u5bf9\u5e94\u771f\u5b9e\u6545\u969c），\u53ec\u56de\u7387\u4e0d\u4f4e\u4e8e 95%（\u771f\u5b9e\u6545\u969c\u4e0d\u6f0f\u62a5），\u8bef\u62a5\u7387\u4f4e\u4e8e 5%，\u4e09\u8005\u51b2\u7a81\u65f6\u4f18\u5148\u4fdd\u53ec\u56de\u3002<\/p>\n

      \u9608\u503c\u91cd\u6821\u51c6\u662f\u6539\u8fdb\u4e2d\u6700\u9891\u7e41\u7684\u52a8\u4f5c\u3002\u7edf\u8ba1\u9608\u503c\u968f\u6570\u636e\u6f02\u79fb，\u6bcf\u5468\u7528\u6700\u8fd1 30 \u5929\u6570\u636e\u91cd\u7b97\u53c2\u8003\u5206\u5e03；\u4e1a\u52a1\u9608\u503c\u968f SLO \u53d8\u66f4，SLO \u6bcf\u5b63\u5ea6\u8bc4\u5ba1\u65f6\u540c\u6b65\u66f4\u65b0\u3002\u6821\u51c6\u5fc5\u987b\u7559\u4e0b\u5ba1\u8ba1\u8f68\u8ff9：\u9608\u503c\u4ece 3 \u4e2a\u6807\u51c6\u5dee\u8c03\u5230 2.5 \u4e2a\u6807\u51c6\u5dee，\u8981\u8bb0\u5f55\u539f\u56e0\u4e0e\u6548\u679c（\u8bef\u62a5\u4ece 8% \u964d\u5230 4%），\u5426\u5219\u4e09\u4e2a\u6708\u540e\u6ca1\u4eba\u8bb0\u5f97\u9608\u503c\u4e3a\u4f55\u5982\u6b64\u8bbe\u5b9a，\u8c03\u6574\u4f1a\u9000\u5316\u4e3a\u76f2\u6539\u3002<\/p>\n

      \u8bc4\u4f30\u96c6\u66f4\u65b0\u652f\u6491\u79bb\u7ebf\u76d1\u63a7\u7684\u957f\u671f\u6709\u6548\u6027\u3002\u9ec4\u91d1\u8bc4\u4f30\u96c6\u6bcf\u6708\u8865\u5145\u65b0\u4e1a\u52a1\u573a\u666f\u7684\u63d0\u793a\u8bcd，\u5254\u9664\u5df2\u5931\u6548\u573a\u666f，\u4fdd\u8bc1\u8bc4\u4f30\u96c6\u4e0e\u7ebf\u4e0a\u5206\u5e03\u5bf9\u9f50；PSI \u7684\u53c2\u8003\u5206\u5e03\u6bcf\u6708\u6eda\u52a8\u66f4\u65b0\u4e00\u6b21，\u907f\u514d\u57fa\u7ebf\u672c\u8eab\u6f02\u79fb\u3002\u6539\u8fdb\u5faa\u73af\u7684\u6700\u540e\u4e00\u73af\u662f\u628a\u4f18\u5316\u5199\u56de\u6d41\u7a0b：\u9608\u503c\u8c03\u6574\u3001Runbook \u8865\u5145\u3001\u89c4\u5219\u589e\u5220\u90fd\u8d70\u8bc4\u5ba1\u5408\u5e76，\u5f62\u6210"\u89c2\u6d4b\u3001\u51b3\u7b56\u3001\u6267\u884c\u3001\u6c89\u6dc0"\u7684\u95ed\u73af，\u8ba9\u76d1\u63a7\u4f53\u7cfb\u968f\u4e1a\u52a1\u6f14\u8fdb\u800c\u4e0d\u662f\u50f5\u5316\u6210\u4e00\u5957\u9759\u6001\u914d\u7f6e\u3002<\/p>\n

      # \u6765\u6e90：\u81ea\u5b9e\u73b0 \/ alert_quality.py<\/span>
      \ndef<\/span> evaluate<\/span>(<\/span>alerts,<\/span> incidents,<\/span> tp_window_min=<\/span>15<\/span>)<\/span>:<\/span>
      \n """\u7528\u544a\u8b66\u4e0e\u771f\u5b9e\u6545\u969c\u5bf9\u7167\u8ba1\u7b97\u7cbe\u786e\u7387\u4e0e\u53ec\u56de\u7387\u3002
      \n \u547d\u4e2d\u5224\u5b9a：\u544a\u8b66\u65f6\u95f4\u843d\u5728\u6545\u969c\u53d1\u751f\u524d 15 \u5206\u949f\u7a97\u53e3\u5185，\u4e14\u4e0d\u665a\u4e8e\u6545\u969c\u53d1\u751f\u3002"""<\/span>
      \n window =<\/span> tp_window_min *<\/span> 60<\/span>
      \n hit_alerts =<\/span> 0<\/span>
      \n for<\/span> alert in<\/span> alerts:<\/span>
      \n if<\/span> any<\/span>(<\/span>0<\/span> <=<\/span> inc[<\/span>"start"<\/span>]<\/span> –<\/span> alert[<\/span>"time"<\/span>]<\/span> <=<\/span> window for<\/span> inc in<\/span> incidents)<\/span>:<\/span>
      \n hit_alerts +=<\/span> 1<\/span>
      \n fp =<\/span> len<\/span>(<\/span>alerts)<\/span> –<\/span> hit_alerts
      \n # \u53ec\u56de\u7387\u6309"\u88ab\u81f3\u5c11\u4e00\u6761\u544a\u8b66\u547d\u4e2d\u7684\u6545\u969c\u6570"\u8ba1\u7b97，\u9632\u6b62\u591a\u6761\u544a\u8b66\u91cd\u53e0\u9ad8\u4f30<\/span>
      \n detected =<\/span> set<\/span>(<\/span>)<\/span>
      \n for<\/span> idx,<\/span> inc in<\/span> enumerate<\/span>(<\/span>incidents)<\/span>:<\/span>
      \n for<\/span> alert in<\/span> alerts:<\/span>
      \n if<\/span> 0<\/span> <=<\/span> inc[<\/span>"start"<\/span>]<\/span> –<\/span> alert[<\/span>"time"<\/span>]<\/span> <=<\/span> window:<\/span>
      \n detected.<\/span>add(<\/span>idx)<\/span>
      \n fn =<\/span> len<\/span>(<\/span>incidents)<\/span> –<\/span> len<\/span>(<\/span>detected)<\/span>
      \n precision =<\/span> hit_alerts \/<\/span> len<\/span>(<\/span>alerts)<\/span> if<\/span> alerts else<\/span> 0.0<\/span>
      \n recall =<\/span> len<\/span>(<\/span>detected)<\/span> \/<\/span> len<\/span>(<\/span>incidents)<\/span> if<\/span> incidents else<\/span> 0.0<\/span>
      \n return<\/span> {<\/span>"tp"<\/span>:<\/span> hit_alerts,<\/span> "fp"<\/span>:<\/span> fp,<\/span> "fn"<\/span>:<\/span> fn,<\/span> "precision"<\/span>:<\/span> precision,<\/span> "recall"<\/span>:<\/span> recall}<\/span><\/p>\n

      if<\/span> __name__ ==<\/span> "__main__"<\/span>:<\/span>
      \n # \u6a21\u62df\u4e00\u4e2a\u6708：8 \u6761\u544a\u8b66\u5bf9\u5e94 3 \u8d77\u771f\u5b9e\u6545\u969c，3 \u6761\u544a\u8b66\u4e3a\u8bef\u62a5<\/span>
      \n alerts =<\/span> [<\/span>{<\/span>"time"<\/span>:<\/span> t}<\/span> for<\/span> t in<\/span> (<\/span>10<\/span>,<\/span> 20<\/span>,<\/span> 30<\/span>,<\/span> 40<\/span>,<\/span> 50<\/span>,<\/span> 60<\/span>,<\/span> 200<\/span>,<\/span> 240<\/span>)<\/span>]<\/span>
      \n incidents =<\/span> [<\/span>{<\/span>"start"<\/span>:<\/span> 10<\/span>}<\/span>,<\/span> {<\/span>"start"<\/span>:<\/span> 30<\/span>}<\/span>,<\/span> {<\/span>"start"<\/span>:<\/span> 50<\/span>}<\/span>]<\/span>
      \n result =<\/span> evaluate(<\/span>alerts,<\/span> incidents)<\/span>
      \n print<\/span>(<\/span>
      \n f"\u7cbe\u786e\u7387 <\/span>{<\/span>result[<\/span>'precision'<\/span>]<\/span>:<\/span>.0%<\/span>}<\/span><\/span>，\u53ec\u56de\u7387 <\/span>{<\/span>result[<\/span>'recall'<\/span>]<\/span>:<\/span>.0%<\/span>}<\/span><\/span>，"<\/span><\/span>
      \n f"\u8bef\u62a5 <\/span>{<\/span>result[<\/span>'fp'<\/span>]<\/span>}<\/span><\/span> \u6761，\u6f0f\u62a5 <\/span>{<\/span>result[<\/span>'fn'<\/span>]<\/span>}<\/span><\/span> \u8d77"<\/span><\/span>
      \n )<\/span><\/p>\n

      \u603b\u7ed3<\/h3>\n

      \u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\u7684\u843d\u5730\u8def\u5f84\u53ef\u4ee5\u6982\u62ec\u4e3a\u4e00\u6761\u4e3b\u7ebf：\u4ee5 99.9% \u53ef\u7528\u6027 SLO \u4e3a\u951a，\u628a P99 \u9996 token \u5ef6\u8fdf 500ms\u3001\u9519\u8bef\u7387 1% \u8fd9\u7c7b\u5177\u4f53\u76ee\u6807\u9010\u5c42\u8f6c\u5316\u4e3a\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u3001\u65e5\u5fd7\u4e0e\u544a\u8b66\u56db\u5957\u5b50\u7cfb\u7edf\u3002\u6307\u6807\u5c42\u7528\u76f4\u65b9\u56fe\u4e0e\u53cc\u901a\u9053\u91c7\u96c6\u8986\u76d6\u5728\u7ebf\u5ef6\u8fdf\u4e0e\u79bb\u7ebf\u8d28\u91cf，\u5f02\u5e38\u68c0\u6d4b\u4ee5 3 \u4e2a\u6807\u51c6\u5dee\u52a0\u591a\u65b9\u6cd5\u6295\u7968\u63a7\u5236\u8bef\u62a5\u7387\u5728 5% \u4ee5\u5185，\u65e5\u5fd7\u5c42\u4ee5\u7ed3\u6784\u5316 JSON \u652f\u6491\u5168\u94fe\u8def\u8fd8\u539f，\u544a\u8b66\u5c42\u4ee5\u5206\u7ea7\u3001\u8def\u7531\u4e0e\u53bb\u91cd\u628a\u65e5\u5747\u901a\u77e5\u4ece 40 \u6761\u538b\u5230 12 \u6761\u3002\u5e73\u53f0\u627f\u8f7d\u5168\u90e8\u6570\u636e，\u6d41\u7a0b\u4e0e\u6cbb\u7406\u4fdd\u8bc1\u4f53\u7cfb\u53ef\u7ef4\u62a4，\u6708\u5ea6\u8bc4\u5ba1\u6301\u7eed\u6821\u51c6\u9608\u503c\u4e0e\u8bc4\u4f30\u96c6\u3002\u76d1\u63a7\u7684\u6700\u7ec8\u68c0\u9a8c\u6807\u51c6\u53ea\u6709\u4e00\u6761：\u4efb\u4f55\u4e00\u6b21\u6545\u969c，\u4ece\u53d1\u751f\u5230\u88ab\u8bc6\u522b\u3001\u5230\u6b62\u635f\u52a8\u4f5c\u6267\u884c\u5b8c\u6bd5，\u5168\u90e8\u73af\u8282\u90fd\u6709\u6570\u636e\u652f\u6491，\u4e14\u8fc7\u7a0b\u53ef\u590d\u76d8\u3001\u7ed3\u679c\u53ef\u6539\u8fdb\u3002<\/p>\n

      \u5916\u90e8\u5f15\u7528<\/h3>\n
        \n
      • Prometheus \u5b98\u65b9\u6587\u6863：\u6307\u6807\u6a21\u578b\u4e0e TSDB \u5b58\u50a8：https:\/\/prometheus.io\/docs\/introduction\/overview\/<\/li>\n
      • prometheus_client Python \u5ba2\u6237\u7aef：https:\/\/github.com\/prometheus\/client_python<\/li>\n
      • Prometheus Alertmanager \u914d\u7f6e\u6587\u6863：https:\/\/prometheus.io\/docs\/alerting\/latest\/alertmanager\/<\/li>\n
      • Grafana \u5b98\u65b9\u6587\u6863：https:\/\/grafana.com\/docs\/<\/li>\n
      • Elasticsearch \u7d22\u5f15\u751f\u547d\u5468\u671f\u7ba1\u7406（ILM）：https:\/\/www.elastic.co\/guide\/en\/elasticsearch\/reference\/current\/index-lifecycle-management.html<\/li>\n
      • StatsD \u6807\u51c6\u534f\u8bae：https:\/\/github.com\/statsd\/statsd<\/li>\n
      • scikit-learn IsolationForest \u6587\u6863：https:\/\/scikit-learn.org\/stable\/modules\/generated\/sklearn.ensemble.IsolationForest.html<\/li>\n
      • Google SRE Workbook（\u9519\u8bef\u9884\u7b97\u4e0e SLO）：https:\/\/sre.google\/workbook\/<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"

        \u76ee\u5f55
        \n\u6a21\u578b\u76d1\u63a7\u6982\u8ff0\u6027\u80fd\u6307\u6807\u5f02\u5e38\u68c0\u6d4b\u65e5\u5fd7\u8bb0\u5f55\u544a\u8b66\u4f53\u7cfb\u76d1\u63a7\u5e73\u53f0\u5b9e\u8df5\u5efa\u8bae\u4e0e\u6700\u4f73\u5b9e\u8df5
        \n\u6458\u8981 \u672c\u6587\u4ee5 7B \u53c2\u6570\u5927\u8bed\u8a00\u6a21\u578b\u5728\u7ebf\u63a8\u7406\u670d\u52a1\u4e3a\u4f8b，\u6784\u5efa\u4e00\u5957\u8986\u76d6\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7\u8bb0\u5f55\u7684\u76d1\u63a7\u544a\u8b66\u4f53\u7cfb\u3002 \u751f\u4ea7\u6807\u51c6\u8bbe\u5b9a\u4e3a P99 \u9996 token \u5ef6\u8fdf\u4f4e\u4e8e 500ms\u3001\u5355\u5361\u751f\u6210\u541e\u5410\u9ad8\u4e8e 800 tokens\/s\u3001\u9519\u8bef\u7387\u4f4e\u4e8e 1%\u3002 \u5f02\u5e38\u68c0\u6d4b\u4ee5 3 \u4e2a\u6807\u51c6\u5dee\u4f5c\u4e3a\u7edf\u8ba1\u544a\u8b66\u9608\u503c，\u5c06\u544a\u8b66\u8bef\u62a5\u7387\u63a7\u5236\u5728 5% \u4ee5\u5185\u3002 1. \u6a21\u578b\u76d1\u63a7\u6982\u8ff0
        \n\u5927\u8bed\u8a00\u6a21\u578b\u63a8\u7406\u670d\u52a1\u4e0e\u4f20\u7edf Web \u670d<\/p>\n","protected":false},"author":2,"featured_media":110861,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[50,956],"topic":[],"class_list":["post-110862","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-50","tag-956"],"yoast_head":"\n\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/110862.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u76ee\u5f55 \u6a21\u578b\u76d1\u63a7\u6982\u8ff0\u6027\u80fd\u6307\u6807\u5f02\u5e38\u68c0\u6d4b\u65e5\u5fd7\u8bb0\u5f55\u544a\u8b66\u4f53\u7cfb\u76d1\u63a7\u5e73\u53f0\u5b9e\u8df5\u5efa\u8bae\u4e0e\u6700\u4f73\u5b9e\u8df5 \u6458\u8981 \u672c\u6587\u4ee5 7B \u53c2\u6570\u5927\u8bed\u8a00\u6a21\u578b\u5728\u7ebf\u63a8\u7406\u670d\u52a1\u4e3a\u4f8b，\u6784\u5efa\u4e00\u5957\u8986\u76d6\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7\u8bb0\u5f55\u7684\u76d1\u63a7\u544a\u8b66\u4f53\u7cfb\u3002 \u751f\u4ea7\u6807\u51c6\u8bbe\u5b9a\u4e3a P99 \u9996 token \u5ef6\u8fdf\u4f4e\u4e8e 500ms\u3001\u5355\u5361\u751f\u6210\u541e\u5410\u9ad8\u4e8e 800 tokens\/s\u3001\u9519\u8bef\u7387\u4f4e\u4e8e 1%\u3002 \u5f02\u5e38\u68c0\u6d4b\u4ee5 3 \u4e2a\u6807\u51c6\u5dee\u4f5c\u4e3a\u7edf\u8ba1\u544a\u8b66\u9608\u503c，\u5c06\u544a\u8b66\u8bef\u62a5\u7387\u63a7\u5236\u5728 5% \u4ee5\u5185\u3002 1. \u6a21\u578b\u76d1\u63a7\u6982\u8ff0 \u5927\u8bed\u8a00\u6a21\u578b\u63a8\u7406\u670d\u52a1\u4e0e\u4f20\u7edf Web \u670d\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/110862.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-09-29T07:24:48+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/09\/20260929072446-6abb67be48c76.png\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"43 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/110862.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/110862.html\",\"name\":\"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-09-29T07:24:48+00:00\",\"dateModified\":\"2026-09-29T07:24:48+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/110862.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/110862.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/110862.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/110862.html","og_locale":"zh_CN","og_type":"article","og_title":"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u76ee\u5f55 \u6a21\u578b\u76d1\u63a7\u6982\u8ff0\u6027\u80fd\u6307\u6807\u5f02\u5e38\u68c0\u6d4b\u65e5\u5fd7\u8bb0\u5f55\u544a\u8b66\u4f53\u7cfb\u76d1\u63a7\u5e73\u53f0\u5b9e\u8df5\u5efa\u8bae\u4e0e\u6700\u4f73\u5b9e\u8df5 \u6458\u8981 \u672c\u6587\u4ee5 7B \u53c2\u6570\u5927\u8bed\u8a00\u6a21\u578b\u5728\u7ebf\u63a8\u7406\u670d\u52a1\u4e3a\u4f8b，\u6784\u5efa\u4e00\u5957\u8986\u76d6\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7\u8bb0\u5f55\u7684\u76d1\u63a7\u544a\u8b66\u4f53\u7cfb\u3002 \u751f\u4ea7\u6807\u51c6\u8bbe\u5b9a\u4e3a P99 \u9996 token \u5ef6\u8fdf\u4f4e\u4e8e 500ms\u3001\u5355\u5361\u751f\u6210\u541e\u5410\u9ad8\u4e8e 800 tokens\/s\u3001\u9519\u8bef\u7387\u4f4e\u4e8e 1%\u3002 \u5f02\u5e38\u68c0\u6d4b\u4ee5 3 \u4e2a\u6807\u51c6\u5dee\u4f5c\u4e3a\u7edf\u8ba1\u544a\u8b66\u9608\u503c，\u5c06\u544a\u8b66\u8bef\u62a5\u7387\u63a7\u5236\u5728 5% \u4ee5\u5185\u3002 1. \u6a21\u578b\u76d1\u63a7\u6982\u8ff0 \u5927\u8bed\u8a00\u6a21\u578b\u63a8\u7406\u670d\u52a1\u4e0e\u4f20\u7edf Web \u670d","og_url":"https:\/\/www.wsisp.com\/helps\/110862.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-09-29T07:24:48+00:00","og_image":[{"url":"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/09\/20260929072446-6abb67be48c76.png"}],"author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"43 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/110862.html","url":"https:\/\/www.wsisp.com\/helps\/110862.html","name":"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-09-29T07:24:48+00:00","dateModified":"2026-09-29T07:24:48+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/110862.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/110862.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/110862.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u6a21\u578b\u76d1\u63a7\u4e0e\u544a\u8b66\uff1a\u6027\u80fd\u6307\u6807\u3001\u5f02\u5e38\u68c0\u6d4b\u4e0e\u65e5\u5fd7"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/110862","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=110862"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/110862\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media\/110861"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=110862"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=110862"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=110862"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=110862"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}