![\"\u5c01\u9762\u4fe1\u606f\u56fe\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/09\/20260831171644-6a95b6fc26020.webp\")
<\/p>\n<\/p>\n
\u63a8\u7406\u670d\u52a1\u51fa\u73b0\u6392\u961f\u3001\u663e\u5b58\u4e0d\u8db3\u6216\u7f51\u5173\u8d85\u65f6\u65f6，\u5148\u628a\u6d41\u91cf\u548c\u6545\u969c\u57df\u9694\u5f00\u3002\u964d\u7ea7\u4e0d\u662f\u7b80\u5355\u6362\u6a21\u578b：\u9700\u8981\u533a\u5206\u53ef\u6392\u961f\u8bf7\u6c42\u3001\u53ef\u8fd4\u56de\u7f13\u5b58\u7684\u8bf7\u6c42，\u4ee5\u53ca\u5e94\u76f4\u63a5\u63d0\u793a\u7a0d\u540e\u91cd\u8bd5\u7684\u8bf7\u6c42\u3002<\/p>\n
\u6392\u67e5\u6545\u969c\u7b2c\u4e00\u6b65\u662f\u901a\u8fc7\u6807\u51c6\u547d\u4ee4\u884c\u63d0\u53d6\u771f\u5b9e\u7684\u73b0\u573a\u8bca\u65ad\u6307\u6807\u3002\u5229\u7528\u547d\u4ee4\u884c\u6293\u53d6 Pod \u4e8b\u4ef6\u4e0e GPU \u5360\u7528\u53c2\u6570：<\/p>\n
kubectl get pods -n llm-prod -l app=vllm-inference -o wide
\nkubectl exec -it -n llm-prod vllm-inference-6789b5894-q9k2x — nvidia-smi –query-gpu=timestamp,name,memory.used,memory.total,utilization.gpu –format=csv -l 1
\ncurl -s -o \/dev\/null -w "HTTPStatus: %{http_code} | TotalTime: %{time_total}s\\\\n" -X POST http:\/\/gateway.internal\/v1\/chat\/completions -H "Content-Type: application\/json" -d '{"model":"llama3-70b","prompt":"ping"}'<\/p>\n
\u63a7\u5236\u53f0\u6355\u83b7\u5230\u7684\u54cd\u5e94\u7ed3\u679c\u5448\u73b0\u660e\u663e\u7684\u8d44\u6e90\u74f6\u9888：<\/p>\n
HTTPStatus: 504 | TotalTime: 15.002s
\nGPU Timestamp, name, memory.used [MiB], memory.total [MiB], utilization.gpu [%]
\n2026-08-31 02:30:11, NVIDIA A100-SXM4-80GB, 81500 MiB, 81920 MiB, 100 %<\/p>\n
\u663e\u5b58\u5df2\u65e0\u53ef\u5206\u914d\u7a7a\u95f4，\u63a8\u7406\u7ebf\u7a0b\u5728\u7b49\u5f85 Cuda Malloc \u91ca\u653e\u7a7a\u95f4，\u800c\u4e0a\u6e38\u8bf7\u6c42\u4ecd\u5728\u6301\u7eed\u6d8c\u5165\u3002<\/p>\n
\u9762\u5bf9\u5927\u6a21\u578b\u63a8\u7406\u8fc7\u7a0b\u4e2d\u7684\u5ef6\u8fdf\u6296\u52a8，\u964d\u7ea7\u65b9\u6848\u4e0d\u80fd\u4f9d\u8d56\u4eba\u5de5\u767b\u5f55\u96c6\u7fa4\u624b\u52a8\u4fee\u6539 Service Label，\u5e94\u7531 Sidecar \u4ee3\u7406\u5c42\u6216 API \u7f51\u5173\u5c42\u5b9e\u73b0\u79d2\u7ea7\u81ea\u52a8\u5207\u6d41\u3002<\/p>\n
\u8def\u7531\u89c4\u5219\u901a\u5e38\u5206\u4e3a\u4e09\u7ea7\u6cbb\u7406\u7b56\u7565：\u4f18\u5148\u8def\u7531\u81f3 70B \u4e3b\u6a21\u578b；\u4e00\u65e6\u4e3b\u6a21\u578b\u8fde\u7eed 3 \u6b21\u54cd\u5e94\u8d85\u65f6\u6216\u8fd4\u56de 50x \u9519\u8bef，\u81ea\u52a8\u89e6\u53d1\u7194\u65ad\u5e76\u964d\u7ea7\u81f3 8B \u5907\u7528\u8f7b\u91cf\u6a21\u578b；\u82e5\u5907\u7528\u6a21\u578b\u4e5f\u5904\u4e8e\u9ad8\u8d1f\u8f7d\u72b6\u6001，\u76f4\u63a5\u8fd4\u56de\u5e26\u6709\u8bed\u4e49\u515c\u5e95\u63d0\u793a\u7684\u9759\u6001 JSON \u54cd\u5e94\u3002<\/p>\n
\u5728 Sidecar \u6216\u7f51\u5173\u5c42\u6ce8\u5165\u8f7b\u91cf\u7ea7 Proxy \u903b\u8f91，\u7528\u4e8e\u5b9e\u65f6\u63a2\u6d4b\u4e0b\u6e38 vLLM \u5b9e\u4f8b\u7684\u670d\u52a1\u72b6\u6001\u3002\u5f53\u9519\u8bef\u7387\u5230\u8fbe\u9884\u8bbe\u9608\u503c\u65f6，\u81ea\u52a8\u4fee\u6539\u5185\u5b58\u4e2d\u7684 upstream \u76ee\u6807\u5730\u5740，\u5b9e\u73b0\u65e0\u611f\u77e5\u670d\u52a1\u964d\u7ea7\u3002<\/p>\n
package downgrade<\/p>\n
import (
\n"context"
\n"fmt"
\n"net\/http"
\n"net\/http\/httputil"
\n"net\/url"
\n"sync\/atomic"
\n"time"
\n)<\/p>\n
type ModelProxy struct {
\nprimaryURL *url.URL
\nfallbackURL *url.URL
\nfailureCount int64
\nthreshold int64
\nisDowngraded int32
\n}<\/p>\n
func NewModelProxy(primary, fallback string, failureThreshold int64) (*ModelProxy, error) {
\npURL, err := url.Parse(primary)
\nif err != nil {
\nreturn nil, fmt.Errorf("invalid primary url: %w", err)
\n}
\nfURL, err := url.Parse(fallback)
\nif err != nil {
\nreturn nil, fmt.Errorf("invalid fallback url: %w", err)
\n}<\/p>\n
return &ModelProxy{
\nprimaryURL: pURL,
\nfallbackURL: fURL,
\nthreshold: failureThreshold,
\nisDowngraded: 0,
\n}, nil
\n}<\/p>\n
func (p *ModelProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
\ndowngraded := atomic.LoadInt32(&p.isDowngraded) == 1<\/p>\n
targetURL := p.primaryURL
\nif downgraded {
\ntargetURL = p.fallbackURL
\nr.Header.Set("X-Model-Downgraded", "true")
\n}<\/p>\n
proxy := httputil.NewSingleHostReverseProxy(targetURL)
\nproxy.Transport = &http.Transport{
\nResponseHeaderTimeout: 3 * time.Second,
\n}<\/p>\n
proxy.ErrorHandler = func(rw http.ResponseWriter, req *http.Request, err error) {
\nif !downgraded {
\nfailures := atomic.AddInt64(&p.failureCount, 1)
\nif failures >= p.threshold {
\natomic.StoreInt32(&p.isDowngraded, 1)
\n\/\/ \u5f00\u542f\u5f02\u6b65\u63a2\u6d3b\u6062\u590d\u534f\u7a0b
\ngo p.recoverHealthCheck()
\n}
\n}<\/p>\n
rw.Header().Set("Content-Type", "application\/json")
\nrw.WriteHeader(http.StatusServiceUnavailable)
\n_, _ = rw.Write([]byte(`{"error":{"code":"service_downgraded","message":"Primary model overloaded, fallback active."}}`))
\n}<\/p>\n
proxy.ServeHTTP(w, r)
\n}<\/p>\n
func (p *ModelProxy) recoverHealthCheck() {
\nticker := time.NewTicker(5 * time.Second)
\ndefer ticker.Stop()<\/p>\n
for range ticker.C {
\nctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
\nreq, _ := http.NewRequestWithContext(ctx, "GET", p.primaryURL.String()+"\/health", nil)
\nresp, err := http.DefaultClient.Do(req)
\ncancel()<\/p>\n
if err == nil && resp.StatusCode == http.StatusOK {
\natomic.StoreInt32(&p.isDowngraded, 0)
\natomic.StoreInt64(&p.failureCount, 0)
\nreturn
\n}
\n}
\n}<\/p>\n
\u4ee3\u7801\u5b9e\u73b0\u7684\u6838\u5fc3\u5728\u4e8e\u901a\u8fc7 atomic \u4fdd\u8bc1\u591a\u534f\u7a0b\u5e76\u53d1\u8bf7\u6c42\u4e0b\u7684\u65e0\u9501\u5207\u6d41\u6548\u7387\u3002\u5c06 ResponseHeaderTimeout \u9650\u5236\u5728 3 \u79d2\u4ee5\u5185，\u907f\u514d\u4e0b\u6e38 GPU \u5361\u6b7b\u62d6\u57ae\u4ee3\u7406\u5c42\u7684\u8fde\u63a5\u6c60\u3002<\/p>\n
\u4ee3\u7801\u90e8\u7f72\u4e0a\u7ebf\u524d，\u5e94\u5728\u6d4b\u8bd5\u73af\u5883\u4e2d\u6ce8\u5165\u8d85\u65f6\u6545\u969c\u8fdb\u884c\u95ed\u73af\u9a8c\u8bc1\u3002\u5229\u7528\u6d4b\u8bd5\u811a\u672c\u6a21\u62df\u9ad8\u538b\u8bf7\u6c42\u5e76\u6253\u6ee1 GPU \u663e\u5b58，\u89c2\u6d4b Proxy \u7684\u54cd\u5e94 Header \u4e0e\u5207\u6d41\u52a8\u4f5c：<\/p>\n
# \u6a21\u62df\u5e76\u53d1\u9ad8\u538b\u8bf7\u6c42
\nhey -z 20s -c 40 -m POST -H "Content-Type: application\/json" -d '{"prompt":"Generate long context string…"}' http:\/\/localhost:8080\/v1\/chat\/completions<\/p>\n
# \u5b9e\u65f6\u76d1\u6d4b\u964d\u7ea7 Header
\ncurl -i -X POST http:\/\/localhost:8080\/v1\/chat\/completions -H "Content-Type: application\/json" -d '{"prompt":"test"}' | grep -E "(HTTP\/|X-Model-Downgraded)"<\/p>\n
\u8fd4\u56de\u7684\u8bca\u65ad\u65e5\u5fd7\u786e\u8ba4\u6d41\u91cf\u5df2\u987a\u7545\u5207\u6362：<\/p>\n
HTTP\/1.1 200 OK
\nX-Model-Downgraded: true<\/p>\n
\u539f\u672c\u53ef\u80fd\u5bfc\u81f4\u7cfb\u7edf\u54cd\u5e94\u6302\u8d77\u7684\u5ef6\u8fdf，\u88ab\u63a7\u5236\u5728 3 \u79d2\u4ee5\u5185\u5e76\u81ea\u52a8\u964d\u7ea7\u4e3a\u5907\u7528\u8282\u70b9\u7684\u54cd\u5e94\u3002<\/p>\n
\u751f\u4ea7\u73af\u5883\u843d\u5730\u65f6\u9700\u8981\u9488\u5bf9\u4ee5\u4e0b\u5173\u952e\u7ec6\u8282\u8fdb\u884c\u9632\u62a4：<\/p>\n
\u9996\u5148，\u964d\u7ea7\u540e\u7684 8B \u5c0f\u6a21\u578b\u8282\u70b9\u82e5\u627f\u63a5\u5168\u90e8\u6d41\u91cf，\u53ef\u80fd\u56e0\u8d44\u6e90\u8fc7\u8f7d\u89e6\u53d1\u4e8c\u6b21\u6545\u969c\u3002\u56e0\u6b64\u5e94\u5728\u964d\u7ea7\u8282\u70b9\u524d\u4e25\u683c\u914d\u7f6e Rate Limiter \u9650\u5236\u603b QPS\u3002<\/p>\n
\u5176\u6b21，\u63a2\u6d3b\u673a\u5236（Health Check）\u8fc7\u4e8e\u9891\u7e41\u53ef\u80fd\u5bfc\u81f4\u521a\u6062\u590d\u7684\u4e3b\u8282\u70b9\u518d\u6b21\u88ab\u7a81\u53d1\u6d41\u91cf\u51b2\u57ae\u3002\u5de5\u7a0b\u5b9e\u8df5\u4e2d\u5e94\u5f15\u5165\u9000\u907f\u63a2\u6d3b\u673a\u5236，\u5e76\u5728\u5207\u56de\u4e3b\u8282\u70b9\u65f6\u91c7\u7528\u9636\u68af\u5f0f\u6d41\u91cf\u6062\u590d\u7b56\u7565（\u5982\u6309 10%\u300130%\u3001\u6309\u68c0\u67e5\u7ed3\u679c\u786e\u8ba4 \u9010\u6b65\u653e\u884c）\u3002<\/p>\n
\u5c06\u964d\u7ea7\u65ad\u8def\u5668\u903b\u8f91\u914d\u7f6e\u4e8e Helm \u7684 ConfigMap，\u5e76\u7ed3\u5408 Kubernetes \u7684 Readiness Probe \u52a8\u6001\u63a2\u9488，\u80fd\u591f\u4fdd\u969c\u5728 GPU \u663e\u5b58\u788e\u7247\u5316\u6216\u6a21\u578b\u63a8\u65ad\u5361\u6b7b\u65f6，\u7ebf\u4e0a\u4e1a\u52a1\u4ecd\u7136\u53ef\u4ee5\u5e73\u7a33\u8fc7\u6e21\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u4e91\u539f\u751f\u4ea4\u4ed8\u670d\u52a1\u5f02\u5e38\u65f6\u5982\u4f55\u5206\u5c42\u964d\u7ea7\u63a8\u7406\u670d\u52a1\u51fa\u73b0\u6392\u961f\u3001\u663e\u5b58\u4e0d\u8db3\u6216\u7f51\u5173\u8d85\u65f6\u65f6，\u5148\u628a\u6d41\u91cf\u548c\u6545\u969c\u57df\u9694\u5f00\u3002\u964d\u7ea7\u4e0d\u662f\u7b80\u5355\u6362\u6a21\u578b：\u9700\u8981\u533a\u5206\u53ef\u6392\u961f\u8bf7\u6c42\u3001\u53ef\u8fd4\u56de\u7f13\u5b58\u7684\u8bf7\u6c42，\u4ee5\u53ca\u5e94\u76f4\u63a5\u63d0\u793a\u7a0d\u540e\u91cd\u8bd5\u7684\u8bf7\u6c42\u3002
\n\u73b0\u573a\u6545\u969c\u4e0e\u7f51\u5173\u8d85\u65f6：GPU \u663e\u5b58\u6ea2\u51fa\u5f15\u53d1\u7684 504 \u8fde\u9501\u54cd\u5e94\u3002
\n\u6392\u67e5\u6545\u969c\u7b2c\u4e00\u6b65\u662f\u901a\u8fc7\u6807\u51c6\u547d\u4ee4\u884c\u63d0\u53d6\u771f\u5b9e\u7684\u73b0\u573a\u8bca\u65ad\u6307\u6807\u3002\u5229\u7528\u547d\u4ee4\u884c\u6293\u53d6 Pod \u4e8b\u4ef6\u4e0e GPU \u5360\u7528\u53c2\u6570：
\nkubectl get pods -n llm-prod<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[9680,66,120,763,50],"topic":[],"class_list":["post-98502","post","type-post","status-publish","format-standard","hentry","category-server","tag-k8","tag-ai","tag-kubernetes","tag-763","tag-50"],"yoast_head":"\n