![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114151-69a81a7f34f41.png\")
<\/p>\n\u9996\u5148\u521b\u5efa\u865a\u62df\u73af\u5883<\/p>\n
conda create -n ai_dev<\/p>\n
\u6fc0\u6d3b\u865a\u62df\u73af\u5883<\/p>\n
conda activate ai_dev<\/p>\n
\u4f7f\u7528conda\u5b89\u88c5pytorch<\/p>\n
conda \u4f1a\u81ea\u52a8\u5904\u7406 CUDA \u7248\u672c\u517c\u5bb9\u548c\u4f9d\u8d56，\u907f\u514d pip \u5b89\u88c5\u65f6\u7684\u7248\u672c\u4e0d\u5339\u914d\u95ee\u9898<\/p>\n
conda install pytorch==2.2.0 torchvision==0.17.0 torchaudio==2.2.0 pytorch-cuda=12.1 -c pytorch -c nvidia<\/p>\n
pytorch\u5b89\u88c5\u6210\u529f<\/p>\n
<\/p>\n
\u5728 ai_dev \u73af\u5883\u4e2d\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4，\u68c0\u67e5 PyTorch \u548c CUDA \u662f\u5426\u53ef\u7528：<\/p>\n
python -c "import torch; print('PyTorch\u7248\u672c:', torch.__version__); print('CUDA\u53ef\u7528:', torch.cuda.is_available())"<\/p>\n
\u6b64\u5904\u6211\u62a5\u4e86\u9519\u8bef<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114152-69a81a808ebd8.png\")
<\/p>\n
\u964d\u7ea7numpy<\/p>\n
conda \u4f1a\u81ea\u52a8\u5904\u7406\u4f9d\u8d56\u51b2\u7a81，\u5982\u679c opencv-python \u9700\u8981 numpy>=2，\u5b83\u4f1a\u63d0\u793a\u4f60\u662f\u5426\u964d\u7ea7 opencv-python，\u9009\u62e9 y \u5373\u53ef\u3002<\/p>\n
conda install "numpy<2"<\/p>\n
\u6267\u884c\u4ee5\u4e0b\u547d\u4ee4，\u68c0\u67e5 NumPy \u7248\u672c\u548c PyTorch \u521d\u59cb\u5316\u662f\u5426\u6b63\u5e38：<\/p>\n
python -c "
\nimport numpy
\nimport torch
\nprint('NumPy\u7248\u672c:', numpy.__version__)
\nprint('PyTorch\u7248\u672c:', torch.__version__)
\nprint('CUDA\u53ef\u7528:', torch.cuda.is_available())
\n"<\/p>\n
\u73af\u5883\u5df2\u7ecf\u5c31\u7eea：<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114153-69a81a81ca2fd.png\")
<\/p>\n
\u5b89\u88c5git-lfs<\/p>\n
sudo apt update && sudo apt install -y git-lfs<\/p>\n
\u521b\u5efa\u76ee\u5f55\u5e76\u4e0b\u8f7d\u5f00\u6e90\u6a21\u578b<\/p>\n
# 1. \u521b\u5efa\u76ee\u5f55\u7ed3\u6784（\u5728\u4f60\u7684\u7528\u6237\u76ee\u5f55\u4e0b）
\nmkdir -p ~\/ai_project ~\/models\/llm ~\/models\/sdxl
\ncd ~\/ai_project<\/p>\n
# 2. \u4e0b\u8f7d Qwen-7B-Chat（\u5f00\u6e90LLM，\u6587\u672c\u6838\u5fc3）
\ngit lfs install
\ngit clone https:\/\/www.modelscope.cn\/qwen\/Qwen-7B-Chat.git \/home\/ubuntu\/models\/llm\/Qwen-7B-Chat<\/p>\n
\u67e5\u770b\u6a21\u578b\u90e8\u7f72\u4f4d\u7f6e<\/p>\n
ls ~\/models\/llm\/Qwen-7B-Chat-Int4\/<\/p>\n
\u6a21\u578b\u6743\u91cd\u6587\u4ef6：<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114154-69a81a822df23.png\")
<\/p>\n
\u786e\u4fdd\u5728 ai_project \u76ee\u5f55\u4e0b：<\/p>\n
cd ~\/ai_project<\/p>\n
\u521b\u5efa llm_deploy.py \u6587\u4ef6<\/p>\n
vim llm_deploy.py<\/p>\n
# \u96a7\u9053\u901a\u7528\u95ee\u9898\u95ee\u7b54（\u5f7b\u5e95\u622a\u65ad\u65e0\u5173\u5185\u5bb9）
\ndef general_qa(self, query):
\n prompt = f"""
\n \u4f60\u662f\u4e13\u4e1a\u7684\u96a7\u9053\u5de5\u7a0b\u54a8\u8be2\u52a9\u624b，\u4ec5\u56de\u7b54\u96a7\u9053\u76f8\u5173\u4e13\u4e1a\u95ee\u9898，\u56de\u7b54\u8981\u7b80\u6d01\u3001\u4e13\u4e1a\u3001\u6613\u61c2\u3002
\n \u7528\u6237\u95ee\u9898：{query}
\n \u5f3a\u5236\u8981\u6c42：\u53ea\u8f93\u51fa\u95ee\u9898\u7684\u6838\u5fc3\u56de\u7b54，\u7981\u6b62\u7eed\u5199\u4efb\u4f55\u65e0\u5173\u5185\u5bb9\u3001\u6d4b\u8bd5\u9898\u3001\u5176\u4ed6\u9886\u57df\u77e5\u8bc6！\u56de\u7b54\u5b8c\u7acb\u5373\u505c\u6b62\u3002
\n """
\n start_time = time.time()
\n messages = [{"role": "user", "content": prompt}]
\n text = self.tokenizer.apply_chat_template(
\n messages,
\n tokenize=False,
\n add_generation_prompt=True
\n )
\n model_inputs = self.tokenizer([text], return_tensors="pt").to(DEVICE)<\/p>\n
# \u79fb\u9664\u4e86\u4e0d\u652f\u6301\u7684 stop_sequence \u53c2\u6570
\n generated_ids = self.model.generate(
\n **model_inputs,
\n generation_config=self.model.generation_config
\n )
\n generated_ids = [
\n output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
\n ]
\n response = self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]<\/p>\n
# \u540e\u5904\u7406：\u622a\u65ad\u6240\u6709\u65e0\u5173\u5185\u5bb9，\u53ea\u4fdd\u7559\u6838\u5fc3\u56de\u7b54
\n stop_keywords = [
\n "\u51c6\u5676\u5c14\u76c6\u5730", "\u4e0b\u5217\u54ea\u4e2a\u9009\u9879", "\u4ee5\u4e0b\u54ea\u4e2a\u9009\u9879", "\u8bba\u6587\u7684\u5173\u952e\u8bcd",
\n "Photoshop", "\u7f16\u7a0b\u8bed\u8a00", "\u9009\u62e9\u9898", "1. ", "2. ", "3. ", "4. ",
\n "\u5473\u7cbe\u7684\u4e3b\u8981\u6210\u5206\u662f\u4ec0\u4e48？", "iMore\u8be6\u7ec6\u89e3\u91ca\u4e00\u4e0b", "\u5b9a\u4e49\u201c\u4eba\u5de5\u667a\u80fd\u201d",
\n "\u7701\u7565\u53f7\u7684\u4f5c\u7528\u662f\u4ec0\u4e48", "\u6162\u6027\u963b\u585e\u6027\u80ba\u75be\u75c5\u7684\u8bca\u65ad\u4f9d\u636e", "\u6d77\u5916\u5e02\u573a",
\n "\u667a\u80fd\u5065\u5eb7\u624b\u73af", "\u4ea7\u54c1\u540d\u79f0", "\u76ee\u6807\u53d7\u4f17", "\u9884\u7b97", "\u65b9\u6cd5", "\u89e3\u91ca",
\n "\u9884\u8ba1\u6548\u679c", "\u60f3\u51fa\u4e00\u79cd\u65b0\u7684\u65b9\u5f0f", "\u63a8\u5e7f\u4e00\u6b3e\u65b0\u4ea7\u54c1", "\u4e3a\u4ec0\u4e48\u8fd9\u79cd\u65b9\u6cd5\u6709\u6548"
\n ]
\n for kw in stop_keywords:
\n if kw in response:
\n response = response.split(kw)[0].strip()
\n break<\/p>\n
# \u6e05\u7406\u9996\u5c3e\u591a\u4f59\u7a7a\u683c\/\u6362\u884c
\n response = response.strip()<\/p>\n
mem_used = torch.cuda.max_memory_allocated(DEVICE) \/ 1024 \/ 1024 \/ 1024
\n print(f"\u901a\u7528\u95ee\u7b54\u63a8\u7406\u8017\u65f6：{time.time()-start_time:.2f}s，\u663e\u5b58\u5360\u7528：{mem_used:.2f}G")
\n return response<\/p>\n
\u6267\u884c ls \u547d\u4ee4，\u7ec8\u7aef\u663e\u793a llm_deploy.py \u5c31\u8bf4\u660e\u6587\u4ef6\u521b\u5efa\u6210\u529f<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114154-69a81a825e477.png\")
<\/p>\n
\u8fd0\u884cLLM<\/p>\n
\u6267\u884c\u547d\u4ee4（\u6307\u5b9a\u7528\u5361 0，\u907f\u514d\u663e\u5b58\u51b2\u7a81）：<\/p>\n
CUDA_VISIBLE_DEVICES=0 python llm_deploy.py<\/p>\n
\u8fd9\u91cc\u62a5\u9519\u6ca1\u6709\u5b89\u88c5transformers\u4f9d\u8d56：<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114154-69a81a827317d.png\")
<\/p>\n
\u5b89\u88c5transformers：<\/p>\n
python -m pip install transformers==4.37.2 torchvision==0.17.2 –break-system-packages<\/p>\n
\u518d\u6b21\u8fd0\u884cLLM<\/p>\n
\u8fd9\u91cc\u62a5\u9519<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114154-69a81a82942a1.png\")
<\/p>\n
\u5b89\u88c5\u8fd9\u4e2a\u5305：<\/p>\n
python -m pip install tiktoken –break-system-packages<\/p>\n
\u5728\u5b89\u88c5\u53e6\u5916\u4e24\u4e2a\u4f9d\u8d56\u5305<\/p>\n
python -m pip install einops transformers_stream_generator –break-system-packages<\/p>\n
\u5b89\u88c5torch<\/p>\n
python -m pip install torch==2.3.0 torchvision==0.18.0 –break-system-packages<\/p>\n
\u5b89\u88c5transformers_stream_generator<\/p>\n
python -m pip install transformers_stream_generator –break-system-packages<\/p>\n
\u56e0\u4e3a\u52a0\u8f7d GPTQ \u91cf\u5316\u6a21\u578b\u9700\u8981 auto-gptq \u5305，\u6211\u4eec\u5148\u628a\u5b83\u88c5\u4e0a：<\/p>\n
\u5b89\u88c5gptq<\/p>\n
python -m pip install auto-gptq –break-system-packages<\/p>\n
\u5b89\u88c5\u4e00\u4e2agptq\u4f9d\u8d56\u5e93optimum<\/p>\n
python -m pip install optimum –break-system-packages<\/p>\n
\u73b0\u5728\u5927\u6a21\u578b\u5df2\u7ecf\u53ef\u4ee5\u542f\u52a8\u4e86，\u6211\u4eec\u5c01\u88c5\u4e00\u4e2aapi\u5b9e\u73b0\u524d\u540e\u7aef\u5206\u79bb\u3002<\/p>\n
\u5c01\u88c5fastapi<\/p>\n
# \u786e\u4fdd\u4f60\u5728 ai_dev \u73af\u5883\u4e2d
\nconda activate ai_dev<\/p>\n
# \u4f7f\u7528 conda \u5b89\u88c5 fastapi \u548c uvicorn
\nconda install fastapi uvicorn -c conda-forge<\/p>\n
\u5148\u786e\u4fdd\u4f60\u5728 ~\/ai_project \u8fd9\u4e2a\u6587\u4ef6\u5939\u91cc：<\/p>\n
cd ~\/ai_project<\/p>\n
\u521b\u5efa\u5e76\u6253\u5f00 tunnel_llm_api.py \u6587\u4ef6：<\/p>\n
vim tunnel_llm_api.py<\/p>\n
\u7c98\u8d34\u4ee3\u7801<\/p>\n
import torch
\nimport json
\nimport time
\nfrom fastapi import FastAPI, HTTPException
\nfrom pydantic import BaseModel
\nfrom transformers import AutoModelForCausalLM, AutoTokenizer
\nfrom transformers.generation import GenerationConfig<\/p>\n
# ========== \u521d\u59cb\u5316FastAPI ==========
\napp = FastAPI(title="\u96a7\u9053LLM API\u670d\u52a1", version="1.0")<\/p>\n
# ========== \u6a21\u578b\u914d\u7f6e ==========
\nDEVICE = "cuda:0"
\nMODEL_PATH = "\/home\/ubuntu\/models\/llm\/Qwen-7B-Chat"
\nTORCH_DTYPE = torch.float16
\nMAX_NEW_TOKENS = 2048<\/p>\n
# ========== \u5168\u5c40\u52a0\u8f7d\u6a21\u578b（\u542f\u52a8\u65f6\u52a0\u8f7d\u4e00\u6b21） ==========
\nprint("\u5f00\u59cb\u52a0\u8f7dLLM\u6a21\u578b…")
\ntokenizer = AutoTokenizer.from_pretrained(
\n MODEL_PATH, trust_remote_code=True, resume_download=True
\n)
\nmodel = AutoModelForCausalLM.from_pretrained(
\n MODEL_PATH,
\n device_map=DEVICE,
\n torch_dtype=TORCH_DTYPE,
\n trust_remote_code=True,
\n resume_download=True
\n)
\n# \u901a\u7528\u95ee\u7b54\u914d\u7f6e
\ngeneral_gen_config = GenerationConfig(
\n temperature=0.7, top_p=0.9, repetition_penalty=1.1, max_new_tokens=MAX_NEW_TOKENS, do_sample=True
\n)
\n# \u7ed3\u6784\u5316\u4efb\u52a1\u914d\u7f6e
\nstructured_gen_config = GenerationConfig(
\n temperature=0.01, top_p=0.9, repetition_penalty=1.2, max_new_tokens=MAX_NEW_TOKENS, do_sample=False
\n)
\nprint("\u6a21\u578b\u52a0\u8f7d\u5b8c\u6210！")<\/p>\n
# ========== \u6570\u636e\u6a21\u578b（API\u5165\u53c2） ==========
\nclass QueryRequest(BaseModel):
\n query: str
\n type: str = "general" # general:\u901a\u7528\u95ee\u7b54, intent:\u610f\u56fe\u8bc6\u522b, warn:\u9884\u8b66, disease:\u75c5\u5bb3, collision:\u78b0\u649e<\/p>\n
# ========== \u8f85\u52a9\u51fd\u6570 ==========
\ndef extract_complete_json(text):
\n start_idx = text.find('{')
\n if start_idx == -1:
\n return None
\n brace_count = 0
\n end_idx = -1
\n for i in range(start_idx, len(text)):
\n if text[i] == '{':
\n brace_count += 1
\n elif text[i] == '}':
\n brace_count -= 1
\n if brace_count == 0:
\n end_idx = i
\n break
\n if end_idx == -1:
\n return None
\n return text[start_idx:end_idx+1]<\/p>\n
def general_qa(query):
\n prompt = f"""
\n \u4f60\u662f\u4e13\u4e1a\u7684\u96a7\u9053\u5de5\u7a0b\u54a8\u8be2\u52a9\u624b，\u4ec5\u56de\u7b54\u96a7\u9053\u76f8\u5173\u4e13\u4e1a\u95ee\u9898，\u56de\u7b54\u8981\u7b80\u6d01\u3001\u4e13\u4e1a\u3001\u6613\u61c2\u3002
\n \u7528\u6237\u95ee\u9898：{query}
\n \u5f3a\u5236\u8981\u6c42：\u53ea\u8f93\u51fa\u95ee\u9898\u7684\u6838\u5fc3\u56de\u7b54，\u7981\u6b62\u7eed\u5199\u4efb\u4f55\u65e0\u5173\u5185\u5bb9！
\n """
\n messages = [{"role": "user", "content": prompt}]
\n text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
\n model_inputs = tokenizer([text], return_tensors="pt").to(DEVICE)
\n generated_ids = model.generate(**model_inputs, generation_config=general_gen_config)
\n generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]
\n response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]<\/p>\n
# \u622a\u65ad\u65e0\u5173\u5185\u5bb9
\n stop_keywords = ["\u51c6\u5676\u5c14\u76c6\u5730", "\u4e0b\u5217\u54ea\u4e2a\u9009\u9879", "Photoshop", "1. "]
\n for kw in stop_keywords:
\n if kw in response:
\n response = response.split(kw)[0].strip()
\n return response.strip()<\/p>\n
def generate_structured(prompt):
\n messages = [{"role": "user", "content": prompt}]
\n text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
\n model_inputs = tokenizer([text], return_tensors="pt").to(DEVICE)
\n generated_ids = model.generate(**model_inputs, generation_config=structured_gen_config)
\n generated_ids = [output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)]
\n response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]<\/p>\n
try:
\n json_str = extract_complete_json(response)
\n if json_str is None:
\n raise ValueError("\u672a\u627e\u5230\u5b8c\u6574JSON")
\n return json.loads(json_str)
\n except Exception as e:
\n return {"error": "\u683c\u5f0f\u9519\u8bef", "detail": str(e)}<\/p>\n
# ========== API\u63a5\u53e3 ==========
\n@app.post("\/api\/chat")
\nasync def chat(request: QueryRequest):
\n try:
\n start_time = time.time()
\n if request.type == "general":
\n # \u901a\u7528\u95ee\u7b54
\n result = {"answer": general_qa(request.query)}
\n elif request.type == "intent":
\n # \u610f\u56fe\u8bc6\u522b
\n prompt = f"""
\n \u89d2\u8272：\u96a7\u9053\u573a\u666f\u610f\u56fe\u8bc6\u522b\u6a21\u578b
\n \u8f93\u5165：{request.query}
\n \u8f93\u51fa\u683c\u5f0f：{{"intent":"traffic_query","slot":{{"tunnel_name":"\u9e4f\u9e44\u96a7\u9053","date":"\u4eca\u5929"}}}}
\n """
\n result = generate_structured(prompt)
\n elif request.type == "warn":
\n # \u9884\u8b66\u751f\u6210（\u793a\u4f8b\u53c2\u6570，\u53ef\u4ecequery\u63d0\u53d6）
\n prompt = f"""
\n \u89d2\u8272：\u96a7\u9053\u9884\u8b66\u751f\u6210\u6a21\u578b
\n \u8f93\u5165：\u6307\u6807=\u4e00\u6c27\u5316\u78b3，\u5f53\u524d\u503c=200.22ppm，\u9608\u503c=150ppm
\n \u8f93\u51fa\u683c\u5f0f：{{"warn_text":"","suggestion":""}}
\n """
\n result = generate_structured(prompt)
\n elif request.type == "disease":
\n # \u75c5\u5bb3\u9884\u6d4b
\n prompt = f"""
\n \u89d2\u8272：\u96a7\u9053\u75c5\u5bb3\u9884\u6d4b\u6a21\u578b
\n \u8f93\u5165：\u75c5\u5bb3\u7c7b\u578b=\u6a2a\u5411\u88c2\u7f1d，\u5f53\u524d\u9762\u79ef=0.4540\u33a1，\u9884\u6d4b\u65f6\u95f4=12\u4e2a\u6708
\n \u8f93\u51fa\u683c\u5f0f：{{"future_area":"","growth_rate":"","level":"","secondary_risk":"","evolution_type":""}}
\n """
\n result = generate_structured(prompt)
\n elif request.type == "collision":
\n # \u78b0\u649e\u89e3\u6790
\n prompt = f"""
\n \u89d2\u8272：\u96a7\u9053\u78b0\u649e\u89e3\u6790\u6a21\u578b
\n \u8f93\u5165：{request.query}
\n \u8f93\u51fa\u683c\u5f0f：{{"car_type1":"SUV","car_type2":"\u8d27\u8f66","angle":"\u4fa7\u9762","speed":"40km\/h","time_steps":[1,3,5,10]}}
\n """
\n result = generate_structured(prompt)
\n else:
\n raise HTTPException(status_code=400, detail="\u4e0d\u652f\u6301\u7684\u7c7b\u578b")<\/p>\n
return {
\n "code": 200,
\n "msg": "success",
\n "data": result,
\n "time_cost": f"{time.time()-start_time:.2f}s"
\n }
\n except Exception as e:
\n raise HTTPException(status_code=500, detail=f"\u670d\u52a1\u5668\u9519\u8bef：{str(e)}")<\/p>\n
# ========== \u542f\u52a8\u670d\u52a1 ==========
\nif __name__ == "__main__":
\n # \u542f\u52a8API\u670d\u52a1，\u5141\u8bb8\u5916\u90e8\u8bbf\u95ee（0.0.0.0）
\n import uvicorn
\n uvicorn.run(
\n app="tunnel_llm_api:app",
\n host="0.0.0.0", # \u5141\u8bb8\u6240\u6709IP\u8bbf\u95ee
\n port=8000, # \u7aef\u53e3
\n workers=1, # \u5355\u8fdb\u7a0b（\u6a21\u578b\u52a0\u8f7d\u4e00\u6b21）
\n reload=False # \u751f\u4ea7\u73af\u5883\u5173\u95ed\u70ed\u91cd\u8f7d
\n )<\/p>\n
\u5728\u76ee\u5f55\u4e0b\u6267\u884c\u4ee3\u7801：<\/p>\n
CUDA_VISIBLE_DEVICES=0 nohup python tunnel_llm_api.py > llm_api.log 2>&1 &<\/p>\n
\u67e5\u770b\u6a21\u578b\u52a0\u8f7d\u5b9e\u65f6\u65e5\u5fd7：<\/p>\n
tail -f llm_api.log<\/p>\n
\u6a21\u578b\u52a0\u8f7d\u6210\u529f：<\/p>\n
![\"\"](\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/03\/20260304114154-69a81a82b2687.png\")
<\/p>\n
\u5728\u53e6\u4e00\u4e2a\u7ec8\u7aef\u7a97\u53e3（\u4fdd\u6301\u5f53\u524d\u65e5\u5fd7\u7a97\u53e3\u6253\u5f00，\u65b9\u4fbf\u76d1\u63a7）\u6267\u884c：<\/p>\n
conda activate ai_dev
\ncd ~\/ai_project<\/p>\n
\u5b89\u88c5curl\u5de5\u5177<\/p>\n
sudo apt update && sudo apt install -y curl<\/p>\n
# \u6d4b\u8bd5\u901a\u7528\u95ee\u7b54
\ncurl -X POST "http:\/\/localhost:8000\/api\/chat" -H "Content-Type: application\/json" -d '{"query":"\u96a7\u9053\u901a\u98ce\u7cfb\u7edf\u6709\u54ea\u4e9b\u7c7b\u578b？","type":"general"}'<\/p>\n
\u56db\u3001\u524d\u7aef\u90e8\u7f72<\/p>\n
\u9996\u5148\u68c0\u67e5\u8fdb\u7a0b\u662f\u5426\u8fd8\u5728<\/p>\n
# \u68c0\u67e5 API \u670d\u52a1\u8fdb\u7a0b
\nps -ef | grep tunnel_llm_api.py
\n# \u5982\u679c\u6ca1\u6709\u8fdb\u7a0b，\u91cd\u65b0\u542f\u52a8
\nconda activate ai_dev
\ncd ~\/ai_project
\nCUDA_VISIBLE_DEVICES=0 nohup python tunnel_llm_api.py > llm_api.log 2>&1 &<\/p>\n
Streamlit \u662f\u5feb\u901f\u642d\u5efa Python Web \u5e94\u7528\u7684\u5de5\u5177，\u65e0\u9700\u5199\u524d\u7aef\u4ee3\u7801，\u5148\u5b89\u88c5：<\/p>\n
# \u786e\u4fdd\u4f60\u5728 ai_dev \u73af\u5883\u4e2d
\nconda activate ai_dev<\/p>\n
# \u4f7f\u7528 conda \u5b89\u88c5 streamlit
\nconda install streamlit -c conda-forge<\/p>\n
\u5728 ~\/ai_project \u76ee\u5f55\u4e0b\u521b\u5efa llm_chat_app.py \u6587\u4ef6：<\/p>\n
# \u8fdb\u5165\u9879\u76ee\u76ee\u5f55
\ncd ~\/ai_project<\/p>\n
# \u7528 vim \u521b\u5efa\u5e76\u7f16\u8f91\u6587\u4ef6
\nvim llm_chat_app.py<\/p>\n
import streamlit as st
\nimport requests
\nimport time<\/p>\n
# ===================== \u9875\u9762\u57fa\u7840\u914d\u7f6e =====================
\n# \u8bbe\u7f6e\u9875\u9762\u6807\u9898\u3001\u56fe\u6807\u3001\u5e03\u5c40
\nst.set_page_config(
\n page_title="\u96a7\u9053\u5de5\u7a0b\u5927\u6a21\u578b\u52a9\u624b",
\n page_icon="\u26f0\ufe0f",
\n layout="wide", # \u5bbd\u5c4f\u5e03\u5c40
\n initial_sidebar_state="collapsed" # \u6536\u8d77\u4fa7\u8fb9\u680f
\n)<\/p>\n
# ===================== \u6837\u5f0f\u7f8e\u5316（\u53ef\u9009，\u63d0\u5347\u4f53\u9a8c） =====================
\nst.markdown("""
\n <style>
\n \/* \u804a\u5929\u6846\u6837\u5f0f *\/
\n .stChatMessage {
\n padding: 1rem;
\n border-radius: 10px;
\n margin-bottom: 0.8rem;
\n }
\n \/* \u8f93\u5165\u6846\u6837\u5f0f *\/
\n .stChatInput {
\n position: fixed;
\n bottom: 20px;
\n width: 80%;
\n left: 10%;
\n z-index: 999;
\n }
\n \/* \u6807\u9898\u6837\u5f0f *\/
\n h1 {
\n text-align: center;
\n color: #2E86AB;
\n margin-bottom: 2rem;
\n }
\n <\/style>
\n""", unsafe_allow_html=True)<\/p>\n
# ===================== \u521d\u59cb\u5316\u804a\u5929\u5386\u53f2 =====================
\n# \u9996\u6b21\u6253\u5f00\u9875\u9762\u65f6，\u521b\u5efa\u7a7a\u7684\u804a\u5929\u8bb0\u5f55
\nif "messages" not in st.session_state:
\n st.session_state.messages = [
\n {"role": "assistant", "content": "\u4f60\u597d！\u6211\u662f\u96a7\u9053\u5de5\u7a0b\u4e13\u4e1a\u52a9\u624b，\u6709\u4efb\u4f55\u96a7\u9053\u76f8\u5173\u95ee\u9898\u90fd\u53ef\u4ee5\u95ee\u6211～"}
\n ]<\/p>\n
# ===================== \u663e\u793a\u804a\u5929\u5386\u53f2 =====================
\nst.title("\u26f0\ufe0f \u96a7\u9053\u5de5\u7a0b\u5927\u6a21\u578b\u52a9\u624b")<\/p>\n
# \u904d\u5386\u804a\u5929\u8bb0\u5f55，\u663e\u793a\u6bcf\u4e00\u6761\u6d88\u606f
\nfor msg in st.session_state.messages:
\n # \u8bbe\u7f6e\u6d88\u606f\u89d2\u8272（\u7528\u6237\/\u52a9\u624b），\u5bf9\u5e94\u4e0d\u540c\u7684\u5934\u50cf\u548c\u989c\u8272
\n with st.chat_message(msg["role"], avatar="👤" if msg["role"] == "user" else "🤖"):
\n st.markdown(msg["content"])<\/p>\n
# ===================== \u63a5\u6536\u7528\u6237\u8f93\u5165\u5e76\u8c03\u7528API =====================
\n# \u5e95\u90e8\u8f93\u5165\u6846，\u63d0\u793a\u6587\u5b57："\u8bf7\u8f93\u5165\u4f60\u7684\u96a7\u9053\u76f8\u5173\u95ee\u9898…"
\nif user_prompt := st.chat_input("\u8bf7\u8f93\u5165\u4f60\u7684\u96a7\u9053\u76f8\u5173\u95ee\u9898…"):
\n # 1. \u663e\u793a\u7528\u6237\u8f93\u5165\u7684\u6d88\u606f
\n with st.chat_message("user", avatar="👤"):
\n st.markdown(user_prompt)
\n # 2. \u628a\u7528\u6237\u6d88\u606f\u52a0\u5165\u804a\u5929\u5386\u53f2
\n st.session_state.messages.append({"role": "user", "content": user_prompt})<\/p>\n
# 3. \u8c03\u7528\u5df2\u90e8\u7f72\u7684 LLM API \u670d\u52a1
\n with st.spinner("🤖 \u6a21\u578b\u6b63\u5728\u601d\u8003\u4e2d…"):
\n try:
\n # \u8c03\u7528 API \u7684\u6838\u5fc3\u4ee3\u7801
\n response = requests.post(
\n url="http:\/\/192.168.2.199:8000\/api\/chat", # \u4f60\u7684 API \u5730\u5740
\n json={
\n "query": user_prompt, # \u7528\u6237\u8f93\u5165\u7684\u95ee\u9898
\n "type": "general" # \u901a\u7528\u95ee\u7b54\u7c7b\u578b
\n },
\n headers={"Content-Type": "application\/json"},
\n timeout=30 # \u8d85\u65f6\u65f6\u95f430\u79d2（\u6a21\u578b\u63a8\u7406\u9700\u8981\u65f6\u95f4）
\n )<\/p>\n
# \u68c0\u67e5 API \u54cd\u5e94\u72b6\u6001
\n response.raise_for_status()
\n result = response.json()<\/p>\n
# 4. \u5904\u7406 API \u8fd4\u56de\u7ed3\u679c
\n if result["code"] == 200:
\n # \u6210\u529f：\u63d0\u53d6\u6a21\u578b\u56de\u7b54
\n model_answer = result["data"]["answer"]
\n # \u663e\u793a\u63a8\u7406\u8017\u65f6（\u53ef\u9009）
\n model_answer += f"\\\\n\\\\n\u23f1\ufe0f \u63a8\u7406\u8017\u65f6：{result['time_cost']}"
\n else:
\n # \u5931\u8d25：\u663e\u793a\u9519\u8bef\u4fe1\u606f
\n model_answer = f"\u274c \u670d\u52a1\u54cd\u5e94\u9519\u8bef：{result['msg']}"<\/p>\n
except requests.exceptions.ConnectionError:
\n model_answer = "\u274c \u8fde\u63a5\u5931\u8d25！\u8bf7\u68c0\u67e5 API \u670d\u52a1\u662f\u5426\u542f\u52a8，\u6216 IP\/\u7aef\u53e3\u662f\u5426\u6b63\u786e\u3002"
\n except requests.exceptions.Timeout:
\n model_answer = "\u274c \u8bf7\u6c42\u8d85\u65f6！\u6a21\u578b\u63a8\u7406\u65f6\u95f4\u8fc7\u957f，\u8bf7\u7a0d\u540e\u518d\u8bd5\u3002"
\n except Exception as e:
\n model_answer = f"\u274c \u8c03\u7528\u5931\u8d25：{str(e)}"<\/p>\n
# 5. \u663e\u793a\u6a21\u578b\u56de\u7b54
\n with st.chat_message("assistant", avatar="🤖"):
\n st.markdown(model_answer)
\n # 6. \u628a\u6a21\u578b\u56de\u7b54\u52a0\u5165\u804a\u5929\u5386\u53f2
\n st.session_state.messages.append({"role": "assistant", "content": model_answer})<\/p>\n
\u542f\u52a8 Streamlit Web \u670d\u52a1<\/p>\n
\u5728 ~\/ai_project \u76ee\u5f55\u4e0b\u6267\u884c\u542f\u52a8\u547d\u4ee4：<\/p>\n
# \u786e\u4fdd\u5728 ai_dev \u73af\u5883\u4e2d
\nconda activate ai_dev<\/p>\n
# \u542f\u52a8 Streamlit \u670d\u52a1（\u5141\u8bb8\u5185\u7f51\u8bbf\u95ee，\u7aef\u53e38501）
\nstreamlit run llm_chat_app.py –server.address 0.0.0.0 –server.port 8501<\/p>\n
\u6539\u6210\u540e\u53f0\u5e38\u9a7b：<\/p>\n
conda activate ai_dev<\/p>\n
cd ~\/ai_project<\/p>\n
# \u505c\u6b62\u53ef\u80fd\u7684 API \u670d\u52a1\u8fdb\u7a0b
\npkill -f tunnel_llm_api.py
\n# \u505c\u6b62\u53ef\u80fd\u7684 Streamlit \u670d\u52a1\u8fdb\u7a0b
\npkill -f streamlit<\/p>\n
\u540e\u53f0\u542f\u52a8 LLM API \u670d\u52a1（\u5fc5\u987b\u5148\u542f\u52a8，\u56e0\u4e3a Web \u754c\u9762\u4f9d\u8d56\u5b83）<\/p>\n
CUDA_VISIBLE_DEVICES=0 nohup python tunnel_llm_api.py > llm_api.log 2>&1 &<\/p>\n
\u5728\u53e6\u5916\u4e00\u4e2a\u7ec8\u7aef\u6267\u884c<\/p>\n
nohup streamlit run llm_chat_app.py –server.address 0.0.0.0 –server.port 8501 > streamlit.log 2>&1 &<\/p>\n","protected":false},"excerpt":{"rendered":"
\u4e00\u3001\u521b\u5efa\u865a\u62df\u73af\u5883\u9996\u5148\u521b\u5efa\u865a\u62df\u73af\u5883conda create -n ai_dev\u6fc0\u6d3b\u865a\u62df\u73af\u5883conda activate ai_dev\u4f7f\u7528conda\u5b89\u88c5pytorchconda \u4f1a\u81ea\u52a8\u5904\u7406 CUDA \u7248\u672c\u517c\u5bb9\u548c\u4f9d\u8d56，\u907f\u514d pip \u5b89\u88c5\u65f6\u7684\u7248\u672c\u4e0d\u5339\u914d\u95ee\u9898conda install pytorch2.2.0 torchvision0.17.0 torchaudio2.2.0 pytorch-cuda12.1 -c pytorch -c nvidiapytorch\u5b89\u88c5\u6210\u529f<\/p>\n","protected":false},"author":2,"featured_media":80249,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[50],"topic":[],"class_list":["post-80257","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-50"],"yoast_head":"\n