DeepSeek-R1-Distill-Qwen-1.5B\u662f\u4e00\u4e2a\u8d85\u8f7b\u91cf\u7ea7\u7684\u667a\u80fd\u5bf9\u8bdd\u6a21\u578b，\u5b83\u5de7\u5999\u7ed3\u5408\u4e86DeepSeek\u6a21\u578b\u7684\u5f3a\u5927\u63a8\u7406\u80fd\u529b\u548cQwen\u6a21\u578b\u7684\u6210\u719f\u67b6\u6784\u4f18\u52bf\u3002\u7ecf\u8fc7\u7cbe\u5fc3\u7684\u84b8\u998f\u4f18\u5316，\u8fd9\u4e2a1.5B\u53c2\u6570\u7684\u6a21\u578b\u5728\u4fdd\u6301\u6838\u5fc3\u667a\u80fd\u7684\u540c\u65f6，\u5927\u5e45\u964d\u4f4e\u4e86\u8ba1\u7b97\u8d44\u6e90\u9700\u6c42，\u7279\u522b\u9002\u5408\u5728\u6709\u9650\u786c\u4ef6\u73af\u5883\u4e0b\u90e8\u7f72\u3002<\/p>\n
\u672c\u9879\u76ee\u5c55\u793a\u4e86\u5982\u4f55\u5c06\u8fd9\u4e2a\u4f18\u79c0\u7684\u6a21\u578b\u5c01\u88c5\u5230NVIDIA Triton\u63a8\u7406\u670d\u52a1\u5668\u4e2d，\u6253\u9020\u4e00\u4e2a\u9ad8\u6027\u80fd\u3001\u53ef\u6269\u5c55\u7684\u672c\u5730\u5316\u667a\u80fd\u5bf9\u8bdd\u670d\u52a1\u3002\u4e0e\u4f20\u7edf\u7684\u76f4\u63a5\u90e8\u7f72\u65b9\u5f0f\u4e0d\u540c，Triton\u670d\u52a1\u5668\u63d0\u4f9b\u4e86\u751f\u4ea7\u7ea7\u7684\u63a8\u7406\u670d\u52a1\u80fd\u529b，\u5305\u62ec\u6a21\u578b\u7248\u672c\u7ba1\u7406\u3001\u52a8\u6001\u6279\u5904\u7406\u3001\u5e76\u53d1\u63a8\u7406\u7b49\u4f01\u4e1a\u7ea7\u7279\u6027\u3002<\/p>\n
\u6838\u5fc3\u4ef7\u503c\u4eae\u70b9：<\/p>\n
\u5728\u5f00\u59cb\u90e8\u7f72\u4e4b\u524d，\u8bf7\u786e\u4fdd\u4f60\u7684\u7cfb\u7edf\u6ee1\u8db3\u4ee5\u4e0b\u57fa\u672c\u8981\u6c42：<\/p>\n
\u786c\u4ef6\u8981\u6c42：<\/p>\n
\u8f6f\u4ef6\u8981\u6c42：<\/p>\n
\u4ee5\u4e0b\u662f\u5feb\u901f\u90e8\u7f72\u7684\u5b8c\u6574\u6d41\u7a0b，\u53ea\u9700\u6309\u987a\u5e8f\u6267\u884c\u8fd9\u4e9b\u547d\u4ee4：<\/p>\n
# 1. \u62c9\u53d6Triton\u670d\u52a1\u5668\u955c\u50cf
\ndocker pull nvcr.io\/nvidia\/tritonserver:23.10-py3<\/p>\n
# 2. \u521b\u5efa\u6a21\u578b\u5b58\u50a8\u76ee\u5f55
\nmkdir -p triton_models\/deepseek\/1\/model<\/p>\n
# 3. \u4e0b\u8f7d\u6a21\u578b\u6587\u4ef6（\u5047\u8bbe\u6a21\u578b\u5df2\u8f6c\u6362\u4e3aONNX\u683c\u5f0f）
\n# \u5c06\u4f60\u7684\u6a21\u578b\u6587\u4ef6(.onnx)\u548c\u914d\u7f6e\u6587\u4ef6\u653e\u5230triton_models\/deepseek\/1\/model\/\u76ee\u5f55\u4e0b<\/p>\n
# 4. \u521b\u5efa\u6a21\u578b\u914d\u7f6e\u6587\u4ef6
\ncat > triton_models\/deepseek\/1\/config.pbtxt << EOF
\nname: "deepseek"
\nplatform: "onnxruntime_onnx"
\nmax_batch_size: 8
\ninput [
\n {
\n name: "input_ids"
\n data_type: TYPE_INT64
\n dims: [ -1 ]
\n },
\n {
\n name: "attention_mask"
\n data_type: TYPE_INT64
\n dims: [ -1 ]
\n }
\n]
\noutput [
\n {
\n name: "output"
\n data_type: TYPE_FP32
\n dims: [ -1, 50257 ]
\n }
\n]
\nEOF<\/p>\n
# 5. \u542f\u52a8Triton\u670d\u52a1\u5668
\ndocker run -d –gpus=all -p 8000:8000 -p 8001:8001 -p 8002:8002 \\\\
\n -v $(pwd)\/triton_models:\/models \\\\
\n nvcr.io\/nvidia\/tritonserver:23.10-py3 \\\\
\n tritonserver –model-repository=\/models<\/p>\n
\u90e8\u7f72\u9a8c\u8bc1： \u7b49\u5f85\u670d\u52a1\u5668\u542f\u52a8\u540e，\u8bbf\u95ee\u4ee5\u4e0b\u7aef\u70b9\u68c0\u67e5\u670d\u52a1\u72b6\u6001：<\/p>\n
\u5982\u679c\u8fd4\u56de\u72b6\u6001\u6b63\u5e38，\u8bf4\u660e\u90e8\u7f72\u6210\u529f！<\/p>\n
Triton\u670d\u52a1\u5668\u7684\u5f3a\u5927\u4e4b\u5904\u5728\u4e8e\u5176\u7075\u6d3b\u7684\u914d\u7f6e\u80fd\u529b\u3002\u4ee5\u4e0b\u662f\u4e00\u4e9b\u5173\u952e\u914d\u7f6e\u9879\u7684\u8be6\u7ec6\u8bf4\u660e：<\/p>\n
# \u52a8\u6001\u6279\u5904\u7406\u914d\u7f6e – \u5927\u5e45\u63d0\u5347\u541e\u5410\u91cf
\ndynamic_batching {
\n max_queue_delay_microseconds: 100
\n preferred_batch_size: [4, 8]
\n}<\/p>\n
# \u5b9e\u4f8b\u7ec4\u914d\u7f6e – \u5145\u5206\u5229\u7528GPU\u8d44\u6e90
\ninstance_group [
\n {
\n kind: KIND_GPU
\n count: 1
\n gpus: [0]
\n }
\n]<\/p>\n
# \u4f18\u5316\u914d\u7f6e – \u9488\u5bf9\u5bf9\u8bdd\u573a\u666f\u8c03\u4f18
\noptimization {
\n execution_accelerators {
\n gpu_execution_accelerator : [
\n {
\n name : "tensorrt"
\n parameters { key: "precision_mode" value: "FP16" }
\n }
\n ]
\n }
\n}<\/p>\n
\u6839\u636e\u6211\u4eec\u7684\u6d4b\u8bd5\u7ecf\u9a8c，\u4ee5\u4e0b\u914d\u7f6e\u53ef\u4ee5\u5728DeepSeek-R1-Distill-Qwen-1.5B\u6a21\u578b\u4e0a\u83b7\u5f97\u6700\u4f73\u6027\u80fd：<\/p>\n
\u6279\u5904\u7406\u5927\u5c0f\u4f18\u5316：<\/p>\n
\u7cbe\u5ea6\u9009\u62e9：<\/p>\n
\u4e0b\u9762\u662f\u4e00\u4e2a\u5b8c\u6574\u7684Python\u5ba2\u6237\u7aef\u793a\u4f8b，\u5c55\u793a\u4e86\u5982\u4f55\u4e0eTriton\u670d\u52a1\u5668\u8fdb\u884c\u4ea4\u4e92：<\/p>\n
import tritonclient.http as httpclient
\nimport numpy as np
\nfrom transformers import AutoTokenizer<\/p>\n
class DeepSeekTritonClient:
\n def __init__(self, url="localhost:8000"):
\n self.client = httpclient.InferenceServerClient(url=url)
\n self.tokenizer = AutoTokenizer.from_pretrained("\/root\/ds_1.5b")<\/p>\n
def chat(self, message, history=None):
\n # \u51c6\u5907\u8f93\u5165\u6570\u636e
\n inputs = self._prepare_inputs(message, history)<\/p>\n
# \u8bbe\u7f6e\u8f93\u5165\u8f93\u51fa
\n input_ids = httpclient.InferInput("input_ids", inputs["input_ids"].shape, "INT64")
\n attention_mask = httpclient.InferInput("attention_mask", inputs["attention_mask"].shape, "INT64")<\/p>\n
input_ids.set_data_from_numpy(inputs["input_ids"].astype(np.int64))
\n attention_mask.set_data_from_numpy(inputs["attention_mask"].astype(np.int64))<\/p>\n
# \u6267\u884c\u63a8\u7406
\n response = self.client.infer(
\n model_name="deepseek",
\n inputs=[input_ids, attention_mask]
\n )<\/p>\n
# \u5904\u7406\u8f93\u51fa
\n output = response.as_numpy("output")
\n return self._decode_output(output)<\/p>\n
def _prepare_inputs(self, message, history):
\n # \u4f7f\u7528\u6a21\u578b\u539f\u751f\u7684\u804a\u5929\u6a21\u677f
\n messages = history + [{"role": "user", "content": message}] if history else [
\n {"role": "user", "content": message}
\n ]<\/p>\n
text = self.tokenizer.apply_chat_template(
\n messages,
\n tokenize=False,
\n add_generation_prompt=True
\n )<\/p>\n
inputs = self.tokenizer(text, return_tensors="pt")
\n return {
\n "input_ids": inputs["input_ids"].numpy(),
\n "attention_mask": inputs["attention_mask"].numpy()
\n }<\/p>\n
def _decode_output(self, output):
\n # \u83b7\u53d6\u6700\u53ef\u80fd\u7684token
\n next_token = np.argmax(output, axis=-1)
\n return self.tokenizer.decode(next_token[0])<\/p>\n
# \u4f7f\u7528\u793a\u4f8b
\nclient = DeepSeekTritonClient()
\nresponse = client.chat("\u8bf7\u89e3\u91ca\u4e00\u4e0b\u673a\u5668\u5b66\u4e60\u7684\u57fa\u672c\u6982\u5ff5")
\nprint(response)<\/p>\n
\u5bf9\u4e8e\u751f\u4ea7\u73af\u5883，\u4f60\u53ef\u80fd\u8fd8\u9700\u8981\u4ee5\u4e0b\u589e\u5f3a\u529f\u80fd：<\/p>\n
def stream_chat(self, message, history=None, callback=None):
\n """\u6d41\u5f0f\u8f93\u51fa\u5b9e\u73b0"""
\n # \u6a21\u62df\u6d41\u5f0f\u8f93\u51fa
\n full_response = ""
\n for token in self._generate_tokens(message, history):
\n full_response += token
\n if callback:
\n callback(token)
\n return full_response<\/p>\n
def batch_chat(self, messages):
\n """\u6279\u91cf\u5904\u7406\u591a\u4e2a\u6d88\u606f"""
\n results = []
\n for msg in messages:
\n results.append(self.chat(msg))
\n return results<\/p>\n
\u6211\u4eec\u5728\u4e0d\u540c\u786c\u4ef6\u914d\u7f6e\u4e0b\u8fdb\u884c\u4e86\u8be6\u7ec6\u6d4b\u8bd5，\u4ee5\u4e0b\u662f\u5178\u578b\u7ed3\u679c：<\/p>\n
\u6d4b\u8bd5\u73af\u5883：<\/p>\n
\u6027\u80fd\u6570\u636e：<\/p>\n
| 1<\/td>\n | 45<\/td>\n | 22<\/td>\n | 3.8<\/td>\n<\/tr>\n |
| 4<\/td>\n | 120<\/td>\n | 33<\/td>\n | 4.2<\/td>\n<\/tr>\n |
| 8<\/td>\n | 210<\/td>\n | 38<\/td>\n | 4.8<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n5.2 \u4f18\u5316\u5efa\u8bae<\/h4>\n\u6839\u636e\u6d4b\u8bd5\u7ed3\u679c，\u6211\u4eec\u7ed9\u51fa\u4ee5\u4e0b\u4f18\u5316\u5efa\u8bae：<\/p>\n \u5ef6\u8fdf\u654f\u611f\u573a\u666f：<\/p>\n
|