\u60f3\u8bd5\u8bd5\u90a3\u4e2a\u80fd\u770b\u61c2\u56fe\u7247，\u8fd8\u80fd\u50cf\u4eba\u4e00\u6837\u4e00\u6b65\u6b65\u63a8\u7406\u7684AI\u6a21\u578b\u5417？Llama-3.2V-11B-cot\u5c31\u662f\u8fd9\u6837\u4e00\u4e2a\u89c6\u89c9\u8bed\u8a00\u6a21\u578b\u3002\u5b83\u4e0d\u4ec5\u80fd\u7406\u89e3\u56fe\u7247\u5185\u5bb9，\u8fd8\u80fd\u628a\u601d\u8003\u8fc7\u7a0b\u62c6\u89e3\u6210\u201c\u603b\u7ed3\u2192\u63cf\u8ff0\u2192\u63a8\u7406\u2192\u7ed3\u8bba\u201d\u56db\u4e2a\u6b65\u9aa4，\u8ba9AI\u7684\u201c\u8111\u56de\u8def\u201d\u6e05\u6670\u53ef\u89c1\u3002<\/p>\n
\u4e0d\u8fc7，\u76f4\u63a5\u8fd0\u884cPython\u811a\u672c\u867d\u7136\u7b80\u5355，\u4f46\u60f3\u5728\u751f\u4ea7\u73af\u5883\u91cc\u7a33\u5b9a\u3001\u9ad8\u6548\u5730\u7528\u5b83，\u5c31\u5f97\u6362\u4e2a\u601d\u8def\u4e86\u3002\u4eca\u5929，\u6211\u5c31\u5e26\u4f60\u8d70\u4e00\u904d\u5de5\u4e1a\u7ea7\u7684\u90e8\u7f72\u6d41\u7a0b：\u628aLlama-3.2V-11B-cot\u5c01\u88c5\u8fdbNVIDIA Triton\u63a8\u7406\u670d\u52a1\u5668，\u518d\u7ed9\u5b83\u505a\u4e00\u6b21\u5168\u9762\u7684\u6027\u80fd\u201c\u4f53\u68c0\u201d\u3002\u65e0\u8bba\u4f60\u662f\u60f3\u642d\u5efa\u4e00\u4e2a\u9ad8\u53ef\u7528\u7684AI\u670d\u52a1，\u8fd8\u662f\u5355\u7eaf\u597d\u5947\u8fd9\u4e2a\u6a21\u578b\u7684\u6781\u9650\u5728\u54ea\u91cc，\u8fd9\u7bc7\u6559\u7a0b\u90fd\u80fd\u7ed9\u4f60\u7b54\u6848\u3002<\/p>\n
\u5728\u804a\u5177\u4f53\u64cd\u4f5c\u4e4b\u524d，\u6211\u4eec\u5148\u641e\u6e05\u695a\u4e00\u4e2a\u95ee\u9898：\u4e3a\u4ec0\u4e48\u4e0d\u7528\u7b80\u5355\u7684python app.py，\u800c\u8981\u6298\u817eTriton？<\/p>\n
\u60f3\u8c61\u4e00\u4e0b，\u4f60\u5f00\u53d1\u4e86\u4e00\u4e2a\u5f88\u68d2\u7684\u56fe\u7247\u5206\u6790\u5e94\u7528\u3002\u4e00\u5f00\u59cb\u7528\u6237\u4e0d\u591a，\u4e00\u4e2aPython\u8fdb\u7a0b\u8fd8\u80fd\u5e94\u4ed8\u3002\u4f46\u968f\u7740\u7528\u6237\u91cf\u66b4\u6da8，\u95ee\u9898\u5c31\u6765\u4e86：\u8bf7\u6c42\u6392\u961f\u3001\u54cd\u5e94\u53d8\u6162\u3001\u5185\u5b58\u6ea2\u51fa，\u751a\u81f3\u6574\u4e2a\u670d\u52a1\u6302\u6389\u3002\u8fd9\u65f6\u5019，\u4f60\u5c31\u9700\u8981\u4e00\u4e2a\u66f4\u4e13\u4e1a\u7684\u201c\u670d\u52a1\u5458\u201d\u3002<\/p>\n
NVIDIA Triton\u63a8\u7406\u670d\u52a1\u5668\u5c31\u662f\u8fd9\u4e2a\u4e13\u4e1a\u7684\u201c\u670d\u52a1\u5458\u201d\u3002\u5b83\u7684\u6838\u5fc3\u4ef7\u503c\u5728\u4e8e：<\/p>\n
\u7b80\u5355\u8bf4，python app.py\u9002\u5408\u5f00\u53d1\u548c\u5feb\u901f\u9a8c\u8bc1，\u800cTriton\u662f\u4e3a7×24\u5c0f\u65f6\u7a33\u5b9a\u8fd0\u884c\u3001\u670d\u52a1\u5927\u91cf\u7528\u6237\u800c\u751f\u7684\u3002\u63a5\u4e0b\u6765，\u6211\u4eec\u5c31\u4e00\u6b65\u6b65\u628a\u5b83\u201c\u8bf7\u201d\u8fdb\u6765\u3002<\/p>\n
\u5de5\u6b32\u5584\u5176\u4e8b，\u5fc5\u5148\u5229\u5176\u5668\u3002\u90e8\u7f72\u7684\u7b2c\u4e00\u6b65\u662f\u51c6\u5907\u597d\u6218\u573a\u3002<\/p>\n
\u786e\u4fdd\u4f60\u7684\u673a\u5668\u6ee1\u8db3\u4ee5\u4e0b\u6761\u4ef6，\u8fd9\u662f\u540e\u7eed\u6240\u6709\u6b65\u9aa4\u7684\u57fa\u7840：<\/p>\n
\u9996\u5148，\u4f60\u9700\u8981\u62e5\u6709\u539f\u59cb\u7684Llama-3.2V-11B-cot\u6a21\u578b\u3002\u5b83\u901a\u5e38\u5305\u542b\u4ee5\u4e0b\u5173\u952e\u6587\u4ef6：<\/p>\n
\u5047\u8bbe\u4f60\u7684\u6a21\u578b\u6587\u4ef6\u5b58\u653e\u5728 \/home\/user\/llama-3.2v-11b-cot\/ \u76ee\u5f55\u4e0b\u3002\u6211\u4eec\u7684\u76ee\u6807\u662f\u5c06\u8fd9\u4e00\u5957\u6587\u4ef6，\u8f6c\u6362\u6210Triton\u80fd\u591f\u8bc6\u522b\u548c\u670d\u52a1\u7684\u683c\u5f0f\u3002<\/p>\n
Triton\u539f\u751f\u652f\u6301\u591a\u79cd\u540e\u7aef，\u5982PyTorch\u3001TensorRT\u548cONNX\u3002\u5176\u4e2d，ONNX\u683c\u5f0f\u5177\u6709\u5f88\u597d\u7684\u901a\u7528\u6027\u548c\u4f18\u5316\u6f5c\u529b\u3002\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528optimum\u548connxruntime\u5e93\u8fdb\u884c\u8f6c\u6362\u3002<\/p>\n
\u9996\u5148，\u5b89\u88c5\u5fc5\u8981\u7684\u5e93：<\/p>\n
pip install optimum[exporters] onnxruntime-gpu<\/p>\n
\u7136\u540e，\u7f16\u5199\u4e00\u4e2a\u8f6c\u6362\u811a\u672c export_to_onnx.py：<\/p>\n
from optimum.onnxruntime import ORTModelForVision2Seq
\nfrom transformers import AutoProcessor<\/p>\n
# \u5b9a\u4e49\u6a21\u578b\u8def\u5f84
\nmodel_id = "\/home\/user\/llama-3.2v-11b-cot"
\nonnx_path = ".\/llama-3.2v-11b-cot-onnx"<\/p>\n
# \u5bfc\u51fa\u4e3aONNX\u683c\u5f0f
\nmodel = ORTModelForVision2Seq.from_pretrained(model_id, export=True)
\nprocessor = AutoProcessor.from_pretrained(model_id)<\/p>\n
# \u4fdd\u5b58ONNX\u6a21\u578b\u548c\u5904\u7406\u5668
\nmodel.save_pretrained(onnx_path)
\nprocessor.save_pretrained(onnx_path)
\nprint(f"\u6a21\u578b\u5df2\u6210\u529f\u5bfc\u51fa\u81f3: {onnx_path}")<\/p>\n
\u8fd0\u884c\u8fd9\u4e2a\u811a\u672c，\u4f60\u5c06\u5728\u5f53\u524d\u76ee\u5f55\u5f97\u5230 llama-3.2v-11b-cot-onnx \u6587\u4ef6\u5939，\u91cc\u9762\u5305\u542b\u4e86ONNX\u683c\u5f0f\u7684\u6a21\u578b\u6587\u4ef6\u3002\u8fd9\u4e00\u6b65\u5c06\u6a21\u578b\u7684\u8ba1\u7b97\u56fe\u56fa\u5b9a\u4e0b\u6765，\u6709\u5229\u4e8e\u540e\u7eed\u7684\u56fe\u4f18\u5316\u548c\u52a0\u901f\u3002<\/p>\n
Triton\u901a\u8fc7\u4e00\u4e2a\u6e05\u6670\u7684\u76ee\u5f55\u7ed3\u6784\u6765\u7ba1\u7406\u6a21\u578b，\u8fd9\u4e2a\u7ed3\u6784\u53eb\u505a\u201c\u6a21\u578b\u4ed3\u5e93\u201d\u3002\u6211\u4eec\u6765\u4e3aLlama-3.2V-11B-cot\u521b\u5efa\u5b83\u7684\u201c\u5bb6\u201d\u3002<\/p>\n
\u521b\u5efa\u4e00\u4e2a\u76ee\u5f55，\u4f8b\u5982 triton_model_repository，\u5e76\u5728\u5176\u4e2d\u4e3a\u6211\u4eec\u7684\u6a21\u578b\u5efa\u7acb\u5b50\u76ee\u5f55：<\/p>\n
mkdir -p triton_model_repository\/llama_3_2v_11b_cot\/1<\/p>\n
\u8fd9\u91cc\u7684\u7ed3\u6784\u542b\u4e49\u662f：<\/p>\n
\u73b0\u5728，\u5c06\u8f6c\u6362\u597d\u7684ONNX\u6a21\u578b\u6587\u4ef6（\u6216\u539f\u59cbPyTorch\u6a21\u578b\u6587\u4ef6）\u590d\u5236\u5230\u7248\u672c\u76ee\u5f55 1 \u4e2d：<\/p>\n
# \u5982\u679c\u4f60\u4f7f\u7528ONNX\u683c\u5f0f
\ncp -r .\/llama-3.2v-11b-cot-onnx\/* triton_model_repository\/llama_3_2v_11b_cot\/1\/<\/p>\n
# \u5982\u679c\u4f60\u76f4\u63a5\u4f7f\u7528\u539f\u59cbPyTorch\u683c\u5f0f，\u786e\u4fdd\u81f3\u5c11\u5305\u542bpytorch_model.bin\u548cconfig.json
\n# cp \/home\/user\/llama-3.2v-11b-cot\/* triton_model_repository\/llama_3_2v_11b_cot\/1\/<\/p>\n
\u63a5\u4e0b\u6765，\u521b\u5efaTriton\u6a21\u578b\u7684\u6838\u5fc3\u914d\u7f6e\u6587\u4ef6 config.pbtxt，\u653e\u5728 llama_3_2v_11b_cot \u76ee\u5f55\u4e0b（\u6ce8\u610f：\u4e0d\u662f\u7248\u672c\u76ee\u5f551\u91cc\u9762）\u3002<\/p>\n
name: "llama_3_2v_11b_cot"
\nplatform: "onnxruntime_onnx" # \u5982\u679c\u4f7f\u7528ONNX\u683c\u5f0f\u3002\u82e5\u7528PyTorch，\u5219\u6539\u4e3a "pytorch_libtorch"
\nmax_batch_size: 4 # \u6700\u5927\u6279\u5904\u7406\u5927\u5c0f，\u6839\u636e\u4f60\u7684GPU\u663e\u5b58\u8c03\u6574\u3002\u5982\u679c\u6a21\u578b\u4e0d\u652f\u6301\u6279\u5904\u7406，\u8bbe\u4e3a0\u3002<\/p>\n
input [
\n {
\n name: "pixel_values"
\n data_type: TYPE_FP32
\n dims: [3, 336, 336] # \u8f93\u5165\u56fe\u7247\u7684\u5c3a\u5bf8 [\u901a\u9053, \u9ad8, \u5bbd]
\n },
\n {
\n name: "input_ids"
\n data_type: TYPE_INT64
\n dims: [-1] # -1 \u8868\u793a\u53ef\u53d8\u957f\u5ea6\u7ef4\u5ea6
\n },
\n {
\n name: "attention_mask"
\n data_type: TYPE_INT64
\n dims: [-1]
\n }
\n]<\/p>\n
output [
\n {
\n name: "output_0" # ONNX\u5bfc\u51fa\u65f6\u7684\u8f93\u51fa\u540d\u79f0，\u53ef\u80fd\u9700\u8981\u6839\u636e\u5b9e\u9645\u60c5\u51b5\u8c03\u6574
\n data_type: TYPE_FP32
\n dims: [-1, -1] # \u8f93\u51falogits\u7684\u7ef4\u5ea6 [\u6279\u6b21, \u5e8f\u5217\u957f\u5ea6, \u8bcd\u8868\u5927\u5c0f]，\u540e\u4e24\u7ef4\u53ef\u53d8
\n }
\n]<\/p>\n
instance_group [
\n {
\n count: 1 # \u6bcf\u4e2aGPU\u4e0a\u8fd0\u884c\u51e0\u4e2a\u6a21\u578b\u5b9e\u4f8b
\n kind: KIND_GPU
\n gpus: [0] # \u4f7f\u7528\u54ea\u51e0\u5f20GPU，\u4f8b\u5982[0,1]\u8868\u793a\u4f7f\u7528GPU0\u548cGPU1
\n }
\n]<\/p>\n
dynamic_batching {
\n preferred_batch_size: [1, 2, 4]
\n max_queue_delay_microseconds: 500000 # \u8bf7\u6c42\u5728\u961f\u5217\u4e2d\u7b49\u5f85\u6279\u5904\u7406\u7684\u6700\u5927\u65f6\u95f4（\u5fae\u79d2）
\n}<\/p>\n
\u8fd9\u4e2a\u914d\u7f6e\u6587\u4ef6\u544a\u8bc9Triton：<\/p>\n
\u89c6\u89c9\u8bed\u8a00\u6a21\u578b\u7684\u63a8\u7406\u6d41\u7a0b\u6bd4\u7eaf\u6587\u672c\u6a21\u578b\u590d\u6742\u3002\u5b83\u9700\u8981：<\/p>\n
Triton\u901a\u8fc7\u201c\u96c6\u6210\u6a21\u578b\u201d\u529f\u80fd\u6765\u4e32\u8054\u8fd9\u4e9b\u6b65\u9aa4\u3002\u6211\u4eec\u9700\u8981\u4e3a\u9884\u5904\u7406\u548c\u540e\u5904\u7406\u5206\u522b\u7f16\u5199Python\u811a\u672c\u3002<\/p>\n
\u5728\u6a21\u578b\u76ee\u5f55\u4e0b\u521b\u5efa ensemble_model \u5b50\u76ee\u5f55\u548c\u914d\u7f6e\u6587\u4ef6：<\/p>\n
mkdir -p triton_model_repository\/llama_3_2v_11b_cot_ensemble\/1<\/p>\n
\u521b\u5efa\u96c6\u6210\u6a21\u578b\u7684\u914d\u7f6e\u6587\u4ef6 triton_model_repository\/llama_3_2v_11b_cot_ensemble\/config.pbtxt:<\/p>\n
name: "llama_3_2v_11b_cot_ensemble"
\nplatform: "ensemble"
\nmax_batch_size: 4<\/p>\n
input [
\n {
\n name: "IMAGE"
\n data_type: TYPE_UINT8
\n dims: [-1, -1, 3] # \u539f\u59cb\u56fe\u50cf，\u9ad8\u3001\u5bbd\u53ef\u53d8，3\u901a\u9053
\n },
\n {
\n name: "QUESTION"
\n data_type: TYPE_STRING
\n dims: [ -1 ]
\n }
\n]<\/p>\n
output [
\n {
\n name: "ANSWER"
\n data_type: TYPE_STRING
\n dims: [ -1 ]
\n }
\n]<\/p>\n
ensemble_scheduling {
\n step [
\n {
\n model_name: "llama_3_2v_11b_cot_preprocess"
\n model_version: -1 # -1 \u8868\u793a\u4f7f\u7528\u6700\u65b0\u7248\u672c
\n input_map {
\n key: "image"
\n value: "IMAGE"
\n }
\n input_map {
\n key: "question"
\n value: "QUESTION"
\n }
\n output_map {
\n key: "pixel_values"
\n value: "preprocessed_image"
\n }
\n output_map {
\n key: "input_ids"
\n value: "tokenized_text"
\n }
\n output_map {
\n key: "attention_mask"
\n value: "attention_mask"
\n }
\n },
\n {
\n model_name: "llama_3_2v_11b_cot"
\n model_version: -1
\n input_map {
\n key: "pixel_values"
\n value: "preprocessed_image"
\n }
\n input_map {
\n key: "input_ids"
\n value: "tokenized_text"
\n }
\n input_map {
\n key: "attention_mask"
\n value: "attention_mask"
\n }
\n output_map {
\n key: "output_0"
\n value: "model_logits"
\n }
\n },
\n {
\n model_name: "llama_3_2v_11b_cot_postprocess"
\n model_version: -1
\n input_map {
\n key: "logits"
\n value: "model_logits"
\n }
\n output_map {
\n key: "answer"
\n value: "ANSWER"
\n }
\n }
\n ]
\n}<\/p>\n
\u8fd9\u4e2a\u914d\u7f6e\u5b9a\u4e49\u4e86\u4e00\u4e2a\u5de5\u4f5c\u6d41\u6c34\u7ebf：\u9884\u5904\u7406\u6a21\u578b -> \u6838\u5fc3\u6a21\u578b -> \u540e\u5904\u7406\u6a21\u578b\u3002\u63a5\u4e0b\u6765，\u6211\u4eec\u9700\u8981\u5b9e\u73b0\u9884\u5904\u7406\u548c\u540e\u5904\u7406\u8fd9\u4e24\u4e2a\u6a21\u578b\u3002<\/p>\n
\u521b\u5efa\u9884\u5904\u7406\u6a21\u578b： \u5728\u4ed3\u5e93\u4e2d\u521b\u5efa triton_model_repository\/llama_3_2v_11b_cot_preprocess\/1\/ \u76ee\u5f55，\u5e76\u521b\u5efa model.py：<\/p>\n
# triton_model_repository\/llama_3_2v_11b_cot_preprocess\/1\/model.py
\nimport triton_python_backend_utils as pb_utils
\nimport numpy as np
\nfrom PIL import Image
\nimport io
\nimport torch
\nfrom transformers import AutoProcessor
\nimport json<\/p>\n
class TritonPythonModel:
\n def initialize(self, args):
\n # \u521d\u59cb\u5316\u5904\u7406\u5668，\u8fd9\u91cc\u9700\u8981\u52a0\u8f7d\u4f60\u6a21\u578b\u5bf9\u5e94\u7684processor
\n self.processor = AutoProcessor.from_pretrained("\/home\/user\/llama-3.2v-11b-cot")
\n print("Preprocess model initialized.")<\/p>\n
def execute(self, requests):
\n responses = []
\n for request in requests:
\n # 1. \u83b7\u53d6\u539f\u59cb\u8f93\u5165
\n image_input = pb_utils.get_input_tensor_by_name(request, "IMAGE")
\n question_input = pb_utils.get_input_tensor_by_name(request, "QUESTION")<\/p>\n
# \u539f\u59cb\u56fe\u50cf\u6570\u636e (numpy array, dtype=uint8)
\n image_np = image_input.as_numpy()[0] # \u5047\u8bbe\u6279\u6b21\u5927\u5c0f\u4e3a1
\n question_text = question_input.as_numpy()[0].decode('utf-8')<\/p>\n
# 2. \u56fe\u50cf\u9884\u5904\u7406：\u8c03\u6574\u5927\u5c0f\u3001\u5f52\u4e00\u5316\u7b49
\n image_pil = Image.fromarray(image_np)
\n # \u4f7f\u7528\u5904\u7406\u5668\u7684\u56fe\u50cf\u5904\u7406\u529f\u80fd
\n image_tensor = self.processor.image_processor(image_pil, return_tensors="pt")['pixel_values']<\/p>\n
# 3. \u6587\u672c\u9884\u5904\u7406：\u5206\u8bcd
\n text_encoding = self.processor.tokenizer(question_text, return_tensors="pt", padding=True, truncation=True)<\/p>\n
# 4. \u6784\u5efa\u8f93\u51fa\u5f20\u91cf
\n out_pixel_values = pb_utils.Tensor("pixel_values", image_tensor.numpy().astype(np.float32))
\n out_input_ids = pb_utils.Tensor("input_ids", text_encoding['input_ids'].numpy().astype(np.int64))
\n out_attention_mask = pb_utils.Tensor("attention_mask", text_encoding['attention_mask'].numpy().astype(np.int64))<\/p>\n
# 5. \u5c01\u88c5\u54cd\u5e94
\n inference_response = pb_utils.InferenceResponse(output_tensors=[out_pixel_values, out_input_ids, out_attention_mask])
\n responses.append(inference_response)
\n return responses<\/p>\n
def finalize(self):
\n print("Cleaning up preprocess model.")<\/p>\n
\u522b\u5fd8\u4e86\u521b\u5efa\u5bf9\u5e94\u7684 config.pbtxt \u6765\u5b9a\u4e49\u8fd9\u4e2a\u9884\u5904\u7406\u6a21\u578b\u7684\u8f93\u5165\u8f93\u51fa\u3002<\/p>\n
\u521b\u5efa\u540e\u5904\u7406\u6a21\u578b： \u7c7b\u4f3c\u5730，\u521b\u5efa triton_model_repository\/llama_3_2v_11b_cot_postprocess\/1\/model.py，\u8d1f\u8d23\u5c06\u6a21\u578b\u8f93\u51fa\u7684logits\u89e3\u7801\u6210\u6587\u672c\u3002<\/p>\n
# triton_model_repository\/llama_3_2v_11b_cot_postprocess\/1\/model.py
\nimport triton_python_backend_utils as pb_utils
\nimport numpy as np
\nimport torch
\nfrom transformers import AutoProcessor<\/p>\n
class TritonPythonModel:
\n def initialize(self, args):
\n self.processor = AutoProcessor.from_pretrained("\/home\/user\/llama-3.2v-11b-cot")
\n self.tokenizer = self.processor.tokenizer
\n print("Postprocess model initialized.")<\/p>\n
def execute(self, requests):
\n responses = []
\n for request in requests:
\n # \u83b7\u53d6\u6a21\u578b\u8f93\u51fa\u7684logits
\n logits_input = pb_utils.get_input_tensor_by_name(request, "logits")
\n logits_np = logits_input.as_numpy() # [batch_size, seq_len, vocab_size]<\/p>\n
# \u5c06numpy\u8f6c\u6362\u56detorch tensor\u4ee5\u4fbf\u4f7f\u7528transformers\u89e3\u7801
\n logits_tensor = torch.from_numpy(logits_np)<\/p>\n
# \u4f7f\u7528\u8d2a\u5a6a\u89e3\u7801（\u6216beam search\u7b49\u66f4\u590d\u6742\u7684\u89e3\u7801\u7b56\u7565）
\n predicted_token_ids = torch.argmax(logits_tensor, dim=-1)<\/p>\n
# \u5c06token ids\u89e3\u7801\u4e3a\u6587\u672c
\n # \u6ce8\u610f：\u8fd9\u91cc\u9700\u8981\u6839\u636e\u4f60\u7684\u6a21\u578b\u8f93\u51fa\u7ed3\u6784\u8fdb\u884c\u8c03\u6574，\u53ef\u80fd\u53ea\u9700\u8981\u89e3\u7801\u6700\u540e\u4e00\u6bb5
\n generated_text = self.tokenizer.batch_decode(predicted_token_ids, skip_special_tokens=True)<\/p>\n
# \u6784\u5efa\u8f93\u51fa\u5f20\u91cf
\n out_answer = pb_utils.Tensor("answer", np.array(generated_text, dtype=object))<\/p>\n
inference_response = pb_utils.InferenceResponse(output_tensors=[out_answer])
\n responses.append(inference_response)
\n return responses<\/p>\n
def finalize(self):
\n print("Cleaning up postprocess model.")<\/p>\n
\u540c\u6837，\u9700\u8981\u4e3a\u540e\u5904\u7406\u6a21\u578b\u521b\u5efa config.pbtxt\u3002<\/p>\n
\u81f3\u6b64，\u4e00\u4e2a\u5b8c\u6574\u7684Triton\u6a21\u578b\u4ed3\u5e93\u5c31\u51c6\u5907\u597d\u4e86\u3002\u5b83\u5305\u542b\u4e86\u6838\u5fc3\u63a8\u7406\u6a21\u578b\u3001\u9884\u5904\u7406\u548c\u540e\u5904\u7406\u903b\u8f91，\u5bf9\u5916\u63d0\u4f9b\u4e00\u4e2a\u63a5\u6536\u56fe\u7247\u548c\u95ee\u9898\u3001\u8fd4\u56de\u6587\u672c\u7b54\u6848\u7684\u7b80\u6d01\u63a5\u53e3\u3002<\/p>\n
\u4ed3\u5e93\u5efa\u597d\u4e86，\u8ba9\u6211\u4eec\u542f\u52a8\u670d\u52a1\u5668\u5e76\u770b\u770b\u6548\u679c\u3002<\/p>\n
\u8fd9\u662f\u6700\u63a8\u8350\u7684\u65b9\u5f0f，\u80fd\u907f\u514d\u590d\u6742\u7684\u4f9d\u8d56\u95ee\u9898\u3002<\/p>\n
# \u62c9\u53d6Triton Server\u7684Docker\u955c\u50cf（\u9009\u62e9\u4e0e\u4f60CUDA\u7248\u672c\u5339\u914d\u7684tag）
\ndocker pull nvcr.io\/nvidia\/tritonserver:23.10-py3<\/p>\n
# \u8fd0\u884c\u5bb9\u5668，\u6302\u8f7d\u6a21\u578b\u4ed3\u5e93
\ndocker run –gpus=all –rm -p 8000:8000 -p 8001:8001 -p 8002:8002 \\\\
\n -v \/path\/to\/your\/triton_model_repository:\/models \\\\
\n nvcr.io\/nvidia\/tritonserver:23.10-py3 \\\\
\n tritonserver –model-repository=\/models<\/p>\n
\u547d\u4ee4\u89e3\u91ca：<\/p>\n
\u5982\u679c\u4e00\u5207\u987a\u5229，\u4f60\u4f1a\u5728\u65e5\u5fd7\u4e2d\u770b\u5230\u7c7b\u4f3c\u4e0b\u9762\u7684\u8f93\u51fa，\u8868\u660e\u6a21\u578b\u52a0\u8f7d\u6210\u529f：<\/p>\n
…
\nI1230 10:00:00.000000 1 model_repository_manager.cc:1344] successfully loaded 'llama_3_2v_11b_cot' version 1
\nI1230 10:00:00.000001 1 model_repository_manager.cc:1344] successfully loaded 'llama_3_2v_11b_cot_ensemble' version 1
\n…
\n+———————-+———+——–+
\n| Model | Version | Status |
\n+———————-+———+——–+
\n| llama_3_2v_11b_cot | 1 | READY |
\n| llama_3_2v_11b_cot_ensemble | 1 | READY |
\n+———————-+———+——–+
\n…<\/p>\n
\u670d\u52a1\u5668\u8dd1\u8d77\u6765\u4e86，\u6211\u4eec\u5199\u4e2a\u7b80\u5355\u7684Python\u5ba2\u6237\u7aef\u6765\u6d4b\u8bd5\u4e00\u4e0b\u3002\u8fd9\u4e2a\u5ba2\u6237\u7aef\u4f1a\u53d1\u9001\u4e00\u5f20\u56fe\u7247\u548c\u4e00\u4e2a\u95ee\u9898\u7ed9\u6211\u4eec\u7684\u96c6\u6210\u6a21\u578b\u3002<\/p>\n
# test_client.py
\nimport requests
\nimport json
\nimport base64
\nfrom PIL import Image
\nimport io<\/p>\n
# Triton\u670d\u52a1\u5668\u5730\u5740
\nurl = "http:\/\/localhost:8000\/v2\/models\/llama_3_2v_11b_cot_ensemble\/infer"<\/p>\n
# 1. \u51c6\u5907\u56fe\u7247\u548c\u95ee\u9898
\nimage_path = "test_image.jpg" # \u66ff\u6362\u6210\u4f60\u7684\u6d4b\u8bd5\u56fe\u7247\u8def\u5f84
\nquestion = "What is in this image? Please think step by step."<\/p>\n
# \u8bfb\u53d6\u5e76\u7f16\u7801\u56fe\u7247
\nwith open(image_path, "rb") as f:
\n image_bytes = f.read()
\nimage_b64 = base64.b64encode(image_bytes).decode('utf-8')<\/p>\n
# 2. \u6784\u5efa\u8bf7\u6c42\u4f53（\u9075\u5faaTriton\u7684\u63a8\u7406\u534f\u8bae）
\n# \u6ce8\u610f：\u8fd9\u91cc\u6211\u4eec\u76f4\u63a5\u53d1\u9001\u539f\u59cb\u5b57\u8282，\u4e5f\u53ef\u4ee5\u53d1\u9001base64\u7f16\u7801\u7684\u5b57\u7b26\u4e32，\u9700\u8981\u5728\u9884\u5904\u7406\u6a21\u578b\u4e2d\u76f8\u5e94\u89e3\u6790\u3002
\nwith open(image_path, "rb") as f:
\n image_data = f.read()<\/p>\n
# \u6784\u5efa\u7b26\u5408Triton\u8f93\u5165\u683c\u5f0f\u7684\u8bf7\u6c42
\n# \u5bf9\u4e8e\u4e8c\u8fdb\u5236\u56fe\u50cf，\u6211\u4eec\u53ef\u4ee5\u76f4\u63a5\u53d1\u9001bytes
\npayload = {
\n "inputs": [
\n {
\n "name": "IMAGE",
\n "shape": [1], # \u6279\u6b21\u5927\u5c0f\u4e3a1
\n "datatype": "BYTES",
\n "data": [image_data] # \u6ce8\u610f：\u8fd9\u91cc\u9700\u8981\u662f\u5217\u8868，\u5373\u4f7f\u53ea\u6709\u4e00\u4e2a\u5143\u7d20
\n },
\n {
\n "name": "QUESTION",
\n "shape": [1],
\n "datatype": "BYTES",
\n "data": [question.encode('utf-8')]
\n }
\n ],
\n "outputs": [{"name": "ANSWER"}]
\n}<\/p>\n
# 3. \u53d1\u9001\u8bf7\u6c42
\nheaders = {"Content-Type": "application\/json"}
\nresponse = requests.post(url, data=json.dumps(payload), headers=headers)<\/p>\n
# 4. \u89e3\u6790\u54cd\u5e94
\nif response.status_code == 200:
\n result = response.json()
\n answer_bytes = result['outputs'][0]['data'][0]
\n answer = answer_bytes.decode('utf-8')
\n print("\u95ee\u9898:", question)
\n print("\u6a21\u578b\u56de\u7b54:", answer)
\nelse:
\n print("\u8bf7\u6c42\u5931\u8d25:", response.status_code, response.text)<\/p>\n
\u8fd0\u884c\u8fd9\u4e2a\u5ba2\u6237\u7aef\u811a\u672c，\u5982\u679c\u914d\u7f6e\u6b63\u786e，\u4f60\u5c06\u6536\u5230\u6a21\u578b\u6309\u7167SUMMARY\u2192CAPTION\u2192REASONING\u2192CONCLUSION\u683c\u5f0f\u751f\u6210\u7684\u63a8\u7406\u7b54\u6848\u3002<\/p>\n
\u670d\u52a1\u80fd\u8dd1\u901a\u53ea\u662f\u7b2c\u4e00\u6b65，\u6211\u4eec\u8fd8\u5f97\u77e5\u9053\u5b83\u80fd\u8dd1\u591a\u5feb\u3001\u80fd\u625b\u4f4f\u591a\u5927\u538b\u529b\u3002\u8fd9\u5c31\u662f\u6027\u80fd\u538b\u6d4b\u7684\u76ee\u7684\u3002<\/p>\n
NVIDIA Triton\u81ea\u5e26\u4e00\u4e2a\u5f3a\u5927\u7684\u6027\u80fd\u5206\u6790\u5de5\u5177 perf_analyzer\u3002\u6211\u4eec\u53ef\u4ee5\u7528\u5b83\u6765\u6a21\u62df\u5927\u91cf\u5e76\u53d1\u8bf7\u6c42，\u6d4b\u8bd5\u670d\u52a1\u7684\u541e\u5410\u91cf\u3001\u5ef6\u8fdf\u7b49\u5173\u952e\u6307\u6807\u3002<\/p>\n
\u9996\u5148，\u786e\u4fdd\u4f60\u7684\u6d4b\u8bd5\u56fe\u7247 test_image.jpg \u548c\u95ee\u9898\u6587\u672c test_question.txt \u5df2\u7ecf\u51c6\u5907\u597d\u3002\u7136\u540e\u8fd0\u884c\u4ee5\u4e0b\u547d\u4ee4：<\/p>\n
# \u8fdb\u5165Triton\u5bb9\u5668\u5185\u90e8\u6267\u884c，\u6216\u8005\u672c\u5730\u5b89\u88c5perf_analyzer
\ndocker exec -it <\u4f60\u7684\u5bb9\u5668ID> \/bin\/bash<\/p>\n
# \u5728\u5bb9\u5668\u5185\u8fd0\u884cperf_analyzer
\nperf_analyzer -m llama_3_2v_11b_cot_ensemble \\\\
\n -u localhost:8000 \\\\
\n –input-data \/path\/to\/test_data.json \\\\
\n –concurrency-range 1:8:2 \\\\ # \u6d4b\u8bd5\u5e76\u53d1\u6570\u4ece1\u52308，\u6b65\u957f\u4e3a2
\n –measurement-mode count_windows \\\\
\n –measurement-request-count 100<\/p>\n
# \u6216\u8005，\u66f4\u7b80\u5355\u5730，\u4f7f\u7528\u5185\u7f6e\u7684\u968f\u673a\u6570\u636e\u5feb\u901f\u6d4b\u8bd5
\nperf_analyzer -m llama_3_2v_11b_cot_ensemble \\\\
\n -u localhost:8000 \\\\
\n –shape IMAGE:1,3,336,336 \\\\ # \u6307\u5b9a\u8f93\u5165\u5f62\u72b6，\u5bf9\u4e8eBYTES\u7c7b\u578b\u53ef\u80fd\u4e0d\u9002\u7528，\u9700\u8981\u51c6\u5907\u771f\u5b9e\u6570\u636e\u6587\u4ef6
\n –shape QUESTION:1 \\\\
\n –concurrency-range 1:4<\/p>\n
\u4e3a\u4e86\u8fdb\u884c\u6709\u610f\u4e49\u7684\u6d4b\u8bd5，\u4f60\u9700\u8981\u521b\u5efa\u4e00\u4e2a\u5305\u542b\u771f\u5b9e\u8f93\u5165\u6570\u636e\u7684JSON\u6587\u4ef6 test_data.json：<\/p>\n
{
\n "data": [
\n {
\n "IMAGE": {"b64": "\/9j\/4AAQSkZJRgABAQAAAQABAAD\/2wBDAA…"}, \/\/ \u4f60\u7684\u6d4b\u8bd5\u56fe\u7247\u7684base64
\n "QUESTION": "What is in this image?"
\n },
\n \/\/ … \u53ef\u4ee5\u6709\u591a\u7ec4\u6d4b\u8bd5\u6570\u636e
\n ]
\n}<\/p>\n
perf_analyzer \u4f1a\u8f93\u51fa\u8be6\u7ec6\u7684\u62a5\u544a，\u5305\u62ec：<\/p>\n
\u5047\u8bbe\u4f60\u5f97\u5230\u4e86\u4e00\u4efd\u7c7b\u4f3c\u4e0b\u9762\u7684\u7b80\u5316\u62a5\u544a：<\/p>\n
*** Measurement Results ***
\n Concurrency: 4
\n Throughput: 12.5 infer\/sec
\n Avg Latency: 312.5 ms
\n P99 Latency: 520 ms
\n GPU Utilization: 85%<\/p>\n
\u5982\u4f55\u89e3\u8bfb？<\/p>\n
\u5e38\u89c1\u7684\u4f18\u5316\u65b9\u5411：<\/p>\n
\u8c03\u6574\u6279\u5904\u7406\u5927\u5c0f (max_batch_size)： \u8fd9\u662f\u63d0\u5347\u541e\u5410\u91cf\u6700\u6709\u6548\u7684\u624b\u6bb5\u3002\u5728 config.pbtxt \u4e2d\u589e\u5927 max_batch_size（\u6bd4\u5982\u4ece4\u6539\u4e3a8），\u5e76\u8c03\u6574 dynamic_batching \u4e2d\u7684 preferred_batch_size\u3002\u6ce8\u610f：\u8fd9\u4f1a\u5bfc\u81f4\u5355\u4e2a\u8bf7\u6c42\u5ef6\u8fdf\u589e\u52a0（\u56e0\u4e3a\u8981\u7b49\u51d1\u6279），\u4f46\u603b\u4f53\u541e\u5410\u91cf\u4f1a\u4e0a\u5347\u3002\u4f60\u9700\u8981\u6839\u636e\u4e1a\u52a1\u573a\u666f\u6743\u8861（\u91cd\u541e\u5410\u8fd8\u662f\u91cd\u5ef6\u8fdf）\u3002<\/p>\n<\/li>\n
\u4f7f\u7528\u66f4\u5feb\u7684\u6a21\u578b\u683c\u5f0f：<\/p>\n
\u4f18\u5316\u9884\u5904\u7406\/\u540e\u5904\u7406：<\/p>\n
\u589e\u52a0\u6a21\u578b\u5b9e\u4f8b (instance_group)： \u5728 config.pbtxt \u7684 instance_group \u4e2d，\u53ef\u4ee5\u8bbe\u7f6e count: 2 \u751a\u81f3\u66f4\u591a\u3002\u8fd9\u4f1a\u5728\u540c\u4e00\u4e2aGPU\u4e0a\u521b\u5efa\u6a21\u578b\u7684\u591a\u4e2a\u526f\u672c，\u5141\u8bb8Triton\u5728\u5b83\u4eec\u4e4b\u95f4\u5e76\u884c\u5904\u7406\u8bf7\u6c42，\u5c24\u5176\u6709\u5229\u4e8e\u5904\u7406\u5927\u91cf\u77ed\u65f6\u8bf7\u6c42\u3002\u4f46\u8fd9\u4f1a\u6210\u500d\u589e\u52a0\u663e\u5b58\u6d88\u8017\u3002<\/p>\n<\/li>\n
\u4f7f\u7528\u591aGPU： \u5982\u679c\u4f60\u6709\u591a\u4e2aGPU，\u53ef\u4ee5\u5728 instance_group \u7684 gpus \u5217\u8868\u4e2d\u6307\u5b9a [0,1]，\u5e76\u5c06 count \u8bbe\u7f6e\u4e3a\u6bcf\u4e2aGPU\u4e0a\u7684\u5b9e\u4f8b\u6570\u3002Triton\u4f1a\u81ea\u52a8\u5728GPU\u95f4\u8fdb\u884c\u8d1f\u8f7d\u5747\u8861\u3002<\/p>\n<\/li>\n
\u4f18\u5316\u662f\u4e00\u4e2a\u8fed\u4ee3\u8fc7\u7a0b：\u4fee\u6539\u914d\u7f6e -> \u538b\u6d4b -> \u5206\u6790\u7ed3\u679c -> \u518d\u4fee\u6539\u3002\u76ee\u6807\u662f\u627e\u5230\u6ee1\u8db3\u4f60\u4e1a\u52a1\u9700\u6c42（\u5982：\u5e73\u5747\u5ef6\u8fdf<500ms，\u541e\u5410\u91cf>20 infer\/sec）\u4e0b\u7684\u6700\u4f18\u914d\u7f6e\u3002<\/p>\n
\u8d70\u5b8c\u8fd9\u4e00\u6574\u5957\u6d41\u7a0b，\u4f60\u5df2\u7ecf\u4e0d\u662f\u4ec5\u4ec5\u5728\u201c\u8fd0\u884c\u201d\u4e00\u4e2aAI\u6a21\u578b，\u800c\u662f\u5728\u201c\u90e8\u7f72\u201d\u4e00\u4e2a\u751f\u4ea7\u7ea7\u7684AI\u63a8\u7406\u670d\u52a1\u3002\u6211\u4eec\u6765\u56de\u987e\u4e00\u4e0b\u5173\u952e\u6b65\u9aa4\u548c\u6536\u83b7：<\/p>\n
\u90e8\u7f72\u8fd9\u6837\u4e00\u4e2a\u590d\u6742\u7684\u6a21\u578b\u786e\u5b9e\u6bd4\u8dd1\u901a\u4e00\u4e2aDemo\u8981\u7e41\u7410，\u4f46\u8fd9\u4efd\u6295\u5165\u662f\u503c\u5f97\u7684\u3002\u5b83\u610f\u5473\u7740\u4f60\u7684AI\u80fd\u529b\u4ece\u201c\u73a9\u5177\u201d\u9636\u6bb5\u8fc8\u5411\u4e86\u201c\u5de5\u5177\u201d\u9636\u6bb5，\u53ef\u4ee5\u66f4\u7a33\u5b9a\u3001\u66f4\u9ad8\u6548\u5730\u670d\u52a1\u4e8e\u771f\u5b9e\u7684\u5e94\u7528\u548c\u7528\u6237\u3002\u4e0b\u6b21\u5f53\u4f60\u9700\u8981\u5904\u7406\u6d77\u91cf\u7684\u56fe\u7247\u7406\u89e3\u4efb\u52a1\u65f6，\u8fd9\u4e2a\u57fa\u4e8eTriton\u7684\u670d\u52a1\u5c06\u4f1a\u6210\u4e3a\u4f60\u53ef\u9760\u7684\u57fa\u77f3\u3002<\/p>\n
\u83b7\u53d6\u66f4\u591aAI\u955c\u50cf<\/p>\n
\u60f3\u63a2\u7d22\u66f4\u591aAI\u955c\u50cf\u548c\u5e94\u7528\u573a\u666f？\u8bbf\u95ee CSDN\u661f\u56fe\u955c\u50cf\u5e7f\u573a，\u63d0\u4f9b\u4e30\u5bcc\u7684\u9884\u7f6e\u955c\u50cf，\u8986\u76d6\u5927\u6a21\u578b\u63a8\u7406\u3001\u56fe\u50cf\u751f\u6210\u3001\u89c6\u9891\u751f\u6210\u3001\u6a21\u578b\u5fae\u8c03\u7b49\u591a\u4e2a\u9886\u57df，\u652f\u6301\u4e00\u952e\u90e8\u7f72\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
Llama-3.2V-11B-cot\u90e8\u7f72\u6559\u7a0b：NVIDIA Triton\u63a8\u7406\u670d\u52a1\u5668\u5c01\u88c5\u4e0e\u6027\u80fd\u538b\u6d4b
\n\u60f3\u8bd5\u8bd5\u90a3\u4e2a\u80fd\u770b\u61c2\u56fe\u7247，\u8fd8\u80fd\u50cf\u4eba\u4e00\u6837\u4e00\u6b65\u6b65\u63a8\u7406\u7684AI\u6a21\u578b\u5417？Llama-3.2V-11B-cot\u5c31\u662f\u8fd9\u6837\u4e00\u4e2a\u89c6\u89c9\u8bed\u8a00\u6a21\u578b\u3002\u5b83\u4e0d\u4ec5\u80fd\u7406\u89e3\u56fe\u7247\u5185\u5bb9，\u8fd8\u80fd\u628a\u601d\u8003\u8fc7\u7a0b\u62c6\u89e3\u6210\u201c\u603b\u7ed3\u2192\u63cf\u8ff0\u2192\u63a8\u7406\u2192\u7ed3\u8bba\u201d\u56db\u4e2a\u6b65\u9aa4，\u8ba9AI\u7684\u201c\u8111\u56de\u8def\u201d\u6e05\u6670\u53ef\u89c1\u3002
\n\u4e0d\u8fc7，\u76f4\u63a5\u8fd0\u884cPython\u811a\u672c\u867d\u7136\u7b80\u5355，\u4f46\u60f3\u5728\u751f\u4ea7\u73af<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[7591,9521,9327],"topic":[],"class_list":["post-85083","post","type-post","status-publish","format-standard","hentry","category-server","tag-ai","tag-9521","tag-9327"],"yoast_head":"\n