{"id":112394,"date":"2026-10-03T13:32:52","date_gmt":"2026-10-03T05:32:52","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/112394.html"},"modified":"2026-10-03T13:32:52","modified_gmt":"2026-10-03T05:32:52","slug":"llm-ops-%e8%af%84%e6%b5%8b%e4%b8%8e%e5%8f%af%e8%a7%82%e6%b5%8b%e5%ae%9e%e6%88%98%ef%bc%881%ef%bc%89%ef%bc%9allm-%e5%ba%94%e7%94%a8%e4%b8%ba%e4%bb%80%e4%b9%88%e5%bf%85%e9%a1%bb%e6%9c%89%e8%af%84","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/112394.html","title":{"rendered":"LLM Ops \u8bc4\u6d4b\u4e0e\u53ef\u89c2\u6d4b\u5b9e\u6218\uff081\uff09\uff1aLLM \u5e94\u7528\u4e3a\u4ec0\u4e48\u5fc5\u987b\u6709\u8bc4\u6d4b\u96c6\uff1a\u544a\u522b\u51ed\u611f\u89c9\u9a8c\u6536"},"content":{"rendered":"

\u95ee\u9898\u80cc\u666f<\/h3>\n

\u8fd9\u662f\u300aLLM Ops \u8bc4\u6d4b\u4e0e\u53ef\u89c2\u6d4b\u5b9e\u6218\u300b\u7cfb\u5217\u7684\u7b2c\u4e00\u7bc7，\u4e4b\u540e\u5341\u7bc7\u7684\u8bc4\u6d4b\u3001\u95e8\u7981\u3001\u8ffd\u8e2a\u3001\u7070\u5ea6\u90fd\u4f1a\u7ad9\u5728\u4eca\u5929\u8fd9\u5757\u5730\u57fa\u4e0a\u3002\u5148\u8bf4\u4e00\u4e2a\u5927\u591a\u6570\u56e2\u961f\u90fd\u7ecf\u5386\u8fc7\u7684\u573a\u666f：\u5ba2\u670d\u56e2\u961f\u4e0a\u7ebf\u4e86\u4e00\u4e2a\u57fa\u4e8e RAG \u7684\u552e\u540e\u653f\u7b56\u95ee\u7b54\u673a\u5668\u4eba，\u6f14\u793a\u4f1a\u4e0a\u8d1f\u8d23\u4eba\u8fde\u95ee\u4e94\u4e2a\u95ee\u9898，\u56db\u4e2a\u7b54\u5f97\u6f02\u4eae，\u4e8e\u662f\u5ba3\u5e03"\u6548\u679c\u4e0d\u9519，\u4e0a\u7ebf"\u3002\u4e24\u5468\u540e\u6295\u8bc9\u5806\u79ef：\u9000\u6b3e\u65f6\u6548\u95ee\u9898\u7b54\u9519\u3001\u628a"\u4e0d\u652f\u6301\u4e03\u5929\u65e0\u7406\u7531"\u7684\u5546\u54c1\u8bf4\u6210\u652f\u6301\u3001\u9047\u5230\u65b9\u8a00\u8f93\u5165\u76f4\u63a5\u80e1\u7f16\u3002\u56de\u5934\u590d\u76d8，\u6ca1\u6709\u4eba\u80fd\u8bf4\u6e05"\u4e0a\u7ebf\u90a3\u5929\u7684\u673a\u5668\u4eba\u5230\u5e95\u884c\u4e0d\u884c"\u2014\u2014\u56e0\u4e3a\u6839\u672c\u6ca1\u4eba\u7cfb\u7edf\u5730\u91cf\u8fc7\u3002\u4f20\u7edf\u8f6f\u4ef6\u6709\u5355\u5143\u6d4b\u8bd5\u548c\u65ad\u8a00，\u8f93\u5165\u786e\u5b9a\u8f93\u51fa\u786e\u5b9a；LLM \u5e94\u7528\u8f93\u51fa\u7684\u662f\u4e00\u6761\u6982\u7387\u5206\u5e03，"\u611f\u89c9\u4e0d\u9519"\u672c\u8d28\u4e0a\u662f\u62ff\u4e94\u516d\u4e2a\u6837\u672c\u53bb\u731c\u4e00\u4e2a\u5206\u5e03\u7684\u5f62\u72b6，\u8fd9\u79cd\u9a8c\u6536\u65b9\u5f0f\u5728\u7edf\u8ba1\u5b66\u4e0a\u5c31\u4e0d\u6210\u7acb\u3002\u672c\u7bc7\u56de\u7b54\u4e09\u4e2a\u95ee\u9898：\u8bc4\u6d4b\u96c6\u5230\u5e95\u5728\u5de5\u7a0b\u4e0a\u626e\u6f14\u4ec0\u4e48\u89d2\u8272；\u4e00\u4e2a\u5c0f\u5230"\u4e00\u5468\u5c31\u80fd\u5efa\u8d77\u6765"\u7684\u8bc4\u6d4b\u4f53\u7cfb\u957f\u4ec0\u4e48\u6837；\u4e0d\u5efa\u8bc4\u6d4b\u96c6\u65f6\u4f60\u5b9e\u9645\u5728\u7528\u4ec0\u4e48\u66ff\u4ee3\u54c1，\u4ee5\u53ca\u90a3\u4e2a\u66ff\u4ee3\u54c1\u574f\u5728\u54ea\u91cc\u3002<\/p>\n

\u6838\u5fc3\u539f\u7406<\/h3>\n

\u7b2c\u4e00\u5c42，\u8bc4\u6d4b\u96c6\u662f"\u6548\u679c\u7684\u7248\u672c\u63a7\u5236"\u3002\u4ee3\u7801\u6709 git，\u914d\u7f6e\u7684\u6bcf\u4e00\u6b21\u53d8\u66f4\u6709 diff \u53ef\u67e5，\u800c prompt\u3001\u6a21\u578b\u7248\u672c\u3001\u68c0\u7d22\u53c2\u6570\u8fd9\u4e09\u6837\u4e1c\u897f\u7684\u53d8\u66f4\u5728\u4f20\u7edf\u6d41\u7a0b\u91cc\u5b8c\u5168\u6ca1\u6709 diff：\u6539\u4e86 prompt \u7684\u7b2c\u4e09\u884c\u63aa\u8f9e，\u4f60\u4e0d\u77e5\u9053\u5b83\u5bf9"\u591a\u8f6e\u8ffd\u95ee"\u7c7b\u95ee\u9898\u9020\u6210\u4e86\u4ec0\u4e48\u5f71\u54cd，\u56e0\u4e3a\u6ca1\u6709\u4efb\u4f55\u57fa\u7ebf\u53ef\u4ee5\u5bf9\u7167\u3002\u8bc4\u6d4b\u96c6\u505a\u7684\u4e8b\u60c5，\u5c31\u662f\u628a"\u6548\u679c"\u53d8\u6210\u4e00\u4e2a\u53ef\u4ee5\u5bf9\u4e24\u6b21\u53d8\u66f4\u505a\u5dee\u5206\u7684\u6807\u91cf（\u6216\u4e00\u7ec4\u6807\u91cf）\u3002\u6ca1\u6709\u5b83，\u56de\u6eda\u90fd\u65e0\u6cd5\u51b3\u7b56\u2014\u2014\u4f60\u60f3\u9000\u56de\u65e7\u7248\u672c，\u5374\u62ff\u4e0d\u51fa"\u65e7\u7248\u672c\u5230\u5e95\u5dee\u5728\u54ea\u3001\u5dee\u591a\u5c11"\u7684\u8bc1\u636e\u3002<\/p>\n

\u7b2c\u4e8c\u5c42，LLM \u7684\u975e\u786e\u5b9a\u6027\u8ba9\u5c0f\u6837\u672c\u5f7b\u5e95\u5931\u771f\u3002\u540c\u4e00\u8f93\u5165\u6e29\u5ea6\u4e0d\u4e3a\u96f6\u65f6\u8f93\u51fa\u4f1a\u53d8，\u4e8e\u662f\u5355\u6b21\u95ee\u7b54\u7ed3\u679c\u5e26\u6709\u91c7\u6837\u566a\u58f0；\u5f53\u4f60\u53ea\u51ed\u5341\u6765\u4e2a\u6848\u4f8b\u5224\u65ad\u7248\u672c\u597d\u574f，\u566a\u58f0\u7ecf\u5e38\u6bd4\u771f\u5b9e\u6548\u5e94\u5927\u3002\u628a\u8fd9\u4ef6\u4e8b\u6a21\u62df\u51fa\u6765\u6700\u76f4\u89c2：\u5047\u8bbe\u65b0\u7248\u672c\u771f\u5b9e\u901a\u8fc7\u7387 0.75\u3001\u65e7\u7248\u672c 0.72（\u65b0\u7248\u672c\u786e\u5b9e\u66f4\u597d），\u7528\u56fa\u5b9a\u968f\u673a\u79cd\u5b50\u6309\u4e0d\u540c\u6837\u672c\u91cf\u5404\u505a 400 \u6b21\u62bd\u6837\u8bc4\u6d4b，\u7edf\u8ba1"\u628a\u66f4\u597d\u7684\u65b0\u7248\u8bef\u5224\u6210\u6301\u5e73\u6216\u9000\u6b65"\u7684\u6bd4\u4f8b\u3002<\/p>\n

\u7b2c\u4e09\u5c42，\u4e0d\u5efa\u8bc4\u6d4b\u96c6\u4e0d\u7b49\u4e8e\u6ca1\u6709\u8bc4\u6d4b，\u53ea\u662f\u8bc4\u6d4b\u6807\u51c6\u52a3\u5316\u4e86\u3002\u56e2\u961f\u4f1a\u81ea\u53d1\u627e\u5230\u4ee3\u7406\u6307\u6807：\u6f14\u793a\u65f6\u7684\u901a\u8fc7\u7387\u3001\u4ea7\u54c1\u7ecf\u7406\u7684\u624b\u5de5\u62bd\u6d4b\u3001\u4e0a\u7ebf\u540e\u524d\u51e0\u6761\u7528\u6237\u53cd\u9988\u3001\u7ade\u5bf9 demo \u7684\u5bf9\u6bd4\u5370\u8c61\u3002\u8fd9\u4e9b\u4ee3\u7406\u6709\u4e09\u4e2a\u5171\u540c\u75c5\u7076\u2014\u2014\u6837\u672c\u5c0f\u4e14\u504f（\u4e13\u6311\u80fd\u6253\u7684\u95ee）\u3001\u6807\u51c6\u6f02\u79fb（\u6bcf\u4e2a\u4eba\u5fc3\u91cc"\u597d"\u7684\u5b9a\u4e49\u4e0d\u540c，\u540c\u4e00\u6761\u56de\u7b54\u4e24\u4e2a\u4eba\u7ed9\u51fa\u76f8\u53cd\u7ed3\u8bba）\u3001\u4e0d\u53ef\u7d2f\u79ef（\u4eba\u7684\u5370\u8c61\u65e0\u6cd5\u53c2\u4e0e\u4e0b\u4e00\u6b21\u7248\u672c\u6bd4\u8f83）\u3002\u8bc4\u6d4b\u96c6\u5de5\u7a0b\u7684\u5168\u90e8\u52a8\u673a，\u5c31\u662f\u628a\u8fd9\u5957\u52a3\u5316\u7684\u9690\u5f0f\u8bc4\u6d4b\u6362\u6210\u53ef\u590d\u73b0\u3001\u53ef\u7d2f\u79ef\u3001\u53ef\u5ba1\u8ba1\u7684\u663e\u5f0f\u8bc4\u6d4b\u3002<\/p>\n

\u7b2c\u56db\u5c42，\u8bc4\u6d4b\u96c6\u7684\u6700\u5c0f\u53ef\u884c\u5f62\u6001\u3002\u4e0d\u9700\u8981\u4e00\u6b65\u5230\u4f4d，\u4e09\u6837\u4e1c\u897f\u51d1\u9f50\u5c31\u7b97\u7acb\u8d77\u6765\u4e86：\u4e00\u4e2a 200 \u5230 500 \u6761\u3001\u6309\u4e1a\u52a1\u610f\u56fe\u5206\u5c42\u62bd\u6837\u7684\u7528\u4f8b\u96c6（\u610f\u56fe\u5982：\u653f\u7b56\u95ee\u7b54\u3001\u8ba2\u5355\u67e5\u8be2\u3001\u8fb9\u754c\u62d2\u7b54\u3001\u591a\u8f6e\u8ffd\u95ee），\u6bcf\u6761\u5e26\u53c2\u8003\u7b54\u6848\u6216\u8bc4\u5206\u6807\u51c6；\u4e00\u5957\u81ea\u52a8\u6253\u5206\u6d41\u7a0b（\u4e0b\u4e00\u7bc7\u5efa\u6570\u636e\u96c6，\u7b2c\u4e09\u7bc7\u8bb2\u6253\u5206）；\u4e00\u6761 CI \u91cc\u7684\u56de\u5f52\u57fa\u7ebf\u2014\u2014\u6bcf\u6b21 prompt \u6216\u6a21\u578b\u53d8\u66f4\u8dd1\u4e00\u904d，\u51fa\u5206\u5bf9\u6bd4\u3002\u5148\u6709\u7c97\u7cd9\u4f46\u7a33\u5b9a\u7684\u5c3a\u5b50，\u518d\u8c08\u7cbe\u7ec6\u7684\u5c3a\u5b50\u3002<\/p>\n

\u5b9e\u9a8c\u4e00：\u51ed\u611f\u89c9\u9a8c\u6536\u5728\u7edf\u8ba1\u4e0a\u4e3a\u4ec0\u4e48\u4f1a\u7ffb\u8f66<\/h3>\n

\u672c\u673a\u65e0\u7b2c\u4e09\u65b9\u4f9d\u8d56，\u4ee5\u4e0b\u7528 Python \u6807\u51c6\u5e93\u505a\u786e\u5b9a\u6027\u6a21\u62df（\u56fa\u5b9a\u79cd\u5b50），\u6f14\u793a\u62bd\u6837\u89c4\u6a21\u4e0e\u7ed3\u8bba\u7a33\u5b9a\u6027\u7684\u5173\u7cfb；\u751f\u4ea7\u73af\u5883\u8bf7\u7528\u771f\u5b9e\u8bc4\u6d4b\u5e73\u53f0\u8dd1\u771f\u5b9e\u7528\u4f8b\u3002<\/p>\n

import<\/span> random<\/p>\n

TRUE_A =<\/span> 0.72<\/span> # \u65e7\u7248\u672c\u771f\u5b9e\u901a\u8fc7\u7387<\/span>
\nTRUE_B =<\/span> 0.75<\/span> # \u65b0\u7248\u672c\u771f\u5b9e\u901a\u8fc7\u7387(\u786e\u5b9e\u66f4\u597d)<\/span>
\nTRIALS =<\/span> 400<\/span><\/p>\n

def<\/span> trial<\/span>(<\/span>n,<\/span> seed)<\/span>:<\/span>
\n rng =<\/span> random.<\/span>Random(<\/span>seed)<\/span>
\n flips =<\/span> 0<\/span>
\n gaps =<\/span> [<\/span>]<\/span>
\n for<\/span> _ in<\/span> range<\/span>(<\/span>TRIALS)<\/span>:<\/span>
\n a =<\/span> sum<\/span>(<\/span>1<\/span> for<\/span> _ in<\/span> range<\/span>(<\/span>n)<\/span> if<\/span> rng.<\/span>random(<\/span>)<\/span> <<\/span> TRUE_A)<\/span> \/<\/span> n
\n b =<\/span> sum<\/span>(<\/span>1<\/span> for<\/span> _ in<\/span> range<\/span>(<\/span>n)<\/span> if<\/span> rng.<\/span>random(<\/span>)<\/span> <<\/span> TRUE_B)<\/span> \/<\/span> n
\n if<\/span> b <=<\/span> a:<\/span>
\n flips +=<\/span> 1<\/span>
\n gaps.<\/span>append(<\/span>abs<\/span>(<\/span>b –<\/span> a)<\/span>)<\/span>
\n return<\/span> flips,<\/span> sum<\/span>(<\/span>gaps)<\/span> \/<\/span> len<\/span>(<\/span>gaps)<\/span><\/p>\n

print<\/span>(<\/span>"\u8bc4\u6d4b\u96c6\u6837\u672c\u91cf\u5bf9\u7ed3\u8bba\u7a33\u5b9a\u6027\u7684\u5f71\u54cd(\u6bcf\u6863 400 \u6b21\u91cd\u590d\u5b9e\u9a8c, \u79cd\u5b50=42):"<\/span>)<\/span>
\nprint<\/span>(<\/span>"\u6837\u672c\u91cf n | \u8bef\u5224\u7387(\u628a\u66f4\u597d\u7684\u65b0\u7248\u5224\u6210\u6301\u5e73\u6216\u9000\u6b65) | \u5e73\u5747\u8868\u89c2\u5dee\u8ddd"<\/span>)<\/span>
\nfor<\/span> n in<\/span> (<\/span>20<\/span>,<\/span> 50<\/span>,<\/span> 100<\/span>,<\/span> 300<\/span>,<\/span> 1000<\/span>)<\/span>:<\/span>
\n flips,<\/span> mean_gap =<\/span> trial(<\/span>n,<\/span> seed=<\/span>42<\/span>)<\/span>
\n print<\/span>(<\/span>"n=%4d | %5.1f%% | %.3f"<\/span> %<\/span> (<\/span>n,<\/span> flips \/<\/span> TRIALS *<\/span> 100<\/span>,<\/span> mean_gap)<\/span>)<\/span><\/p>\n

\u8fd0\u884c\u8f93\u51fa：<\/p>\n

\u8bc4\u6d4b\u96c6\u6837\u672c\u91cf\u5bf9\u7ed3\u8bba\u7a33\u5b9a\u6027\u7684\u5f71\u54cd(\u6bcf\u6863 400 \u6b21\u91cd\u590d\u5b9e\u9a8c, \u79cd\u5b50=42):
\n\u6837\u672c\u91cf n | \u8bef\u5224\u7387(\u628a\u66f4\u597d\u7684\u65b0\u7248\u5224\u6210\u6301\u5e73\u6216\u9000\u6b65) | \u5e73\u5747\u8868\u89c2\u5dee\u8ddd
\nn= 20 | 47.0% | 0.114
\nn= 50 | 40.2% | 0.077
\nn= 100 | 32.8% | 0.059
\nn= 300 | 24.2% | 0.038
\nn=1000 | 6.5% | 0.031<\/p>\n

\u8fd9\u7ec4\u6570\u5b57\u503c\u5f97\u88f1\u8d77\u6765\u8d34\u5728\u8bc4\u5ba1\u4f1a\u4e0a：\u4e24\u4e2a\u7248\u672c\u771f\u5b9e\u5dee\u8ddd 3 \u4e2a\u767e\u5206\u70b9\u65f6，20 \u6761\u7528\u4f8b\u7684\u8bc4\u6d4b\u6709 47% \u7684\u6982\u7387\u5f97\u51fa\u53cd\u5411\u7ed3\u8bba\u2014\u2014\u548c\u629b\u786c\u5e01\u4e00\u6837；\u54ea\u6015 100 \u6761\u4ecd\u6709\u8fd1\u4e09\u5206\u4e4b\u4e00\u7684\u8bef\u5224\u7387\u3002\u6ce8\u610f 3 \u4e2a\u767e\u5206\u70b9\u672c\u5c31\u662f"\u52c9\u5f3a\u503c\u5f97\u5207\u6362"\u7684\u5dee\u8ddd，\u5982\u679c\u4f60\u7684\u771f\u5b9e\u63d0\u5347\u53ea\u6709 1 \u4e2a\u70b9，\u9700\u8981\u4e0a\u5343\u6761\u7528\u4f8b\u624d\u770b\u5f97\u89c1\u3002\u8fd9\u4e5f\u89e3\u91ca\u4e86\u4e3a\u4ec0\u4e48\u6f14\u793a\u4f1a\u5f0f\u7684\u9a8c\u6536\u603b\u80fd"\u6210\u529f"：\u6f14\u793a\u8005\u6311\u7684\u6848\u4f8b\u3001\u6311\u7684\u8f6e\u6b21，\u76f8\u5f53\u4e8e\u5728 47% \u8bef\u5224\u7387\u4e4b\u4e0a\u518d\u505a\u4e00\u8f6e\u5bf9\u5df1\u6709\u5229\u7684\u7b5b\u9009\u3002<\/p>\n

\u5b9e\u9a8c\u4e8c：\u8981\u591a\u5927\u7684\u5c3a\u5b50\u624d\u914d\u8bf4"\u63d0\u5347\u4e86"<\/h3>\n

\u6362\u4e00\u4e2a\u89d2\u5ea6，\u7528\u6b63\u6001\u8fd1\u4f3c\u7b97\u7f6e\u4fe1\u533a\u95f4\u534a\u5bbd（\u7eaf\u6807\u51c6\u5e93 math \u5373\u53ef），\u56de\u7b54"n \u6761\u7528\u4f8b\u7684\u901a\u8fc7\u7387\u4f30\u8ba1，\u8bef\u5dee\u5230\u5e95\u6709\u591a\u5927"\u3002<\/p>\n

import<\/span> math<\/p>\n

BASE =<\/span> 0.72<\/span>
\nZ =<\/span> 1.96<\/span><\/p>\n

def<\/span> half_width<\/span>(<\/span>p,<\/span> n)<\/span>:<\/span>
\n return<\/span> Z *<\/span> math.<\/span>sqrt(<\/span>p *<\/span> (<\/span>1<\/span> –<\/span> p)<\/span> \/<\/span> n)<\/span><\/p>\n

print<\/span>(<\/span>"\u901a\u8fc7\u7387 0.72 \u7684\u70b9\u4f30\u8ba1, \u4e0d\u540c\u6837\u672c\u91cf\u4e0b\u7684 95% \u7f6e\u4fe1\u533a\u95f4(\u6b63\u6001\u8fd1\u4f3c):"<\/span>)<\/span>
\nfor<\/span> n in<\/span> (<\/span>20<\/span>,<\/span> 50<\/span>,<\/span> 100<\/span>,<\/span> 200<\/span>,<\/span> 400<\/span>,<\/span> 800<\/span>)<\/span>:<\/span>
\n h =<\/span> half_width(<\/span>BASE,<\/span> n)<\/span>
\n print<\/span>(<\/span>"n=%4d \u534a\u5bbd=%.3f \u533a\u95f4=[%.3f, %.3f]"<\/span> %<\/span> (<\/span>n,<\/span> h,<\/span> BASE –<\/span> h,<\/span> BASE +<\/span> h)<\/span>)<\/span><\/p>\n

target =<\/span> 0.03<\/span>
\nn =<\/span> 1<\/span>
\nwhile<\/span> half_width(<\/span>BASE,<\/span> n)<\/span> ><\/span> target:<\/span>
\n n +=<\/span> 1<\/span>
\nprint<\/span>(<\/span>"\u8981\u628a\u7f6e\u4fe1\u533a\u95f4\u534a\u5bbd\u538b\u5230 0.03 \u4ee5\u5185, \u5355\u7248\u672c\u6700\u5c11\u9700\u8981 %d \u4e2a\u7528\u4f8b"<\/span> %<\/span> n)<\/span>
\nprint<\/span>(<\/span>"\u4e24\u4e2a\u7248\u672c\u5404\u81ea\u72ec\u7acb\u8bc4\u6d4b\u65f6, \u6bd4\u8f83\u7684\u6709\u6548\u5dee\u8ddd\u7ea6\u4e3a\u534a\u5bbd\u7684 sqrt(2) \u500d"<\/span>)<\/span>
\nprint<\/span>(<\/span>"\u53ef\u5206\u8fa8\u5dee\u8ddd\u4e0b\u9650\u7ea6 %.3f"<\/span> %<\/span> (<\/span>target *<\/span> math.<\/span>sqrt(<\/span>2<\/span>)<\/span>)<\/span>)<\/span><\/p>\n

\u8fd0\u884c\u8f93\u51fa：<\/p>\n

\u901a\u8fc7\u7387 0.72 \u7684\u70b9\u4f30\u8ba1, \u4e0d\u540c\u6837\u672c\u91cf\u4e0b\u7684 95% \u7f6e\u4fe1\u533a\u95f4(\u6b63\u6001\u8fd1\u4f3c):
\nn= 20 \u534a\u5bbd=0.197 \u533a\u95f4=[0.523, 0.917]
\nn= 50 \u534a\u5bbd=0.124 \u533a\u95f4=[0.596, 0.844]
\nn= 100 \u534a\u5bbd=0.088 \u533a\u95f4=[0.632, 0.808]
\nn= 200 \u534a\u5bbd=0.062 \u533a\u95f4=[0.658, 0.782]
\nn= 400 \u534a\u5bbd=0.044 \u533a\u95f4=[0.676, 0.764]
\nn= 800 \u534a\u5bbd=0.031 \u533a\u95f4=[0.689, 0.751]
\n\u8981\u628a\u7f6e\u4fe1\u533a\u95f4\u534a\u5bbd\u538b\u5230 0.03 \u4ee5\u5185, \u5355\u7248\u672c\u6700\u5c11\u9700\u8981 861 \u4e2a\u7528\u4f8b
\n\u4e24\u4e2a\u7248\u672c\u5404\u81ea\u72ec\u7acb\u8bc4\u6d4b\u65f6, \u6bd4\u8f83\u7684\u6709\u6548\u5dee\u8ddd\u7ea6\u4e3a\u534a\u5bbd\u7684 sqrt(2) \u500d
\n\u53ef\u5206\u8fa8\u5dee\u8ddd\u4e0b\u9650\u7ea6 0.042<\/p>\n

20 \u6761\u7528\u4f8b\u6d4b\u51fa\u7684"\u901a\u8fc7\u7387 72%"，\u771f\u5b9e\u533a\u95f4\u662f [52%, 92%]\u2014\u2014\u8fd9\u4e2a\u5206\u6570\u9664\u4e86\u81ea\u6211\u5b89\u6170\u6beb\u65e0\u4fe1\u606f\u91cf\u3002\u8981\u628a\u4e0d\u786e\u5b9a\u5ea6\u538b\u5230 \u00b13 \u4e2a\u70b9，\u5355\u7248\u672c\u9700\u8981\u8fd1 900 \u6761；\u4e24\u7248\u5bf9\u6bd4\u65f6\u53ef\u5206\u8fa8\u5dee\u8ddd\u8fdb\u4e00\u6b65\u653e\u5927\u5230 4 \u4e2a\u70b9\u4e0a\u4e0b\u3002\u5de5\u7a0b\u4e0a\u4ece\u4e2d\u5f97\u5230\u4e24\u6761\u786c\u7ed3\u8bba：\u5176\u4e00，\u8bc4\u6d4b\u96c6\u89c4\u6a21\u662f\u6309"\u4f60\u60f3\u5206\u8fa8\u591a\u5927\u7684\u5dee\u8ddd"\u5012\u63a8\u51fa\u6765\u7684，\u4e0d\u662f\u62cd\u8111\u888b\u5b9a 100 \u6761；\u5176\u4e8c，\u540c\u4e00\u6279\u7528\u4f8b\u8981\u6253\u5728\u4e24\u4e2a\u7248\u672c\u4e0a\u505a\u914d\u5bf9\u6bd4\u8f83，\u6bd4\u4e24\u4e2a\u72ec\u7acb\u6837\u672c\u6bd4\u8f83\u7075\u654f\u5f97\u591a\u2014\u2014\u8fd9\u6b63\u662f\u7b2c\u56db\u7bc7\u56de\u5f52\u6d4b\u8bd5\u91c7\u7528"\u540c\u6848\u4f8b\u96c6\u9010\u6761 diff"\u800c\u4e0d\u662f\u53ea\u770b\u603b\u5206\u7684\u539f\u56e0\u3002<\/p>\n

\u5de5\u7a0b\u843d\u5730\u8def\u5f84<\/h3>\n

\u7b2c\u4e00\u6b65，\u4ece\u7ebf\u4e0a\u65e5\u5fd7\u51b7\u542f\u52a8\u3002\u6ca1\u6709\u8bc4\u6d4b\u96c6\u7684\u56e2\u961f\u51e0\u4e4e\u4e00\u5b9a\u6709\u5bf9\u8bdd\u65e5\u5fd7：\u5ba2\u670d\u7cfb\u7edf\u3001API \u8bf7\u6c42\u8bb0\u5f55\u3001\u53cd\u9988\u8868\u3002\u6309\u4e1a\u52a1\u610f\u56fe\u5206\u6876\u62bd\u6837 300 \u6761，\u4eba\u5de5\u8fc7\u4e00\u904d\u6311\u51fa"\u7b54\u9519\u7684\u3001\u53ef\u7591\u7684\u3001\u5178\u578b\u7684"\u4e09\u7c7b，\u6807\u6ce8\u9884\u671f\u884c\u4e3a（\u5e94\u8be5\u5f15\u7528\u54ea\u6761\u653f\u7b56\u3001\u5e94\u8be5\u62d2\u7b54\u3001\u5e94\u8be5\u8ffd\u95ee\u4ec0\u4e48）\u3002\u4e24\u4e09\u4e2a\u4eba\u5de5\u65e5\u5c31\u80fd\u7acb\u8d77\u7b2c\u4e00\u7248，\u5148\u8dd1\u901a\u94fe\u8def\u518d\u8ffd\u6c42\u89c4\u6a21\u3002<\/p>\n

\u7b2c\u4e8c\u6b65，\u8bc4\u5206\u6807\u51c6\u5199\u5230\u5b57\u7b26\u4e32\u7ea7\u3002\u6bcf\u6761\u7528\u4f8b\u9664\u4e86\u53c2\u8003\u7b54\u6848，\u8fd8\u8981\u5199"\u5224\u5206\u8981\u70b9"：\u5fc5\u987b\u5305\u542b\u9000\u6b3e\u65f6\u6548\u4e09\u4e2a\u5b57\u3001\u5fc5\u987b\u4e0d\u627f\u8bfa\u5177\u4f53\u5230\u8d26\u65e5\u3001\u51fa\u73b0\u7535\u8bdd\u53f7\u7801\u5373\u5224\u9519\u3002\u8981\u70b9\u5f0f\u6807\u51c6\u8ba9\u4e0d\u540c\u6807\u6ce8\u5458\u7ed3\u8bba\u8d8b\u4e8e\u4e00\u81f4，\u4e5f\u662f\u7b2c\u4e09\u7bc7 LLM-as-judge \u63d0\u793a\u8bcd\u7684\u539f\u6599\u3002<\/p>\n

\u7b2c\u4e09\u6b65，\u5206\u6570\u8fdb CI，\u57fa\u7ebf\u8fdb\u6863\u6848\u3002\u6bcf\u6b21 prompt\/\u6a21\u578b\/\u68c0\u7d22\u53c2\u6570\u53d8\u66f4\u81ea\u52a8\u8dd1\u8bc4\u6d4b，\u5206\u6570\u4e0e\u53d8\u66f4\u53f7\u7ed1\u5b9a\u5b58\u6863\u3002\u8bc4\u6d4b\u96c6\u672c\u8eab\u4e5f\u662f\u4ee3\u7801：\u8fdb\u7248\u672c\u5e93\u3001\u6709 owner\u3001\u6709 review，\u88ab\u4e1a\u52a1\u65b9\u6295\u8bc9"\u7b54\u975e\u6240\u95ee"\u7684\u6848\u4f8b\u5148\u8fdb\u76d1\u63a7\u540d\u5355\u3001\u5b63\u5ea6\u8bc4\u5ba1\u540e\u664b\u5347\u4e3a\u7528\u4f8b\u2014\u2014\u6570\u636e\u96c6\u8981\u968f\u4e1a\u52a1\u751f\u957f，\u5426\u5219\u534a\u5e74\u540e\u5b83\u5c31\u6d4b\u4e0d\u5230\u771f\u5b9e\u5206\u5e03\u4e86\u3002<\/p>\n

\u5e38\u89c1\u9677\u9631<\/h3>\n

\u5176\u4e00，\u7528\u901a\u7528 benchmark（MMLU\u3001GSM8K \u8fd9\u7c7b）\u4ee3\u66ff\u4e1a\u52a1\u8bc4\u6d4b：\u699c\u4e0a\u5206\u6570\u4e0e\u4f60\u7684\u552e\u540e\u95ee\u7b54\u8d28\u91cf\u51e0\u4e4e\u4e0d\u76f8\u5173，\u6a21\u578b\u9009\u578b\u5fc5\u987b\u4ee5\u81ea\u5df1\u7684\u8bc4\u6d4b\u96c6\u4e3a\u51c6\u3002\u5176\u4e8c，\u8bc4\u6d4b\u96c6\u4e0e\u8bad\u7ec3\/\u63d0\u793a\u793a\u4f8b\u540c\u6e90：\u628a\u8c03 prompt \u65f6\u7528\u8fc7\u7684\u6848\u4f8b\u7559\u5728\u8bc4\u6d4b\u96c6\u91cc，\u7b49\u4e8e\u5f00\u5377\u8003\u8bd5，\u5206\u6570\u865a\u9ad8\u4e14\u63a9\u76d6\u771f\u95ee\u9898\u2014\u2014\u8c03 prompt \u7528\u7684\u6848\u4f8b\u548c\u8bc4\u6d4b\u6848\u4f8b\u8981\u7269\u7406\u9694\u79bb\u3002\u5176\u4e09，\u53ea\u6d4b\u5355\u8f6e：\u771f\u5b9e\u7528\u6237\u5927\u91cf\u8ffd\u95ee\u3001\u6539\u53e3\u3001\u5939\u5e26\u65b9\u8a00，\u5355\u8f6e\u901a\u8fc7\u7387\u6f02\u4eae\u7684\u591a\u8f6e\u4e00\u584c\u7cca\u6d82，\u8bc4\u6d4b\u96c6\u4e2d\u591a\u8f6e\u5bf9\u8bdd\u7528\u4f8b\u5360\u6bd4\u5efa\u8bae\u4e0d\u4f4e\u4e8e\u4e09\u6210\u3002\u5176\u56db，\u628a\u8bc4\u6d4b\u5f53\u4e00\u6b21\u6027\u6d3b\u52a8：\u4e3a\u9009\u578b\u5efa\u5b8c\u96c6\u5c31\u6254，\u4e4b\u540e prompt \u6539\u4e86\u4e03\u7248\u6ca1\u4eba\u91cd\u8dd1，\u8bc4\u6d4b\u96c6\u6700\u5927\u7684\u4ef7\u503c\u6070\u6070\u5728"\u6301\u7eed"\u4e24\u4e2a\u5b57\u3002\u5176\u4e94，\u8bc4\u5206\u6807\u51c6\u53e3\u5934\u5316："\u56de\u7b54\u8981\u81ea\u7136"\u8fd9\u79cd\u5f62\u5bb9\u8bcd\u65e0\u6cd5\u590d\u6838，\u4e24\u4e2a\u4eba\u6253\u51fa\u7684\u5206\u5dee\u5230 0.4 \u4ee5\u4e0a（\u6ee1\u5206 1 \u5206\u5236），\u5c3a\u5b50\u672c\u8eab\u5c31\u662f\u566a\u58f0\u6e90\u3002<\/p>\n

\u843d\u5730\u6e05\u5355<\/h3>\n