{"id":89913,"date":"2026-08-04T06:25:03","date_gmt":"2026-08-03T22:25:03","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/89913.html"},"modified":"2026-08-04T06:25:03","modified_gmt":"2026-08-03T22:25:03","slug":"l5-%e5%ae%9a%e5%be%8b%e8%b7%a8%e6%9e%b6%e6%9e%84%e9%aa%8c%e8%af%81%e5%ae%9e%e9%aa%8c%e6%96%b9%e6%a1%88","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/89913.html","title":{"rendered":"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848"},"content":{"rendered":"<h2>L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848<\/h2>\n<h3>\u2014\u2014Mamba \u4e0e RWKV \u975e Transformer \u67b6\u6784\u6d4b\u8bd5\u6b65\u9aa4<\/h3>\n<p>\u6587\u6863\u5b9a\u4f4d&#xff1a;\u672c\u65b9\u6848\u4e3a L5 \u8870\u51cf\u4e0d\u53ef\u6d88\u9664\u5b9a\u5f8b&#xff08;\u4e0b\u79f0&#034;L5 \u5b9a\u5f8b&#034;&#xff09;\u7684\u8de8\u67b6\u6784\u666e\u9002\u6027\u9a8c\u8bc1\u63d0\u4f9b\u5b8c\u6574\u53ef\u6267\u884c\u6b65\u9aa4\u3002L5 \u5b9a\u5f8b\u82e5\u4ec5\u5728 Transformer \u4e0a\u6210\u7acb&#xff0c;\u5219\u5e94\u964d\u7ea7\u4e3a&#034;Transformer \u7ecf\u9a8c\u89c4\u5f8b&#034;\u800c\u975e&#034;\u666e\u904d\u5b9a\u5f8b&#034;&#xff1b;\u82e5\u5728\u975e Transformer \u67b6\u6784\u4e0a\u540c\u6837\u6210\u7acb&#xff0c;\u5219\u5177\u5907\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u3002\u672c\u65b9\u6848\u662f L5 \u80fd\u5426\u79f0\u4e3a&#034;\u5b9a\u5f8b&#034;\u7684\u51b3\u5b9a\u6027\u5b9e\u9a8c\u3002<\/p>\n<p>\u524d\u7f6e\u9605\u8bfb&#xff1a;<\/p>\n<ul>\n<li>L5\u5b9a\u5f8b\u53ef\u8bc1\u4f2a\u6027\u6d4b\u8bd5\u65b9\u6848_\u8d44\u6e90\u7ea6\u675f\u7248.md&#xff08;\u7eaf API \u65b9\u6848&#xff0c;\u672c\u65b9\u6848\u4e3a\u5176\u672c\u5730\u589e\u5f3a\u7248\u7684\u5b9e\u9a8c 4 \u5c55\u5f00&#xff09;<\/li>\n<li>L5\u8870\u51cf\u5047\u8bf4\u6a21\u578b\u5b9e\u8bc1\u9a8c\u8bc1\u5b9e\u9a8c\u8bbe\u8ba1\u65b9\u6848.md<\/li>\n<li>\u57fa\u4e8e\u56db\u4efd\u4ea4\u53c9\u6587\u6863\u7684\u5927\u6a21\u578b\u4eba\u673a\u534f\u540c\u4e94\u5b9a\u5f8b\u4f53\u7cfb\u5b8c\u6574\u8bba\u6587.txt<\/li>\n<\/ul>\n<hr \/>\n<h3>\u6458\u8981<\/h3>\n<p>L5 \u8870\u51cf\u4e0d\u53ef\u6d88\u9664\u5b9a\u5f8b\u7684\u6838\u5fc3\u6570\u5b66\u5f62\u5f0f S_c &#061; S\u00b7e^(-\u03bbC)\u00b7(1-\u03b2H) \u662f\u4ece Transformer \u81ea\u6ce8\u610f\u529b\u67b6\u6784\u63a8\u5bfc\u51fa\u7684\u3002\u8be5\u5b9a\u5f8b\u7684&#034;\u666e\u904d\u5b9a\u5f8b&#034;\u5730\u4f4d\u4f9d\u8d56\u4e8e\u4e00\u4e2a\u672a\u7ecf\u9a8c\u8bc1\u7684\u524d\u63d0&#xff1a;\u8870\u51cf\u662f\u6ce8\u610f\u529b\u673a\u5236\u7684\u56fa\u6709\u5c5e\u6027&#xff0c;\u800c\u975e Transformer \u7279\u5b9a\u5b9e\u73b0\u7684\u5076\u7136\u73b0\u8c61\u3002\u672c\u65b9\u6848\u8bbe\u8ba1\u5728 Mamba&#xff08;\u72b6\u6001\u7a7a\u95f4\u6a21\u578b&#xff09;\u3001RWKV&#xff08;\u7ebf\u6027 RNN&#xff09;\u4e24\u7c7b\u975e Transformer \u67b6\u6784\u4e0a\u7684\u5bf9\u7167\u5b9e\u9a8c&#xff0c;\u901a\u8fc7\u4e09\u5957\u67b6\u6784\u7279\u5f02\u6027\u8870\u51cf\u6d4b\u91cf\u6cd5 &#043; \u4e00\u5957\u901a\u7528\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u7684\u4ea4\u53c9\u9a8c\u8bc1&#xff0c;\u5224\u5b9a L5 \u662f\u5426\u5177\u6709\u8de8\u67b6\u6784\u666e\u9002\u6027\u3002\u65b9\u6848\u5305\u542b 7 \u4e2a\u6267\u884c\u6b65\u9aa4\u3001\u660e\u786e\u7684\u9884\u6ce8\u518c\u5224\u5b9a\u6807\u51c6\u3001\u8bda\u5b9e\u7684\u65b9\u6cd5\u5b66\u5c40\u9650\u6027\u58f0\u660e&#xff0c;\u9884\u8ba1\u5728\u5355\u5f20 RTX 4090 \u4e0a 2 \u5468\u5185\u53ef\u5b8c\u6210 1.4B \u89c4\u6a21\u5bf9\u9f50\u7ec4\u7684\u5b8c\u6574\u5bf9\u7167\u3002<\/p>\n<p>\u5173\u952e\u8bcd&#xff1a;L5 \u8870\u51cf\u5b9a\u5f8b&#xff1b;\u8de8\u67b6\u6784\u9a8c\u8bc1&#xff1b;Mamba&#xff1b;RWKV&#xff1b;\u72b6\u6001\u7a7a\u95f4\u6a21\u578b&#xff1b;\u7ebf\u6027 RNN&#xff1b;\u6ce8\u610f\u529b\u8870\u51cf&#xff1b;\u53ef\u8bc1\u4f2a\u6027<\/p>\n<hr \/>\n<h3>\u7b2c\u4e00\u7ae0 \u5b9e\u9a8c\u76ee\u7684\u4e0e\u5b9a\u4f4d<\/h3>\n<h4>1.1 \u95ee\u9898\u80cc\u666f<\/h4>\n<p>L5 \u5b9a\u5f8b\u7684\u63d0\u51fa\u57fa\u4e8e Transformer \u81ea\u6ce8\u610f\u529b\u67b6\u6784\u7684\u4ee5\u4e0b\u89c2\u5bdf&#xff1a;<\/p>\n<ul>\n<li>\u6ce8\u610f\u529b\u6743\u91cd\u968f\u4f4d\u7f6e\u8ddd\u79bb\u6307\u6570\u8870\u51cf<\/li>\n<li>\u4e0a\u4e0b\u6587\u957f\u5ea6 C \u589e\u52a0\u5bfc\u81f4\u4fe1\u606f\u71b5 H(w)&#061;log\u00a9 \u4e0d\u53ef\u9006\u589e\u957f<\/li>\n<li>\u5269\u4f59\u53ef\u4fe1\u6b65\u6570 S_c \u968f C \u5355\u8c03\u4e0b\u964d<\/li>\n<\/ul>\n<p>\u4f46\u9879\u76ee \u4e94\u5b9a\u5f8b\u4f53\u7cfb\u5b8c\u6574\u8bba\u6587 \u00a77.1 \u81ea\u5df1\u627f\u8ba4&#xff1a;<\/p>\n<p>\u201c\u4f53\u7cfb\u4ec5\u9488\u5bf9\u4e3b\u6d41 Transformer \u67b6\u6784\u5927\u6a21\u578b&#xff0c;\u672a\u8986\u76d6\u7ebf\u6027\u6ce8\u610f\u529b\u3001\u65e0\u6ce8\u610f\u529b\u65b0\u578b\u57fa\u7840\u6a21\u578b\u3002\u201d<\/p>\n<p>\u8fd9\u610f\u5473\u7740 L5 \u7684&#034;\u666e\u904d\u5b9a\u5f8b&#034;\u5730\u4f4d\u5b58\u5728\u672a\u88ab\u9a8c\u8bc1\u7684\u7f3a\u53e3\u3002\u5982\u679c Mamba\/RWKV \u7b49\u975e Transformer \u67b6\u6784\u4e0d\u5b58\u5728\u540c\u6837\u7684\u8870\u51cf&#xff0c;\u5219 L5 \u5e94\u964d\u7ea7\u4e3a&#034;Transformer \u7ecf\u9a8c\u89c4\u5f8b&#034;\u3002<\/p>\n<h4>1.2 \u5b9e\u9a8c\u5b9a\u4f4d<\/h4>\n<p>\u672c\u5b9e\u9a8c\u662f L5\u5b9a\u5f8b\u53ef\u8bc1\u4f2a\u6027\u6d4b\u8bd5\u65b9\u6848_\u8d44\u6e90\u7ea6\u675f\u7248.md \u4e2d\u5b9e\u9a8c 4 \u7684\u672c\u5730\u589e\u5f3a\u7248\u5b8c\u6574\u5c55\u5f00\u3002\u7eaf API \u65b9\u6848\u65e0\u6cd5\u4e25\u683c\u9a8c\u8bc1\u8de8\u67b6\u6784\u666e\u9002\u6027&#xff08;\u6240\u6709\u95ed\u6e90 API \u5747\u4e3a Transformer \u53d8\u4f53&#xff09;&#xff0c;\u672c\u65b9\u6848\u901a\u8fc7\u672c\u5730\u90e8\u7f72\u975e Transformer \u67b6\u6784\u6a21\u578b\u5b8c\u6210\u4e25\u683c\u9a8c\u8bc1\u3002<\/p>\n<h4>1.3 \u6838\u5fc3\u7814\u7a76\u95ee\u9898<\/h4>\n<p>\u4e3b\u95ee\u9898&#xff1a;L5 \u8870\u51cf\u5b9a\u5f8b\u662f\u5426\u5728\u975e Transformer \u67b6\u6784&#xff08;Mamba\/RWKV&#xff09;\u4e0a\u4f9d\u7136\u6210\u7acb&#xff1f;<\/p>\n<p>\u5b50\u95ee\u9898&#xff1a;<\/p>\n<li>Mamba\/RWKV \u7684 S_c \u662f\u5426\u968f\u4e0a\u4e0b\u6587\u957f\u5ea6 C \u589e\u52a0\u800c\u4e0b\u964d&#xff1f;<\/li>\n<li>\u82e5\u4e0b\u964d&#xff0c;\u8870\u51cf\u51fd\u6570\u5f62\u5f0f\u662f\u5426\u4ecd\u4e3a\u6307\u6570&#xff1f;<\/li>\n<li>\u82e5\u4e3a\u6307\u6570&#xff0c;\u8870\u51cf\u7cfb\u6570 \u03bb \u662f\u5426\u4e0e Transformer \u63a5\u8fd1&#xff1f;<\/li>\n<li>\u4e09\u79cd\u67b6\u6784\u7684\u8870\u51cf\u673a\u5236\u662f\u5426\u672c\u8d28\u7b49\u4ef7&#xff1f;<\/li>\n<hr \/>\n<h3>\u7b2c\u4e8c\u7ae0 \u6838\u5fc3\u6280\u672f\u6311\u6218<\/h3>\n<h4>2.1 \u8870\u51cf\u8f7d\u4f53\u7684\u67b6\u6784\u5dee\u5f02<\/h4>\n<p>L5 \u5b9a\u5f8b\u7684 \u03bb \u662f\u4ece Transformer \u7684 attention weights \u6d4b\u91cf\u7684\u3002\u4f46\u4e09\u7c7b\u67b6\u6784\u7684\u8870\u51cf\u8f7d\u4f53\u5b8c\u5168\u4e0d\u540c&#xff1a;<\/p>\n<table>\n<tr>\u67b6\u6784\u6ce8\u610f\u529b\u673a\u5236\u8870\u51cf\u8f7d\u4f53\u662f\u5426\u53ef\u76f4\u63a5\u63d0\u53d6<\/tr>\n<tbody>\n<tr>\n<td>Transformer<\/td>\n<td>\u81ea\u6ce8\u610f\u529b O(n\u00b2)<\/td>\n<td>attention weights \u77e9\u9635<\/td>\n<td>\u2705 \u53ef\u76f4\u63a5\u63d0\u53d6<\/td>\n<\/tr>\n<tr>\n<td>Mamba<\/td>\n<td>\u65e0\u6ce8\u610f\u529b&#xff08;SSM&#xff09;<\/td>\n<td>hidden state \u5bf9\u65e9\u671f\u4fe1\u606f\u7684\u4fdd\u7559<\/td>\n<td>\u274c \u9700\u95f4\u63a5\u6d4b\u91cf<\/td>\n<\/tr>\n<tr>\n<td>RWKV<\/td>\n<td>\u65e0\u6ce8\u610f\u529b&#xff08;\u7ebf\u6027 RNN&#xff09;<\/td>\n<td>state \u5411\u91cf\u7684\u6f14\u5316\u8870\u51cf<\/td>\n<td>&#x1f7e1; state \u53ef\u8bfb\u4f46\u9700\u89e3\u8bfb<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u76f4\u63a5\u5957\u7528 Transformer \u7684 attention \u6d4b\u91cf\u6cd5\u4f1a\u5931\u8d25\u2014\u2014Mamba\/RWKV \u6839\u672c\u6ca1\u6709 attention weights \u53ef\u63d0\u53d6\u3002<\/p>\n<h4>2.2 \u89e3\u51b3\u65b9\u6848&#xff1a;\u4e09\u5957\u7b49\u4ef7\u6d4b\u91cf\u6cd5 &#043; \u4e00\u5957\u901a\u7528\u6cd5<\/h4>\n<p>\u672c\u65b9\u6848\u8bbe\u8ba1\u56db\u5957\u6d4b\u91cf\u65b9\u6cd5\u4ea4\u53c9\u9a8c\u8bc1&#xff1a;<\/p>\n<table>\n<tr>\u65b9\u6cd5\u9002\u7528\u67b6\u6784\u6d4b\u91cf\u5bf9\u8c61\u89d2\u8272<\/tr>\n<tbody>\n<tr>\n<td>\u65b9\u6cd5 A&#xff1a;attention \u77e9\u9635\u76f4\u63a5\u63d0\u53d6<\/td>\n<td>Transformer<\/td>\n<td>attention weights \u8870\u51cf<\/td>\n<td>\u57fa\u7ebf<\/td>\n<\/tr>\n<tr>\n<td>\u65b9\u6cd5 B&#xff1a;hidden state \u63a2\u9488\u6cd5<\/td>\n<td>Mamba<\/td>\n<td>hidden state \u6270\u52a8\u8870\u51cf<\/td>\n<td>\u67b6\u6784\u7279\u5f02\u6027<\/td>\n<\/tr>\n<tr>\n<td>\u65b9\u6cd5 C&#xff1a;state \u5411\u91cf\u76f4\u63a5\u8bfb\u53d6<\/td>\n<td>RWKV<\/td>\n<td>state \u5411\u91cf\u6f14\u5316\u8870\u51cf<\/td>\n<td>\u67b6\u6784\u7279\u5f02\u6027<\/td>\n<\/tr>\n<tr>\n<td>\u65b9\u6cd5 D&#xff1a;\u9ed1\u76d2\u63a2\u6d4b\u6cd5<\/td>\n<td>\u4e09\u67b6\u6784\u901a\u7528<\/td>\n<td>\u53ec\u56de\u7387\u8870\u51cf<\/td>\n<td>\u4ea4\u53c9\u9a8c\u8bc1\u515c\u5e95<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5173\u952e\u8bbe\u8ba1&#xff1a;\u65b9\u6cd5 D \u662f\u4e09\u67b6\u6784\u901a\u7528\u7684&#xff0c;\u5176\u6d4b\u91cf\u7ed3\u679c\u53ef\u76f4\u63a5\u6a2a\u5411\u5bf9\u6bd4&#xff0c;\u907f\u514d\u65b9\u6cd5 A\/B\/C \u4e4b\u95f4\u7684&#034;\u7b49\u4ef7\u6027&#034;\u4e89\u8bae\u3002\u65b9\u6cd5 A\/B\/C \u4f5c\u4e3a\u673a\u5236\u5c42\u9762\u7684\u8865\u5145\u8bc1\u636e\u3002<\/p>\n<h4>2.3 \u65b9\u6cd5\u5b66\u8bda\u5b9e\u58f0\u660e<\/h4>\n<p>\u4e09\u79cd\u67b6\u6784\u7279\u5f02\u6027\u65b9\u6cd5&#xff08;A\/B\/C&#xff09;\u6d4b\u91cf\u7684&#034;\u8870\u51cf&#034;\u5728\u7269\u7406\u673a\u5236\u4e0a\u4e0d\u5b8c\u5168\u7b49\u4ef7&#xff1a;<\/p>\n<ul>\n<li>Transformer \u6d4b\u7684\u662f\u6ce8\u610f\u529b\u5206\u914d\u6743\u91cd<\/li>\n<li>Mamba \u6d4b\u7684\u662f hidden state \u5bf9\u65e9\u671f\u4fe1\u606f\u7684\u8bb0\u5fc6\u4fdd\u7559<\/li>\n<li>RWKV \u6d4b\u7684\u662f state \u5411\u91cf\u7684\u6f14\u5316\u8870\u51cf<\/li>\n<\/ul>\n<p>\u628a\u5b83\u4eec\u90fd\u6620\u5c04\u5230\u540c\u4e00\u4e2a \u03bb \u662f\u65b9\u6cd5\u5b66\u59a5\u534f\u3002\u672c\u65b9\u6848\u4ee5\u65b9\u6cd5 D&#xff08;\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff09;\u4f5c\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e&#xff0c;\u65b9\u6cd5 A\/B\/C \u4f5c\u4e3a\u673a\u5236\u5c42\u9762\u7684\u8865\u5145\u8bc1\u636e\u3002\u8fd9\u4e00\u8bbe\u8ba1\u9009\u62e9\u9700\u5728\u7ed3\u679c\u62a5\u544a\u4e2d\u660e\u786e\u58f0\u660e\u3002<\/p>\n<hr \/>\n<h3>\u7b2c\u4e09\u7ae0 \u6b65\u9aa4 1&#xff1a;\u786c\u4ef6\u4e0e\u73af\u5883\u51c6\u5907<\/h3>\n<h4>3.1 GPU \u9700\u6c42<\/h4>\n<table>\n<tr>\u6a21\u578b\u89c4\u6a21\u663e\u5b58\u9700\u6c42\u63a8\u8350 GPU\u9002\u7528\u573a\u666f<\/tr>\n<tbody>\n<tr>\n<td>1.4B-1.5B&#xff08;\u89c4\u6a21\u5bf9\u9f50\u7ec4&#xff09;<\/td>\n<td>~8GB<\/td>\n<td>RTX 3090 \/ 4090<\/td>\n<td>\u63a8\u8350&#xff0c;\u907f\u514d\u89c4\u6a21\u5dee\u5f02\u6c61\u67d3\u7ed3\u8bba<\/td>\n<\/tr>\n<tr>\n<td>2.8B-3B<\/td>\n<td>~12GB<\/td>\n<td>RTX 4090 \/ A5000<\/td>\n<td>\u6269\u5c55\u9a8c\u8bc1<\/td>\n<\/tr>\n<tr>\n<td>7B-8B&#xff08;\u5927\u89c4\u6a21\u7ec4&#xff09;<\/td>\n<td>~24GB<\/td>\n<td>A100 40GB \/ 2\u00d73090<\/td>\n<td>\u53ef\u9009&#xff0c;Mamba \u65e0 7B \u516c\u5f00\u6743\u91cd<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u63a8\u8350\u4f7f\u7528 1.4B \u89c4\u6a21\u5bf9\u9f50\u7ec4\u3002\u89c4\u6a21\u5dee\u5f02\u4f1a\u6c61\u67d3\u7ed3\u8bba&#xff08;\u5927\u6a21\u578b\u53ef\u80fd\u56e0\u8bad\u7ec3\u6570\u636e\u66f4\u591a\u800c\u8868\u73b0\u4e0d\u540c&#xff09;&#xff0c;\u89c4\u6a21\u5bf9\u9f50\u80fd\u63a7\u5236\u8fd9\u4e00\u6df7\u6dc6\u53d8\u91cf\u3002<\/p>\n<h4>3.2 \u73af\u5883\u5b89\u88c5<\/h4>\n<p><span class=\"token comment\"># \u57fa\u7840\u73af\u5883<\/span><br \/>\nconda create <span class=\"token parameter variable\">-n<\/span> l5-cross-arch <span class=\"token assign-left variable\">python<\/span><span class=\"token operator\">&#061;<\/span><span class=\"token number\">3.11<\/span><br \/>\nconda activate l5-cross-arch<\/p>\n<p><span class=\"token comment\"># PyTorch&#xff08;\u6309 CUDA \u7248\u672c\u9009\u62e9&#xff0c;\u4ee5\u4e0b\u4e3a CUDA 12.1 \u793a\u4f8b&#xff09;<\/span><br \/>\npip <span class=\"token function\">install<\/span> <span class=\"token assign-left variable\">torch<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">2.3<\/span>.0 &#8211;index-url https:\/\/download.pytorch.org\/whl\/cu121<\/p>\n<p><span class=\"token comment\"># Transformer \u57fa\u7840\u5e93<\/span><br \/>\npip <span class=\"token function\">install<\/span> <span class=\"token assign-left variable\">transformers<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">4.42<\/span>.0 <span class=\"token assign-left variable\">accelerate<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">0.30<\/span>.0<\/p>\n<p><span class=\"token comment\"># Mamba&#xff08;\u9700 CUDA \u7f16\u8bd1&#xff0c;Linux \u73af\u5883\u8f83\u7a33\u5b9a&#xff09;<\/span><br \/>\npip <span class=\"token function\">install<\/span> causal-conv1d<span class=\"token operator\">&gt;&#061;<\/span><span class=\"token number\">1.2<\/span>.0<br \/>\npip <span class=\"token function\">install<\/span> mamba-ssm<\/p>\n<p><span class=\"token comment\"># RWKV<\/span><br \/>\npip <span class=\"token function\">install<\/span> <span class=\"token assign-left variable\">rwkv<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">0.8<\/span>.27<\/p>\n<p><span class=\"token comment\"># \u5b9e\u9a8c\u4f9d\u8d56<\/span><br \/>\npip <span class=\"token function\">install<\/span> numpy scipy scikit-learn matplotlib tqdm<\/p>\n<p><span class=\"token comment\"># HuggingFace Hub&#xff08;\u7528\u4e8e\u6a21\u578b\u4e0b\u8f7d&#xff09;<\/span><br \/>\npip <span class=\"token function\">install<\/span> huggingface_hub<\/p>\n<p>\u6ce8\u610f\u4e8b\u9879&#xff1a;<\/p>\n<ul>\n<li>mamba-ssm \u5728 Windows \u4e0a\u7f16\u8bd1\u56f0\u96be&#xff0c;\u5efa\u8bae\u4f7f\u7528 Linux \u6216 WSL2<\/li>\n<li>\u82e5 CUDA \u7248\u672c\u4e0d\u5339\u914d&#xff0c;causal-conv1d \u7f16\u8bd1\u4f1a\u5931\u8d25&#xff0c;\u9700\u6838\u5bf9 nvcc &#8211;version \u4e0e PyTorch CUDA \u7248\u672c\u4e00\u81f4<\/li>\n<li>RWKV \u7684 rwkv \u5e93\u7248\u672c\u66f4\u65b0\u8f83\u5feb&#xff0c;\u5efa\u8bae\u9501\u5b9a 0.8.27 \u4fdd\u8bc1 API \u517c\u5bb9<\/li>\n<\/ul>\n<h4>3.3 \u6a21\u578b\u6743\u91cd\u4e0b\u8f7d<\/h4>\n<p>\u63a8\u8350\u4f7f\u7528\u89c4\u6a21\u5bf9\u9f50\u7ec4&#xff08;~1.4B&#xff09;&#xff0c;\u4e09\u4e2a\u6a21\u578b\u89c4\u6a21\u63a5\u8fd1&#xff0c;\u4fbf\u4e8e\u516c\u5e73\u5bf9\u6bd4&#xff1a;<\/p>\n<table>\n<tr>\u67b6\u6784\u6a21\u578b\u89c4\u6a21HuggingFace \u8def\u5f84<\/tr>\n<tbody>\n<tr>\n<td>Transformer<\/td>\n<td>Pythia-1.4B<\/td>\n<td>1.4B<\/td>\n<td>EleutherAI\/pythia-1.4b<\/td>\n<\/tr>\n<tr>\n<td>Mamba<\/td>\n<td>Mamba-2-1.3B<\/td>\n<td>1.3B<\/td>\n<td>state-spaces\/mamba2-1.3b<\/td>\n<\/tr>\n<tr>\n<td>RWKV<\/td>\n<td>RWKV-7-World-1B5<\/td>\n<td>1.5B<\/td>\n<td>RWKV\/rwkv-7-world-1b5<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p><span class=\"token comment\"># download_models.py<\/span><br \/>\n<span class=\"token keyword\">from<\/span> huggingface_hub <span class=\"token keyword\">import<\/span> snapshot_download<\/p>\n<p>models <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n    <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;EleutherAI\/pythia-1.4b&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;state-spaces\/mamba2-1.3b&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n    <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;RWKV\/rwkv-7-world-1b5&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n<span class=\"token punctuation\">}<\/span><\/p>\n<p><span class=\"token keyword\">for<\/span> name<span class=\"token punctuation\">,<\/span> repo_id <span class=\"token keyword\">in<\/span> models<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u4e0b\u8f7d <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>name<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>repo_id<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    snapshot_download<span class=\"token punctuation\">(<\/span>repo_id<span class=\"token operator\">&#061;<\/span>repo_id<span class=\"token punctuation\">,<\/span> local_dir<span class=\"token operator\">&#061;<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;.\/models\/<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>name<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u6a21\u578b\u9009\u62e9\u7406\u7531&#xff1a;<\/p>\n<ul>\n<li>Pythia-1.4B&#xff1a;EleutherAI \u7684\u5f00\u6e90 Transformer&#xff0c;\u67b6\u6784\u6807\u51c6&#xff0c;\u4fbf\u4e8e\u63d0\u53d6 attention<\/li>\n<li>Mamba-2-1.3B&#xff1a;State Spaces \u5b98\u65b9 Mamba-2 \u5b9e\u73b0&#xff0c;\u652f\u6301 hidden state \u63d0\u53d6<\/li>\n<li>RWKV-7-World-1B5&#xff1a;RWKV \u6700\u65b0 v7 \u7248\u672c&#xff0c;\u652f\u6301 state \u8bfb\u53d6<\/li>\n<\/ul>\n<hr \/>\n<h3>\u7b2c\u56db\u7ae0 \u6b65\u9aa4 2&#xff1a;\u7edf\u4e00\u6a21\u578b\u63a8\u7406\u63a5\u53e3<\/h3>\n<h4>4.1 \u8bbe\u8ba1\u539f\u5219<\/h4>\n<p>\u4e09\u4e2a\u67b6\u6784\u7684\u539f\u751f API \u5b8c\u5168\u4e0d\u540c&#xff1a;<\/p>\n<ul>\n<li>Transformer \u7528 transformers.AutoModelForCausalLM<\/li>\n<li>Mamba \u7528 mamba_ssm.MambaLMHeadModel<\/li>\n<li>RWKV \u7528 rwkv.model.RWKV &#043; PIPELINE<\/li>\n<\/ul>\n<p>\u5fc5\u987b\u5c01\u88c5\u6210\u7edf\u4e00\u63a5\u53e3&#xff0c;\u624d\u80fd\u7528\u540c\u4e00\u5957\u5b9e\u9a8c\u4ee3\u7801\u6d4b\u8bd5\u4e09\u4e2a\u67b6\u6784&#xff0c;\u907f\u514d\u63a5\u53e3\u5dee\u5f02\u6c61\u67d3\u7ed3\u8bba\u3002<\/p>\n<h4>4.2 \u7edf\u4e00\u63a5\u53e3\u8bbe\u8ba1<\/h4>\n<p><span class=\"token comment\"># file: unified_model_interface.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">from<\/span> transformers <span class=\"token keyword\">import<\/span> AutoModelForCausalLM<span class=\"token punctuation\">,<\/span> AutoTokenizer<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> Optional<span class=\"token punctuation\">,<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">UnifiedModel<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7edf\u4e00\u6a21\u578b\u63a5\u53e3&#xff1a;\u5c4f\u853d\u67b6\u6784\u5dee\u5f02&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> arch<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> model_path<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#039;cuda&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>arch <span class=\"token operator\">&#061;<\/span> arch  <span class=\"token comment\"># &#039;transformer&#039; | &#039;mamba&#039; | &#039;rwkv&#039;<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>device <span class=\"token operator\">&#061;<\/span> device<br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>tokenizer <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><\/p>\n<p>        <span class=\"token keyword\">if<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            self<span class=\"token punctuation\">.<\/span>_load_transformer<span class=\"token punctuation\">(<\/span>model_path<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            self<span class=\"token punctuation\">.<\/span>_load_mamba<span class=\"token punctuation\">(<\/span>model_path<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            self<span class=\"token punctuation\">.<\/span>_load_rwkv<span class=\"token punctuation\">(<\/span>model_path<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">raise<\/span> ValueError<span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u672a\u77e5\u67b6\u6784: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>arch<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_load_transformer<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> path<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u52a0\u8f7d\u6807\u51c6 Transformer&#034;&#034;&#034;<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>tokenizer <span class=\"token operator\">&#061;<\/span> AutoTokenizer<span class=\"token punctuation\">.<\/span>from_pretrained<span class=\"token punctuation\">(<\/span>path<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> AutoModelForCausalLM<span class=\"token punctuation\">.<\/span>from_pretrained<span class=\"token punctuation\">(<\/span><br \/>\n            path<span class=\"token punctuation\">,<\/span> torch_dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>float16<span class=\"token punctuation\">,<\/span> device_map<span class=\"token operator\">&#061;<\/span>self<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">,<\/span><br \/>\n            output_attentions<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span>  <span class=\"token comment\"># \u5173\u952e&#xff1a;\u5f00\u542f attention \u8f93\u51fa<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_load_mamba<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> path<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u52a0\u8f7d Mamba-2&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> mamba_ssm<span class=\"token punctuation\">.<\/span>models<span class=\"token punctuation\">.<\/span>mixer_seq_simple <span class=\"token keyword\">import<\/span> MambaLMHeadModel<br \/>\n        self<span class=\"token punctuation\">.<\/span>tokenizer <span class=\"token operator\">&#061;<\/span> AutoTokenizer<span class=\"token punctuation\">.<\/span>from_pretrained<span class=\"token punctuation\">(<\/span>path<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> MambaLMHeadModel<span class=\"token punctuation\">.<\/span>from_pretrained<span class=\"token punctuation\">(<\/span><br \/>\n            path<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>float16<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>self<span class=\"token punctuation\">.<\/span>device<br \/>\n        <span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_load_rwkv<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> path<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u52a0\u8f7d RWKV-7&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> rwkv<span class=\"token punctuation\">.<\/span>model <span class=\"token keyword\">import<\/span> RWKV<br \/>\n        <span class=\"token keyword\">from<\/span> rwkv<span class=\"token punctuation\">.<\/span>utils <span class=\"token keyword\">import<\/span> PIPELINE<br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> RWKV<span class=\"token punctuation\">(<\/span>model<span class=\"token operator\">&#061;<\/span>path<span class=\"token punctuation\">,<\/span> strategy<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#039;cuda fp16&#039;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>pipeline <span class=\"token operator\">&#061;<\/span> PIPELINE<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;rwkv_vocab_v20230424&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>tokenizer <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>pipeline<span class=\"token punctuation\">.<\/span>tokenizer<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">generate<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> prompt<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                 temperature<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7edf\u4e00\u751f\u6210\u63a5\u53e3&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> self<span class=\"token punctuation\">.<\/span>arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_generate_transformer<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> self<span class=\"token punctuation\">.<\/span>arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_generate_mamba<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> self<span class=\"token punctuation\">.<\/span>arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_generate_rwkv<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_transformer<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        inputs <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> return_tensors<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;pt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            outputs <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span><br \/>\n                <span class=\"token operator\">**<\/span>inputs<span class=\"token punctuation\">,<\/span><br \/>\n                max_new_tokens<span class=\"token operator\">&#061;<\/span>max_new_tokens<span class=\"token punctuation\">,<\/span><br \/>\n                temperature<span class=\"token operator\">&#061;<\/span>temperature <span class=\"token keyword\">if<\/span> temperature <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">1e-5<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                do_sample<span class=\"token operator\">&#061;<\/span>temperature <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                output_attentions<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                return_dict_in_generate<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">.<\/span>decode<span class=\"token punctuation\">(<\/span><br \/>\n            outputs<span class=\"token punctuation\">.<\/span>sequences<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span>inputs<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;input_ids&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            skip_special_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_mamba<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        inputs <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> return_tensors<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;pt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            out <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span><br \/>\n                input_ids<span class=\"token operator\">&#061;<\/span>inputs<span class=\"token punctuation\">.<\/span>input_ids<span class=\"token punctuation\">,<\/span><br \/>\n                max_length<span class=\"token operator\">&#061;<\/span>inputs<span class=\"token punctuation\">.<\/span>input_ids<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span><br \/>\n                temperature<span class=\"token operator\">&#061;<\/span>temperature <span class=\"token keyword\">if<\/span> temperature <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">1e-5<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                top_k<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span> <span class=\"token keyword\">if<\/span> temperature <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">50<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                do_sample<span class=\"token operator\">&#061;<\/span>temperature <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">.<\/span>decode<span class=\"token punctuation\">(<\/span><br \/>\n            out<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span>inputs<span class=\"token punctuation\">.<\/span>input_ids<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            skip_special_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_rwkv<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token punctuation\">,<\/span> temperature<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> rwkv<span class=\"token punctuation\">.<\/span>utils <span class=\"token keyword\">import<\/span> PIPELINE<br \/>\n        out <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>pipeline<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span><br \/>\n            prompt<span class=\"token punctuation\">,<\/span> token_count<span class=\"token operator\">&#061;<\/span>max_new_tokens<span class=\"token punctuation\">,<\/span><br \/>\n            args<span class=\"token operator\">&#061;<\/span>PIPELINE<span class=\"token punctuation\">.<\/span>Args<span class=\"token punctuation\">(<\/span><br \/>\n                temperature<span class=\"token operator\">&#061;<\/span>temperature <span class=\"token keyword\">if<\/span> temperature <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">0.01<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                top_p<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span> <span class=\"token keyword\">if<\/span> temperature <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token number\">0.7<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> out<\/p>\n<h4>4.3 \u63a5\u53e3\u9a8c\u8bc1<\/h4>\n<p>\u63a5\u53e3\u5c01\u88c5\u540e&#xff0c;\u5fc5\u987b\u9a8c\u8bc1\u4e09\u4e2a\u67b6\u6784\u5728\u76f8\u540c\u8f93\u5165\u4e0b\u80fd\u4ea7\u751f\u5408\u7406\u8f93\u51fa&#xff1a;<\/p>\n<p><span class=\"token comment\"># \u9a8c\u8bc1\u811a\u672c<\/span><br \/>\ntest_prompt <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;\u8bf7\u7528\u4e00\u53e5\u8bdd\u89e3\u91ca\u4ec0\u4e48\u662f\u6ce8\u610f\u529b\u673a\u5236\u3002&#034;<\/span><br \/>\n<span class=\"token keyword\">for<\/span> arch <span class=\"token keyword\">in<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    model <span class=\"token operator\">&#061;<\/span> UnifiedModel<span class=\"token punctuation\">(<\/span>arch<span class=\"token punctuation\">,<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;.\/models\/<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>arch<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    output <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span>test_prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token number\">100<\/span><span class=\"token punctuation\">,<\/span> temperature<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;[<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>arch<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">] <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>output<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">200]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#8230;&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u82e5\u4e09\u4e2a\u67b6\u6784\u5747\u80fd\u751f\u6210\u8fde\u8d2f\u6587\u672c&#xff0c;\u5219\u63a5\u53e3\u9a8c\u8bc1\u901a\u8fc7&#xff0c;\u53ef\u8fdb\u5165\u4e0b\u4e00\u6b65\u3002<\/p>\n<hr \/>\n<h3>\u7b2c\u4e94\u7ae0 \u6b65\u9aa4 3&#xff1a;\u67b6\u6784\u7279\u5f02\u6027\u7684\u8870\u51cf\u6d4b\u91cf<\/h3>\n<p>\u8fd9\u662f\u6700\u5173\u952e\u7684\u6b65\u9aa4\u3002\u4e09\u79cd\u67b6\u6784\u7684\u8870\u51cf\u8f7d\u4f53\u4e0d\u540c&#xff0c;\u9700\u8981\u5206\u522b\u5b9e\u73b0\u6d4b\u91cf\u65b9\u6cd5\u3002<\/p>\n<h4>5.1 \u65b9\u6cd5 A&#xff1a;Transformer \u8870\u51cf\u6d4b\u91cf&#xff08;\u57fa\u7ebf&#xff09;<\/h4>\n<p>\u539f\u7406&#xff1a;\u76f4\u63a5\u63d0\u53d6 attention \u77e9\u9635&#xff0c;\u6d4b\u91cf\u4f4d\u7f6e\u8ddd\u79bb d \u4e0e\u5e73\u5747\u6ce8\u610f\u529b\u6743\u91cd w(d) \u7684\u5173\u7cfb&#xff0c;\u62df\u5408 w(d) &#061; A\u00b7e^(-\u03bb\u00b7d)\u3002<\/p>\n<p><span class=\"token comment\"># file: decay_measurement\/transformer_decay.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">TransformerDecayMeter<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;Transformer \u6ce8\u610f\u529b\u8870\u51cf\u6d4b\u91cf&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">:<\/span> UnifiedModel<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> model<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">measure_decay<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> text<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u63d0\u53d6 attention \u77e9\u9635&#xff0c;\u6d4b\u91cf\u4f4d\u7f6e\u8ddd\u79bb vs \u6ce8\u610f\u529b\u6743\u91cd&#034;&#034;&#034;<\/span><br \/>\n        inputs <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">(<\/span>text<span class=\"token punctuation\">,<\/span> return_tensors<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;pt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            outputs <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">(<\/span><span class=\"token operator\">**<\/span>inputs<span class=\"token punctuation\">,<\/span> output_attentions<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># outputs.attentions: tuple of (layers, batch, heads, seq, seq)<\/span><br \/>\n        attentions <span class=\"token operator\">&#061;<\/span> outputs<span class=\"token punctuation\">.<\/span>attentions<br \/>\n        <span class=\"token comment\"># \u5e73\u5747\u6240\u6709\u5c42\u548c\u6240\u6709\u5934<\/span><br \/>\n        avg_attn <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>stack<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">list<\/span><span class=\"token punctuation\">(<\/span>attentions<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>dim<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (seq, seq)<\/span><br \/>\n        avg_attn <span class=\"token operator\">&#061;<\/span> avg_attn<span class=\"token punctuation\">.<\/span>cpu<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>numpy<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u8ddd\u79bb-\u6743\u91cd\u66f2\u7ebf<\/span><br \/>\n        seq_len <span class=\"token operator\">&#061;<\/span> avg_attn<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        distances <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        weights <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">for<\/span> j <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                d <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">abs<\/span><span class=\"token punctuation\">(<\/span>i <span class=\"token operator\">&#8211;<\/span> j<span class=\"token punctuation\">)<\/span><br \/>\n                distances<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">)<\/span><br \/>\n                weights<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>avg_attn<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> j<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_fit_exponential<span class=\"token punctuation\">(<\/span>distances<span class=\"token punctuation\">,<\/span> weights<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_fit_exponential<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> distances<span class=\"token punctuation\">,<\/span> weights<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u62df\u5408 w(d) &#061; A * exp(-\u03bb * d)&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> scipy<span class=\"token punctuation\">.<\/span>optimize <span class=\"token keyword\">import<\/span> curve_fit<\/p>\n<p>        distances <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>distances<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        weights <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>weights<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">def<\/span> <span class=\"token function\">exp_decay<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">,<\/span> lam<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> A <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span>lam <span class=\"token operator\">*<\/span> d<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            popt<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> curve_fit<span class=\"token punctuation\">(<\/span>exp_decay<span class=\"token punctuation\">,<\/span> distances<span class=\"token punctuation\">,<\/span> weights<span class=\"token punctuation\">,<\/span> p0<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                              maxfev<span class=\"token operator\">&#061;<\/span><span class=\"token number\">10000<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;amplitude&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;attention_weights&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;transformer&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">except<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;attention_weights&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;error&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;fit_failed&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<h4>5.2 \u65b9\u6cd5 B&#xff1a;Mamba \u8870\u51cf\u6d4b\u91cf&#xff08;hidden state \u63a2\u9488\u6cd5&#xff09;<\/h4>\n<p>\u539f\u7406&#xff1a;Mamba \u6ca1\u6709 attention&#xff0c;\u4f46\u6709 hidden state\u3002\u5728\u5e8f\u5217\u4e0d\u540c\u4f4d\u7f6e\u63d2\u5165\u63a2\u9488\u6807\u8bb0&#xff0c;\u6d4b\u91cf\u8be5\u63a2\u9488\u5bf9\u540e\u7eed hidden state \u7684\u6270\u52a8\u5982\u4f55\u968f\u8ddd\u79bb\u8870\u51cf\u3002\u6270\u52a8\u8870\u51cf\u7b26\u5408\u6307\u6570\u5f62\u5f0f\u5219\u652f\u6301 L5\u3002<\/p>\n<p>\u65b9\u6cd5&#xff1a;\u7528&#034;\u6270\u52a8\u6cd5&#034;\u2014\u2014\u6bd4\u8f83\u6709\u63a2\u9488 vs \u65e0\u63a2\u9488\u7684 hidden state \u5dee\u5f02\u3002\u63a2\u9488\u4f4d\u7f6e\u8d8a\u9760\u524d&#xff0c;\u5176\u5bf9\u540e\u7eed\u4f4d\u7f6e\u7684\u6270\u52a8\u5e94\u8be5\u8d8a\u5c0f&#xff08;\u82e5\u5b58\u5728\u8870\u51cf&#xff09;\u3002<\/p>\n<p><span class=\"token comment\"># file: decay_measurement\/mamba_decay.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> List<span class=\"token punctuation\">,<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">MambaDecayMeter<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;Mamba hidden state \u8870\u51cf\u6d4b\u91cf&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">:<\/span> UnifiedModel<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> model<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">measure_decay<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> text<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        Mamba \u8870\u51cf\u6d4b\u91cf&#xff1a;hidden state \u63a2\u9488\u6cd5<br \/>\n        \u5728\u5e8f\u5217\u4e0d\u540c\u4f4d\u7f6e\u63d2\u5165\u63a2\u9488&#xff0c;\u6d4b\u91cf\u540e\u7eed hidden state \u7684\u6270\u52a8\u8870\u51cf<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        n_probes <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><br \/>\n        positions <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>linspace<span class=\"token punctuation\">(<\/span><span class=\"token number\">0.05<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.95<\/span><span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">)<\/span><\/p>\n<p>        tokens <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">(<\/span>text<span class=\"token punctuation\">,<\/span> return_tensors<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;pt&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n        input_ids <span class=\"token operator\">&#061;<\/span> tokens<span class=\"token punctuation\">.<\/span>input_ids<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        seq_len <span class=\"token operator\">&#061;<\/span> input_ids<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        probe_token_id <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_get_probe_token_id<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        probe_positions <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>positions <span class=\"token operator\">*<\/span> seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>astype<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u6784\u9020\u5e26\u63a2\u9488\u7684\u8f93\u5165<\/span><br \/>\n        modified_ids <span class=\"token operator\">&#061;<\/span> input_ids<span class=\"token punctuation\">.<\/span>clone<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            modified_ids<span class=\"token punctuation\">[<\/span>pos<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> probe_token_id<\/p>\n<p>        <span class=\"token comment\"># \u6d4b\u91cf\u6bcf\u4e2a\u63a2\u9488\u5bf9\u540e\u7eed hidden state \u7684\u5f71\u54cd<\/span><br \/>\n        decay_curve <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_measure_probe_influence<span class=\"token punctuation\">(<\/span>modified_ids<span class=\"token punctuation\">,<\/span> probe_positions<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_fit_exponential<span class=\"token punctuation\">(<\/span>probe_positions<span class=\"token punctuation\">,<\/span> decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_get_probe_token_id<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u83b7\u53d6\u4e00\u4e2a\u7a00\u6709 token \u4f5c\u4e3a\u63a2\u9488&#034;&#034;&#034;<\/span><br \/>\n        rare_tokens <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;\u2581&lt;GAP&gt;&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;\u2581&lt;PROBE&gt;&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;&lt;unused0&gt;&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> tok <span class=\"token keyword\">in<\/span> rare_tokens<span class=\"token punctuation\">:<\/span><br \/>\n            ids <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span>tok<span class=\"token punctuation\">,<\/span> add_special_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>ids<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> ids<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039; 7777777&#039;<\/span><span class=\"token punctuation\">,<\/span> add_special_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_measure_probe_influence<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> input_ids<span class=\"token punctuation\">:<\/span> torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">,<\/span><br \/>\n                                   probe_positions<span class=\"token punctuation\">:<\/span> List<span class=\"token punctuation\">[<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> np<span class=\"token punctuation\">.<\/span>ndarray<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        \u6d4b\u91cf\u6bcf\u4e2a\u63a2\u9488\u5bf9\u540e\u7eed\u4f4d\u7f6e\u7684 hidden state \u5f71\u54cd<br \/>\n        \u65b9\u6cd5&#xff1a;\u6270\u52a8\u6cd5\u2014\u2014\u6bd4\u8f83\u6709\u63a2\u9488 vs \u65e0\u63a2\u9488\u7684 hidden state \u5dee\u5f02<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        input_ids <span class=\"token operator\">&#061;<\/span> input_ids<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u57fa\u7ebf&#xff1a;\u65e0\u63a2\u9488\u7684 hidden state&#xff08;\u7528\u666e\u901a token \u66ff\u6362\u63a2\u9488\u4f4d\u7f6e&#xff09;<\/span><br \/>\n        baseline_ids <span class=\"token operator\">&#061;<\/span> input_ids<span class=\"token punctuation\">.<\/span>clone<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            baseline_ids<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> pos<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>tokenizer<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span><br \/>\n                <span class=\"token string\">&#039; the&#039;<\/span><span class=\"token punctuation\">,<\/span> add_special_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            baseline_out <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">(<\/span>baseline_ids<span class=\"token punctuation\">,<\/span> output_hidden_states<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            probe_out <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">(<\/span>input_ids<span class=\"token punctuation\">,<\/span> output_hidden_states<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u53d6\u6700\u540e\u4e00\u5c42 hidden states<\/span><br \/>\n        baseline_hidden <span class=\"token operator\">&#061;<\/span> baseline_out<span class=\"token punctuation\">.<\/span>hidden_states<span class=\"token punctuation\">[<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (seq, dim)<\/span><br \/>\n        probe_hidden <span class=\"token operator\">&#061;<\/span> probe_out<span class=\"token punctuation\">.<\/span>hidden_states<span class=\"token punctuation\">[<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u6bcf\u4e2a\u63a2\u9488\u4f4d\u7f6e\u4e4b\u540e\u7684 hidden state \u5dee\u5f02&#xff08;\u5f71\u54cd\u6269\u6563&#xff09;<\/span><br \/>\n        decay_curve <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> probe_pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            window <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">,<\/span> baseline_hidden<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> probe_pos<span class=\"token punctuation\">)<\/span><br \/>\n            diffs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token keyword\">for<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> window<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">if<\/span> probe_pos <span class=\"token operator\">&#043;<\/span> step <span class=\"token operator\">&lt;<\/span> baseline_hidden<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    diff <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>norm<span class=\"token punctuation\">(<\/span><br \/>\n                        probe_hidden<span class=\"token punctuation\">[<\/span>probe_pos <span class=\"token operator\">&#043;<\/span> step<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> baseline_hidden<span class=\"token punctuation\">[<\/span>probe_pos <span class=\"token operator\">&#043;<\/span> step<span class=\"token punctuation\">]<\/span><br \/>\n                    <span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                    diffs<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>diff<span class=\"token punctuation\">)<\/span><\/p>\n<p>            decay_curve<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>diffs<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">if<\/span> diffs <span class=\"token keyword\">else<\/span> <span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_fit_exponential<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> positions<span class=\"token punctuation\">,<\/span> decay_curve<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u62df\u5408\u5f71\u54cd\u5f3a\u5ea6 vs \u63a2\u9488\u4f4d\u7f6e&#xff08;\u8ddd\u672b\u5c3e\u8ddd\u79bb&#xff09;&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> scipy<span class=\"token punctuation\">.<\/span>optimize <span class=\"token keyword\">import<\/span> curve_fit<\/p>\n<p>        max_pos <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>positions<span class=\"token punctuation\">)<\/span><br \/>\n        distances <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>max_pos <span class=\"token operator\">&#8211;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>positions<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> max_pos<br \/>\n        weights <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">def<\/span> <span class=\"token function\">exp_decay<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">,<\/span> lam<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> A <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span>lam <span class=\"token operator\">*<\/span> d<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            popt<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> curve_fit<span class=\"token punctuation\">(<\/span>exp_decay<span class=\"token punctuation\">,<\/span> distances<span class=\"token punctuation\">,<\/span> weights<span class=\"token punctuation\">,<\/span> p0<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                              maxfev<span class=\"token operator\">&#061;<\/span><span class=\"token number\">10000<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;amplitude&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;hidden_state_probe&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">except<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;hidden_state_probe&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;error&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;fit_failed&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<h4>5.3 \u65b9\u6cd5 C&#xff1a;RWKV \u8870\u51cf\u6d4b\u91cf&#xff08;state \u5411\u91cf\u76f4\u63a5\u8bfb\u53d6&#xff09;<\/h4>\n<p>\u539f\u7406&#xff1a;RWKV \u7684 state \u5411\u91cf\u662f\u663e\u5f0f\u7684&#xff0c;\u968f\u5e8f\u5217\u63a8\u8fdb\u800c\u66f4\u65b0&#xff0c;\u5185\u542b decay factor\u3002\u6d4b\u91cf\u63a2\u9488 token \u6ce8\u5165\u540e&#xff0c;state \u5411\u91cf\u7684\u6270\u52a8\u5982\u4f55\u968f\u8ddd\u79bb\u8870\u51cf\u3002<\/p>\n<p>\u4f18\u52bf&#xff1a;RWKV \u7684 state \u53ef\u76f4\u63a5\u8bfb\u53d6&#xff0c;\u6bd4 Mamba \u7684 hidden state \u66f4&#034;\u900f\u660e&#034;\u3002<\/p>\n<p><span class=\"token comment\"># file: decay_measurement\/rwkv_decay.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> List<span class=\"token punctuation\">,<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">RWKVDecayMeter<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;RWKV state \u5411\u91cf\u8870\u51cf\u6d4b\u91cf&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">:<\/span> UnifiedModel<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> model<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">measure_decay<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> text<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        RWKV \u8870\u51cf\u6d4b\u91cf&#xff1a;\u76f4\u63a5\u8bfb\u53d6 state \u5411\u91cf<br \/>\n        \u6d4b\u91cf\u63a2\u9488 token \u6ce8\u5165\u540e&#xff0c;state \u5411\u91cf\u7684\u6270\u52a8\u8870\u51cf<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        n_probes <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><br \/>\n        positions <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>linspace<span class=\"token punctuation\">(<\/span><span class=\"token number\">0.05<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.95<\/span><span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">)<\/span><\/p>\n<p>        tokens <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>pipeline<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span>text<span class=\"token punctuation\">)<\/span><br \/>\n        seq_len <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>tokens<span class=\"token punctuation\">)<\/span><br \/>\n        probe_positions <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>positions <span class=\"token operator\">*<\/span> seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>astype<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u63d2\u5165\u63a2\u9488 token<\/span><br \/>\n        probe_token <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_get_probe_token<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        modified_tokens <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">(<\/span>tokens<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            modified_tokens<span class=\"token punctuation\">[<\/span>pos<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> probe_token<\/p>\n<p>        <span class=\"token comment\"># \u6d4b\u91cf state \u6f14\u5316<\/span><br \/>\n        decay_curve <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_measure_state_evolution<span class=\"token punctuation\">(<\/span>modified_tokens<span class=\"token punctuation\">,<\/span> probe_positions<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>_fit_exponential<span class=\"token punctuation\">(<\/span>probe_positions<span class=\"token punctuation\">,<\/span> decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_get_probe_token<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u83b7\u53d6\u63a2\u9488 token id&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token number\">0<\/span>  <span class=\"token comment\"># \u7528 &lt;pad&gt; \u6216\u7a00\u6709 token<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_measure_state_evolution<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> tokens<span class=\"token punctuation\">:<\/span> List<span class=\"token punctuation\">[<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                                   probe_positions<span class=\"token punctuation\">:<\/span> List<span class=\"token punctuation\">[<\/span><span class=\"token builtin\">int<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> np<span class=\"token punctuation\">.<\/span>ndarray<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        \u9010 token \u524d\u5411&#xff0c;\u8bb0\u5f55 state \u5411\u91cf<br \/>\n        \u6d4b\u91cf\u63a2\u9488 token \u6ce8\u5165\u540e&#xff0c;state \u5411\u91cf\u7684\u6270\u52a8\u5982\u4f55\u968f\u8ddd\u79bb\u8870\u51cf<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u57fa\u7ebf&#xff1a;\u65e0\u63a2\u9488\u7684 state<\/span><br \/>\n        baseline_tokens <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">(<\/span>tokens<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            baseline_tokens<span class=\"token punctuation\">[<\/span>pos<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">100<\/span>  <span class=\"token comment\"># \u666e\u901a token<\/span><\/p>\n<p>        decay_curve <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> probe_pos <span class=\"token keyword\">in<\/span> probe_positions<span class=\"token punctuation\">:<\/span><br \/>\n            diff <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_measure_state_diff_after_probe<span class=\"token punctuation\">(<\/span><br \/>\n                tokens<span class=\"token punctuation\">,<\/span> baseline_tokens<span class=\"token punctuation\">,<\/span> probe_pos<br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n            decay_curve<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>diff<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_measure_state_diff_after_probe<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> probe_tokens<span class=\"token punctuation\">,<\/span> baseline_tokens<span class=\"token punctuation\">,<\/span><br \/>\n                                          probe_pos<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6d4b\u91cf\u63a2\u9488\u540e state \u7684\u6270\u52a8\u8870\u51cf&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u8dd1\u5230\u63a2\u9488\u4f4d\u7f6e<\/span><br \/>\n        state_probe <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><br \/>\n        state_base <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>probe_pos <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            _<span class=\"token punctuation\">,<\/span> state_probe <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>forward<span class=\"token punctuation\">(<\/span><br \/>\n                tokens<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span>probe_tokens<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> state<span class=\"token operator\">&#061;<\/span>state_probe<br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n            _<span class=\"token punctuation\">,<\/span> state_base <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>forward<span class=\"token punctuation\">(<\/span><br \/>\n                tokens<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span>baseline_tokens<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> state<span class=\"token operator\">&#061;<\/span>state_base<br \/>\n            <span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u6d4b\u91cf\u540e\u7eed window \u6b65\u7684 state \u5dee\u5f02<\/span><br \/>\n        window <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>probe_tokens<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> probe_pos <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        diffs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> window<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> probe_pos <span class=\"token operator\">&#043;<\/span> step <span class=\"token operator\">&gt;&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>probe_tokens<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">break<\/span><br \/>\n            _<span class=\"token punctuation\">,<\/span> state_probe <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>forward<span class=\"token punctuation\">(<\/span><br \/>\n                tokens<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span>probe_tokens<span class=\"token punctuation\">[<\/span>probe_pos <span class=\"token operator\">&#043;<\/span> step<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> state<span class=\"token operator\">&#061;<\/span>state_probe<br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n            _<span class=\"token punctuation\">,<\/span> state_base <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>forward<span class=\"token punctuation\">(<\/span><br \/>\n                tokens<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span>baseline_tokens<span class=\"token punctuation\">[<\/span>probe_pos <span class=\"token operator\">&#043;<\/span> step<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> state<span class=\"token operator\">&#061;<\/span>state_base<br \/>\n            <span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token comment\"># \u8ba1\u7b97 state \u5dee\u5f02\u7684\u8303\u6570<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>state_probe<span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">dict<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                diff <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><br \/>\n                    torch<span class=\"token punctuation\">.<\/span>norm<span class=\"token punctuation\">(<\/span>state_probe<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> state_base<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                    <span class=\"token keyword\">for<\/span> k <span class=\"token keyword\">in<\/span> state_probe<span class=\"token punctuation\">.<\/span>keys<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                diff <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>norm<span class=\"token punctuation\">(<\/span>state_probe <span class=\"token operator\">&#8211;<\/span> state_base<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            diffs<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>diff<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> np<span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>diffs<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">if<\/span> diffs <span class=\"token keyword\">else<\/span> <span class=\"token number\">0.0<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_fit_exponential<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> positions<span class=\"token punctuation\">,<\/span> decay_curve<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> scipy<span class=\"token punctuation\">.<\/span>optimize <span class=\"token keyword\">import<\/span> curve_fit<\/p>\n<p>        max_pos <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>positions<span class=\"token punctuation\">)<\/span><br \/>\n        distances <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>max_pos <span class=\"token operator\">&#8211;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>positions<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> max_pos<br \/>\n        weights <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>decay_curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">def<\/span> <span class=\"token function\">exp_decay<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">,<\/span> lam<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> A <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span>lam <span class=\"token operator\">*<\/span> d<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            popt<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> curve_fit<span class=\"token punctuation\">(<\/span>exp_decay<span class=\"token punctuation\">,<\/span> distances<span class=\"token punctuation\">,<\/span> weights<span class=\"token punctuation\">,<\/span> p0<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                              maxfev<span class=\"token operator\">&#061;<\/span><span class=\"token number\">10000<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;amplitude&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;state_vector&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">except<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;state_vector&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;error&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;fit_failed&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<h4>5.4 \u65b9\u6cd5 D&#xff1a;\u901a\u7528\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff08;\u4e09\u67b6\u6784\u7edf\u4e00&#xff0c;\u4ea4\u53c9\u9a8c\u8bc1&#xff09;<\/h4>\n<p>\u539f\u7406&#xff1a;\u5728\u4e0a\u4e0b\u6587\u4e0d\u540c\u4f4d\u7f6e\u653e\u7f6e\u5173\u952e\u4fe1\u606f&#xff0c;\u6d4b\u91cf\u6a21\u578b\u53ec\u56de\u7387\u3002\u53ec\u56de\u7387\u968f\u4f4d\u7f6e\u7684\u53d8\u5316\u95f4\u63a5\u53cd\u6620\u4fe1\u606f\u4fdd\u7559\u80fd\u529b\u3002\u8fd9\u662f Lost in the Middle \u7684\u7ecf\u5178\u8303\u5f0f&#xff0c;\u4e0e\u67b6\u6784\u65e0\u5173&#xff0c;\u53ef\u76f4\u63a5\u6a2a\u5411\u5bf9\u6bd4\u3002<\/p>\n<p>\u89d2\u8272&#xff1a;\u4f5c\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e&#xff0c;\u907f\u514d\u65b9\u6cd5 A\/B\/C \u7684&#034;\u7b49\u4ef7\u6027&#034;\u4e89\u8bae\u3002<\/p>\n<p><span class=\"token comment\"># file: decay_measurement\/universal_blackbox.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">UniversalBlackBoxProbe<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n    \u901a\u7528\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff08;\u4e09\u67b6\u6784\u7edf\u4e00&#xff09;<br \/>\n    \u5728\u4e0a\u4e0b\u6587\u4e0d\u540c\u4f4d\u7f6e\u653e\u5173\u952e\u4fe1\u606f&#xff0c;\u6d4b\u53ec\u56de\u7387<br \/>\n    \u4e0e\u67b6\u6784\u65e0\u5173&#xff0c;\u53ef\u76f4\u63a5\u6a2a\u5411\u5bf9\u6bd4<br \/>\n    &#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">:<\/span> UnifiedModel<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> model<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">measure_recall_curve<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                              n_trials<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6d4b\u91cf\u53ec\u56de\u7387\u66f2\u7ebf&#xff08;\u4e09\u67b6\u6784\u901a\u7528&#xff09;&#034;&#034;&#034;<\/span><br \/>\n        all_recalls <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token keyword\">for<\/span> trial <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>n_trials<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token comment\"># \u6784\u9020\u5e26\u63a2\u9488\u7684\u4e0a\u4e0b\u6587<\/span><br \/>\n            probe_data <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_build_probe_context<span class=\"token punctuation\">(<\/span>context_k<span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">)<\/span><br \/>\n            prompt <span class=\"token operator\">&#061;<\/span> probe_data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;context&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#039;\\\\n\\\\n&#039;<\/span> <span class=\"token operator\">&#043;<\/span> probe_data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;query&#039;<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>            <span class=\"token comment\"># \u7528\u7edf\u4e00\u63a5\u53e3\u751f\u6210&#xff08;\u5c4f\u853d\u67b6\u6784\u5dee\u5f02&#xff09;<\/span><br \/>\n            response <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span> temperature<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            <span class=\"token comment\"># \u6d4b\u91cf\u6bcf\u4e2a\u4f4d\u7f6e\u7684\u53ec\u56de\u7387<\/span><br \/>\n            recalls <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token keyword\">for<\/span> i<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">(<\/span>pos<span class=\"token punctuation\">,<\/span> marker_content<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">enumerate<\/span><span class=\"token punctuation\">(<\/span><span class=\"token builtin\">zip<\/span><span class=\"token punctuation\">(<\/span><br \/>\n                probe_data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;positions&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> probe_data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;markers&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                recalled <span class=\"token operator\">&#061;<\/span> marker_content <span class=\"token keyword\">in<\/span> response<br \/>\n                recalls<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span><span class=\"token number\">1.0<\/span> <span class=\"token keyword\">if<\/span> recalled <span class=\"token keyword\">else<\/span> <span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            all_recalls<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>recalls<span class=\"token punctuation\">)<\/span><\/p>\n<p>        mean_recall <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>all_recalls<span class=\"token punctuation\">,<\/span> axis<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#039;positions&#039;<\/span><span class=\"token punctuation\">:<\/span> probe_data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;positions&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;recall_curve&#039;<\/span><span class=\"token punctuation\">:<\/span> mean_recall<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;context_k&#039;<\/span><span class=\"token punctuation\">:<\/span> context_k<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>arch<br \/>\n        <span class=\"token punctuation\">}<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_build_probe_context<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6784\u9020\u5e26\u63a2\u9488\u6807\u8bb0\u7684\u4e0a\u4e0b\u6587&#034;&#034;&#034;<\/span><br \/>\n        positions <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>linspace<span class=\"token punctuation\">(<\/span><span class=\"token number\">0.05<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.95<\/span><span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># \u6bcf\u4e2a\u63a2\u9488\u6709\u552f\u4e00\u5185\u5bb9&#xff0c;\u4fbf\u4e8e\u68c0\u7d22<\/span><br \/>\n        markers <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><br \/>\n            <span class=\"token string-interpolation\"><span class=\"token string\">f&#034;ZKPROBE<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>i<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">02d<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span><span class=\"token builtin\">hash<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>i<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">%<\/span> <span class=\"token number\">10000<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">04d<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><br \/>\n            <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>n_probes<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token comment\"># \u7528\u4e2d\u6027\u6587\u672c\u586b\u5145\u81f3\u76ee\u6807\u957f\u5ea6<\/span><br \/>\n        filler <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_generate_filler_text<span class=\"token punctuation\">(<\/span>context_k<span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">,<\/span> positions<span class=\"token punctuation\">,<\/span> markers<span class=\"token punctuation\">)<\/span><\/p>\n<p>        query <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;\u8bf7\u5217\u51fa\u4e0a\u4e0b\u6587\u4e2d\u6240\u6709 ZKPROBE \u6807\u8bb0\u7684\u5185\u5bb9&#xff0c;\u683c\u5f0f&#xff1a;ZKPROBE \u7f16\u53f7: \u5185\u5bb9&#034;<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#039;context&#039;<\/span><span class=\"token punctuation\">:<\/span> filler<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;query&#039;<\/span><span class=\"token punctuation\">:<\/span> query<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;positions&#039;<\/span><span class=\"token punctuation\">:<\/span> positions<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;markers&#039;<\/span><span class=\"token punctuation\">:<\/span> markers<br \/>\n        <span class=\"token punctuation\">}<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_filler_text<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">,<\/span> n_probes<span class=\"token punctuation\">,<\/span> positions<span class=\"token punctuation\">,<\/span> markers<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u751f\u6210\u5e26\u63a2\u9488\u6807\u8bb0\u7684\u586b\u5145\u6587\u672c&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u76ee\u6807 token \u6570&#xff08;\u7c97\u7565\u4f30\u7b97&#xff0c;4 \u5b57\u7b26 \u2248 1 token&#xff09;<\/span><br \/>\n        target_chars <span class=\"token operator\">&#061;<\/span> context_k <span class=\"token operator\">*<\/span> <span class=\"token number\">1000<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">4<\/span><\/p>\n<p>        <span class=\"token comment\"># \u7528\u91cd\u590d\u7684\u4e2d\u6027\u6587\u672c\u586b\u5145<\/span><br \/>\n        base_text <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span><br \/>\n            <span class=\"token string\">&#034;The quick brown fox jumps over the lazy dog. &#034;<\/span><br \/>\n            <span class=\"token string\">&#034;This is neutral filler text for context length control. &#034;<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">100<\/span><\/p>\n<p>        <span class=\"token comment\"># \u5728\u6307\u5b9a\u4f4d\u7f6e\u63d2\u5165\u63a2\u9488<\/span><br \/>\n        text_len <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>base_text<span class=\"token punctuation\">)<\/span><br \/>\n        result <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;&#034;<\/span><br \/>\n        last_pos <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> pos<span class=\"token punctuation\">,<\/span> marker <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">zip<\/span><span class=\"token punctuation\">(<\/span>positions<span class=\"token punctuation\">,<\/span> markers<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            target_pos <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">(<\/span>pos <span class=\"token operator\">*<\/span> target_chars <span class=\"token operator\">\/<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> target_pos <span class=\"token operator\">&gt;<\/span> text_len<span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token comment\"># \u6269\u5c55\u586b\u5145\u6587\u672c<\/span><br \/>\n                base_text <span class=\"token operator\">&#061;<\/span> base_text <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span>target_pos <span class=\"token operator\">\/\/<\/span> text_len <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                text_len <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>base_text<span class=\"token punctuation\">)<\/span><br \/>\n            target_pos <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>target_pos<span class=\"token punctuation\">,<\/span> text_len <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            result <span class=\"token operator\">&#043;&#061;<\/span> base_text<span class=\"token punctuation\">[<\/span>last_pos<span class=\"token punctuation\">:<\/span>target_pos<span class=\"token punctuation\">]<\/span><br \/>\n            result <span class=\"token operator\">&#043;&#061;<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#034; [SPECIAL_MARKER: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>marker<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">] &#034;<\/span><\/span><br \/>\n            last_pos <span class=\"token operator\">&#061;<\/span> target_pos<br \/>\n        result <span class=\"token operator\">&#043;&#061;<\/span> base_text<span class=\"token punctuation\">[<\/span>last_pos<span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> result<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">fit_lambda_from_recall<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> recall_curves<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        \u4ece\u53ec\u56de\u7387\u66f2\u7ebf\u62df\u5408 \u03bb<br \/>\n        \u5047\u8bbe&#xff1a;\u53ec\u56de\u7387 R(d) \u4e0e\u4fe1\u606f\u4fdd\u7559\u5f3a\u5ea6\u6b63\u76f8\u5173<br \/>\n        R(d) \u2248 A \u00b7 e^(-\u03bb\u00b7d) &#043; noise<br \/>\n        \u5176\u4e2d d \u662f\u8ddd\u4e0a\u4e0b\u6587\u672b\u5c3e\u7684\u5f52\u4e00\u5316\u8ddd\u79bb<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> scipy<span class=\"token punctuation\">.<\/span>optimize <span class=\"token keyword\">import<\/span> curve_fit<\/p>\n<p>        all_distances <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        all_recalls <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token keyword\">for<\/span> curve <span class=\"token keyword\">in<\/span> recall_curves<span class=\"token punctuation\">:<\/span><br \/>\n            positions <span class=\"token operator\">&#061;<\/span> curve<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;positions&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            recalls <span class=\"token operator\">&#061;<\/span> curve<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;recall_curve&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token comment\"># \u8ddd\u79bb\u672b\u5c3e\u7684\u5f52\u4e00\u5316\u8ddd\u79bb<\/span><br \/>\n            distances <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1.0<\/span> <span class=\"token operator\">&#8211;<\/span> positions<br \/>\n            all_distances<span class=\"token punctuation\">.<\/span>extend<span class=\"token punctuation\">(<\/span>distances<span class=\"token punctuation\">)<\/span><br \/>\n            all_recalls<span class=\"token punctuation\">.<\/span>extend<span class=\"token punctuation\">(<\/span>recalls<span class=\"token punctuation\">)<\/span><\/p>\n<p>        all_distances <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>all_distances<span class=\"token punctuation\">)<\/span><br \/>\n        all_recalls <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>array<span class=\"token punctuation\">(<\/span>all_recalls<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">def<\/span> <span class=\"token function\">exp_decay<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">,<\/span> A<span class=\"token punctuation\">,<\/span> lam<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> A <span class=\"token operator\">*<\/span> np<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span>lam <span class=\"token operator\">*<\/span> d<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            popt<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> curve_fit<span class=\"token punctuation\">(<\/span>exp_decay<span class=\"token punctuation\">,<\/span> all_distances<span class=\"token punctuation\">,<\/span> all_recalls<span class=\"token punctuation\">,<\/span><br \/>\n                              p0<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> maxfev<span class=\"token operator\">&#061;<\/span><span class=\"token number\">10000<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;amplitude&#039;<\/span><span class=\"token punctuation\">:<\/span> popt<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;blackbox_recall&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">:<\/span> curve<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;arch&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;unknown&#039;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">except<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;blackbox_recall&#039;<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;error&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;fit_failed&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<hr \/>\n<h3>\u7b2c\u516d\u7ae0 \u6b65\u9aa4 4&#xff1a;S_c \u6d4b\u91cf&#xff08;\u4e09\u67b6\u6784\u7edf\u4e00&#xff09;<\/h3>\n<h4>6.1 \u8bbe\u8ba1\u539f\u5219<\/h4>\n<p>S_c&#xff08;\u5269\u4f59\u53ef\u4fe1\u6b65\u6570&#xff09;\u6d4b\u91cf\u7684\u662f\u8f93\u51fa\u8d28\u91cf&#xff0c;\u4e0e\u67b6\u6784\u65e0\u5173\u3002\u56e0\u6b64\u4e09\u67b6\u6784\u4f7f\u7528\u5b8c\u5168\u4e00\u81f4\u7684 S_c \u6d4b\u91cf\u65b9\u6cd5&#xff0c;\u786e\u4fdd\u6a2a\u5411\u5bf9\u6bd4\u7684\u516c\u5e73\u6027\u3002<\/p>\n<h4>6.2 \u7edf\u4e00\u7684 S_c \u6d4b\u91cf\u5b9e\u73b0<\/h4>\n<p><span class=\"token comment\"># file: sc_measurement.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> json<br \/>\n<span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> Dict<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">SCMeasurement<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7edf\u4e00\u7684 S_c \u6d4b\u91cf&#xff08;\u4e0e\u7eaf API \u7248\u4e00\u81f4&#xff0c;\u4e09\u67b6\u6784\u901a\u7528&#xff09;&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model<span class=\"token punctuation\">:<\/span> UnifiedModel<span class=\"token punctuation\">,<\/span> ground_truth<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>model <span class=\"token operator\">&#061;<\/span> model<br \/>\n        self<span class=\"token punctuation\">.<\/span>ground_truth <span class=\"token operator\">&#061;<\/span> ground_truth<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">measure_s_c<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> prompt<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> n_steps<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        \u6d4b\u91cf\u5269\u4f59\u53ef\u4fe1\u6b65\u6570<br \/>\n        \u8fd4\u56de\u9996\u6b21\u51fa\u73b0\u4e0d\u53ef\u4fe1\u8f93\u51fa\u7684\u6b65\u6570&#xff08;\u5168\u90e8\u53ef\u4fe1\u5219\u8fd4\u56de n_steps&#xff09;<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        output <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>model<span class=\"token punctuation\">.<\/span>generate<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> max_new_tokens<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2048<\/span><span class=\"token punctuation\">,<\/span> temperature<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">for<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> n_steps <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            step_output <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_extract_step<span class=\"token punctuation\">(<\/span>output<span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token keyword\">not<\/span> self<span class=\"token punctuation\">.<\/span>_is_credible<span class=\"token punctuation\">(<\/span>step_output<span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> step <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span>  <span class=\"token comment\"># \u4e0a\u4e00\u6b65\u662f\u6700\u540e\u4e00\u4e2a\u53ef\u4fe1\u6b65<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> n_steps<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_extract_step<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> output<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u4ece\u8f93\u51fa\u4e2d\u63d0\u53d6\u6307\u5b9a\u6b65\u9aa4\u7684\u5185\u5bb9&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u5047\u8bbe\u8f93\u51fa\u683c\u5f0f\u4e3a JSON \u6570\u7ec4&#xff0c;\u6bcf\u6b65\u4e00\u4e2a\u5bf9\u8c61<\/span><br \/>\n        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            parsed <span class=\"token operator\">&#061;<\/span> json<span class=\"token punctuation\">.<\/span>loads<span class=\"token punctuation\">(<\/span>output<span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">and<\/span> step <span class=\"token operator\">&lt;&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> json<span class=\"token punctuation\">.<\/span>dumps<span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">[<\/span>step <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">dict<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">and<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span> <span class=\"token keyword\">in<\/span> parsed<span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> json<span class=\"token punctuation\">.<\/span>dumps<span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">[<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">except<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">pass<\/span><br \/>\n        <span class=\"token comment\"># \u56de\u9000&#xff1a;\u7528\u6b63\u5219\u5339\u914d<\/span><br \/>\n        <span class=\"token keyword\">import<\/span> re<br \/>\n        pattern <span class=\"token operator\">&#061;<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">rf&#039;&#034;step&#034;:\\\\s*<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">.*?(?&#061;&#034;step&#034;:|\\\\Z)&#039;<\/span><\/span><br \/>\n        <span class=\"token keyword\">match<\/span> <span class=\"token operator\">&#061;<\/span> re<span class=\"token punctuation\">.<\/span>search<span class=\"token punctuation\">(<\/span>pattern<span class=\"token punctuation\">,<\/span> output<span class=\"token punctuation\">,<\/span> re<span class=\"token punctuation\">.<\/span>DOTALL<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token keyword\">match<\/span><span class=\"token punctuation\">.<\/span>group<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">if<\/span> <span class=\"token keyword\">match<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token string\">&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_is_credible<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> step_output<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">bool<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n        \u4e25\u683c\u7684\u53ef\u4fe1\u5ea6\u5224\u5b9a&#xff08;\u9884\u6ce8\u518c&#xff0c;\u4e0d\u53ef\u4e8b\u540e\u4fee\u6539&#xff09;<br \/>\n        \u4e0e\u7eaf API \u7248\u5b8c\u5168\u4e00\u81f4<br \/>\n        &#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u6761\u4ef6 1&#xff1a;\u683c\u5f0f\u5408\u89c4&#xff08;\u5fc5\u987b\u4e3a\u6709\u6548 JSON&#xff09;<\/span><br \/>\n        <span class=\"token keyword\">try<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            parsed <span class=\"token operator\">&#061;<\/span> json<span class=\"token punctuation\">.<\/span>loads<span class=\"token punctuation\">(<\/span>step_output<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">except<\/span> json<span class=\"token punctuation\">.<\/span>JSONDecodeError<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><\/p>\n<p>        <span class=\"token comment\"># \u6761\u4ef6 2&#xff1a;\u4e8b\u5b9e\u51c6\u786e&#xff08;\u4e0e ground_truth \u6838\u5bf9&#xff09;<\/span><br \/>\n        expected <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>ground_truth<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> <span class=\"token keyword\">not<\/span> self<span class=\"token punctuation\">.<\/span>_check_factual_accuracy<span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">,<\/span> expected<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><\/p>\n<p>        <span class=\"token comment\"># \u6761\u4ef6 3&#xff1a;\u903b\u8f91\u4e00\u81f4&#xff08;\u4e0e\u524d\u5e8f\u6b65\u9aa4\u65e0\u77db\u76fe&#xff09;<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> <span class=\"token keyword\">not<\/span> self<span class=\"token punctuation\">.<\/span>_check_logical_consistency<span class=\"token punctuation\">(<\/span>parsed<span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">True<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_check_factual_accuracy<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> output<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">,<\/span> expected<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">bool<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u4e8b\u5b9e\u6838\u67e5&#xff1a;\u5173\u952e\u5b57\u6bb5\u5fc5\u987b\u5339\u914d&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> key <span class=\"token keyword\">in<\/span> expected<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> key <span class=\"token keyword\">not<\/span> <span class=\"token keyword\">in<\/span> output<span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> self<span class=\"token punctuation\">.<\/span>_normalize<span class=\"token punctuation\">(<\/span>output<span class=\"token punctuation\">[<\/span>key<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">!&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_normalize<span class=\"token punctuation\">(<\/span>expected<span class=\"token punctuation\">[<\/span>key<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">True<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_normalize<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> value<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6807\u51c6\u5316\u5b57\u7b26\u4e32\u4ee5\u4fbf\u6bd4\u8f83&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>value<span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">return<\/span> value<span class=\"token punctuation\">.<\/span>strip<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>lower<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">(<\/span>value<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>strip<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>lower<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_check_logical_consistency<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> output<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">,<\/span> step<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">bool<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u903b\u8f91\u4e00\u81f4\u6027\u68c0\u67e5&#xff08;\u7b80\u5316\u7248&#xff0c;\u53ef\u6269\u5c55&#xff09;&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u68c0\u67e5\u5fc5\u8981\u5b57\u6bb5\u662f\u5426\u5b58\u5728<\/span><br \/>\n        required_fields <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;conclusion&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#039;evidence&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> field <span class=\"token keyword\">in<\/span> required_fields<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> field <span class=\"token keyword\">not<\/span> <span class=\"token keyword\">in<\/span> output <span class=\"token keyword\">or<\/span> <span class=\"token keyword\">not<\/span> output<span class=\"token punctuation\">[<\/span>field<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">False<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token boolean\">True<\/span><\/p>\n<h4>6.3 Ground Truth \u6784\u5efa<\/h4>\n<p>\u6bcf\u4e2a\u6d4b\u8bd5\u4efb\u52a1\u5728\u5b9e\u9a8c\u524d\u9884\u6784\u5efa Ground Truth&#xff0c;\u4e0d\u53ef\u4fee\u6539&#xff1a;<\/p>\n<p><span class=\"token comment\"># file: ground_truth_builder.py<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">build_ground_truth<\/span><span class=\"token punctuation\">(<\/span>task_id<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">,<\/span> n_steps<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;<br \/>\n    \u4e3a\u6bcf\u4e2a\u6d4b\u8bd5\u4efb\u52a1\u9884\u6784\u5efa Ground Truth<br \/>\n    \u5728\u5b9e\u9a8c\u524d\u56fa\u5b9a&#xff0c;\u4e0d\u53ef\u4fee\u6539<br \/>\n    &#034;&#034;&#034;<\/span><br \/>\n    ground_truth <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> n_steps <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        ground_truth<span class=\"token punctuation\">[<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#039;conclusion&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;expected_conclusion_for_step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;evidence&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;expected_evidence_for_step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;key_facts&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;fact_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">_1&#039;<\/span><\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;fact_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">_2&#039;<\/span><\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> ground_truth<\/p>\n<hr \/>\n<h3>\u7b2c\u4e03\u7ae0 \u6b65\u9aa4 5&#xff1a;\u5b8c\u6574\u5b9e\u9a8c\u6267\u884c<\/h3>\n<h4>7.1 \u5b9e\u9a8c\u77e9\u9635<\/h4>\n<table>\n<tr>\u5b9e\u9a8c\u76ee\u7684\u53d8\u91cf\u6837\u672c\u91cf<\/tr>\n<tbody>\n<tr>\n<td>5.1 S_c \u8870\u51cf\u5bf9\u7167<\/td>\n<td>\u6d4b\u4e09\u67b6\u6784 S_c \u662f\u5426\u968f C \u4e0b\u964d<\/td>\n<td>\u67b6\u6784 \u00d7 \u4e0a\u4e0b\u6587\u957f\u5ea6<\/td>\n<td>3 \u00d7 5 \u00d7 10 &#061; 150<\/td>\n<\/tr>\n<tr>\n<td>5.2 \u03bb \u76f4\u63a5\u6d4b\u91cf<\/td>\n<td>\u6d4b\u4e09\u67b6\u6784\u8870\u51cf\u7cfb\u6570<\/td>\n<td>\u67b6\u6784<\/td>\n<td>3 \u00d7 20 &#061; 60<\/td>\n<\/tr>\n<tr>\n<td>5.3 \u9ed1\u76d2\u63a2\u6d4b\u4ea4\u53c9\u9a8c\u8bc1<\/td>\n<td>\u7528\u7edf\u4e00\u65b9\u6cd5\u6a2a\u5411\u5bf9\u6bd4<\/td>\n<td>\u67b6\u6784 \u00d7 \u4e0a\u4e0b\u6587\u957f\u5ea6<\/td>\n<td>3 \u00d7 3 \u00d7 5 &#061; 45<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>7.2 \u5b8c\u6574\u5b9e\u9a8c\u6267\u884c\u6846\u67b6<\/h4>\n<p><span class=\"token comment\"># file: run_cross_arch_experiment.py<\/span><\/p>\n<p><span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">from<\/span> typing <span class=\"token keyword\">import<\/span> Dict<span class=\"token punctuation\">,<\/span> List<br \/>\n<span class=\"token keyword\">from<\/span> scipy<span class=\"token punctuation\">.<\/span>stats <span class=\"token keyword\">import<\/span> spearmanr<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">CrossArchExperiment<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u8de8\u67b6\u6784\u5bf9\u7167\u5b9e\u9a8c&#034;&#034;&#034;<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> model_configs<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>models <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> config <span class=\"token keyword\">in<\/span> model_configs<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            self<span class=\"token punctuation\">.<\/span>models<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> UnifiedModel<span class=\"token punctuation\">(<\/span>arch<span class=\"token operator\">&#061;<\/span>arch<span class=\"token punctuation\">,<\/span> model_path<span class=\"token operator\">&#061;<\/span>config<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;path&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">run_full_experiment<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u8fd0\u884c\u5b8c\u6574\u5b9e\u9a8c&#034;&#034;&#034;<\/span><br \/>\n        results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><\/p>\n<p>        <span class=\"token comment\"># \u5b9e\u9a8c 5.1&#xff1a;S_c \u8870\u51cf\u5bf9\u7167<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;[\u5b9e\u9a8c 5.1] S_c \u8870\u51cf\u5bf9\u7167&#8230;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;sc_decay&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_run_sc_decay_comparison<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u5b9e\u9a8c 5.2&#xff1a;\u03bb \u76f4\u63a5\u6d4b\u91cf<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;[\u5b9e\u9a8c 5.2] \u03bb \u76f4\u63a5\u6d4b\u91cf&#8230;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;lambda_measure&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_run_lambda_comparison<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u5b9e\u9a8c 5.3&#xff1a;\u9ed1\u76d2\u63a2\u6d4b\u4ea4\u53c9\u9a8c\u8bc1<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;[\u5b9e\u9a8c 5.3] \u9ed1\u76d2\u63a2\u6d4b\u4ea4\u53c9\u9a8c\u8bc1&#8230;&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;blackbox_probe&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_run_blackbox_comparison<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u751f\u6210\u7ed3\u8bba<\/span><br \/>\n        results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_generate_verdict<span class=\"token punctuation\">(<\/span>results<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> results<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_run_sc_decay_comparison<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u5b9e\u9a8c 5.1&#xff1a;\u4e09\u67b6\u6784 S_c \u8870\u51cf\u5bf9\u7167&#034;&#034;&#034;<\/span><br \/>\n        context_lengths <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">8<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">16<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">32<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">64<\/span><span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># K tokens<\/span><br \/>\n        complexity <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span>  <span class=\"token comment\"># 10 \u6b65\u63a8\u7406\u4efb\u52a1<\/span><br \/>\n        n_trials <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">10<\/span><\/p>\n<p>        results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> model <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>models<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            s_c_curves <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>            <span class=\"token keyword\">for<\/span> context_k <span class=\"token keyword\">in<\/span> context_lengths<span class=\"token punctuation\">:<\/span><br \/>\n                s_c_list <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n                <span class=\"token keyword\">for<\/span> trial <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>n_trials<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    prompt <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_build_reasoning_task<span class=\"token punctuation\">(<\/span>context_k<span class=\"token punctuation\">,<\/span> complexity<span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">)<\/span><br \/>\n                    ground_truth <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_build_ground_truth<span class=\"token punctuation\">(<\/span>complexity<span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">)<\/span><\/p>\n<p>                    sc_meter <span class=\"token operator\">&#061;<\/span> SCMeasurement<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> ground_truth<span class=\"token punctuation\">)<\/span><br \/>\n                    s_c <span class=\"token operator\">&#061;<\/span> sc_meter<span class=\"token punctuation\">.<\/span>measure_s_c<span class=\"token punctuation\">(<\/span>prompt<span class=\"token punctuation\">,<\/span> n_steps<span class=\"token operator\">&#061;<\/span>complexity<span class=\"token punctuation\">)<\/span><br \/>\n                    s_c_list<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>s_c<span class=\"token punctuation\">)<\/span><\/p>\n<p>                s_c_curves<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">{<\/span><br \/>\n                    <span class=\"token string\">&#039;context_k&#039;<\/span><span class=\"token punctuation\">:<\/span> context_k<span class=\"token punctuation\">,<\/span><br \/>\n                    <span class=\"token string\">&#039;s_c_median&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>median<span class=\"token punctuation\">(<\/span>s_c_list<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                    <span class=\"token string\">&#039;s_c_ci&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>percentile<span class=\"token punctuation\">(<\/span>s_c_list<span class=\"token punctuation\">,<\/span> <span class=\"token number\">2.5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                              <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>percentile<span class=\"token punctuation\">(<\/span>s_c_list<span class=\"token punctuation\">,<\/span> <span class=\"token number\">97.5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span><br \/>\n                <span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            <span class=\"token comment\"># \u8ba1\u7b97 Spearman \u76f8\u5173<\/span><br \/>\n            cs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span>c<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;context_k&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token keyword\">for<\/span> c <span class=\"token keyword\">in<\/span> s_c_curves<span class=\"token punctuation\">]<\/span><br \/>\n            scs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span>c<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;s_c_median&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token keyword\">for<\/span> c <span class=\"token keyword\">in<\/span> s_c_curves<span class=\"token punctuation\">]<\/span><br \/>\n            rho<span class=\"token punctuation\">,<\/span> p <span class=\"token operator\">&#061;<\/span> spearmanr<span class=\"token punctuation\">(<\/span>cs<span class=\"token punctuation\">,<\/span> scs<span class=\"token punctuation\">)<\/span><\/p>\n<p>            results<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;curves&#039;<\/span><span class=\"token punctuation\">:<\/span> s_c_curves<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;spearman_rho&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>rho<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;spearman_p&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>p<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">:<\/span> rho <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">0<\/span> <span class=\"token keyword\">and<\/span> p <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">0.05<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> results<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_run_lambda_comparison<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u5b9e\u9a8c 5.2&#xff1a;\u4e09\u67b6\u6784 \u03bb \u76f4\u63a5\u6d4b\u91cf&#034;&#034;&#034;<\/span><br \/>\n        test_texts <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_generate_test_texts<span class=\"token punctuation\">(<\/span>n<span class=\"token operator\">&#061;<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">,<\/span> length_k<span class=\"token operator\">&#061;<\/span><span class=\"token number\">8<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> model <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>models<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">from<\/span> decay_measurement<span class=\"token punctuation\">.<\/span>transformer_decay <span class=\"token keyword\">import<\/span> TransformerDecayMeter<br \/>\n                meter <span class=\"token operator\">&#061;<\/span> TransformerDecayMeter<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">elif<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">from<\/span> decay_measurement<span class=\"token punctuation\">.<\/span>mamba_decay <span class=\"token keyword\">import<\/span> MambaDecayMeter<br \/>\n                meter <span class=\"token operator\">&#061;<\/span> MambaDecayMeter<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">elif<\/span> arch <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                <span class=\"token keyword\">from<\/span> decay_measurement<span class=\"token punctuation\">.<\/span>rwkv_decay <span class=\"token keyword\">import<\/span> RWKVDecayMeter<br \/>\n                meter <span class=\"token operator\">&#061;<\/span> RWKVDecayMeter<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">)<\/span><\/p>\n<p>            lambdas <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token keyword\">for<\/span> text <span class=\"token keyword\">in<\/span> test_texts<span class=\"token punctuation\">:<\/span><br \/>\n                result <span class=\"token operator\">&#061;<\/span> meter<span class=\"token punctuation\">.<\/span>measure_decay<span class=\"token punctuation\">(<\/span>text<span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token keyword\">if<\/span> result<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">is<\/span> <span class=\"token keyword\">not<\/span> <span class=\"token boolean\">None<\/span> <span class=\"token keyword\">and<\/span> result<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&gt;<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    lambdas<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>result<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>            results<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;lambda_mean&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>lambdas<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">if<\/span> lambdas <span class=\"token keyword\">else<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;lambda_std&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>std<span class=\"token punctuation\">(<\/span>lambdas<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">if<\/span> lambdas <span class=\"token keyword\">else<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;lambda_ci&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>percentile<span class=\"token punctuation\">(<\/span>lambdas<span class=\"token punctuation\">,<\/span> <span class=\"token number\">2.5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                             <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>percentile<span class=\"token punctuation\">(<\/span>lambdas<span class=\"token punctuation\">,<\/span> <span class=\"token number\">97.5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token keyword\">if<\/span> lambdas <span class=\"token keyword\">else<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;n_valid&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>lambdas<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;method&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;arch_specific&#039;<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> results<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_run_blackbox_comparison<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u5b9e\u9a8c 5.3&#xff1a;\u9ed1\u76d2\u63a2\u6d4b\u4ea4\u53c9\u9a8c\u8bc1&#xff08;\u4e09\u67b6\u6784\u7edf\u4e00\u65b9\u6cd5&#xff09;&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token keyword\">from<\/span> decay_measurement<span class=\"token punctuation\">.<\/span>universal_blackbox <span class=\"token keyword\">import<\/span> UniversalBlackBoxProbe<\/p>\n<p>        context_lengths <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">16<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">64<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> model <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>models<span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            probe <span class=\"token operator\">&#061;<\/span> UniversalBlackBoxProbe<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">)<\/span><\/p>\n<p>            arch_results <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token keyword\">for<\/span> context_k <span class=\"token keyword\">in<\/span> context_lengths<span class=\"token punctuation\">:<\/span><br \/>\n                curve <span class=\"token operator\">&#061;<\/span> probe<span class=\"token punctuation\">.<\/span>measure_recall_curve<span class=\"token punctuation\">(<\/span><br \/>\n                    context_k<span class=\"token punctuation\">,<\/span> n_probes<span class=\"token operator\">&#061;<\/span><span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span> n_trials<span class=\"token operator\">&#061;<\/span><span class=\"token number\">5<\/span><br \/>\n                <span class=\"token punctuation\">)<\/span><br \/>\n                arch_results<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>curve<span class=\"token punctuation\">)<\/span><\/p>\n<p>            <span class=\"token comment\"># \u4ece\u53ec\u56de\u7387\u66f2\u7ebf\u62df\u5408 \u03bb<\/span><br \/>\n            lambda_fit <span class=\"token operator\">&#061;<\/span> probe<span class=\"token punctuation\">.<\/span>fit_lambda_from_recall<span class=\"token punctuation\">(<\/span>arch_results<span class=\"token punctuation\">)<\/span><\/p>\n<p>            results<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;curves&#039;<\/span><span class=\"token punctuation\">:<\/span> arch_results<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;lambda_from_blackbox&#039;<\/span><span class=\"token punctuation\">:<\/span> lambda_fit<br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> results<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_verdict<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> results<span class=\"token punctuation\">:<\/span> Dict<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u751f\u6210\u5224\u5b9a\u7ed3\u8bba&#034;&#034;&#034;<\/span><br \/>\n        <span class=\"token comment\"># \u63d0\u53d6\u5404\u67b6\u6784\u7684 S_c \u8870\u51cf\u60c5\u51b5<\/span><br \/>\n        sc_decay <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> data <span class=\"token keyword\">in<\/span> results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;sc_decay&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            sc_decay<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">:<\/span> data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;rho&#039;<\/span><span class=\"token punctuation\">:<\/span> data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;spearman_rho&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;p&#039;<\/span><span class=\"token punctuation\">:<\/span> data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;spearman_p&#039;<\/span><span class=\"token punctuation\">]<\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><\/p>\n<p>        <span class=\"token comment\"># \u63d0\u53d6\u5404\u67b6\u6784\u7684\u9ed1\u76d2\u63a2\u6d4b \u03bb<\/span><br \/>\n        blackbox_lambdas <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> data <span class=\"token keyword\">in<\/span> results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;blackbox_probe&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            lam <span class=\"token operator\">&#061;<\/span> data<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;lambda_from_blackbox&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;lambda&#039;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> lam <span class=\"token keyword\">is<\/span> <span class=\"token keyword\">not<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                blackbox_lambdas<span class=\"token punctuation\">[<\/span>arch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> lam<\/p>\n<p>        <span class=\"token comment\"># \u5224\u5b9a\u903b\u8f91&#xff08;\u4ee5\u9ed1\u76d2\u63a2\u6d4b\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e&#xff09;<\/span><br \/>\n        transformer_declines <span class=\"token operator\">&#061;<\/span> sc_decay<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        mamba_declines <span class=\"token operator\">&#061;<\/span> sc_decay<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        rwkv_declines <span class=\"token operator\">&#061;<\/span> sc_decay<span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>get<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">if<\/span> transformer_declines <span class=\"token keyword\">and<\/span> mamba_declines <span class=\"token keyword\">and<\/span> rwkv_declines<span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token comment\"># \u4e09\u67b6\u6784\u90fd\u8870\u51cf&#xff0c;\u8fdb\u4e00\u6b65\u68c0\u67e5 \u03bb \u662f\u5426\u63a5\u8fd1<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>blackbox_lambdas<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                lam_values <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">list<\/span><span class=\"token punctuation\">(<\/span>blackbox_lambdas<span class=\"token punctuation\">.<\/span>values<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                lam_variance <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span>np<span class=\"token punctuation\">.<\/span>var<span class=\"token punctuation\">(<\/span>lam_values<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token keyword\">if<\/span> lam_variance <span class=\"token operator\">&lt;<\/span> <span class=\"token number\">0.01<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;L5 \u5728\u4e09\u67b6\u6784\u4e0a\u5747\u6210\u7acb\u4e14 \u03bb \u63a5\u8fd1&#xff0c;\u8870\u51cf\u662f\u666e\u904d\u89c4\u5f8b&#034;<\/span><br \/>\n                    falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">False<\/span><br \/>\n                <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#034;L5 \u5728\u4e09\u67b6\u6784\u4e0a\u5747\u6210\u7acb\u4f46 \u03bb \u5dee\u5f02\u5927&#xff08;\u65b9\u5dee&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>lam_variance<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.4f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#xff09;&#xff0c;\u9700\u6309\u67b6\u6784\u6807\u5b9a\u53c2\u6570&#034;<\/span><\/span><br \/>\n                    falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span>  <span class=\"token comment\"># \u90e8\u5206\u6210\u7acb<\/span><br \/>\n            <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;L5 \u5728\u4e09\u67b6\u6784\u4e0a\u5747\u6210\u7acb&#xff08;\u03bb \u6570\u636e\u4e0d\u8db3&#xff0c;\u65e0\u6cd5\u6bd4\u8f83\u53c2\u6570&#xff09;&#034;<\/span><br \/>\n                falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">False<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> transformer_declines <span class=\"token keyword\">and<\/span> <span class=\"token keyword\">not<\/span> <span class=\"token punctuation\">(<\/span>mamba_declines <span class=\"token keyword\">or<\/span> rwkv_declines<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;L5 \u4ec5\u5728 Transformer \u4e0a\u6210\u7acb&#xff0c;\u975e Transformer \u67b6\u6784\u65e0\u8870\u51cf\u2014\u2014L5 \u662f Transformer \u7279\u6709\u89c4\u5f8b&#xff0c;\u975e\u666e\u904d\u5b9a\u5f8b&#034;<\/span><br \/>\n            falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">True<\/span><br \/>\n        <span class=\"token keyword\">elif<\/span> transformer_declines <span class=\"token keyword\">and<\/span> <span class=\"token punctuation\">(<\/span>mamba_declines <span class=\"token operator\">!&#061;<\/span> rwkv_declines<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;L5 \u90e8\u5206\u666e\u9002&#xff0c;\u9700\u6309\u67b6\u6784\u5206\u7c7b\u8ba8\u8bba&#034;<\/span><br \/>\n            falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">None<\/span><br \/>\n        <span class=\"token keyword\">else<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            verdict <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;\u8870\u51cf\u73b0\u8c61\u5728\u4e09\u67b6\u6784\u4e0a\u5747\u4e0d\u663e\u8457&#xff0c;L5 \u7406\u8bba\u57fa\u7840\u5b58\u7591&#034;<\/span><br \/>\n            falsified <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">True<\/span><\/p>\n<p>        <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n            <span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">:<\/span> verdict<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;falsified&#039;<\/span><span class=\"token punctuation\">:<\/span> falsified<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;sc_decay_analysis&#039;<\/span><span class=\"token punctuation\">:<\/span> sc_decay<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;blackbox_lambda_comparison&#039;<\/span><span class=\"token punctuation\">:<\/span> blackbox_lambdas<span class=\"token punctuation\">,<\/span><br \/>\n            <span class=\"token string\">&#039;evidence&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;transformer_declines&#039;<\/span><span class=\"token punctuation\">:<\/span> transformer_declines<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;mamba_declines&#039;<\/span><span class=\"token punctuation\">:<\/span> mamba_declines<span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;rwkv_declines&#039;<\/span><span class=\"token punctuation\">:<\/span> rwkv_declines<br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token punctuation\">}<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_build_reasoning_task<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> complexity<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6784\u5efa\u63a8\u7406\u4efb\u52a1&#xff08;\u4e0e\u7eaf API \u7248\u4e00\u81f4&#xff09;&#034;&#034;&#034;<\/span><br \/>\n        context <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_generate_filler_context<span class=\"token punctuation\">(<\/span>context_k<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#034;&#034;&#034;<br \/>\n\u8bf7\u6309\u6b65\u9aa4\u5b8c\u6210\u4ee5\u4e0b\u63a8\u7406\u4efb\u52a1&#xff0c;\u5171 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>complexity<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u6b65\u3002<br \/>\n\u6bcf\u6b65\u8f93\u51fa\u4e00\u4e2a\u72ec\u7acb\u7ed3\u8bba&#xff0c;\u683c\u5f0f\u4e3a JSON&#xff1a;{{&#034;step&#034;: N, &#034;conclusion&#034;: &#034;&#8230;&#034;, &#034;evidence&#034;: &#034;&#8230;&#034;}}<\/p>\n<p>\u4e0a\u4e0b\u6587&#xff08;\u957f\u5ea6\u7ea6 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>context_k<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">K tokens&#xff09;&#xff1a;<br \/>\n<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>context<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"><\/p>\n<p>\u4efb\u52a1&#xff1a;\u8bf7\u57fa\u4e8e\u4e0a\u4e0b\u6587&#xff0c;\u5b8c\u6210 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>complexity<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u6b65\u903b\u8f91\u63a8\u7406\u3002<br \/>\n&#034;&#034;&#034;<\/span><\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_filler_context<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> context_k<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> <span class=\"token builtin\">str<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u751f\u6210\u586b\u5145\u4e0a\u4e0b\u6587&#034;&#034;&#034;<\/span><br \/>\n        base_text <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;\u8fd9\u662f\u4e00\u6bb5\u7528\u4e8e\u586b\u5145\u4e0a\u4e0b\u6587\u7684\u4e2d\u6027\u6587\u672c&#xff0c;\u4e0d\u542b\u5173\u952e\u4fe1\u606f\u3002&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">100<\/span><br \/>\n        target_chars <span class=\"token operator\">&#061;<\/span> context_k <span class=\"token operator\">*<\/span> <span class=\"token number\">1000<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">2<\/span>  <span class=\"token comment\"># \u4e2d\u6587\u7ea6 2 \u5b57\u7b26\/token<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> base_text<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span>target_chars<span class=\"token punctuation\">]<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_build_ground_truth<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> complexity<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> trial<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> Dict<span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6784\u5efa Ground Truth&#034;&#034;&#034;<\/span><br \/>\n        gt <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> step <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> complexity <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            gt<span class=\"token punctuation\">[<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#039;step_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n                <span class=\"token string\">&#039;conclusion&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;expected_conclusion_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>trial<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><span class=\"token punctuation\">,<\/span><br \/>\n                <span class=\"token string\">&#039;evidence&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#039;expected_evidence_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>trial<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">_<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>step<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#039;<\/span><\/span><br \/>\n            <span class=\"token punctuation\">}<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> gt<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_generate_test_texts<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> n<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> length_k<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> List<span class=\"token punctuation\">[<\/span><span class=\"token builtin\">str<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u751f\u6210\u6d4b\u8bd5\u6587\u672c\u7528\u4e8e \u03bb \u6d4b\u91cf&#034;&#034;&#034;<\/span><br \/>\n        texts <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        base <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;The quick brown fox jumps over the lazy dog. &#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">100<\/span><br \/>\n        target_chars <span class=\"token operator\">&#061;<\/span> length_k <span class=\"token operator\">*<\/span> <span class=\"token number\">1000<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">4<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            text <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>base <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span>target_chars <span class=\"token operator\">\/\/<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>base<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span>target_chars<span class=\"token punctuation\">]<\/span><br \/>\n            texts<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>text <span class=\"token operator\">&#043;<\/span> <span class=\"token string-interpolation\"><span class=\"token string\">f&#034; Unique marker <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>i<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">.&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> texts<\/p>\n<p><span class=\"token keyword\">if<\/span> __name__ <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#039;__main__&#039;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    model_configs <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">{<\/span><br \/>\n        <span class=\"token string\">&#039;transformer&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#039;path&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;.\/models\/transformer&#039;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span>  <span class=\"token comment\"># pythia-1.4b<\/span><br \/>\n        <span class=\"token string\">&#039;mamba&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#039;path&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;.\/models\/mamba&#039;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span>               <span class=\"token comment\"># mamba2-1.3b<\/span><br \/>\n        <span class=\"token string\">&#039;rwkv&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token punctuation\">{<\/span><span class=\"token string\">&#039;path&#039;<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token string\">&#039;.\/models\/rwkv&#039;<\/span><span class=\"token punctuation\">}<\/span><span class=\"token punctuation\">,<\/span>                 <span class=\"token comment\"># rwkv-7-world-1b5<\/span><br \/>\n    <span class=\"token punctuation\">}<\/span><\/p>\n<p>    experiment <span class=\"token operator\">&#061;<\/span> CrossArchExperiment<span class=\"token punctuation\">(<\/span>model_configs<span class=\"token punctuation\">)<\/span><br \/>\n    results <span class=\"token operator\">&#061;<\/span> experiment<span class=\"token punctuation\">.<\/span>run_full_experiment<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\\\\n&#034;<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">60<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u8de8\u67b6\u6784\u5bf9\u7167\u5b9e\u9a8c\u7ed3\u679c&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;&#061;&#034;<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">60<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n\u5224\u5b9a&#xff1a;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n\u5404\u67b6\u6784 S_c \u8870\u51cf\u5206\u6790&#xff1a;&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> arch<span class=\"token punctuation\">,<\/span> data <span class=\"token keyword\">in<\/span> results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;sc_decay_analysis&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>items<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;  <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>arch<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">: declines&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;declines&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">, \u03c1&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;rho&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.3f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">, p&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>data<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;p&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.4f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\\\\n\u5404\u67b6\u6784\u9ed1\u76d2\u63a2\u6d4b \u03bb&#xff1a;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>results<span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;verdict&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token string\">&#039;blackbox_lambda_comparison&#039;<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<hr \/>\n<h3>\u7b2c\u516b\u7ae0 \u6b65\u9aa4 6&#xff1a;\u5224\u5b9a\u6807\u51c6&#xff08;\u9884\u6ce8\u518c&#xff09;<\/h3>\n<h4>8.1 \u9884\u6ce8\u518c\u5224\u5b9a\u77e9\u9635<\/h4>\n<p>\u5b9e\u9a8c\u524d\u516c\u5f00\u627f\u8bfa\u4ee5\u4e0b\u5224\u5b9a\u6807\u51c6&#xff0c;\u5b9e\u9a8c\u540e\u4e0d\u5f97\u4fee\u6539&#xff1a;<\/p>\n<table>\n<tr>\u89c2\u5bdf\u7ed3\u679c\u7edf\u8ba1\u68c0\u9a8c\u7ed3\u8bbaL5 \u5730\u4f4d<\/tr>\n<tbody>\n<tr>\n<td>\u4e09\u67b6\u6784 S_c \u5747\u968f C \u4e0b\u964d&#xff0c;\u4e14\u9ed1\u76d2 \u03bb \u63a5\u8fd1&#xff08;\u65b9\u5dee &lt; 0.01&#xff09;<\/td>\n<td>\u4e09\u7ec4 Spearman \u03c1 &lt; 0, p &lt; 0.05<\/td>\n<td>\u2705 L5 \u662f\u666e\u904d\u5b9a\u5f8b<\/td>\n<td>\u7ef4\u6301&#034;\u5b9a\u5f8b&#034;\u5730\u4f4d<\/td>\n<\/tr>\n<tr>\n<td>\u4e09\u67b6\u6784 S_c \u5747\u968f C \u4e0b\u964d&#xff0c;\u4f46 \u03bb \u5dee\u5f02\u5927&#xff08;\u65b9\u5dee \u2265 0.01&#xff09;<\/td>\n<td>\u4e09\u7ec4\u5747\u663e\u8457\u4e0b\u964d<\/td>\n<td>&#x1f7e1; L5 \u666e\u9002\u4f46\u53c2\u6570\u9700\u6309\u67b6\u6784\u6807\u5b9a<\/td>\n<td>\u7ef4\u6301&#034;\u5b9a\u5f8b&#034;\u4f46\u9700\u4fee\u6b63\u53c2\u6570<\/td>\n<\/tr>\n<tr>\n<td>Transformer \u8870\u51cf&#xff0c;Mamba\/RWKV \u4e0d\u8870\u51cf<\/td>\n<td>\u4ec5 Transformer \u663e\u8457<\/td>\n<td>\u274c L5 \u662f Transformer \u7279\u6709\u89c4\u5f8b<\/td>\n<td>\u964d\u7ea7\u4e3a&#034;Transformer \u7ecf\u9a8c\u89c4\u5f8b&#034;<\/td>\n<\/tr>\n<tr>\n<td>Mamba\/RWKV \u8870\u51cf\u4f46\u5f62\u5f0f\u4e0d\u540c&#xff08;\u975e\u6307\u6570&#xff09;<\/td>\n<td>\u51fd\u6570\u5f62\u5f0f\u4e0d\u5339\u914d<\/td>\n<td>&#x1f7e1; L5 \u9700\u6309\u67b6\u6784\u5206\u7c7b&#xff0c;\u51fd\u6570\u5f62\u5f0f\u9700\u4fee\u6b63<\/td>\n<td>\u90e8\u5206\u6210\u7acb<\/td>\n<\/tr>\n<tr>\n<td>\u4e09\u67b6\u6784\u5747\u4e0d\u8870\u51cf<\/td>\n<td>\u4e09\u7ec4\u5747\u4e0d\u663e\u8457<\/td>\n<td>\u274c \u8870\u51cf\u73b0\u8c61\u4e0d\u666e\u904d&#xff0c;L5 \u7406\u8bba\u57fa\u7840\u5b58\u7591<\/td>\n<td>\u91cd\u5927\u4fee\u6b63<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>8.2 \u4ee5\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e<\/h4>\n<p>\u7531\u4e8e\u65b9\u6cd5 A\/B\/C \u6d4b\u91cf\u7684&#034;\u8870\u51cf&#034;\u5728\u7269\u7406\u673a\u5236\u4e0a\u4e0d\u5b8c\u5168\u7b49\u4ef7&#xff08;\u89c1 \u00a72.3&#xff09;&#xff0c;\u672c\u65b9\u6848\u4ee5\u65b9\u6cd5 D&#xff08;\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff09;\u4f5c\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e&#xff1a;<\/p>\n<ul>\n<li>\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u6d4b\u7684\u662f&#034;\u53ec\u56de\u7387\u8870\u51cf&#034;&#xff0c;\u4e0e\u67b6\u6784\u65e0\u5173&#xff0c;\u53ef\u76f4\u63a5\u6a2a\u5411\u5bf9\u6bd4<\/li>\n<li>\u65b9\u6cd5 A\/B\/C \u4f5c\u4e3a\u673a\u5236\u5c42\u9762\u7684\u8865\u5145\u8bc1\u636e&#xff0c;\u89e3\u91ca&#034;\u4e3a\u4ec0\u4e48\u8870\u51cf&#034;<\/li>\n<\/ul>\n<h4>8.3 \u7edf\u8ba1\u529f\u6548<\/h4>\n<p>\u5047\u8bbe&#xff1a;<br \/>\n&#8211; H0&#xff1a;Mamba\/RWKV \u7684 S_c \u4e0d\u968f C \u4e0b\u964d&#xff08;\u03c1 \u2265 0&#xff09;<br \/>\n&#8211; H1&#xff1a;Mamba\/RWKV \u7684 S_c \u968f C \u4e0b\u964d&#xff08;\u03c1 &lt; 0&#xff0c;\u6548\u5e94\u91cf |\u03c1| \u2265 0.5&#xff09;<br \/>\n&#8211; \u03b1 &#061; 0.05&#xff0c;\u529f\u6548 1-\u03b2 &#061; 0.80<\/p>\n<p>\u6837\u672c\u91cf\u8ba1\u7b97&#xff08;Spearman \u76f8\u5173\u68c0\u9a8c&#xff09;&#xff1a;<br \/>\nn &#061; 29&#xff08;\u6bcf\u7ec4&#xff09;<\/p>\n<p>\u5b9e\u9a8c 5.1&#xff1a;\u6bcf\u67b6\u6784 5 \u4e2a\u4e0a\u4e0b\u6587\u957f\u5ea6 \u00d7 10 \u91cd\u590d &#061; 50 \u6837\u672c &gt; 29 \u2705<br \/>\n\u5b9e\u9a8c 5.2&#xff1a;\u6bcf\u67b6\u6784 20 \u4e2a\u6d4b\u8bd5\u6587\u672c \u2265 29 \u7684\u8981\u6c42\u9700\u8865\u5145\u81f3 30<br \/>\n\u5b9e\u9a8c 5.3&#xff1a;\u6bcf\u67b6\u6784 3 \u4e2a\u4e0a\u4e0b\u6587\u957f\u5ea6 \u00d7 5 \u91cd\u590d &#061; 15 \u6837\u672c &lt; 29&#xff0c;\u9700\u589e\u81f3 10 \u91cd\u590d<\/p>\n<hr \/>\n<h3>\u7b2c\u4e5d\u7ae0 \u6b65\u9aa4 7&#xff1a;\u65b9\u6cd5\u5b66\u6311\u6218\u4e0e\u8bda\u5b9e\u58f0\u660e<\/h3>\n<h4>9.1 \u4e09\u79cd \u03bb \u6d4b\u91cf\u65b9\u6cd5\u7684&#034;\u7b49\u4ef7\u6027&#034;\u95ee\u9898<\/h4>\n<table>\n<tr>\u67b6\u6784\u6d4b\u91cf\u5bf9\u8c61\u7269\u7406\u673a\u5236\u662f\u5426\u771f\u6b63\u7b49\u4ef7<\/tr>\n<tbody>\n<tr>\n<td>Transformer<\/td>\n<td>attention weights \u8870\u51cf<\/td>\n<td>\u6ce8\u610f\u529b\u5206\u914d\u6743\u91cd<\/td>\n<td>\u57fa\u51c6<\/td>\n<\/tr>\n<tr>\n<td>Mamba<\/td>\n<td>hidden state \u6270\u52a8\u8870\u51cf<\/td>\n<td>\u4fe1\u606f\u5728 hidden state \u4e2d\u7684\u4fdd\u7559<\/td>\n<td>&#x1f7e1; \u8fd1\u4f3c\u7b49\u4ef7\u2014\u2014\u6d4b\u7684\u662f\u4fe1\u606f\u4fdd\u7559\u5ea6&#xff0c;\u4f46\u673a\u5236\u4e0d\u540c<\/td>\n<\/tr>\n<tr>\n<td>RWKV<\/td>\n<td>state \u5411\u91cf\u8870\u51cf<\/td>\n<td>state \u5411\u91cf\u7684\u663e\u5f0f\u6f14\u5316<\/td>\n<td>&#x1f7e1; \u8fd1\u4f3c\u7b49\u4ef7\u2014\u2014state \u6709\u663e\u5f0f decay factor&#xff0c;\u4f46\u4e0e attention \u673a\u5236\u4e0d\u540c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8bda\u5b9e\u58f0\u660e&#xff1a;\u4e09\u79cd\u65b9\u6cd5\u6d4b\u91cf\u7684&#034;\u8870\u51cf&#034;\u5728\u7269\u7406\u673a\u5236\u4e0a\u4e0d\u5b8c\u5168\u7b49\u4ef7\u3002Transformer \u6d4b\u7684\u662f\u6ce8\u610f\u529b\u5206\u914d&#xff0c;Mamba \u6d4b\u7684\u662f hidden state \u8bb0\u5fc6\u4fdd\u7559&#xff0c;RWKV \u6d4b\u7684\u662f state \u5411\u91cf\u6f14\u5316\u3002\u628a\u5b83\u4eec\u90fd\u6620\u5c04\u5230\u540c\u4e00\u4e2a \u03bb \u662f\u65b9\u6cd5\u5b66\u59a5\u534f\u3002\u672c\u65b9\u6848\u4ee5\u65b9\u6cd5 D&#xff08;\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff09;\u4f5c\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e&#xff0c;\u65b9\u6cd5 A\/B\/C \u4f5c\u4e3a\u673a\u5236\u5c42\u9762\u7684\u8865\u5145\u8bc1\u636e\u3002\u8fd9\u4e00\u8bbe\u8ba1\u9009\u62e9\u9700\u5728\u7ed3\u679c\u62a5\u544a\u4e2d\u660e\u786e\u58f0\u660e\u3002<\/p>\n<h4>9.2 \u6a21\u578b\u89c4\u6a21\u5bf9\u9f50\u7684\u5c40\u9650<\/h4>\n<p>\u5373\u4f7f\u7528 1.4B \u5bf9\u9f50\u7ec4&#xff0c;\u4e09\u4e2a\u6a21\u578b\u7684\u8bad\u7ec3\u6570\u636e\u3001\u8bad\u7ec3\u65f6\u957f\u3001\u67b6\u6784\u7ec6\u8282\u4ecd\u6709\u5dee\u5f02&#xff1a;<\/p>\n<table>\n<tr>\u7ef4\u5ea6Pythia-1.4BMamba-2-1.3BRWKV-7-1B5<\/tr>\n<tbody>\n<tr>\n<td>\u67b6\u6784<\/td>\n<td>GPT-NeoX<\/td>\n<td>Mamba-2 SSM<\/td>\n<td>RWKV-7 \u7ebf\u6027 RNN<\/td>\n<\/tr>\n<tr>\n<td>\u8bad\u7ec3\u6570\u636e<\/td>\n<td>The Pile<\/td>\n<td>The Pile&#xff08;\u90e8\u5206&#xff09;<\/td>\n<td>World&#xff08;\u591a\u8bed\u8a00&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u8bad\u7ec3 tokens<\/td>\n<td>~300B<\/td>\n<td>~300B<\/td>\n<td>~3T<\/td>\n<\/tr>\n<tr>\n<td>\u4e0a\u4e0b\u6587\u7a97\u53e3<\/td>\n<td>2048<\/td>\n<td>8192<\/td>\n<td>32K&#043;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7406\u60f3\u5bf9\u7167\u5e94\u4f7f\u7528\u76f8\u540c\u6570\u636e\u8bad\u7ec3\u7684\u4e0d\u540c\u67b6\u6784\u6a21\u578b&#xff08;\u5982 Pythia \u7cfb\u5217 vs \u540c\u6570\u636e\u7684 Mamba\/RWKV&#xff09;&#xff0c;\u4f46\u76ee\u524d\u6ca1\u6709\u8fd9\u6837\u7684\u516c\u5f00\u6a21\u578b\u3002\u8fd9\u662f\u56fa\u6709\u5c40\u9650&#xff0c;\u9700\u5728\u7ed3\u679c\u62a5\u544a\u4e2d\u58f0\u660e\u3002<\/p>\n<h4>9.3 \u4e0a\u4e0b\u6587\u7a97\u53e3\u5dee\u5f02\u7684\u5f71\u54cd<\/h4>\n<p>\u4e09\u4e2a\u6a21\u578b\u7684\u4e0a\u4e0b\u6587\u7a97\u53e3\u5dee\u5f02\u663e\u8457&#xff08;Pythia 2K vs RWKV 32K&#043;&#xff09;\u3002\u5728 C&#061;32K\/64K \u65f6&#xff0c;Pythia-1.4B \u53ef\u80fd\u65e0\u6cd5\u5904\u7406\u3002\u89e3\u51b3\u65b9\u6848&#xff1a;<\/p>\n<ul>\n<li>\u5bf9 Transformer \u4f7f\u7528\u652f\u6301\u957f\u4e0a\u4e0b\u6587\u7684\u6a21\u578b&#xff08;\u5982 Llama-3-8B-Instruct&#xff0c;128K \u7a97\u53e3&#xff09;<\/li>\n<li>\u6216\u5728 C \u2264 2K \u8303\u56f4\u5185\u5bf9\u6bd4&#xff08;\u727a\u7272\u957f\u4e0a\u4e0b\u6587\u6570\u636e\u70b9&#xff09;<\/li>\n<\/ul>\n<h4>9.4 \u9ed1\u76d2\u63a2\u6d4b\u6cd5\u7684\u515c\u5e95\u4f5c\u7528<\/h4>\n<p>\u5982\u679c\u67b6\u6784\u7279\u5f02\u6027\u7684 \u03bb \u6d4b\u91cf\u65b9\u6cd5\u5b58\u5728\u4e89\u8bae&#xff0c;\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u662f\u4e09\u67b6\u6784\u901a\u7528\u7684\u515c\u5e95\u65b9\u6848\u3002\u5b83\u6d4b\u7684\u662f&#034;\u53ec\u56de\u7387\u8870\u51cf&#034;&#xff0c;\u4e0e\u67b6\u6784\u65e0\u5173&#xff0c;\u53ef\u76f4\u63a5\u5bf9\u6bd4\u3002\u5efa\u8bae\u4ee5\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u4e3a\u4e3b\u7ed3\u8bba&#xff0c;\u67b6\u6784\u7279\u5f02\u6027\u6d4b\u91cf\u4e3a\u8865\u5145\u8bc1\u636e\u3002<\/p>\n<h4>9.5 \u7ed3\u679c\u89e3\u8bfb\u539f\u5219<\/h4>\n<p>\u7ed3\u679c\u89e3\u8bfb\u539f\u5219&#xff08;\u9884\u6ce8\u518c&#xff09;&#xff1a;<\/p>\n<p>1. \u82e5\u9ed1\u76d2\u63a2\u6d4b\u663e\u793a\u4e09\u67b6\u6784\u5747\u8870\u51cf\u4e14 \u03bb \u63a5\u8fd1&#xff1a;<br \/>\n   &#8211; L5 \u5728\u5f53\u524d\u6837\u672c\u8303\u56f4\u5185\u5177\u5907\u8de8\u67b6\u6784\u666e\u9002\u6027<br \/>\n   &#8211; \u4f46\u4e0d\u7b49\u4e8e&#034;\u88ab\u8bc1\u660e\u6210\u7acb&#034;&#xff0c;\u4ecd\u9700\u63a5\u53d7\u540e\u7eed\u8bc1\u4f2a\u5c1d\u8bd5<\/p>\n<p>2. \u82e5\u9ed1\u76d2\u63a2\u6d4b\u663e\u793a Mamba\/RWKV \u4e0d\u8870\u51cf&#xff1a;<br \/>\n   &#8211; L5 \u7684\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u88ab\u8bc1\u4f2a<br \/>\n   &#8211; \u5e94\u964d\u7ea7\u4e3a&#034;Transformer \u7ecf\u9a8c\u89c4\u5f8b&#034;<br \/>\n   &#8211; \u4e94\u5b9a\u5f8b\u4f53\u7cfb\u9700\u4fee\u6b63\u9002\u7528\u8303\u56f4\u58f0\u660e<\/p>\n<p>3. \u82e5\u9ed1\u76d2\u63a2\u6d4b\u4e0e\u67b6\u6784\u7279\u5f02\u6027\u6d4b\u91cf\u7ed3\u679c\u4e0d\u4e00\u81f4&#xff1a;<br \/>\n   &#8211; \u4ee5\u9ed1\u76d2\u63a2\u6d4b\u4e3a\u4e3b\u7ed3\u8bba<br \/>\n   &#8211; \u67b6\u6784\u7279\u5f02\u6027\u6d4b\u91cf\u7684\u4e0d\u4e00\u81f4\u9700\u8fdb\u4e00\u6b65\u7814\u7a76<br \/>\n   &#8211; \u53ef\u80fd\u63ed\u793a\u4e0d\u540c\u67b6\u6784\u7684\u8870\u51cf\u673a\u5236\u672c\u8d28\u4e0d\u540c<\/p>\n<p>4. \u82e5\u6240\u6709\u7ed3\u679c\u652f\u6301 L5&#xff1a;<br \/>\n   &#8211; L5 \u5728\u5f53\u524d\u6837\u672c\u8303\u56f4\u5185\u672a\u88ab\u8bc1\u4f2a<br \/>\n   &#8211; \u9700\u6301\u7eed\u63a5\u53d7\u540e\u7eed\u5b9e\u9a8c&#xff08;\u66f4\u5927\u89c4\u6a21\u3001\u66f4\u591a\u67b6\u6784&#xff09;\u7684\u8bc1\u4f2a\u5c1d\u8bd5<\/p>\n<hr \/>\n<h3>\u7b2c\u5341\u7ae0 \u6267\u884c\u65f6\u95f4\u8868<\/h3>\n<h4>10.1 \u9636\u6bb5\u5212\u5206<\/h4>\n<table>\n<tr>\u9636\u6bb5\u5185\u5bb9\u65f6\u95f4\u4ea7\u51fa<\/tr>\n<tbody>\n<tr>\n<td>1<\/td>\n<td>\u73af\u5883\u51c6\u5907 &#043; \u6a21\u578b\u4e0b\u8f7d<\/td>\n<td>1 \u5929<\/td>\n<td>\u53ef\u8fd0\u884c\u7684\u4e09\u6a21\u578b\u73af\u5883<\/td>\n<\/tr>\n<tr>\n<td>2<\/td>\n<td>\u7edf\u4e00\u63a5\u53e3\u5f00\u53d1\u4e0e\u6d4b\u8bd5<\/td>\n<td>2 \u5929<\/td>\n<td>UnifiedModel \u7c7b &#043; \u9a8c\u8bc1\u62a5\u544a<\/td>\n<\/tr>\n<tr>\n<td>3<\/td>\n<td>\u4e09\u67b6\u6784\u8870\u51cf\u6d4b\u91cf\u65b9\u6cd5\u5b9e\u73b0<\/td>\n<td>3 \u5929<\/td>\n<td>\u65b9\u6cd5 A\/B\/C\/D \u4ee3\u7801 &#043; \u5355\u5143\u6d4b\u8bd5<\/td>\n<\/tr>\n<tr>\n<td>4<\/td>\n<td>S_c \u6d4b\u91cf &#043; Ground Truth \u6784\u5efa<\/td>\n<td>2 \u5929<\/td>\n<td>SCMeasurement \u7c7b &#043; GT \u6570\u636e\u96c6<\/td>\n<\/tr>\n<tr>\n<td>5<\/td>\n<td>\u5b8c\u6574\u5b9e\u9a8c\u6267\u884c&#xff08;10 \u6b21\u91cd\u590d&#xff09;<\/td>\n<td>3 \u5929<\/td>\n<td>\u539f\u59cb\u5b9e\u9a8c\u6570\u636e<\/td>\n<\/tr>\n<tr>\n<td>6<\/td>\n<td>\u6570\u636e\u5206\u6790 &#043; \u62a5\u544a\u64b0\u5199<\/td>\n<td>2 \u5929<\/td>\n<td>\u7ed3\u679c\u62a5\u544a &#043; \u7ed3\u8bba\u5224\u5b9a<\/td>\n<\/tr>\n<tr>\n<td>\u603b\u8ba1<\/td>\n<td><\/td>\n<td>~2 \u5468<\/td>\n<td>\u5b8c\u6574\u8de8\u67b6\u6784\u9a8c\u8bc1\u62a5\u544a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>10.2 \u8d44\u6e90\u9700\u6c42<\/h4>\n<table>\n<tr>\u8d44\u6e90\u9700\u6c42\u5907\u6ce8<\/tr>\n<tbody>\n<tr>\n<td>GPU<\/td>\n<td>1\u00d7RTX 4090&#xff08;24GB&#xff09;<\/td>\n<td>1.4B \u89c4\u6a21\u5bf9\u9f50\u7ec4\u8db3\u591f<\/td>\n<\/tr>\n<tr>\n<td>\u5b58\u50a8<\/td>\n<td>~30GB<\/td>\n<td>\u4e09\u6a21\u578b\u6743\u91cd &#043; \u5b9e\u9a8c\u6570\u636e<\/td>\n<\/tr>\n<tr>\n<td>API \u8c03\u7528<\/td>\n<td>\u4e0d\u9700\u8981<\/td>\n<td>\u5168\u90e8\u672c\u5730\u63a8\u7406<\/td>\n<\/tr>\n<tr>\n<td>\u4eba\u5de5\u6807\u6ce8<\/td>\n<td>\u4e0d\u9700\u8981<\/td>\n<td>\u5168\u81ea\u52a8\u5316\u5224\u5b9a<\/td>\n<\/tr>\n<tr>\n<td>\u5f00\u53d1\u73af\u5883<\/td>\n<td>Linux\/WSL2<\/td>\n<td>Mamba \u7f16\u8bd1\u9700\u8981<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>10.3 \u98ce\u9669\u4e0e\u7f13\u89e3<\/h4>\n<table>\n<tr>\u98ce\u9669\u6982\u7387\u5f71\u54cd\u7f13\u89e3\u63aa\u65bd<\/tr>\n<tbody>\n<tr>\n<td>Mamba \u7f16\u8bd1\u5931\u8d25<\/td>\n<td>\u4e2d<\/td>\n<td>\u963b\u585e\u5b9e\u9a8c<\/td>\n<td>\u4f7f\u7528 WSL2 \u6216 Docker&#xff0c;\u9884\u8bbe CUDA \u7248\u672c<\/td>\n<\/tr>\n<tr>\n<td>RWKV state API \u53d8\u5316<\/td>\n<td>\u4f4e<\/td>\n<td>\u65b9\u6cd5 C \u5931\u8d25<\/td>\n<td>\u9501\u5b9a rwkv&#061;&#061;0.8.27 \u7248\u672c<\/td>\n<\/tr>\n<tr>\n<td>\u6a21\u578b\u89c4\u6a21\u5dee\u5f02\u6c61\u67d3\u7ed3\u8bba<\/td>\n<td>\u4e2d<\/td>\n<td>\u7ed3\u8bba\u53ef\u4fe1\u5ea6\u4e0b\u964d<\/td>\n<td>\u7528 1.4B \u5bf9\u9f50\u7ec4&#xff0c;\u62a5\u544a\u4e2d\u58f0\u660e\u5c40\u9650<\/td>\n<\/tr>\n<tr>\n<td>\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u53ec\u56de\u7387\u8fc7\u4f4e<\/td>\n<td>\u4e2d<\/td>\n<td>\u03bb \u62df\u5408\u5931\u8d25<\/td>\n<td>\u589e\u52a0 n_trials&#xff0c;\u8c03\u6574\u63a2\u9488\u6807\u8bb0\u53ef\u8bc6\u522b\u6027<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u7b2c\u5341\u4e00\u7ae0 \u4e0e\u7eaf API \u65b9\u6848\u7684\u5173\u7cfb<\/h3>\n<h4>11.1 \u4e92\u8865\u5173\u7cfb<\/h4>\n<p>\u672c\u65b9\u6848\u662f L5\u5b9a\u5f8b\u53ef\u8bc1\u4f2a\u6027\u6d4b\u8bd5\u65b9\u6848_\u8d44\u6e90\u7ea6\u675f\u7248.md \u4e2d\u5b9e\u9a8c 4 \u7684\u5b8c\u6574\u5c55\u5f00&#xff1a;<\/p>\n<table>\n<tr>\u65b9\u6848\u8986\u76d6\u547d\u9898\u8d44\u6e90\u9700\u6c42\u89d2\u8272<\/tr>\n<tbody>\n<tr>\n<td>\u7eaf API \u65b9\u6848<\/td>\n<td>\u547d\u9898 A\/B\/C &#043; \u547d\u9898 D&#xff08;API \u66ff\u4ee3\u7248&#xff09;<\/td>\n<td>\u4ec5 API \u8c03\u7528<\/td>\n<td>\u6838\u5fc3\u547d\u9898\u9a8c\u8bc1<\/td>\n<\/tr>\n<tr>\n<td>\u672c\u65b9\u6848<\/td>\n<td>\u547d\u9898 D&#xff08;\u4e25\u683c\u8de8\u67b6\u6784\u9a8c\u8bc1&#xff09;<\/td>\n<td>\u672c\u5730 GPU<\/td>\n<td>\u666e\u9002\u6027\u51b3\u5b9a\u6027\u9a8c\u8bc1<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h4>11.2 \u6267\u884c\u987a\u5e8f\u5efa\u8bae<\/h4>\n<p>Phase 1&#xff1a;\u7eaf API \u65b9\u6848&#xff08;8 \u5468&#xff09;<br \/>\n\u251c\u2500\u2500 \u5b9e\u9a8c 1-3&#xff08;\u9a8c\u8bc1\u547d\u9898 A\/B\/C&#xff09;<br \/>\n\u251c\u2500\u2500 \u5b9e\u9a8c 4 API \u66ff\u4ee3\u7248&#xff08;\u521d\u6b65\u666e\u9002\u6027&#xff09;<br \/>\n\u2514\u2500\u2500 \u03bb \u9ed1\u76d2\u63a2\u6d4b\u6cd5<\/p>\n<p>Phase 2&#xff1a;\u672c\u65b9\u6848&#xff08;2 \u5468&#xff0c;\u53ef\u4e0e Phase 1 \u540e\u671f\u5e76\u884c&#xff09;<br \/>\n\u251c\u2500\u2500 \u5b9e\u9a8c 5.1&#xff1a;S_c \u8870\u51cf\u5bf9\u7167<br \/>\n\u251c\u2500\u2500 \u5b9e\u9a8c 5.2&#xff1a;\u03bb \u76f4\u63a5\u6d4b\u91cf<br \/>\n\u2514\u2500\u2500 \u5b9e\u9a8c 5.3&#xff1a;\u9ed1\u76d2\u63a2\u6d4b\u4ea4\u53c9\u9a8c\u8bc1<\/p>\n<p>Phase 3&#xff1a;\u7efc\u5408\u62a5\u544a&#xff08;1 \u5468&#xff09;<br \/>\n\u251c\u2500\u2500 \u6574\u5408\u7eaf API &#043; \u8de8\u67b6\u6784\u7ed3\u679c<br \/>\n\u251c\u2500\u2500 \u751f\u6210\u603b\u4f53\u5224\u5b9a<br \/>\n\u2514\u2500\u2500 \u64b0\u5199\u53ef\u53d1\u8868\u8bba\u6587<\/p>\n<hr \/>\n<h3>\u9644\u5f55 A&#xff1a;\u9879\u76ee\u76ee\u5f55\u7ed3\u6784<\/h3>\n<p>l5_cross_arch_experiment\/<br \/>\n\u251c\u2500\u2500 models\/<br \/>\n\u2502   \u251c\u2500\u2500 transformer\/          # Pythia-1.4B<br \/>\n\u2502   \u251c\u2500\u2500 mamba\/                # Mamba-2-1.3B<br \/>\n\u2502   \u2514\u2500\u2500 rwkv\/                 # RWKV-7-1B5<br \/>\n\u251c\u2500\u2500 src\/<br \/>\n\u2502   \u251c\u2500\u2500 unified_model_interface.py<br \/>\n\u2502   \u251c\u2500\u2500 decay_measurement\/<br \/>\n\u2502   \u2502   \u251c\u2500\u2500 __init__.py<br \/>\n\u2502   \u2502   \u251c\u2500\u2500 transformer_decay.py     # \u65b9\u6cd5 A<br \/>\n\u2502   \u2502   \u251c\u2500\u2500 mamba_decay.py           # \u65b9\u6cd5 B<br \/>\n\u2502   \u2502   \u251c\u2500\u2500 rwkv_decay.py            # \u65b9\u6cd5 C<br \/>\n\u2502   \u2502   \u2514\u2500\u2500 universal_blackbox.py    # \u65b9\u6cd5 D<br \/>\n\u2502   \u251c\u2500\u2500 sc_measurement.py<br \/>\n\u2502   \u251c\u2500\u2500 ground_truth_builder.py<br \/>\n\u2502   \u2514\u2500\u2500 run_cross_arch_experiment.py<br \/>\n\u251c\u2500\u2500 preregistration\/<br \/>\n\u2502   \u2514\u2500\u2500 preregistration.md     # \u9884\u6ce8\u518c\u627f\u8bfa\u4e66<br \/>\n\u251c\u2500\u2500 results\/<br \/>\n\u2502   \u251c\u2500\u2500 raw_data\/<br \/>\n\u2502   \u2514\u2500\u2500 l5_cross_arch_report.md<br \/>\n\u251c\u2500\u2500 requirements.txt<br \/>\n\u2514\u2500\u2500 README.md<\/p>\n<hr \/>\n<h3>\u9644\u5f55 B&#xff1a;\u9884\u6ce8\u518c\u627f\u8bfa\u4e66&#xff08;\u8de8\u67b6\u6784\u5b9e\u9a8c\u4e13\u7528&#xff09;<\/h3>\n<p># L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u9884\u6ce8\u518c\u627f\u8bfa\u4e66<\/p>\n<p>## \u5b9e\u9a8c\u4fe1\u606f<br \/>\n&#8211; \u5b9e\u9a8c\u5f00\u59cb\u65e5\u671f&#xff1a;____<br \/>\n&#8211; \u9884\u6ce8\u518c\u516c\u5f00\u5730\u5740&#xff1a;____<br \/>\n&#8211; \u7814\u7a76\u8005&#xff1a;____<br \/>\n&#8211; \u6a21\u578b\u7248\u672c&#xff1a;Pythia-1.4B \/ Mamba-2-1.3B \/ RWKV-7-1B5<\/p>\n<p>## \u8bc1\u4f2a\u6761\u4ef6&#xff08;\u5b9e\u9a8c\u524d\u56fa\u5b9a&#xff0c;\u4e0d\u53ef\u4fee\u6539&#xff09;<\/p>\n<p>1. L5 \u666e\u9002\u6027\u8bc1\u4f2a\u6761\u4ef6&#xff1a;<br \/>\n   \u82e5\u9ed1\u76d2\u63a2\u6d4b\u663e\u793a Mamba\/RWKV \u7684 S_c \u4e0d\u968f C \u4e0b\u964d&#xff08;Spearman p &gt; 0.05&#xff09;&#xff0c;<br \/>\n   \u4e14 Transformer \u663e\u8457\u4e0b\u964d&#xff08;p &lt; 0.05&#xff09;&#xff0c;<br \/>\n   \u5219 L5 \u7684\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u88ab\u8bc1\u4f2a&#xff0c;\u5e94\u964d\u7ea7\u4e3a&#034;Transformer \u7ecf\u9a8c\u89c4\u5f8b&#034;\u3002<\/p>\n<p>2. \u03bb \u4e00\u81f4\u6027\u8bc1\u4f2a\u6761\u4ef6&#xff1a;<br \/>\n   \u82e5\u4e09\u67b6\u6784\u9ed1\u76d2\u63a2\u6d4b \u03bb \u65b9\u5dee &gt; 0.01&#xff0c;<br \/>\n   \u5219 L5 \u666e\u9002\u4f46\u53c2\u6570\u9700\u6309\u67b6\u6784\u6807\u5b9a\u3002<\/p>\n<p>3. \u51fd\u6570\u5f62\u5f0f\u8bc1\u4f2a\u6761\u4ef6&#xff1a;<br \/>\n   \u82e5 Mamba\/RWKV \u7684\u8870\u51cf\u66f2\u7ebf\u66f4\u9002\u5408\u5e42\u5f8b\/\u5bf9\u6570\u800c\u975e\u6307\u6570&#xff08;AIC \u5dee\u5f02 \u2265 10&#xff09;&#xff0c;<br \/>\n   \u5219 L5 \u51fd\u6570\u5f62\u5f0f\u9700\u6309\u67b6\u6784\u4fee\u6b63\u3002<\/p>\n<p>## \u4e3b\u7ed3\u8bba\u4f9d\u636e<br \/>\n&#8211; \u4ee5\u65b9\u6cd5 D&#xff08;\u9ed1\u76d2\u63a2\u6d4b\u6cd5&#xff09;\u4e3a\u4e3b\u7ed3\u8bba\u4f9d\u636e<br \/>\n&#8211; \u65b9\u6cd5 A\/B\/C \u4e3a\u673a\u5236\u5c42\u9762\u8865\u5145\u8bc1\u636e<\/p>\n<p>## \u5b9e\u9a8c\u540e\u7981\u6b62\u884c\u4e3a<br \/>\n&#8211; [ ] \u4fee\u6539\u5224\u5b9a\u9608\u503c<br \/>\n&#8211; [ ] \u5254\u9664&#034;\u4e0d\u5229&#034;\u6570\u636e<br \/>\n&#8211; [ ] \u589e\u52a0\u4e8b\u540e\u5bf9\u7167<br \/>\n&#8211; [ ] \u91cd\u65b0\u5b9a\u4e49&#034;\u4e0d\u53ef\u4fe1&#034;<br \/>\n&#8211; [ ] \u4fee\u6539 Ground Truth<br \/>\n&#8211; [ ] \u5207\u6362\u4e3b\u7ed3\u8bba\u4f9d\u636e\u65b9\u6cd5<\/p>\n<p>## \u5b9e\u9a8c\u540e\u5141\u8bb8\u884c\u4e3a<br \/>\n&#8211; [ ] \u62a5\u544a\u6240\u6709\u7ed3\u679c&#xff08;\u5305\u62ec\u8bc1\u4f2a\u7ed3\u679c&#xff09;<br \/>\n&#8211; [ ] \u8fdb\u884c\u9884\u8bbe\u7684\u654f\u611f\u6027\u5206\u6790<br \/>\n&#8211; [ ] \u8865\u5145\u67b6\u6784\u7279\u5f02\u6027\u6d4b\u91cf\u7684\u673a\u5236\u89e3\u91ca<\/p>\n<p>## \u7b7e\u540d<br \/>\n\u7814\u7a76\u8005\u7b7e\u540d&#xff1a;____  \u65e5\u671f&#xff1a;____<\/p>\n<hr \/>\n<h3>\u53c2\u8003\u6587\u732e<\/h3>\n<li>Popper, K. (1959). The Logic of Scientific Discovery. Hutchinson. &#xff08;\u6ce2\u666e\u5c14\u53ef\u8bc1\u4f2a\u6027\u6807\u51c6&#xff09;<\/li>\n<li>Gu, A., &amp; Dao, T. (2023). Mamba: Linear-Time Sequence Modeling with Selective State Spaces. arXiv:2312.00752. &#xff08;Mamba \u67b6\u6784&#xff09;<\/li>\n<li>Dao, T., &amp; Gu, A. (2024). Transformers are SSMs: Generalized Models and Efficient Algorithms Through Structured State Space Duality. arXiv:2405.21060. &#xff08;Mamba-2 \u67b6\u6784&#xff09;<\/li>\n<li>Peng, B., et al. (2023). RWKV: Reinventing RNNs for the Transformer Era. arXiv:2305.13048. &#xff08;RWKV \u67b6\u6784&#xff09;<\/li>\n<li>Liu, Y., et al. (2023). Lost in the Middle: How Language Models Use Long Contexts. arXiv:2307.03172. &#xff08;\u9ed1\u76d2\u63a2\u6d4b\u6cd5\u8303\u5f0f&#xff09;<\/li>\n<li>Biderman, S., et al. (2023). Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling. arXiv:2304.01373. &#xff08;Pythia \u6a21\u578b&#xff09;<\/li>\n<li>Katharopoulos, A., et al. (2020). Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention. arXiv:2006.16236. &#xff08;\u7ebf\u6027\u6ce8\u610f\u529b\u5bf9\u7167&#xff09;<\/li>\n<hr \/>\n<p>\u6587\u6863\u7248\u672c&#xff1a;v1.0 \u521b\u5efa\u65e5\u671f&#xff1a;2026-08-03 \u4f5c\u8005&#xff1a;AI \u5185\u5bb9\u5de5\u4e1a\u5316\u7406\u8bba\u7ec4 \u72b6\u6001&#xff1a;\u53ef\u6267\u884c\u5b9e\u9a8c\u65b9\u6848&#xff0c;\u5f85\u9884\u6ce8\u518c\u540e\u6267\u884c \u5173\u8054\u6587\u6863&#xff1a;<\/p>\n<ul>\n<li>L5\u5b9a\u5f8b\u53ef\u8bc1\u4f2a\u6027\u6d4b\u8bd5\u65b9\u6848_\u8d44\u6e90\u7ea6\u675f\u7248.md&#xff08;\u7eaf API \u65b9\u6848&#xff0c;\u672c\u65b9\u6848\u4e3a\u5176\u5b9e\u9a8c 4 \u5c55\u5f00&#xff09;<\/li>\n<li>L5\u8870\u51cf\u5047\u8bf4\u6a21\u578b\u5b9e\u8bc1\u9a8c\u8bc1\u5b9e\u9a8c\u8bbe\u8ba1\u65b9\u6848.md<\/li>\n<li>\u4f53\u7cfb\u6210\u719f\u5ea6\u6df1\u5ea6\u63a8\u6f14_L5\u8870\u51cf\u81f3\u8de8\u6a21\u6001\u8bc1\u660e\u7684\u5b8c\u6574\u6f14\u7ece.md<\/li>\n<li>\u57fa\u4e8e\u56db\u4efd\u4ea4\u53c9\u6587\u6863\u7684\u5927\u6a21\u578b\u4eba\u673a\u534f\u540c\u4e94\u5b9a\u5f8b\u4f53\u7cfb\u5b8c\u6574\u8bba\u6587.txt<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848<br \/>\n\u2014\u2014Mamba \u4e0e RWKV \u975e Transformer \u67b6\u6784\u6d4b\u8bd5\u6b65\u9aa4 \u6587\u6863\u5b9a\u4f4d&#xff1a;\u672c\u65b9\u6848\u4e3a L5 \u8870\u51cf\u4e0d\u53ef\u6d88\u9664\u5b9a\u5f8b&#xff08;\u4e0b\u79f0\\&#8221;L5 \u5b9a\u5f8b\\&#8221;&#xff09;\u7684\u8de8\u67b6\u6784\u666e\u9002\u6027\u9a8c\u8bc1\u63d0\u4f9b\u5b8c\u6574\u53ef\u6267\u884c\u6b65\u9aa4\u3002L5 \u5b9a\u5f8b\u82e5\u4ec5\u5728 Transformer \u4e0a\u6210\u7acb&#xff0c;\u5219\u5e94\u964d\u7ea7\u4e3a\\&#8221;Transformer \u7ecf\u9a8c\u89c4\u5f8b\\&#8221;\u800c\u975e\\&#8221;\u666e\u904d\u5b9a\u5f8b\\&#8221;&#xff1b;\u82e5\u5728\u975e Transformer \u67b6\u6784\u4e0a\u540c\u6837\u6210\u7acb&#xff0c;\u5219\u5177\u5907\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u3002\u672c\u65b9\u6848\u662f<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[188],"topic":[],"class_list":["post-89913","post","type-post","status-publish","format-standard","hentry","category-server","tag-188"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title># L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/89913.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 \u2014\u2014Mamba \u4e0e RWKV \u975e Transformer \u67b6\u6784\u6d4b\u8bd5\u6b65\u9aa4 \u6587\u6863\u5b9a\u4f4d&#xff1a;\u672c\u65b9\u6848\u4e3a L5 \u8870\u51cf\u4e0d\u53ef\u6d88\u9664\u5b9a\u5f8b&#xff08;\u4e0b\u79f0&quot;L5 \u5b9a\u5f8b&quot;&#xff09;\u7684\u8de8\u67b6\u6784\u666e\u9002\u6027\u9a8c\u8bc1\u63d0\u4f9b\u5b8c\u6574\u53ef\u6267\u884c\u6b65\u9aa4\u3002L5 \u5b9a\u5f8b\u82e5\u4ec5\u5728 Transformer \u4e0a\u6210\u7acb&#xff0c;\u5219\u5e94\u964d\u7ea7\u4e3a&quot;Transformer \u7ecf\u9a8c\u89c4\u5f8b&quot;\u800c\u975e&quot;\u666e\u904d\u5b9a\u5f8b&quot;&#xff1b;\u82e5\u5728\u975e Transformer \u67b6\u6784\u4e0a\u540c\u6837\u6210\u7acb&#xff0c;\u5219\u5177\u5907\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u3002\u672c\u65b9\u6848\u662f\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/89913.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-08-03T22:25:03+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"23 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/89913.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/89913.html\",\"name\":\"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-08-03T22:25:03+00:00\",\"dateModified\":\"2026-08-03T22:25:03+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/89913.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/89913.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/89913.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/89913.html","og_locale":"zh_CN","og_type":"article","og_title":"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 \u2014\u2014Mamba \u4e0e RWKV \u975e Transformer \u67b6\u6784\u6d4b\u8bd5\u6b65\u9aa4 \u6587\u6863\u5b9a\u4f4d&#xff1a;\u672c\u65b9\u6848\u4e3a L5 \u8870\u51cf\u4e0d\u53ef\u6d88\u9664\u5b9a\u5f8b&#xff08;\u4e0b\u79f0\"L5 \u5b9a\u5f8b\"&#xff09;\u7684\u8de8\u67b6\u6784\u666e\u9002\u6027\u9a8c\u8bc1\u63d0\u4f9b\u5b8c\u6574\u53ef\u6267\u884c\u6b65\u9aa4\u3002L5 \u5b9a\u5f8b\u82e5\u4ec5\u5728 Transformer \u4e0a\u6210\u7acb&#xff0c;\u5219\u5e94\u964d\u7ea7\u4e3a\"Transformer \u7ecf\u9a8c\u89c4\u5f8b\"\u800c\u975e\"\u666e\u904d\u5b9a\u5f8b\"&#xff1b;\u82e5\u5728\u975e Transformer \u67b6\u6784\u4e0a\u540c\u6837\u6210\u7acb&#xff0c;\u5219\u5177\u5907\u666e\u904d\u5b9a\u5f8b\u5730\u4f4d\u3002\u672c\u65b9\u6848\u662f","og_url":"https:\/\/www.wsisp.com\/helps\/89913.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-08-03T22:25:03+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"23 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/89913.html","url":"https:\/\/www.wsisp.com\/helps\/89913.html","name":"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-08-03T22:25:03+00:00","dateModified":"2026-08-03T22:25:03+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/89913.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/89913.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/89913.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"# L5 \u5b9a\u5f8b\u8de8\u67b6\u6784\u9a8c\u8bc1\u5b9e\u9a8c\u65b9\u6848"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/89913","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=89913"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/89913\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=89913"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=89913"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=89913"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=89913"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}