{"id":117264,"date":"2026-10-23T00:16:53","date_gmt":"2026-10-22T16:16:53","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/117264.html"},"modified":"2026-10-23T00:16:53","modified_gmt":"2026-10-22T16:16:53","slug":"%e6%a8%a1%e5%9e%8b%e7%a8%b3%e5%ae%9a%e6%80%a7%e3%80%81z-loss%e3%80%81%e6%b3%a8%e6%84%8f%e5%8a%9b-softmax-%e7%a8%b3%e5%ae%9a%e5%8c%96%e4%b8%8e%e8%bd%af%e6%88%aa%e6%96%ad","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/117264.html","title":{"rendered":"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad"},"content":{"rendered":"<h2>\u6a21\u578b\u7684\u7a33\u5b9a\u6027\u662f\u4ec0\u4e48\u610f\u601d&#xff1f;<\/h2>\n<h3>\u6982\u8ff0<\/h3>\n<p>\u6a21\u578b\u7a33\u5b9a\u6027\u6307\u7684\u662f&#xff1a;\u8bad\u7ec3\u8fc7\u7a0b\u4e2d&#xff0c;\u635f\u5931\u548c\u68af\u5ea6\u4e0d\u8981\u51fa\u73b0\u5267\u70c8\u7684\u5c16\u5cf0\u6216\u7206\u70b8&#xff0c;\u80fd\u591f\u5e73\u7a33\u6536\u655b\u3002<\/p>\n<p>\u5728 LLM \u8bad\u7ec3\u4e2d&#xff0c;\u5e38\u89c1\u7684\u4e0d\u7a33\u5b9a\u73b0\u8c61\u5305\u62ec&#xff1a;<\/p>\n<ul>\n<li>\u635f\u5931\u66f2\u7ebf\u770b\u8d77\u6765\u6b63\u5e38&#xff0c;\u4f46\u68af\u5ea6\u8303\u6570&#xff08;gradient norm&#xff09;\u7a81\u7136\u51fa\u73b0\u5de8\u5927\u5c16\u5cf0&#xff1b;<\/li>\n<li>\u68af\u5ea6\u8303\u6570\u6301\u7eed\u589e\u5927&#xff0c;\u6700\u7ec8\u7206\u70b8&#xff0c;\u8bad\u7ec3\u5d29\u6e83&#xff1b;<\/li>\n<li>\u635f\u5931\u7a81\u7136\u98d9\u5347&#xff0c;\u6a21\u578b\u65e0\u6cd5\u6062\u590d\u3002<\/li>\n<\/ul>\n<p>OLMo2 \u8bba\u6587\u5c55\u793a\u8fc7\u4e00\u5f20\u56fe&#xff1a;\u84dd\u8272\u66f2\u7ebf\u7684\u68af\u5ea6 L2 \u8303\u6570\u5e03\u6ee1\u5c16\u5cf0&#xff0c;\u8303\u6570\u503c\u5b8c\u5168\u5931\u63a7&#xff1b;\u6a59\u8272\u66f2\u7ebf\u5219\u59cb\u7ec8\u4fdd\u6301\u5728\u4f4e\u4f4d&#xff0c;\u8fd9\u624d\u662f\u7406\u60f3\u72b6\u6001\u3002<\/p>\n<h3>\u54ea\u4e9b\u7ec4\u4ef6\u5bb9\u6613\u5f15\u53d1\u4e0d\u7a33\u5b9a&#xff1f;<\/h3>\n<p>Transformer \u4e2d\u4efb\u4f55\u90e8\u5206\u90fd\u53ef\u80fd\u51fa\u95ee\u9898&#xff0c;\u4f46softmax \u662f\u516c\u8ba4\u7684\u201c\u95ee\u9898\u513f\u7ae5\u201d&#xff1a;<\/p>\n<ul>\n<li>\u6307\u6570\u8fd0\u7b97 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">exe^x<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6644em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">x<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u53ef\u80fd\u4ea7\u751f\u6781\u5927\u503c&#xff1b;<\/li>\n<li>\u9664\u6cd5\u53ef\u80fd\u4ea7\u751f\u9664\u96f6\u6216\u6570\u503c\u6ea2\u51fa&#xff1b;<\/li>\n<li>\u4e24\u4e2a softmax \u6a21\u5757&#xff1a;\n<ul>\n<li>\u81ea\u6ce8\u610f\u529b softmax&#xff1a;logits <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">QKT\/dkQK^T\/\\\\sqrt{d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1072em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\">\/<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u53ef\u80fd\u8fc7\u5927&#xff0c;\u5bfc\u81f4 softmax \u9971\u548c\u3001\u68af\u5ea6\u6d88\u5931&#xff1b;<\/li>\n<li>\u8f93\u51fa\u5c42 softmax&#xff1a;logits \u53ef\u80fd\u7206\u70b8&#xff0c;\u4f7f\u4ea4\u53c9\u71b5\u635f\u5931\u4e0d\u7a33\u5b9a\u3002<\/li>\n<\/ul>\n<\/li>\n<\/ul>\n<p>\u4e0b\u9762\u5206\u522b\u8bb2\u4e09\u79cd\u7a33\u5b9a\u5316\u6280\u672f\u3002<\/p>\n<h2>z-Loss \u6280\u672f\u2014\u2014\u89e3\u51b3\u8f93\u51fa\u5c42\u7684 softmax<\/h2>\n<h3>\u6982\u8ff0<\/h3>\n<p>z-Loss \u662f\u4e00\u79cd\u6b63\u5219\u5316\u9879&#xff0c;\u52a0\u5230\u603b\u635f\u5931\u4e2d&#xff0c;\u76ee\u7684\u662f\u9632\u6b62 softmax \u7684\u5f52\u4e00\u5316\u56e0\u5b50 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ZZ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span> \u8fc7\u5927\u6216\u8fc7\u5c0f\u3002<\/p>\n<p>\u8f93\u51fa\u5c42 softmax&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">pi&#061;ezi\u2211jezj&#061;eziZ<br \/>\np_i &#061; \\\\frac{e^{z_i}}{\\\\sum_j e^{z_j}} &#061; \\\\frac{e^{z_i}}{Z}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4632em;vertical-align: -1.1218em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6065em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: -0.044em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: -0.044em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.0274em;vertical-align: -0.686em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: -0.044em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Z&#061;\u2211jezjZ &#061; \\\\sum_j e^{z_j}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1858em;vertical-align: -0.4358em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: -0.044em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0572em\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u662f\u5f52\u4e00\u5316\u56e0\u5b50&#xff08;partition function&#xff09;\u3002<\/p>\n<p>\u5f53\u67d0\u4e9b logits \u975e\u5e38\u5927\u65f6&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ZZ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span> \u4f1a\u7206\u70b8&#xff0c;\u5bfc\u81f4\u6570\u503c\u4e0d\u7a33\u5b9a\u3002<\/p>\n<p>z-Loss \u60e9\u7f5a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z\\\\log Z<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span> \u504f\u79bb 0&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Lz&#061;\u03bb\u22c5(log\u2061Z)2<br \/>\n\\\\mathcal{L}_z &#061; \\\\lambda \\\\cdot (\\\\log Z)^2<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u603b\u635f\u5931&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Ltotal&#061;LCE&#043;\u03bb\u22c5(log\u2061Z)2<br \/>\n\\\\mathcal{L}_{total} &#061; \\\\mathcal{L}_{CE} &#043; \\\\lambda \\\\cdot (\\\\log Z)^2<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mord mathnormal mtight\">o<\/span><span class=\"mord mathnormal mtight\">t<\/span><span class=\"mord mathnormal mtight\">a<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em\">C<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0576em\">E<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>PaLM \u4f7f\u7528 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb&#061;10\u22124\\\\lambda &#061; 10^{-4}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<\/p>\n<h3>\u6570\u503c\u4f8b\u5b50<\/h3>\n<p>\u8bbe\u8f93\u51fa\u5c42 logits \u4e3a&#xff1a;<\/p>\n<p>z &#061; [10, 5, 0]<\/p>\n<p>\u8ba1\u7b97 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ZZ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span>&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Z&#061;e10&#043;e5&#043;e0&#061;22026.47&#043;148.41&#043;1&#061;22175.88<br \/>\nZ &#061; e^{10} &#043; e^5 &#043; e^0 &#061; 22026.47 &#043; 148.41 &#043; 1 &#061; 22175.88<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">10<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">5<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8641em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">22026.47<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">148.41<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">22175.88<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8ba1\u7b97 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z\\\\log Z<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span>&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z&#061;log\u2061(22175.88)\u224810.007<br \/>\n\\\\log Z &#061; \\\\log(22175.88) \\\\approx 10.007<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">22175.88<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">10.007<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>z-Loss (<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\u03bb&#061;10\u22124\\\\lambda &#061; 10^{-4}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>)&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Lz&#061;10\u22124\u00d7(10.007)2&#061;10\u22124\u00d7100.14&#061;0.010014<br \/>\n\\\\mathcal{L}_z &#061; 10^{-4} \\\\times (10.007)^2 &#061; 10^{-4} \\\\times 100.14 &#061; 0.010014<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">10.007<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">100.14<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.010014<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5bf9\u6bd4\u53e6\u4e00\u7ec4\u8f83\u5c0f\u7684 logits&#xff1a;<\/p>\n<p>z &#061; [2, 1, 0]<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Z&#061;e2&#043;e1&#043;e0&#061;7.389&#043;2.718&#043;1&#061;11.107<br \/>\nZ &#061; e^2 &#043; e^1 &#043; e^0 &#061; 7.389 &#043; 2.718 &#043; 1 &#061; 11.107<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8641em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">0<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">7.389<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">2.718<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">11.107<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z&#061;log\u2061(11.107)\u22482.408<br \/>\n\\\\log Z &#061; \\\\log(11.107) \\\\approx 2.408<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">11.107<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">2.408<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Lz&#061;10\u22124\u00d7(2.408)2&#061;10\u22124\u00d75.798&#061;0.0005798<br \/>\n\\\\mathcal{L}_z &#061; 10^{-4} \\\\times (2.408)^2 &#061; 10^{-4} \\\\times 5.798 &#061; 0.0005798<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1141em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">2.408<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9474em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">4<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">5.798<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.0005798<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5bf9\u6bd4&#xff1a;<\/p>\n<table>\n<tr>logits<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ZZ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z\\\\log Z<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span>z-Loss<\/tr>\n<tbody>\n<tr>\n<td align=\"left\">[10, 5, 0]<\/td>\n<td align=\"left\">22175.88<\/td>\n<td align=\"left\">10.007<\/td>\n<td align=\"left\">0.010014<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">[2, 1, 0]<\/td>\n<td align=\"left\">11.107<\/td>\n<td align=\"left\">2.408<\/td>\n<td align=\"left\">0.0005798<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>logits \u8d8a\u5927&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">log\u2061Z\\\\log Z<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span> \u8d8a\u5927&#xff0c;z-Loss \u8d8a\u5927\u3002<\/p>\n<p>\u8bad\u7ec3\u65f6&#xff0c;\u8fd9\u4e2a\u60e9\u7f5a\u9879\u4f1a\u8feb\u4f7f\u6a21\u578b\u4e0d\u8981\u628a logits \u63a8\u5f97\u8fc7\u5927&#xff0c;\u4ece\u800c\u4fdd\u6301 softmax \u6570\u503c\u7a33\u5b9a\u3002<\/p>\n<h2>\u89e3\u51b3\u6ce8\u610f\u529b\u5c42\u7684 softmax \u6709\u4ec0\u4e48\u4f5c\u7528&#xff1f;\u2014\u2014QK \u5f52\u4e00\u5316<\/h2>\n<h3>\u6982\u8ff0<\/h3>\n<p>\u6807\u51c6\u6ce8\u610f\u529b&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">logits&#061;QKTdk<br \/>\n\\\\text{logits} &#061; \\\\frac{QK^T}{\\\\sqrt{d_k}}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord text\"><span class=\"mord\">logits<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4483em;vertical-align: -0.93em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7136\u540e softmax\u3002<\/p>\n<p>\u5982\u679c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">QQ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">Q<\/span><\/span><\/span><\/span><\/span>\u3001<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">KK<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><\/span><\/span> \u7684\u6570\u503c\u5f88\u5927&#xff0c;\u70b9\u79ef\u4f1a\u975e\u5e38\u5927&#xff0c;softmax \u4f1a\u53d8\u5f97\u6781\u7aef\u5c16\u9510&#xff08;\u63a5\u8fd1 one-hot&#xff09;&#xff0c;\u68af\u5ea6\u63a5\u8fd1 0&#xff0c;\u8bad\u7ec3\u56f0\u96be\u3002<\/p>\n<p>QK \u5f52\u4e00\u5316&#xff08;QK-Norm&#xff09;\u7684\u505a\u6cd5\u662f&#xff1a;\u5728\u8ba1\u7b97\u70b9\u79ef\u4e4b\u524d&#xff0c;\u5148\u5bf9 Q \u548c K \u5206\u522b\u505a LayerNorm\/RMSNorm&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Q\u2032&#061;Norm(Q),K\u2032&#061;Norm(K)<br \/>\nQ&#039; &#061; \\\\text{Norm}(Q), \\\\quad K&#039; &#061; \\\\text{Norm}(K)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9963em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0519em;vertical-align: -0.25em\"><\/span><span class=\"mord text\"><span class=\"mord\">Norm<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mclose\">)<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 1em\"><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord text\"><span class=\"mord\">Norm<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">logits&#061;Q\u2032K\u2032Tdk<br \/>\n\\\\text{logits} &#061; \\\\frac{Q&#039;K&#039;^T}{\\\\sqrt{d_k}}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8889em;vertical-align: -0.1944em\"><\/span><span class=\"mord text\"><span class=\"mord\">logits<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4483em;vertical-align: -0.93em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7519em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8fd9\u6837 Q\u3001K \u7684\u6570\u503c\u88ab\u6807\u51c6\u5316&#xff0c;\u70b9\u79ef\u4e0d\u4f1a\u5931\u63a7&#xff0c;softmax \u8f93\u5165\u4fdd\u6301\u5728\u5408\u7406\u8303\u56f4\u3002<\/p>\n<h3>\u6570\u503c\u4f8b\u5b50<\/h3>\n<p>\u8bbe <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">dk&#061;4d_k &#061; 4<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">4<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u67d0\u4e2a Query \u548c Key \u5411\u91cf\u4e3a&#xff1a;<\/p>\n<p>Q &#061; [10, 20, 30, 40]<br \/>\nK &#061; [5, 10, 15, 20]<\/p>\n<p>\u4e0d\u505a\u5f52\u4e00\u5316&#xff1a;<\/p>\n<p>\u70b9\u79ef&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Q\u22c5K&#061;10\u00d75&#043;20\u00d710&#043;30\u00d715&#043;40\u00d720&#061;50&#043;200&#043;450&#043;800&#061;1500<br \/>\nQ \\\\cdot K &#061; 10 \\\\times 5 &#043; 20 \\\\times 10 &#043; 30 \\\\times 15 &#043; 40 \\\\times 20 &#061; 50 &#043; 200 &#043; 450 &#043; 800 &#061; 1500<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">15<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">40<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">50<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">200<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">450<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">800<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1500<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7f29\u653e&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">1500\/4&#061;1500\/2&#061;750<br \/>\n1500\/\\\\sqrt{4} &#061; 1500\/2 &#061; 750<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.2061em;vertical-align: -0.25em\"><\/span><span class=\"mord\">1500\/<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9561em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\">4<\/span><\/span><\/span><span class=\"\" style=\"top: -2.9161em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.0839em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">1500\/2<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">750<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>softmax \u8f93\u5165\u4e3a 750&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">e750e^{750}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8141em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">750<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u76f4\u63a5\u6ea2\u51fa&#xff0c;\u6570\u503c\u4e0d\u7a33\u5b9a\u3002<\/p>\n<p>\u505a QK \u5f52\u4e00\u5316&#xff1a;<\/p>\n<p>\u5bf9 Q \u505a LayerNorm&#xff1a;<\/p>\n<ul>\n<li>\u5747\u503c&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(10&#043;20&#043;30&#043;40)\/4&#061;25(10 &#043; 20 &#043; 30 &#043; 40)\/4 &#061; 25<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">40<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">25<\/span><\/span><\/span><\/span><\/span><\/li>\n<li>\u6807\u51c6\u5dee&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">((10\u221225)2&#043;(20\u221225)2&#043;(30\u221225)2&#043;(40\u221225)2)\/4&#061;(225&#043;25&#043;25&#043;225)\/4&#061;125\u224811.18\\\\sqrt{((10 &#8211; 25)^2 &#043; (20 &#8211; 25)^2 &#043; (30 &#8211; 25)^2 &#043; (40 &#8211; 25)^2)\/4} &#061; \\\\sqrt{(225 &#043; 25 &#043; 25 &#043; 225)\/4} &#061; \\\\sqrt{125} \\\\approx 11.18<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.24em;vertical-align: -0.305em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.935em\"><span class=\"svg-align\" style=\"top: -3.2em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mopen\">((<\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">40<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><\/span><\/span><span class=\"\" style=\"top: -2.895em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 1.28em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.305em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.24em;vertical-align: -0.305em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.935em\"><span class=\"svg-align\" style=\"top: -3.2em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mopen\">(<\/span><span class=\"mord\">225<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">25<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">225<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><\/span><\/span><span class=\"\" style=\"top: -2.895em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 1.28em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.305em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\">125<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">11.18<\/span><\/span><\/span><\/span><\/span><\/li>\n<li>\u5f52\u4e00\u5316&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Q\u2032&#061;[(10\u221225)\/11.18,(20\u221225)\/11.18,(30\u221225)\/11.18,(40\u221225)\/11.18]<br \/>\nQ&#039; &#061; [(10 &#8211; 25)\/11.18, (20 &#8211; 25)\/11.18, (30 &#8211; 25)\/11.18, (40 &#8211; 25)\/11.18]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9963em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[(<\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/11.18<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/11.18<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/11.18<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">40<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">25<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/11.18<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">&#061;[\u22121.342,\u22120.447,0.447,1.342]<br \/>\n&#061; [-1.342, -0.447, 0.447, 1.342]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.3669em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">1.342<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.447<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">0.447<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<p>\u5bf9 K \u505a LayerNorm&#xff1a;<\/p>\n<ul>\n<li>\u5747\u503c&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(5&#043;10&#043;15&#043;20)\/4&#061;12.5(5 &#043; 10 &#043; 15 &#043; 20)\/4 &#061; 12.5<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">15<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">20<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">12.5<\/span><\/span><\/span><\/span><\/span><\/li>\n<li>\u6807\u51c6\u5dee&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">((5\u221212.5)2&#043;(10\u221212.5)2&#043;(15\u221212.5)2&#043;(20\u221212.5)2)\/4&#061;(56.25&#043;6.25&#043;6.25&#043;56.25)\/4&#061;31.25\u22485.590\\\\sqrt{((5 &#8211; 12.5)^2 &#043; (10 &#8211; 12.5)^2 &#043; (15 &#8211; 12.5)^2 &#043; (20 &#8211; 12.5)^2)\/4} &#061; \\\\sqrt{(56.25 &#043; 6.25 &#043; 6.25 &#043; 56.25)\/4} &#061; \\\\sqrt{31.25} \\\\approx 5.590<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.24em;vertical-align: -0.305em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.935em\"><span class=\"svg-align\" style=\"top: -3.2em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mopen\">((<\/span><span class=\"mord\">5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">15<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7401em\"><span class=\"\" style=\"top: -2.989em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><\/span><\/span><span class=\"\" style=\"top: -2.895em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 1.28em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.305em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.24em;vertical-align: -0.305em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.935em\"><span class=\"svg-align\" style=\"top: -3.2em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"mord\" style=\"padding-left: 1em\"><span class=\"mopen\">(<\/span><span class=\"mord\">56.25<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">6.25<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">6.25<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\">56.25<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/4<\/span><\/span><\/span><span class=\"\" style=\"top: -2.895em\"><span class=\"pstrut\" style=\"height: 3.2em\"><\/span><span class=\"hide-tail\" style=\"min-width: 1.02em;height: 1.28em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.305em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\">31.25<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">5.590<\/span><\/span><\/span><\/span><\/span><\/li>\n<li>\u5f52\u4e00\u5316&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">K\u2032&#061;[(5\u221212.5)\/5.590,(10\u221212.5)\/5.590,(15\u221212.5)\/5.590,(20\u221212.5)\/5.590]<br \/>\nK&#039; &#061; [(5 &#8211; 12.5)\/5.590, (10 &#8211; 12.5)\/5.590, (15 &#8211; 12.5)\/5.590, (20 &#8211; 12.5)\/5.590]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8019em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[(<\/span><span class=\"mord\">5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/5.590<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">10<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/5.590<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">15<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/5.590<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">20<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">12.5<\/span><span class=\"mclose\">)<\/span><span class=\"mord\">\/5.590<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">&#061;[\u22121.342,\u22120.447,0.447,1.342]<br \/>\n&#061; [-1.342, -0.447, 0.447, 1.342]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.3669em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">1.342<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.447<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">0.447<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span><\/span><\/li>\n<\/ul>\n<p>\u70b9\u79ef&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Q\u2032\u22c5K\u2032&#061;(\u22121.342)(\u22121.342)&#043;(\u22120.447)(\u22120.447)&#043;(0.447)(0.447)&#043;(1.342)(1.342)<br \/>\nQ&#039; \\\\cdot K&#039; &#061; (-1.342)(-1.342) &#043; (-0.447)(-0.447) &#043; (0.447)(0.447) &#043; (1.342)(1.342)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9963em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8019em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8019em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2032<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">)<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.447<\/span><span class=\"mclose\">)<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.447<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0.447<\/span><span class=\"mclose\">)<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0.447<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">)<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">1.342<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">&#061;1.801&#043;0.200&#043;0.200&#043;1.801&#061;4.002<br \/>\n&#061; 1.801 &#043; 0.200 &#043; 0.200 &#043; 1.801 &#061; 4.002<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.3669em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1.801<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">0.200<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">0.200<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1.801<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">4.002<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7f29\u653e&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">4.002\/2&#061;2.001<br \/>\n4.002\/2 &#061; 2.001<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">4.002\/2<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">2.001<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>softmax \u8f93\u5165\u4e3a 2.001&#xff0c;\u5b8c\u5168\u5728\u5408\u7406\u8303\u56f4\u5185\u3002<\/p>\n<p>\u5bf9\u6bd4&#xff1a;<\/p>\n<table>\n<tr>\u65b9\u6848\u70b9\u79ef\u7f29\u653e\u540e\u6570\u503c\u72b6\u6001<\/tr>\n<tbody>\n<tr>\n<td align=\"left\">\u4e0d\u5f52\u4e00\u5316<\/td>\n<td align=\"left\">1500<\/td>\n<td align=\"left\">750<\/td>\n<td align=\"left\">\u6ea2\u51fa&#xff0c;\u4e0d\u7a33\u5b9a<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">QK \u5f52\u4e00\u5316<\/td>\n<td align=\"left\">4.002<\/td>\n<td align=\"left\">2.001<\/td>\n<td align=\"left\">\u7a33\u5b9a<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h3>\u4f5c\u7528<\/h3>\n<ul>\n<li>\u63a7\u5236 softmax \u8f93\u5165\u7684\u6570\u503c\u8303\u56f4&#xff0c;\u9632\u6b62\u9971\u548c\u548c\u68af\u5ea6\u6d88\u5931&#xff1b;<\/li>\n<li>\u5141\u8bb8\u4f7f\u7528\u66f4\u6fc0\u8fdb\u7684\u5b66\u4e60\u7387&#xff0c;\u4f18\u5316\u5668\u53d1\u6325\u66f4\u5927\u4f5c\u7528&#xff1b;<\/li>\n<li>Gemma2\u3001DCLM\u3001OLMo2\u3001Chameleon \u7b49\u6a21\u578b\u91c7\u7528&#xff1b;<\/li>\n<li>NVIDIA \u56e2\u961f\u5b9e\u9a8c&#xff1a;QK \u5f52\u4e00\u5316\u80fd\u63d0\u5347\u6548\u679c&#xff0c;\u800c\u8f6f\u622a\u65ad\u53cd\u800c\u4f7f\u56f0\u60d1\u5ea6\u53d8\u5dee&#xff08;11.19 \u2192 \u66f4\u5dee&#xff09;\u3002<\/li>\n<\/ul>\n<h2>\u5bf9\u8f93\u5165 softmax \u7684 logits \u8fdb\u884c\u8f6f\u622a\u65ad<\/h2>\n<h3>\u6982\u8ff0<\/h3>\n<p>\u8f6f\u622a\u65ad&#xff08;Soft-clipping \/ Logit-capping&#xff09;\u5728 softmax \u4e4b\u524d&#xff0c;\u7528 tanh \u628a logits \u538b\u7f29\u5230\u6709\u754c\u8303\u56f4&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">logitsclipped&#061;cap\u22c5tanh\u2061(logitscap)<br \/>\n\\\\text{logits}_{\\\\text{clipped}} &#061; \\\\text{cap} \\\\cdot \\\\tanh\\\\left(\\\\frac{\\\\text{logits}}{\\\\text{cap}}\\\\right)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0747em;vertical-align: -0.3802em\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">logits<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.242em\"><span class=\"\" style=\"top: -2.4559em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">clipped<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3802em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6389em;vertical-align: -0.1944em\"><\/span><span class=\"mord text\"><span class=\"mord\">cap<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4em;vertical-align: -0.95em\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3714em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">cap<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">logits<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8804em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d cap \u662f\u9608\u503c&#xff0c;Gemma2\/OLMo2 \u4f7f\u7528 cap &#061; 30.0\u3002<\/p>\n<p>\u5f53 logits \u8fdc\u5927\u4e8e cap \u65f6&#xff0c;tanh \u9971\u548c\u5230 1&#xff0c;\u8f93\u51fa\u63a5\u8fd1 cap&#xff1b;\u5f53 logits \u5f88\u5c0f\u65f6&#xff0c;tanh \u8fd1\u4f3c\u7ebf\u6027&#xff0c;\u51e0\u4e4e\u4e0d\u6539\u53d8\u3002<\/p>\n<h3>\u6570\u503c\u4f8b\u5b50<\/h3>\n<p>\u8bbe cap &#061; 30\u3002<\/p>\n<p>logits &#061; 100&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">100\/30&#061;3.333<br \/>\n100\/30 &#061; 3.333<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">100\/30<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">3.333<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">tanh\u2061(3.333)\u22480.9974<br \/>\n\\\\tanh(3.333) \\\\approx 0.9974<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">3.333<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.9974<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">30\u00d70.9974&#061;29.92<br \/>\n30 \\\\times 0.9974 &#061; 29.92<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.9974<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">29.92<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u539f\u672c 100 \u88ab\u538b\u7f29\u5230 29.92\u3002<\/p>\n<p>logits &#061; 10&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">10\/30&#061;0.333<br \/>\n10\/30 &#061; 0.333<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">10\/30<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.333<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">tanh\u2061(0.333)\u22480.3215<br \/>\n\\\\tanh(0.333) \\\\approx 0.3215<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0.333<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.3215<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">30\u00d70.3215&#061;9.645<br \/>\n30 \\\\times 0.3215 &#061; 9.645<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0.3215<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">9.645<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u539f\u672c 10 \u88ab\u538b\u7f29\u5230 9.645&#xff0c;\u53d8\u5316\u5f88\u5c0f\u3002<\/p>\n<p>logits &#061; 0&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">tanh\u2061(0)&#061;0\\\\tanh(0) &#061; 0<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">30\u00d70&#061;030 \\\\times 0 &#061; 0<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u4e0d\u53d8\u3002<\/p>\n<p>logits &#061; -100&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">\u2212100\/30&#061;\u22123.333-100\/30 &#061; -3.333<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">100\/30<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">3.333<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">tanh\u2061(\u22123.333)\u2248\u22120.9974\\\\tanh(-3.333) \\\\approx -0.9974<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">tanh<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">3.333<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.9974<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">30\u00d7(\u22120.9974)&#061;\u221229.9230 \\\\times (-0.9974) &#061; -29.92<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">30<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">0.9974<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">29.92<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u88ab\u538b\u7f29\u5230 -29.92\u3002<\/p>\n<p>\u5bf9\u6bd4\u8868&#xff1a;<\/p>\n<table>\n<tr>\u539f\u59cb logits\u538b\u7f29\u540e<\/tr>\n<tbody>\n<tr>\n<td align=\"left\">100<\/td>\n<td align=\"left\">29.92<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">50<\/td>\n<td align=\"left\">29.09<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">10<\/td>\n<td align=\"left\">9.645<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">0<\/td>\n<td align=\"left\">0<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">-10<\/td>\n<td align=\"left\">-9.645<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">-100<\/td>\n<td align=\"left\">-29.92<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h3>\u4f5c\u7528<\/h3>\n<ul>\n<li>\u9632\u6b62\u6781\u7aef logits \u5bfc\u81f4 softmax \u9971\u548c&#xff1b;<\/li>\n<li>\u4fdd\u6301\u68af\u5ea6\u6709\u6548&#xff1b;<\/li>\n<li>Gemma2\u3001OLMo2 \u91c7\u7528&#xff1b;<\/li>\n<li>\u4f46 NVIDIA \u56e2\u961f\u5b9e\u9a8c\u53d1\u73b0&#xff1a;\u57fa\u7ebf\u56f0\u60d1\u5ea6 11.19&#xff0c;\u8f6f\u622a\u65ad\u53cd\u800c\u4f7f\u6548\u679c\u53d8\u5dee&#xff1b;QK \u5f52\u4e00\u5316\u5374\u80fd\u63d0\u5347\u6548\u679c\u3002<\/li>\n<\/ul>\n<p>\u4e3a\u4ec0\u4e48\u8f6f\u622a\u65ad\u53ef\u80fd\u53d8\u5dee&#xff1f; \u56e0\u4e3a tanh \u5728\u9971\u548c\u533a\u68af\u5ea6\u63a5\u8fd1 0&#xff0c;\u53cd\u800c\u524a\u5f31\u4e86\u6a21\u578b\u5b66\u4e60\u6781\u7aef\u91cd\u8981 token \u7684\u80fd\u529b&#xff1b;<\/p>\n<p>\u800c QK \u5f52\u4e00\u5316\u662f\u4ece\u6e90\u5934\u63a7\u5236\u6570\u503c&#xff0c;\u4fdd\u7559\u4e86\u7ebf\u6027\u533a\u7684\u68af\u5ea6\u3002<\/p>\n<h2>\u4e09\u79cd\u6280\u672f\u7684\u5bf9\u6bd4<\/h2>\n<table>\n<tr>\u6280\u672f\u4f5c\u7528\u4f4d\u7f6e\u6838\u5fc3\u64cd\u4f5c\u6570\u503c\u4f8b\u5b50\u6548\u679c<\/tr>\n<tbody>\n<tr>\n<td align=\"left\">z-Loss<\/td>\n<td align=\"left\">\u8f93\u51fa\u5c42 softmax<\/td>\n<td align=\"left\">\u60e9\u7f5a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(log\u2061Z)2(\\\\log Z)^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/td>\n<td align=\"left\">logits&#061;[10,5,0] \u2192 z-Loss&#061;0.010<\/td>\n<td align=\"left\">PaLM \u7b49\u91c7\u7528&#xff0c;\u7a33\u5b9a\u635f\u5931<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">QK \u5f52\u4e00\u5316<\/td>\n<td align=\"left\">\u6ce8\u610f\u529b softmax<\/td>\n<td align=\"left\">\u5bf9 Q\u3001K \u505a LayerNorm<\/td>\n<td align=\"left\">\u70b9\u79ef 1500 \u2192 4.002<\/td>\n<td align=\"left\">Gemma2\/OLMo2 \u91c7\u7528&#xff0c;\u5141\u8bb8\u66f4\u5927\u5b66\u4e60\u7387<\/td>\n<\/tr>\n<tr>\n<td align=\"left\">\u8f6f\u622a\u65ad<\/td>\n<td align=\"left\">\u6ce8\u610f\u529b softmax<\/td>\n<td align=\"left\">cap \u00b7 tanh(logits\/cap)<\/td>\n<td align=\"left\">logits&#061;100 \u2192 29.92<\/td>\n<td align=\"left\">Gemma2\/OLMo2 \u91c7\u7528&#xff0c;\u4f46 NVIDIA \u5b9e\u9a8c\u663e\u793a\u53ef\u80fd\u53d8\u5dee<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<h2>\u603b\u7ed3<\/h2>\n<p>\u6a21\u578b\u7a33\u5b9a\u6027&#xff1a;\u8bad\u7ec3\u4e2d\u635f\u5931\u548c\u68af\u5ea6\u4e0d\u51fa\u73b0\u5c16\u5cf0\u3001\u7206\u70b8&#xff0c;\u80fd\u5e73\u7a33\u6536\u655b\u3002softmax \u662f\u6700\u5bb9\u6613\u51fa\u95ee\u9898\u7684\u7ec4\u4ef6\u3002<\/p>\n<p>z-Loss&#xff1a;\u5728\u8f93\u51fa\u5c42 softmax \u4e0a&#xff0c;\u60e9\u7f5a\u5f52\u4e00\u5316\u56e0\u5b50 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ZZ<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><\/span><\/span><\/span><\/span> \u8fc7\u5927&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Lz&#061;\u03bb(log\u2061Z)2\\\\mathcal{L}_z &#061; \\\\lambda(\\\\log Z)^2<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathcal\">L<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1514em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.044em\">z<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">\u03bb<\/span><span class=\"mopen\">(<\/span><span class=\"mop\">lo<span style=\"margin-right: 0.0139em\">g<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">Z<\/span><span class=\"mclose\"><span class=\"mclose\">)<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u6570\u503c\u4e0a&#xff0c;logits \u4ece [2,1,0] \u589e\u5927\u5230 [10,5,0]&#xff0c;z-Loss \u4ece 0.00058 \u589e\u5230 0.010&#xff0c;\u8feb\u4f7f\u6a21\u578b\u4fdd\u6301 logits \u6e29\u548c\u3002<\/p>\n<p>QK \u5f52\u4e00\u5316&#xff1a;\u5728\u6ce8\u610f\u529b softmax \u524d\u5bf9 Q\u3001K \u505a LayerNorm&#xff0c;\u628a\u70b9\u79ef\u4ece 1500 \u538b\u5230 4.002&#xff0c;\u907f\u514d softmax \u9971\u548c&#xff0c;\u5141\u8bb8\u66f4\u5927\u5b66\u4e60\u7387\u3002<\/p>\n<p>\u8f6f\u622a\u65ad&#xff1a;\u7528 cap \u00b7 tanh(logits\/cap) \u628a logits \u538b\u5230 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">(\u2212cap,cap)(-cap, cap)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;100 \u53d8 29.92\u3002\u80fd\u9632\u6b62\u6781\u7aef\u503c&#xff0c;\u4f46\u53ef\u80fd\u524a\u5f31\u6781\u7aef\u91cd\u8981 token \u7684\u5b66\u4e60&#xff0c;NVIDIA \u5b9e\u9a8c\u663e\u793a\u6548\u679c\u53ef\u80fd\u53d8\u5dee\u3002<\/p>\n<p>\u4e00\u53e5\u8bdd&#xff1a;\u7a33\u5b9a\u6027\u6280\u672f\u90fd\u662f\u56f4\u7ed5 softmax \u505a\u6587\u7ae0\u2014\u2014z-Loss \u7ba1\u8f93\u51fa\u5c42\u5f52\u4e00\u5316\u56e0\u5b50&#xff0c;QK \u5f52\u4e00\u5316\u7ba1\u6ce8\u610f\u529b\u8f93\u5165\u8303\u56f4&#xff0c;\u8f6f\u622a\u65ad\u7ba1 logits \u6781\u7aef\u503c&#xff1b;\u4e09\u8005\u5404\u6709\u9002\u7528\u573a\u666f&#xff0c;QK \u5f52\u4e00\u5316\u76ee\u524d\u8bc1\u636e\u6700\u79ef\u6781\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u6a21\u578b\u7684\u7a33\u5b9a\u6027\u662f\u4ec0\u4e48\u610f\u601d&#xff1f;<br \/>\n\u6982\u8ff0<br \/>\n\u6a21\u578b\u7a33\u5b9a\u6027\u6307\u7684\u662f&#xff1a;\u8bad\u7ec3\u8fc7\u7a0b\u4e2d&#xff0c;\u635f\u5931\u548c\u68af\u5ea6\u4e0d\u8981\u51fa\u73b0\u5267\u70c8\u7684\u5c16\u5cf0\u6216\u7206\u70b8&#xff0c;\u80fd\u591f\u5e73\u7a33\u6536\u655b\u3002<br \/>\n\u5728 LLM \u8bad\u7ec3\u4e2d&#xff0c;\u5e38\u89c1\u7684\u4e0d\u7a33\u5b9a\u73b0\u8c61\u5305\u62ec&#xff1a;<br \/>\n\u635f\u5931\u66f2\u7ebf\u770b\u8d77\u6765\u6b63\u5e38&#xff0c;\u4f46\u68af\u5ea6\u8303\u6570&#xff08;gradient norm&#xff09;\u7a81\u7136\u51fa\u73b0\u5de8\u5927\u5c16\u5cf0&#xff1b;\u68af\u5ea6\u8303\u6570\u6301\u7eed\u589e\u5927&#xff0c;\u6700\u7ec8\u7206\u70b8&#xff0c;\u8bad\u7ec3\u5d29\u6e83&#xff1b;\u635f\u5931\u7a81\u7136\u98d9\u5347&amp;#xff0<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[50,100,207,86,427],"topic":[],"class_list":["post-117264","post","type-post","status-publish","format-standard","hentry","category-server","tag-50","tag-100","tag-207","tag-86","tag-427"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/117264.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u6a21\u578b\u7684\u7a33\u5b9a\u6027\u662f\u4ec0\u4e48\u610f\u601d&#xff1f; \u6982\u8ff0 \u6a21\u578b\u7a33\u5b9a\u6027\u6307\u7684\u662f&#xff1a;\u8bad\u7ec3\u8fc7\u7a0b\u4e2d&#xff0c;\u635f\u5931\u548c\u68af\u5ea6\u4e0d\u8981\u51fa\u73b0\u5267\u70c8\u7684\u5c16\u5cf0\u6216\u7206\u70b8&#xff0c;\u80fd\u591f\u5e73\u7a33\u6536\u655b\u3002 \u5728 LLM \u8bad\u7ec3\u4e2d&#xff0c;\u5e38\u89c1\u7684\u4e0d\u7a33\u5b9a\u73b0\u8c61\u5305\u62ec&#xff1a; \u635f\u5931\u66f2\u7ebf\u770b\u8d77\u6765\u6b63\u5e38&#xff0c;\u4f46\u68af\u5ea6\u8303\u6570&#xff08;gradient norm&#xff09;\u7a81\u7136\u51fa\u73b0\u5de8\u5927\u5c16\u5cf0&#xff1b;\u68af\u5ea6\u8303\u6570\u6301\u7eed\u589e\u5927&#xff0c;\u6700\u7ec8\u7206\u70b8&#xff0c;\u8bad\u7ec3\u5d29\u6e83&#xff1b;\u635f\u5931\u7a81\u7136\u98d9\u5347&amp;#xff0\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/117264.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-22T16:16:53+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"4 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/117264.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/117264.html\",\"name\":\"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-10-22T16:16:53+00:00\",\"dateModified\":\"2026-10-22T16:16:53+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/117264.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/117264.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/117264.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/117264.html","og_locale":"zh_CN","og_type":"article","og_title":"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u6a21\u578b\u7684\u7a33\u5b9a\u6027\u662f\u4ec0\u4e48\u610f\u601d&#xff1f; \u6982\u8ff0 \u6a21\u578b\u7a33\u5b9a\u6027\u6307\u7684\u662f&#xff1a;\u8bad\u7ec3\u8fc7\u7a0b\u4e2d&#xff0c;\u635f\u5931\u548c\u68af\u5ea6\u4e0d\u8981\u51fa\u73b0\u5267\u70c8\u7684\u5c16\u5cf0\u6216\u7206\u70b8&#xff0c;\u80fd\u591f\u5e73\u7a33\u6536\u655b\u3002 \u5728 LLM \u8bad\u7ec3\u4e2d&#xff0c;\u5e38\u89c1\u7684\u4e0d\u7a33\u5b9a\u73b0\u8c61\u5305\u62ec&#xff1a; \u635f\u5931\u66f2\u7ebf\u770b\u8d77\u6765\u6b63\u5e38&#xff0c;\u4f46\u68af\u5ea6\u8303\u6570&#xff08;gradient norm&#xff09;\u7a81\u7136\u51fa\u73b0\u5de8\u5927\u5c16\u5cf0&#xff1b;\u68af\u5ea6\u8303\u6570\u6301\u7eed\u589e\u5927&#xff0c;\u6700\u7ec8\u7206\u70b8&#xff0c;\u8bad\u7ec3\u5d29\u6e83&#xff1b;\u635f\u5931\u7a81\u7136\u98d9\u5347&amp;#xff0","og_url":"https:\/\/www.wsisp.com\/helps\/117264.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-10-22T16:16:53+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"4 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/117264.html","url":"https:\/\/www.wsisp.com\/helps\/117264.html","name":"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-10-22T16:16:53+00:00","dateModified":"2026-10-22T16:16:53+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/117264.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/117264.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/117264.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u6a21\u578b\u7a33\u5b9a\u6027\u3001z-Loss\u3001\u6ce8\u610f\u529b softmax \u7a33\u5b9a\u5316\u4e0e\u8f6f\u622a\u65ad"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/117264","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=117264"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/117264\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=117264"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=117264"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=117264"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=117264"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}