{"id":78054,"date":"2026-02-26T04:17:17","date_gmt":"2026-02-25T20:17:17","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/78054.html"},"modified":"2026-02-26T04:17:17","modified_gmt":"2026-02-25T20:17:17","slug":"%e6%96%b0%e5%b9%b4%e4%b8%80%e5%ae%9a%e8%a6%81%e5%ad%a6%e4%bc%9aflashattention-flashattention%e6%9e%81%e7%ae%80%e6%95%99%e7%a8%8b%e4%b8%8e%e5%ae%9e%e7%8e%b0","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/78054.html","title":{"rendered":"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0"},"content":{"rendered":"<h2>FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0<\/h2>\n<h3>\u80cc\u666f<\/h3>\n<p>FlashAttention\u662f\u76ee\u524d\u6700\u6d41\u884c\u7684\u4f18\u5316Transformer\u4e2d\u6ce8\u610f\u529b\u673a\u5236\u7684\u7b97\u6cd5&#xff0c;\u65e8\u5728\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u548c\u51cf\u5c11\u5185\u5b58\u4f7f\u7528\u3002\u4e0e\u5176\u4ed6\u4f18\u5316Attention\u7b97\u6cd5\u76f8\u6bd4&#xff0c;FlashAttention\u662f\u4e00\u79cd\u7cbe\u786e\u8ba1\u7b97Attention\u7684\u7b97\u6cd5&#xff0c;\u800c\u4e0d\u662f\u8fd1\u4f3c\u8ba1\u7b97\u3002\u5b83\u901a\u8fc7\u91cd\u65b0\u7ec4\u7ec7\u8ba1\u7b97\u6d41\u7a0b&#xff0c;\u8fdb\u800c\u51cf\u5c11\u4e86GPU\u4e2d\u5168\u5c40\u5185\u5b58\u7684\u8bbf\u95ee\u6b21\u6570&#xff0c;\u4ece\u800c\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u3002<br \/>\n\u672c\u6587\u8bd5\u56fe\u4ee5\u6700\u7b80\u5355\u7684\u65b9\u5f0f\u4ecb\u7ecd&#xff0c;\u5b9e\u73b0\u4e00\u4e2a\u6700\u7b80\u5355\u7684Flashattention\u7b97\u6cd5(\u4e0d\u8003\u8651\u591a\u5934\u6ce8\u610f\u529b)&#xff0c;\u5e76\u4e14\u53ea\u6d89\u53ca\u7b80\u5355\u7684Python\u4ee3\u7801&#xff0c;\u5f97\u76ca\u4e8etriton&#xff0c;\u57fa\u4e8ePytorch\/Numpy\u7684\u5b9e\u73b0\u53ef\u4ee5\u5f88\u5feb\u5730\u4f7f\u7528triton\u91cd\u5199&#xff0c;\u4ee5\u4fbf\u771f\u6b63\u5730\u5728GPU\u4e0a\u5e76\u884c\u8fd0\u884c&#xff0c;\u4f60\u53ef\u4ee5\u9009\u62e9\u5148\u770b\u672c\u6587&#xff0c;\u518d\u53bb\u770bFlashAttention\u7684\u539f\u59cb\u8bba\u6587&#xff0c;\u6211\u4f1a\u5047\u8bbe\u8bfb\u8005\u6ca1\u6709\u8bfb\u8fc7FlashAttention\u7684\u8bba\u6587&#xff0c;\u5728\u7b26\u53f7\u7684\u9009\u62e9\u4e0a\u5c3d\u53ef\u80fd\u8d34\u8fd1\u539f\u8bba\u6587&#xff0c;\u4ee5\u4fbf\u8bfb\u8005\u5728\u4e24\u8005\u4e4b\u95f4\u4e0d\u4f1a\u4ea7\u751f\u6df7\u6dc6\u3002<\/p>\n<h3>\u7b26\u53f7\u7ea6\u5b9a&#xff1a;<\/h3>\n<ul>\n<li>\n<p>Q, K, V, O: Query, Key, Value, Ouput\u77e9\u9635&#xff0c;\u5f62\u72b6\u4e3a(batch_size, seq_len, d_model) \u5b9e\u9645\u4e0aK\u7684seq_len\u53ef\u4ee5\u548cQ\u7684seq_len\u4e0d\u4e00\u81f4&#xff0c;\u4f46\u662f\u4e3a\u4e86\u7b80\u5316&#xff0c;\u6211\u4eec\u5047\u8bbe\u5b83\u4eec\u4e00\u81f4\u3002<\/p>\n<\/li>\n<li>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mk,im_{k,i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7167em;vertical-align: -0.2861em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>: \u7b2ck\u884c\u524di\u4e2a\u5143\u7d20\u7684\u6700\u5927\u503c\u3002<\/p>\n<\/li>\n<li>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lk,il_{k,i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9805em;vertical-align: -0.2861em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>: \u7b2ck\u884c\u524di\u4e2a\u5143\u7d20\u7684\u6307\u6570\u7684\u548c\u3002<\/p>\n<\/li>\n<li>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Sk,jS_{k,j}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9694em;vertical-align: -0.2861em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><span class=\"mpunct mtight\">,<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>: \u7b2ck\u4e2a\u67e5\u8be2\u548c\u7b2cj\u4e2a\u952e\u4e58\u79ef\u5f97\u5230\u7684\u5206\u6570\u3002<\/p>\n<\/li>\n<li>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oko_{k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>: \u7b2ck\u4e2a\u67e5\u8be2\u5f97\u5230\u7684\u8f93\u51fa\u3002<\/p>\n<\/li>\n<\/ul>\n<h3>(Safe) Softmax:3-pass<\/h3>\n<p>\u6211\u4eec\u4ee5Softmax\u4f5c\u4e3a\u4e00\u4e2a\u5f15\u5b50\u6765\u4ecb\u7ecdFlashAttention\u3002<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">Softmax(x)&#061;[ex1\u2211jexj,ex2\u2211jexj,&#8230;,exn\u2211jexj]<br \/>\nSoftmax(x) &#061; [\\\\frac{e^{x_1}}{\\\\sum_{j}e^{x_j}}, \\\\frac{e^{x_2}}{\\\\sum_{j}e^{x_j}} , &#8230; , \\\\frac{e^{x_n}}{\\\\sum_{j}e^{x_j}}]<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"mord mathnormal\">o<\/span><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4632em;vertical-align: -1.1218em\"><\/span><span class=\"mopen\">[<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6065em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3173em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6065em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3173em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">&#8230;<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6065em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1645em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u5b9e\u9645\u4e0a\u4e3a\u4e86\u6570\u503c\u7a33\u5b9a&#xff0c;\u6211\u4eec\u901a\u5e38\u4f1a\u5728Softmax\u4e2d\u5f15\u5165\u4e00\u4e2a\u504f\u79fb\u91cf&#xff0c;\u901a\u5e38\u662f\u8f93\u5165\u7684\u6700\u5927\u503c&#xff0c;\u8fd9\u6837\u53ef\u4ee5\u907f\u514d\u6307\u6570\u51fd\u6570\u7684\u6ea2\u51fa\u95ee\u9898&#xff0c;\u5e76\u4e14\u5728\u6570\u5b66\u7684\u8ba1\u7b97\u4e0a\u662f\u7b49\u4ef7\u7684&#xff0c;\u56e0\u4e3a&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">exi\u2211jexj&#061;exi\u2212m\u2211jexj\u2212m&#xff0c;m&#061;maxj{xj}<br \/>\n\\\\frac{e^{x_i}}{\\\\sum_{j}e^{x_j}} &#061; \\\\frac{e^{x_i &#8211; m}}{\\\\sum_{j}e^{x_j &#8211; m}}&#xff0c; m &#061; max_{j}\\\\{x_j\\\\}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 2.4632em;vertical-align: -1.1218em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6065em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.5701em;vertical-align: -1.1218em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4483em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.162em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4358em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7134em\"><span class=\"\" style=\"top: -3.0051em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7713em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1218em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mord cjk_fallback\">&#xff0c;<\/span><span class=\"mord mathnormal\">m<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.0361em;vertical-align: -0.2861em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">{<\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">}<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u6240\u4ee5\u6700\u7b80\u5355\u7684\u5b9e\u73b0\u65b9\u5f0f\u5c31\u662f&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">softmax<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">:<\/span> nn<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">,<\/span> o<span class=\"token punctuation\">:<\/span> nn<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  <span class=\"token comment\">## x: (seq_len, d_k)<\/span><br \/>\n  seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> x<span class=\"token punctuation\">.<\/span>shape<br \/>\n  <span class=\"token keyword\">for<\/span> b <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># \u5bf9\u4e8e\u6bcf\u4e2atoken<\/span><br \/>\n    m <span class=\"token operator\">&#061;<\/span> <span class=\"token operator\">&#8211;<\/span>torch<span class=\"token punctuation\">.<\/span>inf<br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      m <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    l <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros_like<span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># \u521d\u59cb\u5316\u6307\u6570\u548c<\/span><br \/>\n    exp_x <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros_like<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      exp_x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> m<span class=\"token punctuation\">)<\/span><br \/>\n      l <span class=\"token operator\">&#043;&#061;<\/span> exp_x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># \u66f4\u65b0\u6307\u6570\u548c<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      a<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> exp_x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">\/<\/span> l <span class=\"token comment\"># \u8ba1\u7b97softmax\u8f93\u51fa<\/span><br \/>\n  <span class=\"token keyword\">return<\/span> a<\/p>\n<p>\u4e0a\u9762\u7528\u4e86\u4e09\u4e2a\u5faa\u73af\u6765\u5b9e\u73b0Softmax&#xff0c;\u56e0\u6b64\u53eb\u505a3-pass Softmax&#xff0c;\u4f7f\u7528torch\u6216\u8005numpy\u7684\u5411\u91cf\u5316\u64cd\u4f5c\u4e5f\u4e0d\u4f1a\u6539\u53d8\u4e09\u6b21\u5faa\u73af\u7684\u672c\u8d28&#xff0c;\u6211\u4eec\u5c06\u8fd9\u4e2a\u8fc7\u7a0b\u5199\u6210\u516c\u5f0f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">m&#061;\u2212\u221e,l&#061;0,a&#061;0for\u00a0i&#061;1,\u2026,dk:m\u2190max\u2061(m,xi)l\u2190l&#043;exi\u2212mfor\u00a0i&#061;1,\u2026,dk:ai\u2190exi\u2212ml<br \/>\n  \\\\begin{aligned}<br \/>\n    m &#061; -\\\\infty, l &#061; 0, a &#061; 0 \\\\\\\\<br \/>\n    \\\\text{for } i&#061;1,\\\\ldots,d_k:&amp; \\\\\\\\<br \/>\n    &amp;\\\\quad m \\\\leftarrow \\\\max(m, x_i) \\\\\\\\<br \/>\n    &amp;\\\\quad l \\\\leftarrow l &#043; e^{x_i &#8211; m} \\\\\\\\<br \/>\n    \\\\text{for } i&#061;1,\\\\ldots,d_k:&amp; \\\\\\\\<br \/>\n    &amp;\\\\quad a_i \\\\leftarrow \\\\frac{e^{x_i &#8211; m}}{l}<br \/>\n  \\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 9.9343em;vertical-align: -4.7172em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.2172em\"><span class=\"\" style=\"top: -7.8255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">\u221e<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">0<\/span><\/span><\/span><span class=\"\" style=\"top: -6.3255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">for\u00a0<\/span><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><\/span><span class=\"\" style=\"top: -4.8255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -3.3255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -1.8255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord text\"><span class=\"mord\">for\u00a0<\/span><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><\/span><span class=\"\" style=\"top: 0.2828em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 4.7172em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.7172em\"><span class=\"\" style=\"top: -6.3255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: -4.8255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 1em\"><\/span><span class=\"mord mathnormal\">m<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">m<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -3.3255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 1em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -1.8255em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: 0.2828em\"><span class=\"pstrut\" style=\"height: 3.4483em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 1em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4483em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7713em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 4.7172em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\nm\u548cl\u90fd\u6709\u9012\u63a8\u7684\u6027\u8d28&#xff0c;\u6211\u4eec\u5b9a\u4e49:<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">mi&#061;max\u2061(mi\u22121,xi)m&#061;max\u2061(m1,m2,\u2026,mdk)&#061;mdkli&#061;\u2211j&#061;1iexj\u2212m&#061;li\u22121&#043;exi\u2212ml&#061;\u2211j&#061;1dkexj\u2212m&#061;ldk<br \/>\n\\\\begin{aligned}<br \/>\nm_i &#061; \\\\max(m_{i-1}, x_i)\\\\\\\\<br \/>\nm &#061; \\\\max(m_1, m_2, \\\\ldots, m_{d_k}) &#061; m_{d_k} \\\\\\\\<br \/>\nl_i &#061; \\\\sum_{j&#061;1}^{i} e^{x_j &#8211; m} &#061; l_{i-1} &#043; e^{x_i &#8211; m}\\\\\\\\<br \/>\nl &#061; \\\\sum_{j&#061;1}^{d_k} e^{x_j &#8211; m} &#061; l_{d_k}<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 10.0922em;vertical-align: -4.7961em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.2961em\"><span class=\"\" style=\"top: -8.3091em\"><span class=\"pstrut\" style=\"height: 3.853em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -6.8091em\"><span class=\"pstrut\" style=\"height: 3.853em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2559em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2559em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.3374em\"><span class=\"pstrut\" style=\"height: 3.853em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.7707em\"><span class=\"pstrut\" style=\"height: 3.853em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.853em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3169em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2559em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 4.7961em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u5fc5\u987b\u904d\u5386\u4e24\u6b21\u884c\u5411\u91cf\u662f\u56e0\u4e3a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u503c\u5fc5\u987b\u4f9d\u8d56<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mm<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><\/span>,\u5982\u679c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u503c\u4e0d\u4f9d\u8d56\u4e8e<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mm<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><\/span>\u800c\u662f\u4f9d\u8d56\u4e8e<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u90a3\u4e48\u6211\u4eec\u5c31\u53ef\u4ee5\u5728\u4e00\u6b21\u5faa\u73af\u4e2d\u540c\u65f6\u66f4\u65b0<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4ece\u800c\u5b9e\u73b0\u5728\u7ebfSoftmax\u3002<\/p>\n<h3>Online Softmax: 2-pass<\/h3>\n<p>\u5728\u7ebfSoftmax\u7684\u6838\u5fc3\u601d\u60f3\u662f\u878d\u5408\u524d\u4e24\u6b65\u8ba1\u7b97&#xff0c;\u521a\u624d\u8bf4<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mm<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ll<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><\/span><\/span>\u90fd\u6709\u9012\u63a8\u7684\u6027\u8d28&#xff0c;\u6bcf\u4e00\u9879<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u90fd\u53ef\u4ee5\u901a\u8fc7<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mi\u22121m_{i-1}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6389em;vertical-align: -0.2083em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u8ba1\u7b97\u51fa\u6765&#xff0c;\u6bcf\u4e00\u9879<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u4e5f\u4e00\u6837&#xff0c;\u5982\u679c\u6211\u4eec\u8981\u538b\u7f29\u524d\u4e24\u6b65\u8ba1\u7b97&#xff0c;\u90a3\u6211\u4eec\u5c31\u9700\u8981\u627e\u5230\u4e00\u79cd\u65b9\u6cd5\u65e2\u53ef\u4ee5\u4fdd\u8bc1\u7ed3\u679c\u6b63\u786e&#xff0c;\u5e76\u4e14\u8ba9<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u53ea\u4f9d\u8d56<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u53c8\u53ef\u4ee5\u4fdd\u8bc1\u4e0d\u7834\u574f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u9012\u63a8\u6027&#xff0c;\u4e0d\u59a8\u5148\u628a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u5b9a\u4e49\u6539\u6210\u5982\u4e0b\u5f62\u5f0f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">li&#061;\u2211j&#061;1iexj\u2212mi<br \/>\nl_i &#061; \\\\sum_{j&#061;1}^{i} e^{x_j &#8211; m_i}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 3.2254em;vertical-align: -1.4138em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u90a3\u4e48\u5f53<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">i&#061;dki&#061;d_k<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u65f6&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">li&#061;ldkl_i&#061;l_{d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9503em;vertical-align: -0.2559em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2559em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u4ecd\u7136\u8868\u793a\u6574\u4e2a\u5e8f\u5217\u7684\u6307\u6570\u548c&#xff0c;\u7ed3\u679c\u6b63\u786e&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u4f9d\u9644\u4e8e<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97&#xff0c;\u5c31\u50cf\u52a8\u6001\u89c4\u5212\u4e2d\u7684\u72b6\u6001\u8f6c\u79fb\u65b9\u7a0b\u4e00\u6837&#xff0c;\u73b0\u5728\u6211\u4eec\u8981\u89e3\u51b3<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u5982\u4f55\u7528<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">li\u22121l_{i-1}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9028em;vertical-align: -0.2083em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u8868\u793a\u7684\u95ee\u9898\u3002<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">li&#061;\u2211j&#061;1iexj\u2212mili\u22121&#061;\u2211j&#061;1i\u22121exj\u2212mi\u22121li\u22121\u00d7emi\u22121&#061;\u2211j&#061;1i\u22121exjli\u22121\u00d7emi\u22121&#043;exi&#061;\u2211j&#061;1iexj(li\u22121\u00d7emi\u22121&#043;exi)e\u2212mi&#061;\u2211j&#061;1iexj\u2212mi&#061;lili\u22121\u00d7emi\u22121\u2212mi&#043;exi\u2212mi&#061;li<br \/>\n\\\\begin{aligned}<br \/>\nl_i &amp;&#061; \\\\sum_{j&#061;1}^{i} e^{x_j &#8211; m_i} \\\\\\\\<br \/>\nl_{i-1} &amp;&#061; \\\\sum_{j&#061;1}^{i-1} e^{x_j &#8211; m_{i-1}} \\\\\\\\<br \/>\nl_{i-1} \\\\times e^{m_{i-1}} &amp;&#061; \\\\sum_{j&#061;1}^{i-1} e^{x_j} \\\\\\\\<br \/>\nl_{i-1} \\\\times e^{m_{i-1}} &#043; e^{x_i} &amp;&#061; \\\\sum_{j&#061;1}^{i} e^{x_j} \\\\\\\\<br \/>\n(l_{i-1} \\\\times e^{m_{i-1}} &#043; e^{x_i}) e^{-m_i} &amp;&#061; \\\\sum_{j&#061;1}^{i} e^{x_j &#8211; m_i} &#061; l_i\\\\\\\\<br \/>\nl_{i-1}\\\\times e^{m_{i-1} &#8211; m_i} &#043; e^{x_i &#8211; m_i} &amp;&#061;l_i<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 19.1272em;vertical-align: -9.3136em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 9.8136em\"><span class=\"\" style=\"top: -11.8136em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -8.2882em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.7627em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -1.2373em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 2.2882em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 4.8419em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 9.3136em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 9.8136em\"><span class=\"\" style=\"top: -11.8136em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -8.2882em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.7627em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -1.2373em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 2.2882em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 4.8419em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 9.3136em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u56e0\u6b64\u6211\u4eec\u5c31\u5f97\u5230\u4e86\u5728\u7ebfSoftmax\u7684\u9012\u63a8\u516c\u5f0f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">mi&#061;max\u2061(mi\u22121,xi)li&#061;li\u22121\u00d7emi\u22121\u2212mi&#043;exi\u2212mi<br \/>\n\\\\begin{aligned}<br \/>\nm_i &amp;&#061; \\\\max(m_{i-1}, x_i)\\\\\\\\<br \/>\nl_i &amp;&#061; l_{i-1}\\\\times e^{m_{i-1} &#8211; m_i} &#043; e^{x_i &#8211; m_i}<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 3em;vertical-align: -1.25em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.75em\"><span class=\"\" style=\"top: -3.91em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.41em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.25em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.75em\"><span class=\"\" style=\"top: -3.91em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -2.41em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.25em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u4ee3\u7801\u662f\u8fd9\u6837\u5199\u7684&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">online_softmax<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">:<\/span> nn<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">,<\/span> a<span class=\"token punctuation\">:<\/span> nn<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  <span class=\"token comment\">## x: (seq_len, d_k)<\/span><br \/>\n  seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> x<span class=\"token punctuation\">.<\/span>shape<br \/>\n  <span class=\"token keyword\">for<\/span> b <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># \u5bf9\u4e8e\u6bcf\u4e2atoken<\/span><br \/>\n    m<span class=\"token punctuation\">,<\/span> l <span class=\"token operator\">&#061;<\/span> <span class=\"token operator\">&#8211;<\/span>torch<span class=\"token punctuation\">.<\/span>inf<span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      m_i <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      l_i <span class=\"token operator\">&#061;<\/span> l <span class=\"token operator\">*<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m <span class=\"token operator\">&#8211;<\/span> m_i<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> m_i<span class=\"token punctuation\">)<\/span><br \/>\n      m<span class=\"token punctuation\">,<\/span> l <span class=\"token operator\">&#061;<\/span> m_i<span class=\"token punctuation\">,<\/span> l_i<br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      a<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">[<\/span>b<span class=\"token punctuation\">,<\/span> i<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#8211;<\/span> m<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> l<br \/>\n  <span class=\"token keyword\">return<\/span> a<\/p>\n<p>\u5f88\u9057\u61be\u4e0d\u80fd\u6700\u540e\u7684O\u4e0d\u80fd\u4e0e\u4e0a\u9762\u7684\u5faa\u73af\u5408\u5e76&#xff0c;\u56e0\u4e3a\u5728\u8ba1\u7b97<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u65f6\u9700\u8981\u7528\u5230\u6700\u7ec8\u7684<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mm<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ll<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u6240\u4ee5\u6211\u4eec\u53ea\u80fd\u5728\u7b2c\u4e8c\u6b21\u5faa\u73af\u4e2d\u8ba1\u7b97<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u56e0\u6b64\u5728\u7ebfSoftmax\u662f2-pass\u7684\u3002\u4f60\u53ef\u80fd\u4f1a\u7591\u60d1&#xff0c;\u867d\u7136\u5c11\u4e00\u6b21\u5faa\u73af&#xff0c;\u4f46\u662f\u5b9e\u9645\u4e0a\u591a\u8ba1\u7b97\u4e86\u4e00\u6b21exp\u51fd\u6570&#xff0c;\u901f\u5ea6\u4e5f\u8bb8\u4f1a\u66f4\u6162&#xff0c;\u6211\u7684\u7406\u89e3\u662f&#xff0c;\u5728CPU\u4e32\u884c\u6267\u884c\u7684\u65f6\u5019\u786e\u5b9e\u53ef\u80fd\u4f1a\u66f4\u6162&#xff0c;\u4f46\u662f\u5728GPU\u4e2d\u4e00\u6b21\u6027\u5c06\u4e00\u6574\u884c\u7684\u6570\u636e\u8bfb\u5165\u9ad8\u901f\u7684\u5171\u4eab\u5185\u5b58\u975e\u5e38\u5360\u7a7a\u95f4&#xff0c;\u6240\u4ee5\u591a\u4e00\u6b21\u5faa\u73af\u5f80\u5f80\u610f\u5473\u7740\u8981\u591a\u4e00\u6b21\u4ece\u5168\u5c40\u5185\u5b58&#xff08;HBM&#xff0c;\u901f\u5ea6\u6162&#xff09;\u4e2d\u8bfb\u53d6\u6570\u636e&#xff0c;\u800c\u5728\u4e00\u4e2a\u5faa\u73af\u4e2d\u53ea\u9700\u8981\u6682\u5b58\u4e24\u4e2a\u53d8\u91cfm\u548cl,\u6240\u6709\u8ba1\u7b97\u90fd\u5728\u9ad8\u901f\u7f13\u5b58\u6216\u5171\u4eab\u5185\u5b58\u4e2d\u5b8c\u6210&#xff0c;\u591a\u8ba1\u7b97\u4e00\u6b21exp\u51fd\u6570\u7684\u5f00\u9500\u5bf9\u4e8e\u4e00\u6b21HBM\u7684\u8bbf\u95ee\u5f00\u9500\u6765\u8bf4\u662f\u5fae\u4e0d\u8db3\u9053\u7684\u3002<\/p>\n<h3>FlashAttention: 1-pass Single-query<\/h3>\n<p>\u4e3a\u4ec0\u4e48FlashAttention\u53ef\u4ee5\u505a1-pass\u5462&#xff0c;\u56e0\u4e3aAttention\u7684\u76ee\u6807\u4e0d\u662f\u6c42Softmax&#xff0c;\u800c\u662f\u6c42\u6700\u7ec8\u7684O\u3002\u6211\u4eec\u5148\u6765\u770b\u770bAttention\u7684\u8ba1\u7b97\u516c\u5f0f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">S&#061;softmax(QKTdk)O&#061;SVO&#061;softmax(QKTdk)V<br \/>\n\\\\begin{aligned}<br \/>\nS &amp;&#061; softmax(\\\\frac{QK^T}{\\\\sqrt{d_k}}) \\\\\\\\<br \/>\nO &amp;&#061; SV \\\\\\\\<br \/>\nO &amp;&#061; softmax(\\\\frac{QK^T}{\\\\sqrt{d_k}})V<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 6.9967em;vertical-align: -3.2483em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.7483em\"><span class=\"\" style=\"top: -5.7483em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><\/span><\/span><span class=\"\" style=\"top: -3.6783em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><\/span><span class=\"\" style=\"top: -1.5em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.2483em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.7483em\"><span class=\"\" style=\"top: -5.7483em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">so<\/span><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span style=\"margin-right: 0.0715em\" class=\"mord mathnormal\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -3.6783em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><\/span><\/span><span class=\"\" style=\"top: -1.5em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">so<\/span><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span style=\"margin-right: 0.0715em\" class=\"mord mathnormal\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose\">)<\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.2483em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u5bf9\u4e8eQ\u4e2d\u7684\u6bcf\u4e00\u4e2a\u67e5\u8be2\u5411\u91cf<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">qk&#061;Q[k,:]\u2208Rdkq_k&#061;Q[k,:] \\\\in R^{d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u6211\u4eec\u6700\u7ec8\u8ba1\u7b97\u5f97\u5230\u5bf9\u5e94\u7684\u8f93\u51fa\u5411\u91cf<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ok&#061;O[k,:]\u2208Rdko_k&#061;O[k,:] \\\\in R^{d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002<br \/>\n\u8ba1\u7b97\u8fc7\u7a0b\u5982\u4e0b&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">ok&#061;softmax(qkKTdk)V<br \/>\no_k &#061; softmax(\\\\frac{q_k K^T}{\\\\sqrt{d_k}})V<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4483em;vertical-align: -0.93em\"><\/span><span class=\"mord mathnormal\">so<\/span><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.0715em\" class=\"mord mathnormal\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose\">)<\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u62c6\u89e3\u6bcf\u4e00\u6b65&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">for\u00a0i&#061;1,\u2026,dk:xi\u2190qkKT[:,i]dkmi\u2190max\u2061(mi\u22121,xi)li\u2190li\u22121\u00d7emi\u22121\u2212mi&#043;exi\u2212miendfori&#061;1,\u2026,dk:ai\u2190exi\u2212mloi\u2190oi\u22121&#043;ai\u00d7V[i,:]endO[k,:]&#061;odk<br \/>\n\\\\begin{aligned}<br \/>\nfor \\\\ i&#061;1,\\\\ldots,d_k:&amp; \\\\\\\\<br \/>\n &amp; x_i \\\\leftarrow \\\\frac{q_k K^T[:, i]}{\\\\sqrt{d_k}} \\\\\\\\<br \/>\n &amp; m_i \\\\leftarrow \\\\max(m_{i-1}, x_i)\\\\\\\\<br \/>\n &amp; l_i \\\\leftarrow l_{i-1}\\\\times e^{m_{i-1} &#8211; m_i} &#043; e^{x_i &#8211; m_i} \\\\\\\\<br \/>\nend&amp;\\\\\\\\<br \/>\nfor i&#061;1,\\\\ldots,d_k:&amp; \\\\\\\\<br \/>\n  &amp; a_i \\\\leftarrow \\\\frac{e^{x_i &#8211; m}}{l} \\\\\\\\<br \/>\n  &amp; o_i \\\\leftarrow o_{i-1} &#043; a_i \\\\times V[i,:] \\\\\\\\<br \/>\n  end&amp;\\\\\\\\<br \/>\nO[k, :] &#061; o_{d_k}<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 17.1827em;vertical-align: -8.3413em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 8.8413em\"><span class=\"\" style=\"top: -11.5197em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">or<\/span><span class=\"mspace\">\u00a0<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><\/span><span class=\"\" style=\"top: -9.3413em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -7.2713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -5.7713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -4.2713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><span class=\"\" style=\"top: -2.7713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">or<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><\/span><span class=\"\" style=\"top: -0.663em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: 1.163em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: 2.663em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><span class=\"\" style=\"top: 4.163em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2559em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 8.3413em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 8.8413em\"><span class=\"\" style=\"top: -11.5197em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: -9.3413em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.0715em\" class=\"mord mathnormal\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">[<\/span><span class=\"mrel\">:<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -7.2713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -5.7713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.2713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: -2.7713em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: -0.663em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4483em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7713em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mathnormal mtight\">m<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 1.163em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: 2.663em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 6.8413em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u8fd9\u662f2-pass Attention\u7684\u5b9e\u73b0\u65b9\u5f0f&#xff0c;\u8fd9\u91cc\u7684\u6700\u540e\u4e00\u6b65\u6211\u89c9\u5f97\u53ef\u80fd\u4e0d\u592a\u597d\u7406\u89e3&#xff0c;\u8981\u8bf4\u660e\u7684\u662f\u5b9e\u9645\u4e0a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">aia_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">S\u2208Rseqlen\u00d7seqlenS \\\\in R^{seq_len \\\\times seq_len}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7713em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7713em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">se<\/span><span class=\"mord mtight\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal mtight\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: -0.0359em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal mtight\">l<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mathnormal mtight\">se<\/span><span class=\"mord mtight\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal mtight\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: -0.0359em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal mtight\">l<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u4e2d\u7684\u7b2c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">kk<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><\/span><\/span><\/span><\/span>\u884c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ii<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span>\u5217\u5143\u7d20&#xff0c;\u800c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">o&#061;O[k,&#xff1a;]&#061;S[k,:]\u00d7V&#061;\u2211i&#061;1seqlenS[k,i]\u00d7V[i,:]&#061;\u2211i&#061;1seqlenai\u00d7V[i,:]o &#061; O[k, &#xff1a;] &#061; S[k, :] \\\\times V &#061; \\\\sum_{i&#061;1}^{seq_len} S[k,i] \\\\times V[i,:] &#061; \\\\sum_{i&#061;1}^{seq_len} a_i \\\\times V[i,:]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">o<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord cjk_fallback\">&#xff1a;<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.104em;vertical-align: -0.2997em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8043em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2029em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">se<\/span><span class=\"mord mtight\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal mtight\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: -0.0359em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal mtight\">l<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.104em;vertical-align: -0.2997em\"><\/span><span class=\"mop\"><span class=\"mop op-symbol small-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8043em\"><span class=\"\" style=\"top: -2.4003em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2029em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">se<\/span><span class=\"mord mtight\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal mtight\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: -0.0359em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal mtight\">l<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\">n<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2997em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span>,\u56e0\u6b64\u53ef\u4ee5\u7528o_i\u8868\u793a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a\u00d7V[i,:]a \\\\times V[i,:]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6667em;vertical-align: -0.0833em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span>\u7684\u524d\u7f00\u548c&#xff0c;\u8fed\u4ee3\u66f4\u65b0\u3002<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">oi&#061;oi\u22121&#043;ai\u00d7V[i,:]&#061;oi\u22121&#043;eix\u2212ml\u00d7V[i,:]&#061;\u2211j&#061;1iexj\u2212ml\u00d7V[j,:]<br \/>\n\\\\begin{aligned}<br \/>\no_i &amp;&#061; o_{i-1} &#043; a_i \\\\times V[i,:]<br \/>\n &#061; o_{i-1} &#043; \\\\frac{e^x_i &#8211; m}{l} \\\\times V[i,:] \\\\\\\\<br \/>\n &amp;&#061; \\\\sum_{j&#061;1}^{i} \\\\frac{e^{x_j} &#8211; m}{l} \\\\times V[j,:]<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 5.8528em;vertical-align: -2.6764em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.1764em\"><span class=\"\" style=\"top: -5.6467em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.849em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.6764em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 3.1764em\"><span class=\"\" style=\"top: -5.6467em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">a<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -2.4413em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">x<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2587em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: -2.849em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.6764em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u6211\u4eec\u7684\u76ee\u6807\u662f\u5c06<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97\u4e5f\u878d\u5408\u5230\u524d\u9762\u53bb&#xff0c;\u56e0\u6b64\u4e0d\u80fd\u4f9d\u8d56\u4e8e\u6700\u7ec8\u7684<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mm<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">m<\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ll<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u6211\u4eec\u9700\u8981\u5c06<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97\u6539\u6210\u4f9d\u8d56\u4e8e<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff0c;\u53ef\u4ee5\u5148\u5c06<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97\u5f0f\u5b50\u8fdb\u884c\u53d8\u5f62&#xff1a;<br \/>\n\u5982\u679c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97\u5f0f\u5b50\u662f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">oi&#061;\u2211j&#061;1iexj\u2212mili\u00d7V[j,:]<br \/>\n\\\\begin{aligned}<br \/>\no_i &#061; \\\\sum_{j&#061;1}^{i} \\\\frac{e^{x_j} &#8211; m_i}{l_i} \\\\times V[j,:]\\\\\\\\<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 3.5254em;vertical-align: -1.5127em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.0127em\"><span class=\"\" style=\"top: -4.0127em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5127em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u90a3\u4e48<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">oi\u22121&#061;\u2211j&#061;1i\u22121exj\u2212mi\u22121li\u22121\u00d7V[j,:]li\u22121\u00d7oi\u22121&#061;\u2211j&#061;1i\u22121exj\u2212mi\u22121\u00d7V[j,:]li\u22121\u00d7oi\u22121\u00d7emi\u22121&#061;\u2211j&#061;1i\u22121exj\u00d7V[j,:]li\u22121\u00d7oi\u22121\u00d7emi\u22121&#043;exi\u00d7V[i,:]&#061;\u2211j&#061;1iexj\u00d7V[j,:]&#061;li\u00d7oi\u00d7emi<br \/>\n\\\\begin{aligned}<br \/>\no_{i-1} &amp;&#061; \\\\sum_{j&#061;1}^{i-1} \\\\frac{e^{x_j} &#8211; m_{i-1}}{l_{i-1}} \\\\times V[j,:] \\\\\\\\<br \/>\nl_{i-1} \\\\times o_{i-1} &amp;&#061; \\\\sum_{j&#061;1}^{i-1} e^{x_j &#8211; m_{i-1}} \\\\times V[j,:] \\\\\\\\<br \/>\nl_{i-1} \\\\times o_{i-1} \\\\times e^{m_{i-1}} &amp;&#061; \\\\sum_{j&#061;1}^{i-1} e^{x_j} \\\\times V[j,:] \\\\\\\\<br \/>\nl_{i-1} \\\\times o_{i-1} \\\\times e^{m_{i-1}} &#043; e^{x_i} \\\\times V[i,:] &amp;&#061; \\\\sum_{j&#061;1}^{i} e^{x_j} \\\\times V[j,:] \\\\\\\\<br \/>\n&amp;&#061; l_i \\\\times o_i \\\\times e^{m_i} \\\\\\\\<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 15.6018em;vertical-align: -7.5509em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 8.0509em\"><span class=\"\" style=\"top: -10.0509em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -6.5254em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 0.5254em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: 3.0792em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.5509em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 8.0509em\"><span class=\"\" style=\"top: -10.0509em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3414em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.6644em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8943em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: -6.5254em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: 0.5254em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-limits\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.8117em\"><span class=\"\" style=\"top: -1.8723em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><span class=\"mrel mtight\">&#061;<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.05em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"\"><span class=\"mop op-symbol large-op\">\u2211<\/span><\/span><\/span><span class=\"\" style=\"top: -4.3em;margin-left: 0em\"><span class=\"pstrut\" style=\"height: 3.05em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.4138em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal mtight\">j<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2819em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span style=\"margin-right: 0.0572em\" class=\"mord mathnormal\">j<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">]<\/span><\/span><\/span><span class=\"\" style=\"top: 3.0792em\"><span class=\"pstrut\" style=\"height: 3.8117em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.5509em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u6240\u4ee5\u6211\u4eec\u53ef\u4ee5\u5f97\u5230\u5982\u4e0b\u7684\u9012\u63a8\u516c\u5f0f&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">oi&#061;1li\u00d7e\u2212mi\u00d7(li\u22121\u00d7oi\u22121\u00d7emi\u22121&#043;exi\u00d7V[i,:])&#061;1li\u00d7(emi\u22121\u2212mi\u00d7li\u22121\u00d7oi\u22121&#043;exi\u2212mi\u00d7V[i,:])<br \/>\n\\\\begin{aligned}<br \/>\no_i &amp;&#061; \\\\frac{1}{l_i} \\\\times e^{-m_i} \\\\times (l_{i-1} \\\\times o_{i-1} \\\\times e^{m_{i-1}} &#043; e^{x_i} \\\\times V[i,:]) \\\\\\\\<br \/>\n&amp;&#061; \\\\frac{1}{l_i}\\\\times(e^{m_{i-1} &#8211; m_i} \\\\times l_{i-1} \\\\times o_{i-1} &#043; e^{x_i &#8211; m_i} \\\\times V[i,:])<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 4.9149em;vertical-align: -2.2074em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.7074em\"><span class=\"\" style=\"top: -4.7074em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.25em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.2074em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.7074em\"><span class=\"\" style=\"top: -4.7074em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7144em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">])<\/span><\/span><\/span><span class=\"\" style=\"top: -2.25em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">])<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 2.2074em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u8fd9\u6837\u6211\u4eec\u5c31\u53ef\u4ee5\u5c06<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8ba1\u7b97\u5408\u5e76\u5230\u7b2c\u4e00\u4e2a\u5faa\u73af\u4e2d\u53bb&#xff0c;\u6700\u7ec8\u7684\u6d41\u7a0b\u5982\u4e0b&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">for\u00a0i&#061;1,\u2026,dk:xi\u2190qkKT[:,i]dkmi\u2190max\u2061(mi\u22121,xi)li\u2190li\u22121\u00d7emi\u22121\u2212mi&#043;exi\u2212mioi&#061;1li\u00d7(emi\u22121\u2212mi\u00d7li\u22121\u00d7oi\u22121&#043;exi\u2212mi\u00d7V[i,:])end<br \/>\n\\\\begin{aligned}<br \/>\nfor \\\\ i&#061;1,\\\\ldots,d_k:&amp; \\\\\\\\<br \/>\n &amp; x_i \\\\leftarrow \\\\frac{q_k K^T[:, i]}{\\\\sqrt{d_k}} \\\\\\\\<br \/>\n &amp; m_i \\\\leftarrow \\\\max(m_{i-1}, x_i)\\\\\\\\<br \/>\n &amp; l_i \\\\leftarrow l_{i-1}\\\\times e^{m_{i-1} &#8211; m_i} &#043; e^{x_i &#8211; m_i} \\\\\\\\<br \/>\n  &amp;o_i &#061; \\\\frac{1}{l_i}\\\\times(e^{m_{i-1} &#8211; m_i} \\\\times l_{i-1} \\\\times o_{i-1} &#043; e^{x_i &#8211; m_i} \\\\times V[i,:])\\\\\\\\<br \/>\nend&amp;\\\\\\\\<br \/>\n\\\\end  {aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 11.2058em;vertical-align: -5.3529em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.8529em\"><span class=\"\" style=\"top: -8.5312em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.1076em\" class=\"mord mathnormal\">f<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">or<\/span><span class=\"mspace\">\u00a0<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><\/span><\/span><span class=\"\" style=\"top: -6.3529em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -4.2829em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -2.7829em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: -0.8014em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: 1.1746em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">n<\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.3529em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.8529em\"><span class=\"\" style=\"top: -8.5312em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><span class=\"\" style=\"top: -6.3529em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0359em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.0715em\" class=\"mord mathnormal\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mopen\">[<\/span><span class=\"mrel\">:<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.2829em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop\">max<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -2.7829em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2190<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.8014em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.836em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2025em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2083em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8213em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">x<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mopen\">[<\/span><span class=\"mord mathnormal\">i<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">:<\/span><span class=\"mclose\">])<\/span><\/span><\/span><span class=\"\" style=\"top: 1.1746em\"><span class=\"pstrut\" style=\"height: 3.5183em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 5.3529em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\npython\u4ee3\u7801\u5982\u4e0b&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">flashattn_naiveQKV<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">.<\/span>shape<br \/>\n  <span class=\"token comment\"># K: (batch_size, seq_len, d_k)<\/span><br \/>\n  <span class=\"token comment\"># V: (batch_size, seq_len, d_k)<\/span><br \/>\n  scale <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span> <span class=\"token operator\">\/<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><br \/>\n  O <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token keyword\">for<\/span> batch <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    Q_<span class=\"token punctuation\">,<\/span> K_<span class=\"token punctuation\">,<\/span> V_ <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># q, k, v: (seq_len, d_k)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> k <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># for each query<\/span><br \/>\n      q <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># (1,d_k)<\/span><br \/>\n      m0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token operator\">&#8211;<\/span>torch<span class=\"token punctuation\">.<\/span>inf<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      l0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n      o0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">)<\/span><br \/>\n      <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        k_i <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        x_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_i<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale <span class=\"token comment\">#  <\/span><br \/>\n        m_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">,<\/span> x_i<span class=\"token punctuation\">)<\/span><br \/>\n        expmm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m0<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n        expxm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x_i<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n        l_i <span class=\"token operator\">&#061;<\/span> l0<span class=\"token operator\">*<\/span>expmm <span class=\"token operator\">&#043;<\/span> expxm<br \/>\n        o0 <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>o0<span class=\"token operator\">*<\/span><span class=\"token punctuation\">(<\/span>l0<span class=\"token operator\">*<\/span>expmm<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> expxm <span class=\"token operator\">*<\/span> V_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> l_i<br \/>\n        m0<span class=\"token punctuation\">,<\/span> l0 <span class=\"token operator\">&#061;<\/span> m_i<span class=\"token punctuation\">,<\/span> l_i<br \/>\n      O<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">,<\/span> k<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o0<br \/>\n  <span class=\"token keyword\">return<\/span> O<\/p>\n<p>\u5728\u8fd9\u91cc\u6709\u4e00\u4e2a\u95ee\u9898&#xff0c;\u5047\u8bbeGPU\u4f1a\u81ea\u52a8\u5e76\u884c&#xff0c;\u8ba9\u6bcf\u4e00\u4e2a\u6267\u884c\u5355\u5143\u5904\u7406\u4e00\u4e2a\u67e5\u8be2&#xff08;\u81f3\u4e8e\u6267\u884c\u5355\u5143\u5177\u4f53\u5982\u4f55\u5206\u914d\u548c\u6267\u884c\u8fd9\u4e2a\u67e5\u8be2\u4efb\u52a1\u6211\u4eec\u6682\u65f6\u5ffd\u7565&#xff09;&#xff0c;\u90a3\u4e48\u6211\u4eec\u5148\u770b\u6700\u5185\u5c42\u5faa\u73af:<\/p>\n<p><span class=\"token comment\"># for every block, do:<\/span><br \/>\nq <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># (1,d_k) load one query<\/span><br \/>\nm0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token operator\">&#8211;<\/span>torch<span class=\"token punctuation\">.<\/span>inf<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\nl0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\no0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  k_i <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token comment\"># load one key<\/span><br \/>\n  x_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_i<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale <span class=\"token comment\">#  <\/span><br \/>\n  m_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">,<\/span> x_i<span class=\"token punctuation\">)<\/span><br \/>\n  expmm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m0<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n  expxm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x_i<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n  l_i <span class=\"token operator\">&#061;<\/span> l0<span class=\"token operator\">*<\/span>expmm <span class=\"token operator\">&#043;<\/span> expxm<br \/>\n  o0 <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>o0<span class=\"token operator\">*<\/span><span class=\"token punctuation\">(<\/span>l0<span class=\"token operator\">*<\/span>expmm<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> expxm <span class=\"token operator\">*<\/span> V_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> l_i <span class=\"token comment\"># load one value<\/span><br \/>\n  m0<span class=\"token punctuation\">,<\/span> l0 <span class=\"token operator\">&#061;<\/span> m_i<span class=\"token punctuation\">,<\/span> l_i<br \/>\nO<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">,<\/span> k<span class=\"token punctuation\">,<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o0<\/p>\n<p>\u5047\u8bbeQ,K,V\u90fd\u5728\u5168\u5c40\u5185\u5b58&#xff08;HBM&#xff09;\u4e2d&#xff0c;\u90a3\u4e48\u4e0a\u9762\u7684\u4ee3\u7801&#xff0c;\u6bcf\u4e2a\u6267\u884c\u5355\u5143\u4f1a\u8bfb\u53d6\u4e00\u4e2a\u67e5\u8be2&#xff0c;\u7136\u540e\u6309\u6279\u6b21\u8bfb\u53d6\u6240\u6709\u7684K\u548cV\u77e9\u9635\u7684\u884c\u5411\u91cf&#xff0c;\u7136\u540e\u5c06\u8f93\u51fa\u5199\u5165&#xff0c;\u8fd9\u4e2a\u6267\u884c\u5355\u5143\u7684HBM IO\u6b21\u6570\u662f&#xff1a;sizeof(q) &#043; sizeof(K) &#043; sizeof(V) &#043; sizeof(o), \u5176\u4e2dsizeof(q) &#061; d_k, sizeof(K) &#061; seq_len * d_k, sizeof(V) &#061; seq_len * d_k, sizeof(o) &#061; d_k, \u56e0\u6b64\u603b\u7684IO\u6b21\u6570\u662f2*seq_len*d_k &#043; 2*d_k\u3002<br \/>\n\u6211\u4eec\u5b8c\u5168\u53ef\u4ee5\u5c06\u91cc\u5916\u5c42\u8c03\u6362\u4f4d\u7f6e&#xff0c;\u5982\u679c\u6211\u4eec\u5148load K[i,:]\u548cV[i,:],\u7136\u540e\u8ba9\u8fd9\u4e2a\u6267\u884c\u5355\u5143\u5904\u7406\u6240\u6709\u7684\u67e5\u8be2&#xff0c;\u5c31\u50cf\u8fd9\u6837&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">flashattn_naiveKVQ<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">.<\/span>shape<br \/>\n  <span class=\"token comment\"># K: (batch_size, seq_len, d_k)<\/span><br \/>\n  <span class=\"token comment\"># V: (batch_size, seq_len, d_k)<\/span><br \/>\n  scale <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span> <span class=\"token operator\">\/<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><br \/>\n  O <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token keyword\">for<\/span> batch <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    Q_<span class=\"token punctuation\">,<\/span> K_<span class=\"token punctuation\">,<\/span> V_ <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># q, k, v: (seq_len, d_k)<\/span><br \/>\n    <span class=\"token comment\"># \u6bcf\u4e2a\u67e5\u8be2\u4f4d\u7f6e\u4e00\u4e2am\u503c<\/span><br \/>\n    m0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>full<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span>torch<span class=\"token punctuation\">.<\/span>inf<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token comment\"># \u6bcf\u4e2a\u67e5\u8be2\u4f4d\u7f6e\u4e00\u4e2al\u503c<\/span><br \/>\n    l0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><br \/>\n    o0 <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">)<\/span>           <span class=\"token comment\"># \u6bcf\u4e2a\u67e5\u8be2\u4f4d\u7f6e\u4e00\u4e2a\u8f93\u51fa\u5411\u91cf<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span> <span class=\"token comment\"># for each query<\/span><br \/>\n      k_i <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><br \/>\n      v_i <span class=\"token operator\">&#061;<\/span> V_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><br \/>\n      <span class=\"token keyword\">for<\/span> k <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        q <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># (1,d_k)<\/span><br \/>\n        x_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_i<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale <span class=\"token comment\">#  <\/span><br \/>\n        m_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> x_i<span class=\"token punctuation\">)<\/span><br \/>\n        expmm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n        expxm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x_i<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n        l_i <span class=\"token operator\">&#061;<\/span> l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span>expmm <span class=\"token operator\">&#043;<\/span> expxm<br \/>\n        o_i <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">(<\/span>l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span>expmm<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> expxm<span class=\"token operator\">*<\/span> v_i<span class=\"token punctuation\">)<\/span><span class=\"token operator\">\/<\/span>l_i<br \/>\n        m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> o0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> m_i<span class=\"token punctuation\">,<\/span> l_i<span class=\"token punctuation\">,<\/span> o_i<br \/>\n    O<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o0<br \/>\n  <span class=\"token keyword\">return<\/span> O<\/p>\n<p>\u95ee\u9898\u5728\u4e8e\u6bcf\u4e00\u6b21\u5bf9Q\u7684\u8fed\u4ee3\u6211\u4eec\u53ea\u4f1a\u66f4\u65b0\u6574\u4e2al0,m0,o0\u4e00\u6b21&#xff0c;\u56e0\u6b64\u6211\u4eec\u9700\u8981\u6682\u5b58\u6240\u6709\u7684l0,m0,o0\u5728\u5168\u5c40\u5185\u5b58(HBM),\u6bcf\u4e2a\u6267\u884c\u5355\u5143\u6267\u884c\u4ee5\u4e0b\u4ee3\u7801&#xff1a;<\/p>\n<p>k_i <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># load one key<\/span><br \/>\nv_i <span class=\"token operator\">&#061;<\/span> V_<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># load one value<\/span><br \/>\n<span class=\"token keyword\">for<\/span> k <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  q <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token comment\"># (1,d_k)<\/span><br \/>\n  x_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_i<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale <span class=\"token comment\">#  <\/span><br \/>\n  m_i <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> x_i<span class=\"token punctuation\">)<\/span><br \/>\n  expmm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n  expxm <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>x_i<span class=\"token operator\">&#8211;<\/span>m_i<span class=\"token punctuation\">)<\/span><br \/>\n  l_i <span class=\"token operator\">&#061;<\/span> l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span>expmm <span class=\"token operator\">&#043;<\/span> expxm<br \/>\n  o_i <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span><span class=\"token punctuation\">(<\/span>l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token operator\">*<\/span>expmm<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> expxm<span class=\"token operator\">*<\/span> v_i<span class=\"token punctuation\">)<\/span><span class=\"token operator\">\/<\/span>l_i<br \/>\n  m0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> l0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> o0<span class=\"token punctuation\">[<\/span>k<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> m_i<span class=\"token punctuation\">,<\/span> l_i<span class=\"token punctuation\">,<\/span> o_i<\/p>\n<p>\u603b\u7684IO\u6b21\u6570\u662fsizeof(k_i) &#043; sizeof(v_i) &#043; sizeof(Q) &#043; sizeof(l0) &#043; sizeof(m0) &#061; 2*d_k &#043; seq_len*d_k &#043; 2 * seq_len,\u56e0\u6b64\u5728seq_len\u548cd_k\u8f83\u5927\u65f6&#xff0c;\u8fd9\u79cd\u65b9\u6cd5\u53ef\u80fd\u4f1a\u83b7\u5f97\u66f4\u597d\u7684\u6027\u80fd&#xff0c;FlashAttentionV1\u4e5f\u91c7\u7528\u8fd9\u79cd\u987a\u5e8f\u6765\u8ba1\u7b97\u3002<br \/>\n\u63a5\u4e0b\u6765\u6211\u4eec\u518d\u52a0\u5165tiling\u7684\u6280\u5de7&#xff0c;\u5c06Q,K,V\u5206\u5757\u5904\u7406&#xff0c;\u7b80\u5355\u6765\u8bf4\u6211\u4eec\u4e00\u6b21\u6027\u5904\u7406Br\u4e2a\u67e5\u8be2\u4e0eBc\u4e2a\u952e&#xff0c;\u503c\u7684\u8ba1\u7b97&#xff0c;\u5728\u8bba\u6587\u4e2d&#xff0c;Br,Bc\u7684\u9009\u53d6\u65b9\u6cd5\u4e3a&#xff1a;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Br&#061;SRAM4dkBr &#061; {\\\\frac{SRAM}{4d_k}}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">r<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.3232em;vertical-align: -0.4509em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8723em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">4<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.394em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal mtight\">SR<\/span><span class=\"mord mathnormal mtight\">A<\/span><span style=\"margin-right: 0.109em\" class=\"mord mathnormal mtight\">M<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.4509em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span>,<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">Bc&#061;min(Br,dk)Bc &#061; min(Br, d_k)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal\">B<\/span><span class=\"mord mathnormal\">c<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">min<\/span><span class=\"mopen\">(<\/span><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">r<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>,\u5176\u4e2dSRAM\u662f\u663e\u5361\u4e2d\u6bcf\u4e2aSM\u7684L1\u7f13\u5b58\u5927\u5c0f\u3002<\/p>\n<h3>FlashAttention: 1-pass Tiled<\/h3>\n<p>\u5728\u4e4b\u524d\u7684\u8ba1\u7b97\u4e2d\u6211\u4eec\u6bcf\u6b21\u8fed\u4ee3\u7684\u5143\u7d20\u8de8\u5ea6\u662f1\u4e2a&#xff0c;\u4e5f\u5c31\u662f\u6bcf\u8ba1\u7b97\u4e00\u4e2a\u5143\u7d20\u5c31\u66f4\u65b0\u4e00\u6b21m,l,o, \u73b0\u5728\u6211\u4eec\u6bcf\u6b21\u8ba1\u7b97Bc\u4e2a\u5143\u7d20\u518d\u66f4\u65b0\u4e00\u6b21m,l,o,\u8fd9\u6837\u7684\u60c5\u51b5\u4f1a\u8ba9o\u7684\u72b6\u6001\u8f6c\u79fb\u516c\u5f0f\u53d8\u5f97\u5f88\u590d\u6742&#xff0c;\u4f46\u662f\u597d\u5728\u5728FlashAttentionV2\u4e2d\u7b80\u5316\u4e86\u8fd9\u4e2a\u72b6\u6001\u8f6c\u79fb\u7684\u8ba1\u7b97&#xff0c;\u56e0\u6b64\u6211\u4eec\u76f4\u63a5\u8df3\u5165FlashAttentionV2\u3002<br \/>\n\u4ecd\u7136\u5148\u8003\u8651\u4e00\u4e2a\u67e5\u8be2q,\u591a\u4e2a\u67e5\u8be2\u53ea\u662f\u5728\u4ee5\u4e0b\u7684\u57fa\u7840\u4e0a\u591a\u4e2a\u884c\u540c\u65f6\u8ba1\u7b97\u3002<br \/>\n\u6211\u4eec\u8bb0<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">S(i)\u2208R1\u00d7BcS^{(i)} \\\\in R^{1 \\\\times Bc}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9271em;vertical-align: -0.0391em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8413em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><span class=\"mbin mtight\">\u00d7<\/span><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span class=\"mord mathnormal mtight\">c<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">q\u2208R1\u00d7dkq\\\\in R^{1 \\\\times d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7335em;vertical-align: -0.1944em\"><\/span><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u4e0e<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ks\u2208RBc\u00d7dkks \\\\in R^{Bc \\\\times d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7335em;vertical-align: -0.0391em\"><\/span><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal\">k<\/span><span class=\"mord mathnormal\">s<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span class=\"mord mathnormal mtight\">c<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u8f6c\u7f6e\u7684\u4e58\u79ef&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">S(i)S^{(i)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.888em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u4e2d\u7684\u6700\u5927\u503c&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0197em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">eS(i)\u2212mie^{S^{(i)}-m_i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0397em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0397em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u5143\u7d20\u548c&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">eS(i)\u2212mie^{S^{(i)}-m_i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0397em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0397em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3281em\"><span class=\"\" style=\"top: -2.357em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.143em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u4e58\u79ef&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">oio_i<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.5806em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">o<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u7684\u7ef4\u5ea6\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">1\u00d7dk1 \\\\times d_k<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">1<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u3002\u67e5\u8be2<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">qq<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal\">q<\/span><\/span><\/span><\/span><\/span>\u7684\u5b8c\u6574\u7684\u5206\u6570\u662f<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">S&#061;{S(1),S(2),&#8230;,S(Bc)}S &#061; \\\\{S^{(1)},S^{(2)},&#8230;,S^{(Bc)}\\\\}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.138em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">{<\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">2<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">&#8230;<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span class=\"mord mathnormal mtight\">c<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">}<\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">V\u2208Rseqlen\u00d7dkV \\\\in R^{seq_len \\\\times d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">se<\/span><span class=\"mord mtight\"><span style=\"margin-right: 0.0359em\" class=\"mord mathnormal mtight\">q<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: -0.0359em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal mtight\">l<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u88ab\u5206\u4e3a<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">V(i)\u2208RBc\u00d7dkV^{(i)} \\\\in R^{Bc \\\\times d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9271em;vertical-align: -0.0391em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span class=\"mord mathnormal mtight\">c<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">m(1)&#061;max(S(1))l(1)&#061;\u2211eS(1)\u2212m(1)O^(1)&#061;eS(1)\u2212m1V(1)Tm(i)&#061;max(m(i\u22121),max(S(i)))l(i)&#061;em(i\u22121)\u2212m(i)&#043;\u2211eS(i)\u2212m(i)O^(i)&#061;eS(i\u22121)\u2212m(i)V(i\u22121)T&#043;eS(i)\u2212m(i)V(i)T&#061;O(i\u22121)em(i\u22121)\u2212m(i)&#043;eS(i)\u2212m(i)V(i)TO&#061;1lO^<br \/>\n\\\\begin{aligned}<br \/>\nm^{(1)} &amp;&#061; max(S^{(1)}) \\\\\\\\<br \/>\nl^{(1)} &amp;&#061; \\\\sum e^{S^{(1)}-m^{(1)}} \\\\\\\\<br \/>\n\\\\hat O^{(1)} &amp;&#061; e^{S^{(1)} &#8211; m^{1}} V^{(1)T} \\\\\\\\<br \/>\nm^{(i)} &amp;&#061; max(m^{(i-1)}, max(S^{(i)}))  \\\\\\\\<br \/>\nl^{(i)} &amp;&#061; e^{m^{(i-1)} &#8211; m^{(i)}} &#043; \\\\sum e^{S^{(i)} &#8211; m^{(i)}}\\\\\\\\<br \/>\n\\\\hat O^{(i)} &amp;&#061; e^{S^{(i-1)} &#8211; m^{(i)}} V^{(i-1)T} &#043; e^{S^{(i)} &#8211; m^{(i)}} V^{(i)T}\\\\\\\\<br \/>\n&amp;&#061; O^{(i-1)} e^{m^{(i-1)} &#8211; m^{(i)}} &#043; e^{S^{(i)} &#8211; m^{(i)}} V^{(i)T} \\\\\\\\<br \/>\nO &amp;&#061; \\\\frac{1}{l}{\\\\hat O}<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 14.632em;vertical-align: -7.066em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.566em\"><span class=\"\" style=\"top: -9.9494em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -8.1997em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -6.26em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.662em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.9123em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.9726em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 0.7771em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: 2.7585em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.066em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.566em\"><span class=\"\" style=\"top: -9.9494em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><span class=\"\" style=\"top: -8.1997em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-symbol large-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -6.26em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em\"><span class=\"\" style=\"top: -2.931em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.662em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">))<\/span><\/span><\/span><span class=\"\" style=\"top: -2.9123em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mop op-symbol large-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.9726em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 0.7771em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 2.7585em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.066em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u5176\u4e2d<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ll<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O^\\\\hat O<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9468em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u5206\u522b\u662f\u6700\u540e\u4e00\u6b21\u8fed\u4ee3\u7684<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">l(i)l^{(i)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.888em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O^(i)\\\\hat O^{(i)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9468em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mim^{i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8247em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8247em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">lil^{i}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8247em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8247em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>,\u5747\u4e3a\u6807\u91cf <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O\u2208RdkO \\\\in R^{d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f\u6700\u7ec8\u7684\u8f93\u51fa\u3002<br \/>\n\u73b0\u5728\u6211\u4eec\u8003\u8651\u591a\u4e2a\u67e5\u8be2\u7684\u60c5\u51b5&#xff1a;<br \/>\n<span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">m(1)&#061;max(S(1))\u2208RBrl(1)&#061;\u2211eS(1)\u2212m(1)\u2208RBrO^(1)&#061;eS(1)\u2212m1V(1)T\u2208RBr\u00d7dkm(i)&#061;max(m(i\u22121),max(S(i)))l(i)&#061;em(i\u22121)\u2212m(i)&#043;\u2211eS(i)\u2212m(i)O^(i)&#061;eS(i\u22121)\u2212m(i)V(i\u22121)T&#043;eS(i)\u2212m(i)V(i)T&#061;O(i\u22121)em(i\u22121)\u2212m(i)&#043;eS(i)\u2212m(i)V(i)TO&#061;1lO^<br \/>\n\\\\begin{aligned}<br \/>\nm^{(1)} &amp;&#061; max(S^{(1)}) \\\\in R^{Br} \\\\\\\\<br \/>\nl^{(1)} &amp;&#061; \\\\sum e^{S^{(1)}-m^{(1)}} \\\\in R^{Br} \\\\\\\\<br \/>\n\\\\hat O^{(1)} &amp;&#061; e^{S^{(1)} &#8211; m^{1}} V^{(1)T} \\\\in R^{Br \\\\times d_k}\\\\\\\\<br \/>\nm^{(i)} &amp;&#061; max(m^{(i-1)}, max(S^{(i)}))  \\\\\\\\<br \/>\nl^{(i)} &amp;&#061; e^{m^{(i-1)} &#8211; m^{(i)}} &#043; \\\\sum e^{S^{(i)} &#8211; m^{(i)}}\\\\\\\\<br \/>\n\\\\hat O^{(i)} &amp;&#061; e^{S^{(i-1)} &#8211; m^{(i)}} V^{(i-1)T} &#043; e^{S^{(i)} &#8211; m^{(i)}} V^{(i)T}\\\\\\\\<br \/>\n&amp;&#061; O^{(i-1)} e^{m^{(i-1)} &#8211; m^{(i)}} &#043; e^{S^{(i)} &#8211; m^{(i)}} V^{(i)T} \\\\\\\\<br \/>\nO &amp;&#061; \\\\frac{1}{l}{\\\\hat O}<br \/>\n\\\\end{aligned}<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 14.632em;vertical-align: -7.066em\"><\/span><span class=\"mord\"><span class=\"mtable\"><span class=\"col-align-r\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.566em\"><span class=\"\" style=\"top: -9.9494em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -8.1997em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -6.26em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.662em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.9123em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.9726em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 0.7771em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><\/span><\/span><span class=\"\" style=\"top: 2.7585em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.066em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"col-align-l\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.566em\"><span class=\"\" style=\"top: -9.9494em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal mtight\">r<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -8.1997em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mop op-symbol large-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal mtight\">r<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -6.26em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em\"><span class=\"\" style=\"top: -2.931em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal mtight\">r<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -4.662em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">ma<\/span><span class=\"mord mathnormal\">x<\/span><span class=\"mopen\">(<\/span><span class=\"mord\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">))<\/span><\/span><\/span><span class=\"\" style=\"top: -2.9123em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mop op-symbol large-op\" style=\"position: relative;top: 0em\">\u2211<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -0.9726em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 0.7771em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">e<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0897em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0576em\" class=\"mord mathnormal mtight\">S<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mbin mtight\">\u2212<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9667em\"><span class=\"\" style=\"top: -2.9667em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5357em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mord\"><span style=\"margin-right: 0.2222em\" class=\"mord mathnormal\">V<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.938em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><span style=\"margin-right: 0.1389em\" class=\"mord mathnormal mtight\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: 2.7585em\"><span class=\"pstrut\" style=\"height: 3.3214em\"><\/span><span class=\"mord\"><span class=\"mord\"><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.3214em\"><span class=\"\" style=\"top: -2.314em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.686em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 7.066em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><br \/>\n\u5176\u4e2d<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">ll<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O^\\\\hat O<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9468em\"><\/span><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u5206\u522b\u662f\u6700\u540e\u4e00\u6b21\u8fed\u4ee3\u7684<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">l(i)l^{(i)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.888em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u548c<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O^(i)\\\\hat O^{(i)}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.9468em\"><\/span><span class=\"mord\"><span class=\"mord accent\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9468em\"><span class=\"\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><\/span><span class=\"\" style=\"top: -3.2523em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"accent-body\" style=\"left: -0.1667em\"><span class=\"mord\">^<\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.888em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">mi,li\u2208RBrm^{i}, l^{i} \\\\in R^{Br}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0191em;vertical-align: -0.1944em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">m<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8247em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0197em\" class=\"mord mathnormal\">l<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8247em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8413em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal mtight\">r<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>, <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">O\u2208RBr\u00d7dkO \\\\in R^{Br \\\\times d_k}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal\">O<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8491em\"><\/span><span class=\"mord\"><span style=\"margin-right: 0.0077em\" class=\"mord mathnormal\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8491em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span style=\"margin-right: 0.0502em\" class=\"mord mathnormal mtight\">B<\/span><span style=\"margin-right: 0.0278em\" class=\"mord mathnormal mtight\">r<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span style=\"margin-right: 0.0315em\" class=\"mord mathnormal mtight\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>\u662f\u6700\u7ec8\u7684\u8f93\u51fa\u3002<br \/>\n\u4ee3\u7801\u5982\u4e0b&#xff1a;<\/p>\n<p>SRAM_SIZE <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1024<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">flashattnV1_block<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">.<\/span>shape<br \/>\n  scale <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span> <span class=\"token operator\">\/<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token comment\"># \u8ba1\u7b97\u5757\u5927\u5c0f&#xff1a;Br\u57fa\u4e8eSRAM\u5927\u5c0f\u548cd_k\u8ba1\u7b97<\/span><br \/>\n  Br <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>SRAM_SIZE <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">4<\/span> <span class=\"token operator\">*<\/span> d_k <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/\/<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">4<\/span> <span class=\"token operator\">*<\/span> d_k<span class=\"token punctuation\">)<\/span><br \/>\n  Bc <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>Br<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># Bc\u5e94\u8be5\u57fa\u4e8e\u5e8f\u5217\u957f\u5ea6\u800c\u4e0d\u662fd_k<\/span><br \/>\n  O <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><\/p>\n<p>  <span class=\"token keyword\">for<\/span> batch <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    Q_<span class=\"token punctuation\">,<\/span> K_<span class=\"token punctuation\">,<\/span> V_ <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><\/p>\n<p>    <span class=\"token comment\"># \u5904\u7406\u6bcf\u4e2a\u67e5\u8be2\u5757<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> q_start <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> Br<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      q_end <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>q_start <span class=\"token operator\">&#043;<\/span> Br<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">)<\/span><br \/>\n      q <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Br, d_k)<\/span><br \/>\n      Br_actual <span class=\"token operator\">&#061;<\/span> q<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token comment\"># \u521d\u59cb\u5316\u7d2f\u52a0\u5668<\/span><br \/>\n      m_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>full<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>Br_actual<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;inf&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n      l_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>Br_actual<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n      o_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>Br_actual<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><\/p>\n<p>      <span class=\"token comment\"># \u5904\u7406\u6bcf\u4e2aKV\u5757<\/span><br \/>\n      <span class=\"token keyword\">for<\/span> kv_start <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> Bc<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        kv_end <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>kv_start <span class=\"token operator\">&#043;<\/span> Bc<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">)<\/span><br \/>\n        k_block <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>kv_start<span class=\"token punctuation\">:<\/span>kv_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Bc_actual, d_k)<\/span><br \/>\n        v_block <span class=\"token operator\">&#061;<\/span> V_<span class=\"token punctuation\">[<\/span>kv_start<span class=\"token punctuation\">:<\/span>kv_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Bc_actual, d_k)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u6ce8\u610f\u529b\u5206\u6570: (Br_actual, Bc_actual)<\/span><br \/>\n        S <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_block<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale<\/p>\n<p>        <span class=\"token comment\"># \u627e\u5230\u5f53\u524d\u5757\u7684\u6700\u5927\u503c<\/span><br \/>\n        m_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>S<span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>values<\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u65b0\u7684\u5168\u5c40\u6700\u5927\u503c<\/span><br \/>\n        m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m_prev<span class=\"token punctuation\">,<\/span> m_block<span class=\"token punctuation\">)<\/span><\/p>\n<p>        exp_m_prev_minus_m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m_prev <span class=\"token operator\">&#8211;<\/span> m_new<span class=\"token punctuation\">)<\/span><br \/>\n        exp_m_block_minus_m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>S <span class=\"token operator\">&#8211;<\/span> m_new<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u7d2f\u52a0\u5668<\/span><br \/>\n        l_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span>exp_m_block_minus_m_new<span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        l_new <span class=\"token operator\">&#061;<\/span> l_prev <span class=\"token operator\">*<\/span> exp_m_prev_minus_m_new <span class=\"token operator\">&#043;<\/span> l_block<\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u8f93\u51fa\u7d2f\u52a0\u5668<\/span><br \/>\n        o_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>exp_m_block_minus_m_new<span class=\"token punctuation\">,<\/span> v_block<span class=\"token punctuation\">)<\/span><br \/>\n        o_new <span class=\"token operator\">&#061;<\/span> o_prev <span class=\"token operator\">*<\/span> exp_m_prev_minus_m_new<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> o_block<\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u72b6\u6001<\/span><br \/>\n        m_prev<span class=\"token punctuation\">,<\/span> l_prev<span class=\"token punctuation\">,<\/span> o_prev <span class=\"token operator\">&#061;<\/span> m_new<span class=\"token punctuation\">,<\/span> l_new<span class=\"token punctuation\">,<\/span> o_new<\/p>\n<p>      O<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">,<\/span> q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o_prev <span class=\"token operator\">\/<\/span> l_prev<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token keyword\">return<\/span> O<\/p>\n<p>\u5728flashAttentionV2\u4e2d&#xff0c;\u72b6\u6001\u8f6c\u79fb\u7684\u8ba1\u7b97\u88ab\u7b80\u5316\u4e86&#xff0c;\u5e76\u4e14\u7531\u4e8e\u6279\u91cf\u8ba1\u7b97Q,K,V\u7684\u5206\u5757&#xff0c;\u4e0d\u9002\u5408\u4fdd\u5b58\u8fc7\u5927\u7684\u4e2d\u95f4\u72b6\u6001&#xff0c;\u56e0\u6b64\u5148\u8fed\u4ee3Q,\u7136\u540e\u8fed\u4ee3K,V\u3002<\/p>\n<p>\u8fd9\u91cc\u4e5f\u63d0\u4f9b\u5148\u8fed\u4ee3K,V\u518d\u8fed\u4ee3Q\u7684\u5b9e\u73b0&#xff1a;<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">flashattnV2_blockKVQ<\/span><span class=\"token punctuation\">(<\/span>Q<span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n  batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">.<\/span>shape<br \/>\n  scale <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span> <span class=\"token operator\">\/<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>d_k<span class=\"token punctuation\">)<\/span><br \/>\n  Br <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span>SRAM_SIZE <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">4<\/span> <span class=\"token operator\">*<\/span> d_k <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/\/<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token number\">4<\/span> <span class=\"token operator\">*<\/span> d_k<span class=\"token punctuation\">)<\/span><br \/>\n  Bc <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>Br<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">)<\/span><br \/>\n  O <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token keyword\">for<\/span> batch <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>batch_size<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    Q_<span class=\"token punctuation\">,<\/span> K_<span class=\"token punctuation\">,<\/span> V_ <span class=\"token operator\">&#061;<\/span> Q<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> V<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span><\/p>\n<p>    <span class=\"token comment\"># \u521d\u59cb\u5316\u6240\u6709\u67e5\u8be2\u4f4d\u7f6e\u7684\u7d2f\u52a0\u5668<\/span><br \/>\n    m_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>full<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">,<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#039;inf&#039;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (seq_len,)<\/span><br \/>\n    l_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span>                   <span class=\"token comment\"># (seq_len,)<\/span><br \/>\n    o_prev <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>seq_len<span class=\"token punctuation\">,<\/span> d_k<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>Q<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span>             <span class=\"token comment\"># (seq_len, d_k)<\/span><\/p>\n<p>    <span class=\"token comment\"># \u5916\u5c42\u5faa\u73af&#xff1a;\u904d\u5386KV\u5757<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> kv_start <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> Bc<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n      kv_end <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>kv_start <span class=\"token operator\">&#043;<\/span> Bc<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">)<\/span><br \/>\n      k_block <span class=\"token operator\">&#061;<\/span> K_<span class=\"token punctuation\">[<\/span>kv_start<span class=\"token punctuation\">:<\/span>kv_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Bc_actual, d_k)<\/span><br \/>\n      v_block <span class=\"token operator\">&#061;<\/span> V_<span class=\"token punctuation\">[<\/span>kv_start<span class=\"token punctuation\">:<\/span>kv_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Bc_actual, d_k)<\/span><br \/>\n      Bc_actual <span class=\"token operator\">&#061;<\/span> k_block<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>      <span class=\"token comment\"># \u5185\u5c42\u5faa\u73af&#xff1a;\u904d\u5386\u67e5\u8be2\u5757<\/span><br \/>\n      <span class=\"token keyword\">for<\/span> q_start <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">,<\/span> Br<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        q_end <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>q_start <span class=\"token operator\">&#043;<\/span> Br<span class=\"token punctuation\">,<\/span> seq_len<span class=\"token punctuation\">)<\/span><br \/>\n        q <span class=\"token operator\">&#061;<\/span> Q_<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Br_actual, d_k)<\/span><br \/>\n        Br_actual <span class=\"token operator\">&#061;<\/span> q<span class=\"token punctuation\">.<\/span>shape<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u6ce8\u610f\u529b\u5206\u6570: (Br_actual, Bc_actual)<\/span><br \/>\n        attn_scores <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">,<\/span> k_block<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> scale<\/p>\n<p>        <span class=\"token comment\"># \u83b7\u53d6\u5f53\u524d\u67e5\u8be2\u5757\u5bf9\u5e94\u7684\u7d2f\u52a0\u5668\u72b6\u6001<\/span><br \/>\n        m_prev_block <span class=\"token operator\">&#061;<\/span> m_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Br_actual,)<\/span><br \/>\n        l_prev_block <span class=\"token operator\">&#061;<\/span> l_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Br_actual,)<\/span><br \/>\n        o_prev_block <span class=\"token operator\">&#061;<\/span> o_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span>  <span class=\"token comment\"># (Br_actual, d_k)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u627e\u5230\u5f53\u524d\u5757\u7684\u6700\u5927\u503c<\/span><br \/>\n        m_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">max<\/span><span class=\"token punctuation\">(<\/span>attn_scores<span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>values  <span class=\"token comment\"># (Br_actual,)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u65b0\u7684\u5168\u5c40\u6700\u5927\u503c<\/span><br \/>\n        m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>maximum<span class=\"token punctuation\">(<\/span>m_prev_block<span class=\"token punctuation\">,<\/span> m_block<span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u8ba1\u7b97\u6307\u6570\u9879<\/span><br \/>\n        exp_m_prev_minus_m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>m_prev_block <span class=\"token operator\">&#8211;<\/span> m_new<span class=\"token punctuation\">)<\/span><br \/>\n        exp_m_block_minus_m_new <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>attn_scores <span class=\"token operator\">&#8211;<\/span> m_new<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u7d2f\u52a0\u5668<\/span><br \/>\n        l_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span>exp_m_block_minus_m_new<span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (Br_actual,)<\/span><br \/>\n        l_new <span class=\"token operator\">&#061;<\/span> l_prev_block <span class=\"token operator\">*<\/span> exp_m_prev_minus_m_new <span class=\"token operator\">&#043;<\/span> l_block<\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u8f93\u51fa\u7d2f\u52a0\u5668<\/span><br \/>\n        o_block <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">(<\/span>exp_m_block_minus_m_new<span class=\"token punctuation\">,<\/span> v_block<span class=\"token punctuation\">)<\/span>  <span class=\"token comment\"># (Br_actual, d_k)<\/span><br \/>\n        o_new <span class=\"token operator\">&#061;<\/span> o_prev_block <span class=\"token operator\">*<\/span> exp_m_prev_minus_m_new<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> o_block<\/p>\n<p>        <span class=\"token comment\"># \u66f4\u65b0\u7d2f\u52a0\u5668\u72b6\u6001<\/span><br \/>\n        m_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> m_new<br \/>\n        l_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> l_new<br \/>\n        o_prev<span class=\"token punctuation\">[<\/span>q_start<span class=\"token punctuation\">:<\/span>q_end<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o_new<\/p>\n<p>    <span class=\"token comment\"># \u6240\u6709KV\u5757\u5904\u7406\u5b8c\u540e&#xff0c;\u8fdb\u884c\u6700\u7ec8\u5f52\u4e00\u5316<\/span><br \/>\n    O<span class=\"token punctuation\">[<\/span>batch<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> o_prev <span class=\"token operator\">\/<\/span> l_prev<span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n  <span class=\"token keyword\">return<\/span> O<\/p>\n<h3>\u603b\u7ed3<\/h3>\n<p>\u4ee5\u4e0a\u5c31\u662fpytorch\u4e32\u884c\u5b9e\u73b0\u7684FlashAttentionV2\u7b97\u6cd5&#xff0c;\u867d\u7136\u5728CPU\u4e0a\u6548\u7387\u4e0d\u9ad8&#xff0c;\u4f46\u662f\u8db3\u4ee5\u63d0\u4f9b\u601d\u8def&#xff0c;\u4f7f\u5176\u5f88\u5bb9\u6613\u5730\u53ef\u4ee5\u91cd\u5199\u6210triton\u4ee3\u7801\u7136\u540e\u5728GPU\u4e0a\u6267\u884c\u3002<br \/>\nFlashAttention\u6700\u91cd\u8981\u7684\u601d\u60f3\u5c31\u662f\u4ece\u964d\u4f4e\u5185\u5b58\u8bbf\u95ee\u6b21\u6570\u6765\u63d0\u5347\u901f\u5ea6\u51fa\u53d1&#xff0c;\u5c06\u9700\u8981\u4f9d\u8d56\u5168\u5c40\u72b6\u6001\u7684\u8ba1\u7b97\u901a\u8fc7\u7c7b\u4f3c\u4e8e\u52a8\u6001\u89c4\u5212\u7684\u65b9\u5f0f\u8fdb\u884c\u8fed\u4ee3\u8ba1\u7b97&#xff0c;\u4ece\u800c\u51cf\u5c11\u8fed\u4ee3\u7684\u6b21\u6570&#xff0c;\u51cf\u5c11\u4e86\u5bf9\u4e8e\u5168\u5c40\u5185\u5b58\u7684\u8bbf\u95ee\u6b21\u6570&#xff0c;\u63d0\u5347\u4e86\u8ba1\u7b97\u6548\u7387&#xff0c;\u8fd9\u662f\u5bf9GPU\u6027\u8d28\u7684\u6df1\u523b\u6d1e\u5bdf\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"<p>FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0<br \/>\n\u80cc\u666f<br \/>\nFlashAttention\u662f\u76ee\u524d\u6700\u6d41\u884c\u7684\u4f18\u5316Transformer\u4e2d\u6ce8\u610f\u529b\u673a\u5236\u7684\u7b97\u6cd5&#xff0c;\u65e8\u5728\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u548c\u51cf\u5c11\u5185\u5b58\u4f7f\u7528\u3002\u4e0e\u5176\u4ed6\u4f18\u5316Attention\u7b97\u6cd5\u76f8\u6bd4&#xff0c;FlashAttention\u662f\u4e00\u79cd\u7cbe\u786e\u8ba1\u7b97Attention\u7684\u7b97\u6cd5&#xff0c;\u800c\u4e0d\u662f\u8fd1\u4f3c\u8ba1\u7b97\u3002\u5b83\u901a\u8fc7\u91cd\u65b0\u7ec4\u7ec7\u8ba1\u7b97\u6d41\u7a0b&#xff0c;\u8fdb\u800c\u51cf\u5c11\u4e86GPU\u4e2d\u5168\u5c40\u5185\u5b58\u7684\u8bbf\u95ee\u6b21\u6570&#xff0c;\u4ece\u800c\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u3002<br \/>\n\u672c\u6587\u8bd5\u56fe\u4ee5\u6700\u7b80\u5355\u7684\u65b9\u5f0f\u4ecb\u7ecd&amp;#xff<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[81,50,207,86],"topic":[],"class_list":["post-78054","post","type-post","status-publish","format-standard","hentry","category-server","tag-python","tag-50","tag-207","tag-86"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/78054.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 \u80cc\u666f FlashAttention\u662f\u76ee\u524d\u6700\u6d41\u884c\u7684\u4f18\u5316Transformer\u4e2d\u6ce8\u610f\u529b\u673a\u5236\u7684\u7b97\u6cd5&#xff0c;\u65e8\u5728\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u548c\u51cf\u5c11\u5185\u5b58\u4f7f\u7528\u3002\u4e0e\u5176\u4ed6\u4f18\u5316Attention\u7b97\u6cd5\u76f8\u6bd4&#xff0c;FlashAttention\u662f\u4e00\u79cd\u7cbe\u786e\u8ba1\u7b97Attention\u7684\u7b97\u6cd5&#xff0c;\u800c\u4e0d\u662f\u8fd1\u4f3c\u8ba1\u7b97\u3002\u5b83\u901a\u8fc7\u91cd\u65b0\u7ec4\u7ec7\u8ba1\u7b97\u6d41\u7a0b&#xff0c;\u8fdb\u800c\u51cf\u5c11\u4e86GPU\u4e2d\u5168\u5c40\u5185\u5b58\u7684\u8bbf\u95ee\u6b21\u6570&#xff0c;\u4ece\u800c\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u3002 \u672c\u6587\u8bd5\u56fe\u4ee5\u6700\u7b80\u5355\u7684\u65b9\u5f0f\u4ecb\u7ecd&amp;#xff\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/78054.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-02-25T20:17:17+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"20 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/78054.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/78054.html\",\"name\":\"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-02-25T20:17:17+00:00\",\"dateModified\":\"2026-02-25T20:17:17+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/78054.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/78054.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/78054.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/78054.html","og_locale":"zh_CN","og_type":"article","og_title":"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 \u80cc\u666f FlashAttention\u662f\u76ee\u524d\u6700\u6d41\u884c\u7684\u4f18\u5316Transformer\u4e2d\u6ce8\u610f\u529b\u673a\u5236\u7684\u7b97\u6cd5&#xff0c;\u65e8\u5728\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u548c\u51cf\u5c11\u5185\u5b58\u4f7f\u7528\u3002\u4e0e\u5176\u4ed6\u4f18\u5316Attention\u7b97\u6cd5\u76f8\u6bd4&#xff0c;FlashAttention\u662f\u4e00\u79cd\u7cbe\u786e\u8ba1\u7b97Attention\u7684\u7b97\u6cd5&#xff0c;\u800c\u4e0d\u662f\u8fd1\u4f3c\u8ba1\u7b97\u3002\u5b83\u901a\u8fc7\u91cd\u65b0\u7ec4\u7ec7\u8ba1\u7b97\u6d41\u7a0b&#xff0c;\u8fdb\u800c\u51cf\u5c11\u4e86GPU\u4e2d\u5168\u5c40\u5185\u5b58\u7684\u8bbf\u95ee\u6b21\u6570&#xff0c;\u4ece\u800c\u63d0\u9ad8\u8ba1\u7b97\u6548\u7387\u3002 \u672c\u6587\u8bd5\u56fe\u4ee5\u6700\u7b80\u5355\u7684\u65b9\u5f0f\u4ecb\u7ecd&amp;#xff","og_url":"https:\/\/www.wsisp.com\/helps\/78054.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-02-25T20:17:17+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"20 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/78054.html","url":"https:\/\/www.wsisp.com\/helps\/78054.html","name":"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-02-25T20:17:17+00:00","dateModified":"2026-02-25T20:17:17+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/78054.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/78054.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/78054.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u65b0\u5e74\u4e00\u5b9a\u8981\u5b66\u4f1aFlashAttention! FlashAttention\u6781\u7b80\u6559\u7a0b\u4e0e\u5b9e\u73b0"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/78054","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=78054"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/78054\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=78054"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=78054"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=78054"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=78054"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}