{"id":107134,"date":"2026-09-19T04:19:50","date_gmt":"2026-09-18T20:19:50","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/107134.html"},"modified":"2026-09-19T04:19:50","modified_gmt":"2026-09-18T20:19:50","slug":"%e6%b7%b1%e5%85%a5%e7%90%86%e8%a7%a3-transformer%ef%bc%9a%e7%90%86%e8%a7%a3%e9%a1%ba%e5%ba%8fpositional-encoding","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/107134.html","title":{"rendered":"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding"},"content":{"rendered":"<h2>\u524d\u8a00<\/h2>\n<p>\u5728\u524d\u9762\u7684\u6587\u7ae0\u4e2d&#xff0c;\u6211\u4eec\u5df2\u7ecf\u9010\u6b65\u89e3\u51b3\u4e86 Transformer \u4e2d\u4e00\u4e2a\u975e\u5e38\u6838\u5fc3\u7684\u95ee\u9898\u3002<\/p>\n<p>\u9996\u5148&#xff0c;\u6211\u4eec\u5728\u7b2c\u4e00\u7bc7\u4e2d\u8ba8\u8bba\u4e86\u4e3a\u4ec0\u4e48\u4f20\u7edf\u7684 RNN\u3001LSTM \u5728\u5904\u7406\u957f\u8ddd\u79bb\u4f9d\u8d56\u548c\u5e76\u884c\u8ba1\u7b97\u65f6\u5b58\u5728\u5c40\u9650&#xff0c;\u56e0\u6b64\u9700\u8981\u4e00\u79cd\u80fd\u591f\u76f4\u63a5\u5efa\u7acb\u4efb\u610f Token \u4e4b\u95f4\u8054\u7cfb\u7684\u673a\u5236\u3002<\/p>\n<p>\u968f\u540e&#xff0c;\u6211\u4eec\u8fdb\u5165 Transformer \u7684\u6838\u5fc3\u3002<\/p>\n<p>\u6211\u4eec\u5148\u7406\u89e3\u4e86 Self-Attention&#xff0c;\u77e5\u9053\u4e86 Query\u3001Key\u3001Value \u662f\u5982\u4f55\u8ba1\u7b97\u7684&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         Q<\/p>\n<p>         &#061;<\/p>\n<p>         X<\/p>\n<p>          W<\/p>\n<p>          Q<\/p>\n<p>         Q&#061;XW_Q <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8778em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.9694em;vertical-align: -0.2861em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">Q<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2861em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         K<\/p>\n<p>         &#061;<\/p>\n<p>         X<\/p>\n<p>          W<\/p>\n<p>          K<\/p>\n<p>         K&#061;XW_K <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0715em\">K<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         V<\/p>\n<p>         &#061;<\/p>\n<p>         X<\/p>\n<p>          W<\/p>\n<p>          V<\/p>\n<p>         V&#061;XW_V <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3283em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.2222em\">V<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7136\u540e\u901a\u8fc7 Scaled Dot-Product Attention \u8ba1\u7b97 Token \u4e4b\u95f4\u7684\u5173\u8054&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         Attention<\/p>\n<p>         \u2061<\/p>\n<p>         (<\/p>\n<p>         Q<\/p>\n<p>         ,<\/p>\n<p>         K<\/p>\n<p>         ,<\/p>\n<p>         V<\/p>\n<p>         )<\/p>\n<p>         &#061;<\/p>\n<p>         softmax<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            Q<\/p>\n<p>             K<\/p>\n<p>             T<\/p>\n<p>             d<\/p>\n<p>             k<\/p>\n<p>          )<\/p>\n<p>         V<\/p>\n<p>         \\\\operatorname{Attention}(Q,K,V)&#061; \\\\operatorname{softmax} \\\\left( \\\\frac{QK^T}{\\\\sqrt{d_k}} \\\\right)V <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">Attention<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4684em;vertical-align: -0.95em\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">softmax<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.5183em\"><span class=\"\" style=\"top: -2.2528em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/p>\n<p>                    <\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.677em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">Q<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8413em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.1389em\">T<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.93em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5728\u4e0a\u4e00\u7bc7\u6587\u7ae0\u4e2d&#xff0c;\u6211\u4eec\u53c8\u8fdb\u4e00\u6b65\u7406\u89e3\u4e86 Multi-Head Attention\u3002<\/p>\n<p>\u5982\u679c\u53ea\u6709\u4e00\u4e2a Attention&#xff0c;\u6a21\u578b\u53ea\u80fd\u5728\u4e00\u4e2a\u8868\u793a\u7a7a\u95f4\u4e2d\u5efa\u7acb Token \u4e4b\u95f4\u7684\u5173\u7cfb&#xff0c;\u800c Multi-Head Attention \u5219\u628a\u8fd9\u79cd\u80fd\u529b\u6269\u5c55\u5230\u4e86\u591a\u4e2a\u4e0d\u540c\u7684\u8868\u793a\u5b50\u7a7a\u95f4&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         MultiHead<\/p>\n<p>         \u2061<\/p>\n<p>         (<\/p>\n<p>         Q<\/p>\n<p>         ,<\/p>\n<p>         K<\/p>\n<p>         ,<\/p>\n<p>         V<\/p>\n<p>         )<\/p>\n<p>         &#061;<\/p>\n<p>         Concat<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            head<\/p>\n<p>            \u2061<\/p>\n<p>           1<\/p>\n<p>          ,<\/p>\n<p>          \u2026<\/p>\n<p>          ,<\/p>\n<p>            head<\/p>\n<p>            \u2061<\/p>\n<p>           h<\/p>\n<p>          )<\/p>\n<p>          W<\/p>\n<p>          O<\/p>\n<p>         \\\\operatorname{MultiHead}(Q,K,V)&#061; \\\\operatorname{Concat} \\\\left( \\\\operatorname{head}_1, \\\\ldots, \\\\operatorname{head}_h \\\\right) W^O <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">MultiHead<\/span><\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">Q<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0715em\">K<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.2222em\">V<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1413em;vertical-align: -0.25em\"><\/span><span class=\"mop\"><span class=\"mord mathrm\">Concat<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\">(<\/span><span class=\"mop\"><span class=\"mop\"><span class=\"mord mathrm\">head<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3011em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\">1<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\">\u2026<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\"><span class=\"mop\"><span class=\"mord mathrm\">head<\/span><\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">h<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\">)<\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">W<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8913em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0278em\">O<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5230\u8fd9\u91cc&#xff0c;\u770b\u8d77\u6765 Transformer \u5df2\u7ecf\u80fd\u591f\u5f88\u597d\u5730\u7406\u89e3\u4e00\u53e5\u8bdd\u4e86\u3002<\/p>\n<p>\u4f46\u662f\u4ed4\u7ec6\u60f3\u4e00\u60f3&#xff0c;\u6211\u4eec\u4f1a\u53d1\u73b0\u4e00\u4e2a\u975e\u5e38\u5947\u602a\u7684\u95ee\u9898&#xff1a;<\/p>\n<p>Transformer \u771f\u7684\u77e5\u9053\u4e00\u4e2a Token \u5728\u53e5\u5b50\u4e2d\u7684\u4f4d\u7f6e\u5417&#xff1f;<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u6211 \u559c\u6b22 \u4f60<\/p>\n<p>\u548c&#xff1a;<\/p>\n<p>\u4f60 \u559c\u6b22 \u6211<\/p>\n<p>\u4e24\u4e2a\u53e5\u5b50\u4f7f\u7528\u7684\u662f\u5b8c\u5168\u76f8\u540c\u7684\u4e09\u4e2a Token&#xff0c;\u53ea\u662f\u987a\u5e8f\u53d1\u751f\u4e86\u53d8\u5316\u3002<\/p>\n<p>\u4f46\u662f\u5b83\u4eec\u8868\u8fbe\u7684\u610f\u601d\u663e\u7136\u4e0d\u540c\u3002<\/p>\n<p>\u5982\u679c\u6a21\u578b\u4e0d\u77e5\u9053 Token \u7684\u4f4d\u7f6e&#xff0c;\u90a3\u4e48\u5b83\u53c8\u600e\u4e48\u77e5\u9053\u8c01\u5728\u524d\u9762\u3001\u8c01\u5728\u540e\u9762\u3001\u8c01\u662f\u7b2c\u4e00\u4e2a\u8bcd\u3001\u8c01\u662f\u7b2c\u4e8c\u4e2a\u8bcd&#xff1f;<\/p>\n<p>\u8fd9\u5c31\u662f\u8fd9\u4e00\u7bc7\u6211\u4eec\u8981\u89e3\u51b3\u7684\u95ee\u9898\u3002<\/p>\n<p>\u7b54\u6848\u5c31\u662f&#xff1a;<\/p>\n<p>Positional Encoding&#xff0c;\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<hr \/>\n<h3>\u76ee\u5f55<\/h3>\n<ul>\n<li>\u4e00\u3001\u4e3a\u4ec0\u4e48 Transformer \u9700\u8981\u4f4d\u7f6e\u7f16\u7801&#xff1f;\n<ul>\n<li>1. Self-Attention \u672c\u8eab\u4e0d\u77e5\u9053\u987a\u5e8f<\/li>\n<li>2. \u4e00\u4e2a\u975e\u5e38\u76f4\u89c2\u7684\u4f8b\u5b50<\/li>\n<\/ul>\n<\/li>\n<li>\u4e8c\u3001\u4f4d\u7f6e\u7f16\u7801\u5230\u5e95\u662f\u4ec0\u4e48&#xff1f;\n<ul>\n<li>1. \u7ed9\u6bcf\u4e00\u4e2a\u4f4d\u7f6e\u4e00\u4e2a\u5411\u91cf<\/li>\n<li>2. \u4e3a\u4ec0\u4e48\u53ef\u4ee5\u76f4\u63a5\u76f8\u52a0&#xff1f;<\/li>\n<\/ul>\n<\/li>\n<li>\u4e09\u3001Transformer \u539f\u8bba\u6587\u4e2d\u7684\u4f4d\u7f6e\u7f16\u7801\n<ul>\n<li>1. <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>           pos<\/p>\n<p>        <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><\/span> \u662f\u4ec0\u4e48&#xff1f;<\/li>\n<li>2. <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\n<p>            i<\/p>\n<p>           i<\/p>\n<p>        <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u662f\u4ec0\u4e48&#xff1f;<\/li>\n<li>3. \u4e3a\u4ec0\u4e48\u8981\u4f7f\u7528\u6b63\u5f26\u548c\u4f59\u5f26&#xff1f;<\/li>\n<\/ul>\n<\/li>\n<li>\u56db\u3001\u4e3a\u4ec0\u4e48\u4e0d\u540c\u7ef4\u5ea6\u8981\u4f7f\u7528\u4e0d\u540c\u9891\u7387&#xff1f;<\/li>\n<li>\u4e94\u3001\u6b63\u5f26\u4f4d\u7f6e\u7f16\u7801\u4e3a\u4ec0\u4e48\u80fd\u591f\u5e2e\u52a9\u6a21\u578b\u7406\u89e3\u76f8\u5bf9\u4f4d\u7f6e&#xff1f;<\/li>\n<li>\u516d\u3001\u7edd\u5bf9\u4f4d\u7f6e\u7f16\u7801\u548c\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801\n<ul>\n<li>1. Sinusoidal Encoding&#xff1a;\u56fa\u5b9a\u4f4d\u7f6e\u7f16\u7801<\/li>\n<li>2. Learned Positional Embedding&#xff1a;\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801<\/li>\n<li>3. \u4e24\u79cd\u65b9\u6cd5\u6709\u4ec0\u4e48\u533a\u522b&#xff1f;<\/li>\n<\/ul>\n<\/li>\n<li>\u4e03\u3001\u4ece\u7edd\u5bf9\u4f4d\u7f6e\u5230\u76f8\u5bf9\u4f4d\u7f6e<\/li>\n<li>\u516b\u3001RoPE&#xff1a;\u73b0\u4ee3\u5927\u8bed\u8a00\u6a21\u578b\u4e2d\u7684\u91cd\u8981\u4f4d\u7f6e\u8868\u793a\u65b9\u6cd5<\/li>\n<li>\u4e5d\u3001\u4f4d\u7f6e\u7f16\u7801\u653e\u5728\u54ea\u91cc&#xff1f;<\/li>\n<li>\u5341\u3001\u628a\u524d\u9762\u4e94\u7bc7\u6587\u7ae0\u4e32\u8d77\u6765<\/li>\n<li>\u5341\u4e00\u3001\u8fd9\u4e00\u7bc7\u6700\u5e94\u8be5\u8bb0\u4f4f\u7684\u5185\u5bb9<\/li>\n<li>\u7ed3\u8bed&#xff1a;Attention \u6709\u4e86&#xff0c;\u987a\u5e8f\u4e5f\u6709\u4e86&#xff0c;\u90a3\u4e48\u4e00\u4e2a Transformer Block \u8fd8\u7f3a\u4ec0\u4e48&#xff1f;<\/li>\n<\/ul>\n<hr \/>\n<h2>\u4e00\u3001\u4e3a\u4ec0\u4e48 Transformer \u9700\u8981\u4f4d\u7f6e\u7f16\u7801&#xff1f;<\/h2>\n<h3>1. Self-Attention \u672c\u8eab\u4e0d\u77e5\u9053\u987a\u5e8f<\/h3>\n<p>\u8fd9\u662f\u7406\u89e3\u4f4d\u7f6e\u7f16\u7801\u6700\u5173\u952e\u7684\u4e00\u70b9\u3002<\/p>\n<p>\u6211\u4eec\u4e4b\u524d\u8bb2 Self-Attention \u7684\u65f6\u5019&#xff0c;\u4e00\u76f4\u5728\u8ba8\u8bba&#xff1a;<\/p>\n<p>\u4e00\u4e2a Token \u5e94\u8be5\u5173\u6ce8\u54ea\u4e9b Token&#xff1f;<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u5c0f\u660e \u559c\u6b22 \u5403 \u82f9\u679c<\/p>\n<p>\u5728 Self-Attention \u4e2d&#xff0c;\u201c\u5403\u201d\u53ef\u4ee5\u5173\u6ce8\u201c\u5c0f\u660e\u201d&#xff0c;\u4e5f\u53ef\u4ee5\u5173\u6ce8\u201c\u82f9\u679c\u201d\u3002<\/p>\n<p>\u4f46\u662f Attention \u505a\u7684\u4e8b\u60c5\u672c\u8d28\u4e0a\u662f\u8ba1\u7b97 Token \u4e4b\u95f4\u7684\u76f8\u5173\u7a0b\u5ea6\u3002<\/p>\n<p>\u5b83\u770b\u5230\u7684\u662f&#xff1a;<\/p>\n<p>Token 1<br \/>\nToken 2<br \/>\nToken 3<br \/>\nToken 4<\/p>\n<p>\u4ee5\u53ca\u5b83\u4eec\u5bf9\u5e94\u7684\u5411\u91cf\u3002<\/p>\n<p>\u5b83\u672c\u8eab\u5e76\u6ca1\u6709\u4e00\u4e2a\u5929\u7136\u7684\u673a\u5236\u544a\u8bc9\u81ea\u5df1&#xff1a;<\/p>\n<p>Token 1 \u662f\u7b2c 1 \u4e2a<br \/>\nToken 2 \u662f\u7b2c 2 \u4e2a<br \/>\nToken 3 \u662f\u7b2c 3 \u4e2a<br \/>\nToken 4 \u662f\u7b2c 4 \u4e2a<\/p>\n<p>\u8fd9\u548c RNN \u6709\u4e00\u4e2a\u975e\u5e38\u660e\u663e\u7684\u533a\u522b\u3002<\/p>\n<p>RNN \u5929\u7136\u6309\u7167\u987a\u5e8f\u5904\u7406&#xff1a;<\/p>\n<p>\u7b2c\u4e00\u4e2a Token<br \/>\n    \u2193<br \/>\n\u7b2c\u4e8c\u4e2a Token<br \/>\n    \u2193<br \/>\n\u7b2c\u4e09\u4e2a Token<br \/>\n    \u2193<br \/>\n\u7b2c\u56db\u4e2a Token<\/p>\n<p>\u56e0\u6b64\u201c\u524d\u540e\u987a\u5e8f\u201d\u672c\u8eab\u5c31\u5b58\u5728\u4e8e RNN \u7684\u8ba1\u7b97\u8fc7\u7a0b\u91cc\u3002<\/p>\n<p>\u800c Transformer \u7684 Self-Attention \u66f4\u50cf\u662f&#xff1a;<\/p>\n<p>Token 1 \u2500\u2510<br \/>\nToken 2 \u2500\u2524<br \/>\nToken 3 \u2500\u253c\u2500\u2500\u2192 \u540c\u65f6\u8fdb\u884c\u5173\u7cfb\u8ba1\u7b97<br \/>\nToken 4 \u2500\u2524<br \/>\nToken 5 \u2500\u2518<\/p>\n<p>\u6240\u6709 Token \u53ef\u4ee5\u5e76\u884c\u53c2\u4e0e\u8ba1\u7b97\u3002<\/p>\n<p>\u8fd9\u5e26\u6765\u4e86 Transformer \u975e\u5e38\u91cd\u8981\u7684\u5e76\u884c\u8ba1\u7b97\u80fd\u529b&#xff0c;\u4f46\u4e5f\u5e26\u6765\u4e86\u4e00\u4e2a\u65b0\u7684\u95ee\u9898&#xff1a;<\/p>\n<p>\u65e2\u7136\u6240\u6709 Token \u53ef\u4ee5\u540c\u65f6\u5904\u7406&#xff0c;\u90a3\u4e48\u987a\u5e8f\u4fe1\u606f\u4ece\u54ea\u91cc\u6765&#xff1f;<\/p>\n<hr \/>\n<h3>2. \u4e00\u4e2a\u975e\u5e38\u76f4\u89c2\u7684\u4f8b\u5b50<\/h3>\n<p>\u5047\u8bbe\u6211\u4eec\u6709\u4e24\u4e2a\u53e5\u5b50&#xff1a;<\/p>\n<p>\u6211 \u559c\u6b22 \u4f60<\/p>\n<p>\u548c&#xff1a;<\/p>\n<p>\u4f60 \u559c\u6b22 \u6211<\/p>\n<p>\u5047\u8bbe\u6682\u65f6\u4e0d\u8003\u8651\u4f4d\u7f6e&#xff0c;\u53ea\u770b Token&#xff1a;<\/p>\n<p>\u53e5\u5b50 A&#xff1a;<\/p>\n<p>\u6211<br \/>\n\u559c\u6b22<br \/>\n\u4f60<\/p>\n<p>\u53e5\u5b50 B&#xff1a;<\/p>\n<p>\u4f60<br \/>\n\u559c\u6b22<br \/>\n\u6211<\/p>\n<p>\u4e24\u4e2a\u53e5\u5b50\u5305\u542b\u5b8c\u5168\u76f8\u540c\u7684 Token\u3002<\/p>\n<p>\u5982\u679c\u6a21\u578b\u53ea\u6709 Token Embedding&#xff0c;\u800c\u6ca1\u6709\u4efb\u4f55\u4f4d\u7f6e\u4fe1\u606f&#xff0c;\u90a3\u4e48\u6a21\u578b\u5f88\u96be\u4ec5\u4ec5\u6839\u636e\u8fd9\u4e9b Token \u5411\u91cf\u77e5\u9053\u5b83\u4eec\u7684\u6392\u5217\u987a\u5e8f\u3002<\/p>\n<p>\u6211\u4eec\u53ef\u4ee5\u628a\u8fd9\u4e2a\u95ee\u9898\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>Token Embedding&#xff1a;<\/p>\n<p>\u6211      \u2192 [\u5411\u91cf]<br \/>\n\u559c\u6b22    \u2192 [\u5411\u91cf]<br \/>\n\u4f60      \u2192 [\u5411\u91cf]<\/p>\n<p>\u8fd9\u4e9b\u5411\u91cf\u53ea\u63cf\u8ff0\u201c\u8fd9\u4e2a Token \u662f\u8c01\u201d&#xff0c;\u5e76\u6ca1\u6709\u63cf\u8ff0&#xff1a;<\/p>\n<p>\u5b83\u73b0\u5728\u4f4d\u4e8e\u4ec0\u4e48\u4f4d\u7f6e&#xff1f;<\/p>\n<p>\u6240\u4ee5\u6211\u4eec\u8fd8\u9700\u8981\u989d\u5916\u7684\u4fe1\u606f&#xff1a;<\/p>\n<p>\u6211      &#043; \u4f4d\u7f6e 1<br \/>\n\u559c\u6b22    &#043; \u4f4d\u7f6e 2<br \/>\n\u4f60      &#043; \u4f4d\u7f6e 3<\/p>\n<p>\u8fd9\u6837\u6a21\u578b\u624d\u80fd\u540c\u65f6\u77e5\u9053&#xff1a;<\/p>\n<p>Token \u662f\u4ec0\u4e48<br \/>\n&#043;<br \/>\nToken \u5728\u54ea\u91cc<\/p>\n<p>\u8fd9\u5c31\u662f\u4f4d\u7f6e\u7f16\u7801\u8981\u89e3\u51b3\u7684\u95ee\u9898\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u9762\u7528\u4e00\u5f20\u56fe\u5bf9\u6bd4 RNN \u4e0e Transformer \u5728\u5904\u7406\u987a\u5e8f\u4fe1\u606f\u65f6\u7684\u672c\u8d28\u533a\u522b&#xff1a;<\/p>\n<p>  #mermaid-svg-XY9xtX0QpmkeuUDk{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-XY9xtX0QpmkeuUDk .error-icon{fill:#552222;}#mermaid-svg-XY9xtX0QpmkeuUDk .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-XY9xtX0QpmkeuUDk .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-XY9xtX0QpmkeuUDk .marker{fill:#333333;stroke:#333333;}#mermaid-svg-XY9xtX0QpmkeuUDk .marker.cross{stroke:#333333;}#mermaid-svg-XY9xtX0QpmkeuUDk svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-XY9xtX0QpmkeuUDk p{margin:0;}#mermaid-svg-XY9xtX0QpmkeuUDk .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster-label text{fill:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster-label span{color:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster-label span p{background-color:transparent;}#mermaid-svg-XY9xtX0QpmkeuUDk .label text,#mermaid-svg-XY9xtX0QpmkeuUDk span{fill:#333;color:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk .node rect,#mermaid-svg-XY9xtX0QpmkeuUDk .node circle,#mermaid-svg-XY9xtX0QpmkeuUDk .node ellipse,#mermaid-svg-XY9xtX0QpmkeuUDk .node polygon,#mermaid-svg-XY9xtX0QpmkeuUDk .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-XY9xtX0QpmkeuUDk .rough-node .label text,#mermaid-svg-XY9xtX0QpmkeuUDk .node .label text,#mermaid-svg-XY9xtX0QpmkeuUDk .image-shape .label,#mermaid-svg-XY9xtX0QpmkeuUDk .icon-shape .label{text-anchor:middle;}#mermaid-svg-XY9xtX0QpmkeuUDk .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-XY9xtX0QpmkeuUDk .rough-node .label,#mermaid-svg-XY9xtX0QpmkeuUDk .node .label,#mermaid-svg-XY9xtX0QpmkeuUDk .image-shape .label,#mermaid-svg-XY9xtX0QpmkeuUDk .icon-shape .label{text-align:center;}#mermaid-svg-XY9xtX0QpmkeuUDk .node.clickable{cursor:pointer;}#mermaid-svg-XY9xtX0QpmkeuUDk .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-XY9xtX0QpmkeuUDk .arrowheadPath{fill:#333333;}#mermaid-svg-XY9xtX0QpmkeuUDk .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-XY9xtX0QpmkeuUDk .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-XY9xtX0QpmkeuUDk .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-XY9xtX0QpmkeuUDk .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-XY9xtX0QpmkeuUDk .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-XY9xtX0QpmkeuUDk .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster text{fill:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk .cluster span{color:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-XY9xtX0QpmkeuUDk .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-XY9xtX0QpmkeuUDk rect.text{fill:none;stroke-width:0;}#mermaid-svg-XY9xtX0QpmkeuUDk .icon-shape,#mermaid-svg-XY9xtX0QpmkeuUDk .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-XY9xtX0QpmkeuUDk .icon-shape p,#mermaid-svg-XY9xtX0QpmkeuUDk .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-XY9xtX0QpmkeuUDk .icon-shape .label rect,#mermaid-svg-XY9xtX0QpmkeuUDk .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-XY9xtX0QpmkeuUDk .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-XY9xtX0QpmkeuUDk .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-XY9xtX0QpmkeuUDk :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Transformer&#xff1a;\u5e76\u884c\u5904\u7406<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Token 1<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u540c\u65f6\u8fdb\u884c\u5173\u7cfb\u8ba1\u7b97<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Token 2<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Token 3<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Token 4<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>RNN&#xff1a;\u4e32\u884c\u5904\u7406<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u7b2c\u4e00\u4e2a Token<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u7b2c\u4e8c\u4e2a Token<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u7b2c\u4e09\u4e2a Token<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u7b2c\u56db\u4e2a Token<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u987a\u5e8f\u4fe1\u606f\u4ece\u54ea\u91cc\u6765&#xff1f;<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u9700\u8981\u5f15\u5165\u4f4d\u7f6e\u7f16\u7801<\/p>\n<p><\/span><\/p>\n<h2>\u4e8c\u3001\u4f4d\u7f6e\u7f16\u7801\u5230\u5e95\u662f\u4ec0\u4e48&#xff1f;<\/h2>\n<h3>1. \u7ed9\u6bcf\u4e00\u4e2a\u4f4d\u7f6e\u4e00\u4e2a\u5411\u91cf<\/h3>\n<p>\u6700\u76f4\u89c2\u7684\u7406\u89e3\u5c31\u662f&#xff1a;<\/p>\n<p>\u7ed9\u53e5\u5b50\u4e2d\u7684\u6bcf\u4e00\u4e2a\u4f4d\u7f6e\u5206\u914d\u4e00\u4e2a\u4f4d\u7f6e\u5411\u91cf\u3002<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u7b2c 0 \u4e2a\u4f4d\u7f6e \u2192 P0<br \/>\n\u7b2c 1 \u4e2a\u4f4d\u7f6e \u2192 P1<br \/>\n\u7b2c 2 \u4e2a\u4f4d\u7f6e \u2192 P2<br \/>\n\u7b2c 3 \u4e2a\u4f4d\u7f6e \u2192 P3<br \/>\n&#8230;<\/p>\n<p>\u4e0e\u6b64\u540c\u65f6&#xff0c;\u6bcf\u4e2a Token \u672c\u8eab\u4e5f\u6709\u4e00\u4e2a Embedding&#xff1a;<\/p>\n<p>\u6211       \u2192 E\u6211<br \/>\n\u559c\u6b22     \u2192 E\u559c\u6b22<br \/>\n\u4f60       \u2192 E\u4f60<\/p>\n<p>\u4e8e\u662f\u6211\u4eec\u628a\u4e24\u79cd\u4fe1\u606f\u7ed3\u5408\u8d77\u6765&#xff1a;<\/p>\n<p>Token Embedding<br \/>\n      &#043;<br \/>\nPosition Encoding<br \/>\n      \u2193<br \/>\nTransformer \u8f93\u5165<\/p>\n<p>\u6570\u5b66\u4e0a\u53ef\u4ee5\u7b80\u5355\u8868\u793a\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         X<\/p>\n<p>         &#061;<\/p>\n<p>         E<\/p>\n<p>         &#043;<\/p>\n<p>         P<\/p>\n<p>         X&#061;E&#043;P <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7667em;vertical-align: -0.0833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d&#xff1a;<\/p>\n<ul>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\n<p>         E<\/p>\n<p>        E<\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><\/span><\/span><\/span><\/span> \u8868\u793a Token Embedding<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\n<p>         P<\/p>\n<p>        P<\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><\/span><\/span><\/span><\/span> \u8868\u793a Position Encoding<\/li>\n<li><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">\n<p>         X<\/p>\n<p>        X<\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><\/span><\/span><\/span> \u8868\u793a\u6700\u7ec8\u8f93\u5165 Transformer \u7684\u8868\u793a<\/li>\n<\/ul>\n<p>\u5bf9\u4e8e\u7b2c <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        i<\/p>\n<p>       i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u4e2a Token&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>          X<\/p>\n<p>          i<\/p>\n<p>         &#061;<\/p>\n<p>          E<\/p>\n<p>          i<\/p>\n<p>         &#043;<\/p>\n<p>          P<\/p>\n<p>          i<\/p>\n<p>         X_i&#061;E_i&#043;P_i <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8fd9\u6837 Transformer \u770b\u5230\u7684\u5c31\u4e0d\u518d\u53ea\u662f\u201c\u6211\u201d\u8fd9\u4e2a Token&#xff0c;\u800c\u662f\u4e00\u4e2a\u540c\u65f6\u643a\u5e26 Token \u5185\u5bb9\u548c\u4f4d\u7f6e\u4fe1\u606f\u7684\u5411\u91cf\u3002<\/p>\n<hr \/>\n<h3>2. \u4e3a\u4ec0\u4e48\u53ef\u4ee5\u76f4\u63a5\u76f8\u52a0&#xff1f;<\/h3>\n<p>\u53ef\u80fd\u6709\u4eba\u4f1a\u4ea7\u751f\u4e00\u4e2a\u7591\u95ee&#xff1a;<\/p>\n<p>Token Embedding \u548c Position Encoding \u662f\u4e24\u79cd\u5b8c\u5168\u4e0d\u540c\u7684\u4fe1\u606f&#xff0c;\u4e3a\u4ec0\u4e48\u53ef\u4ee5\u76f4\u63a5\u76f8\u52a0&#xff1f;<\/p>\n<p>\u5173\u952e\u5c31\u5728\u4e8e&#xff1a;<\/p>\n<p>\u5b83\u4eec\u901a\u5e38\u5177\u6709\u76f8\u540c\u7684\u7ef4\u5ea6\u3002<\/p>\n<p>\u5047\u8bbe\u6a21\u578b\u7684\u9690\u85cf\u7ef4\u5ea6\u662f&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>          d<\/p>\n<p>          model<\/p>\n<p>         &#061;<\/p>\n<p>         512<\/p>\n<p>         d_{\\\\text{model}}&#061;512 <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">512<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u90a3\u4e48&#xff1a;<\/p>\n<p>Token Embedding&#xff1a;<\/p>\n<p>[512 \u7ef4]<\/p>\n<p>\u4f4d\u7f6e\u7f16\u7801&#xff1a;<\/p>\n<p>[512 \u7ef4]<\/p>\n<p>\u4e8e\u662f\u53ef\u4ee5\u76f4\u63a5\u8fdb\u884c\u9010\u5143\u7d20\u76f8\u52a0&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>          X<\/p>\n<p>          i<\/p>\n<p>         &#061;<\/p>\n<p>          E<\/p>\n<p>          i<\/p>\n<p>         &#043;<\/p>\n<p>          P<\/p>\n<p>          i<\/p>\n<p>         X_i&#061;E_i&#043;P_i <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0785em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5f97\u5230\u7684\u4ecd\u7136\u662f\u4e00\u4e2a 512 \u7ef4\u5411\u91cf\u3002<\/p>\n<p>\u8fd9\u6837\u505a\u7684\u597d\u5904\u662f\u975e\u5e38\u7b80\u5355\u3002<\/p>\n<p>\u6211\u4eec\u4e0d\u9700\u8981\u628a&#xff1a;<\/p>\n<p>Token Embedding<br \/>\n&#043;<br \/>\nPosition Encoding<\/p>\n<p>\u62fc\u6210\u4e00\u4e2a\u66f4\u5927\u7684\u5411\u91cf\u3002<\/p>\n<p>\u5426\u5219\u7ef4\u5ea6\u4f1a\u53d8\u6210&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         512<\/p>\n<p>         &#043;<\/p>\n<p>         512<\/p>\n<p>         &#061;<\/p>\n<p>         1024<\/p>\n<p>         512&#043;512&#061;1024 <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">512<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">512<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">1024<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u800c\u76f4\u63a5\u76f8\u52a0\u4e4b\u540e&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         512<\/p>\n<p>         &#043;<\/p>\n<p>         512<\/p>\n<p>         \u2192<\/p>\n<p>         512<\/p>\n<p>         512&#043;512\\\\rightarrow512 <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7278em;vertical-align: -0.0833em\"><\/span><span class=\"mord\">512<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">512<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2192<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">512<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6a21\u578b\u7684\u6574\u4f53\u7ef4\u5ea6\u4fdd\u6301\u4e0d\u53d8\u3002<\/p>\n<p>\u66f4\u91cd\u8981\u7684\u662f&#xff0c;\u7ecf\u8fc7\u540e\u7eed\u7684\u7ebf\u6027\u5c42\u548c Attention \u8ba1\u7b97&#xff0c;\u6a21\u578b\u53ef\u4ee5\u9010\u6e10\u5b66\u4f1a\u5982\u4f55\u4ece\u8fd9\u4e2a\u7ec4\u5408\u5411\u91cf\u4e2d\u5229\u7528 Token \u4fe1\u606f\u548c\u4f4d\u7f6e\u4fe1\u606f\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u9762\u8fd9\u5f20\u56fe\u5c55\u793a\u4e86\u4f4d\u7f6e\u7f16\u7801\u4e0e Token Embedding \u878d\u5408\u540e\u8f93\u5165 Transformer \u7684\u5b8c\u6574\u8fc7\u7a0b&#xff1a;<\/p>\n<p>  #mermaid-svg-QHRXFg3XJ7Zty5Jn{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .error-icon{fill:#552222;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .marker{fill:#333333;stroke:#333333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .marker.cross{stroke:#333333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn p{margin:0;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster-label text{fill:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster-label span{color:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster-label span p{background-color:transparent;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .label text,#mermaid-svg-QHRXFg3XJ7Zty5Jn span{fill:#333;color:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .node rect,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node circle,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node ellipse,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node polygon,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .rough-node .label text,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node .label text,#mermaid-svg-QHRXFg3XJ7Zty5Jn .image-shape .label,#mermaid-svg-QHRXFg3XJ7Zty5Jn .icon-shape .label{text-anchor:middle;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .rough-node .label,#mermaid-svg-QHRXFg3XJ7Zty5Jn .node .label,#mermaid-svg-QHRXFg3XJ7Zty5Jn .image-shape .label,#mermaid-svg-QHRXFg3XJ7Zty5Jn .icon-shape .label{text-align:center;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .node.clickable{cursor:pointer;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .arrowheadPath{fill:#333333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-QHRXFg3XJ7Zty5Jn .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-QHRXFg3XJ7Zty5Jn .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster text{fill:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .cluster span{color:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-QHRXFg3XJ7Zty5Jn rect.text{fill:none;stroke-width:0;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .icon-shape,#mermaid-svg-QHRXFg3XJ7Zty5Jn .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .icon-shape p,#mermaid-svg-QHRXFg3XJ7Zty5Jn .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .icon-shape .label rect,#mermaid-svg-QHRXFg3XJ7Zty5Jn .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-QHRXFg3XJ7Zty5Jn .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-QHRXFg3XJ7Zty5Jn .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-QHRXFg3XJ7Zty5Jn :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Token \u8f93\u5165<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Token Embedding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Position Encoding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u9010\u5143\u7d20\u76f8\u52a0<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>X &#061; E &#043; P<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Transformer<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Multi-Head Attention<\/p>\n<p><\/span><\/p>\n<h2>\u4e09\u3001Transformer \u539f\u8bba\u6587\u4e2d\u7684\u4f4d\u7f6e\u7f16\u7801<\/h2>\n<p>\u8bb2\u5230\u4f4d\u7f6e\u7f16\u7801&#xff0c;\u5c31\u7ed5\u4e0d\u5f00 Transformer \u539f\u8bba\u6587\u4e2d\u7684\u7ecf\u5178\u65b9\u6848&#xff1a;<\/p>\n<p>Sinusoidal Positional Encoding&#xff0c;\u6b63\u5f26\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<p>\u5b83\u4f7f\u7528\u6b63\u5f26\u51fd\u6570\u548c\u4f59\u5f26\u51fd\u6570\u4e3a\u4e0d\u540c\u4f4d\u7f6e\u6784\u9020\u5411\u91cf\u3002<\/p>\n<p>\u5177\u4f53\u516c\u5f0f\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i)}&#061; \\\\sin \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           &#043;<\/p>\n<p>           1<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i&#043;1)}&#061; \\\\cos \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u7b2c\u4e00\u6b21\u770b\u5230\u8fd9\u4e2a\u516c\u5f0f\u53ef\u80fd\u4f1a\u89c9\u5f97\u975e\u5e38\u590d\u6742\u3002<\/p>\n<p>\u4e0d\u8981\u6025\u3002<\/p>\n<p>\u6211\u4eec\u628a\u5b83\u4e00\u70b9\u4e00\u70b9\u62c6\u5f00\u3002<\/p>\n<hr \/>\n<h3>1. <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        p<\/p>\n<p>        o<\/p>\n<p>        s<\/p>\n<p>       pos<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><\/span> \u662f\u4ec0\u4e48&#xff1f;<\/h3>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        p<\/p>\n<p>        o<\/p>\n<p>        s<\/p>\n<p>       pos<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.625em;vertical-align: -0.1944em\"><\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><\/span> \u5c31\u662f Token \u6240\u5728\u7684\u4f4d\u7f6e\u3002<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u6211      \u559c\u6b22      \u5b66\u4e60      Transformer<\/p>\n<p>0        1         2             3<\/p>\n<p>\u90a3\u4e48&#xff1a;<\/p>\n<p>\u6211       \u2192 pos &#061; 0<br \/>\n\u559c\u6b22     \u2192 pos &#061; 1<br \/>\n\u5b66\u4e60     \u2192 pos &#061; 2<br \/>\nTransformer \u2192 pos &#061; 3<\/p>\n<p>\u8fd9\u91cc\u7684\u7f16\u53f7\u4ece 0 \u5f00\u59cb\u8fd8\u662f\u4ece 1 \u5f00\u59cb\u5e76\u4e0d\u662f\u6700\u91cd\u8981\u7684\u3002<\/p>\n<p>\u771f\u6b63\u91cd\u8981\u7684\u662f&#xff1a;<\/p>\n<p>\u4e0d\u540c\u4f4d\u7f6e\u5fc5\u987b\u80fd\u591f\u4ea7\u751f\u4e0d\u540c\u7684\u4f4d\u7f6e\u8868\u793a\u3002<\/p>\n<hr \/>\n<h3>2. <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        i<\/p>\n<p>       i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u662f\u4ec0\u4e48&#xff1f;<\/h3>\n<p><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        i<\/p>\n<p>       i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u53ef\u4ee5\u7406\u89e3\u6210\u4f4d\u7f6e\u5411\u91cf\u4e2d\u7684\u7ef4\u5ea6\u7d22\u5f15\u3002<\/p>\n<p>\u5047\u8bbe&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>          d<\/p>\n<p>          model<\/p>\n<p>         &#061;<\/p>\n<p>         8<\/p>\n<p>         d_{\\\\text{model}}&#061;8 <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8444em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">8<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u90a3\u4e48\u4e00\u4e2a\u4f4d\u7f6e\u7f16\u7801\u53ef\u4ee5\u5199\u6210&#xff1a;<\/p>\n<p>[PE0, PE1, PE2, PE3, PE4, PE5, PE6, PE7]<\/p>\n<p>\u8fd9\u91cc\u7684 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        i<\/p>\n<p>       i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u5c31\u662f\u5728\u63a7\u5236\u6211\u4eec\u6b63\u5728\u8ba1\u7b97\u54ea\u4e00\u7ec4\u7ef4\u5ea6\u3002<\/p>\n<p>\u6b63\u5f26\u51fd\u6570\u8d1f\u8d23\u5076\u6570\u7ef4&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           )<\/p>\n<p>         PE_{(pos,2i)} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u4f59\u5f26\u51fd\u6570\u8d1f\u8d23\u5947\u6570\u7ef4&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           &#043;<\/p>\n<p>           1<\/p>\n<p>           )<\/p>\n<p>         PE_{(pos,2i&#043;1)} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6240\u4ee5\u53ef\u4ee5\u7b80\u5355\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>\u5076\u6570\u7ef4 \u2192 sin<br \/>\n\u5947\u6570\u7ef4 \u2192 cos<\/p>\n<hr \/>\n<h3>3. \u4e3a\u4ec0\u4e48\u8981\u4f7f\u7528\u6b63\u5f26\u548c\u4f59\u5f26&#xff1f;<\/h3>\n<p>\u6211\u4eec\u5f53\u7136\u53ef\u4ee5\u76f4\u63a5\u5199&#xff1a;<\/p>\n<p>\u4f4d\u7f6e 0 \u2192 [0, 0, 0, 0, &#8230;]<br \/>\n\u4f4d\u7f6e 1 \u2192 [1, 1, 1, 1, &#8230;]<br \/>\n\u4f4d\u7f6e 2 \u2192 [2, 2, 2, 2, &#8230;]<\/p>\n<p>\u4f46\u662f\u8fd9\u79cd\u65b9\u5f0f\u6bd4\u8f83\u7b80\u5355\u7c97\u66b4\u3002<\/p>\n<p>Transformer \u539f\u8bba\u6587\u9009\u62e9\u4e86\u4e00\u7ec4\u5177\u6709\u4e0d\u540c\u9891\u7387\u7684\u6b63\u5f26\u548c\u4f59\u5f26\u51fd\u6570\u3002<\/p>\n<p>\u4f60\u53ef\u4ee5\u628a\u5b83\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>\u4f4d\u7f6e<br \/>\n \u2193<br \/>\n\u591a\u4e2a\u4e0d\u540c\u9891\u7387\u7684\u6ce2<br \/>\n \u2193<br \/>\n\u6bcf\u4e2a\u4f4d\u7f6e\u5f97\u5230\u4e00\u4e2a\u72ec\u7279\u7684\u7ec4\u5408<br \/>\n \u2193<br \/>\n\u5f62\u6210\u4f4d\u7f6e\u5411\u91cf<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u7ef4\u5ea6 0&#xff1a;\u53d8\u5316\u6bd4\u8f83\u5feb<br \/>\n\u7ef4\u5ea6 1&#xff1a;\u53d8\u5316\u6bd4\u8f83\u5feb<\/p>\n<p>\u7ef4\u5ea6 2&#xff1a;\u53d8\u5316\u6162\u4e00\u4e9b<br \/>\n\u7ef4\u5ea6 3&#xff1a;\u53d8\u5316\u6162\u4e00\u4e9b<\/p>\n<p>\u7ef4\u5ea6 4&#xff1a;\u53d8\u5316\u66f4\u6162<br \/>\n\u7ef4\u5ea6 5&#xff1a;\u53d8\u5316\u66f4\u6162<br \/>\n&#8230;<\/p>\n<p>\u6700\u7ec8\u4e00\u4e2a\u4f4d\u7f6e\u4f1a\u5f97\u5230\u4e00\u7ec4\u6b63\u5f26\u3001\u4f59\u5f26\u503c\u3002<\/p>\n<p>\u867d\u7136\u6bcf\u4e00\u7ef4\u90fd\u53ea\u662f\u4e00\u4e2a\u7b80\u5355\u7684\u5468\u671f\u51fd\u6570&#xff0c;\u4f46\u662f\u628a\u5f88\u591a\u4e0d\u540c\u9891\u7387\u7ec4\u5408\u8d77\u6765\u4ee5\u540e&#xff0c;\u5c31\u80fd\u591f\u5f62\u6210\u4e30\u5bcc\u7684\u4f4d\u7f6e\u8868\u793a\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u9762\u8fd9\u5f20\u56fe\u5c55\u793a\u4e86\u4f4d\u7f6e\u7f16\u7801\u5728 Transformer \u4e2d\u7684\u5177\u4f53\u4f4d\u7f6e&#xff1a;<\/p>\n<p>  #mermaid-svg-r7xaFm7XtBakr4O2{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-r7xaFm7XtBakr4O2 .error-icon{fill:#552222;}#mermaid-svg-r7xaFm7XtBakr4O2 .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-r7xaFm7XtBakr4O2 .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-r7xaFm7XtBakr4O2 .marker{fill:#333333;stroke:#333333;}#mermaid-svg-r7xaFm7XtBakr4O2 .marker.cross{stroke:#333333;}#mermaid-svg-r7xaFm7XtBakr4O2 svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-r7xaFm7XtBakr4O2 p{margin:0;}#mermaid-svg-r7xaFm7XtBakr4O2 .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster-label text{fill:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster-label span{color:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster-label span p{background-color:transparent;}#mermaid-svg-r7xaFm7XtBakr4O2 .label text,#mermaid-svg-r7xaFm7XtBakr4O2 span{fill:#333;color:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 .node rect,#mermaid-svg-r7xaFm7XtBakr4O2 .node circle,#mermaid-svg-r7xaFm7XtBakr4O2 .node ellipse,#mermaid-svg-r7xaFm7XtBakr4O2 .node polygon,#mermaid-svg-r7xaFm7XtBakr4O2 .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-r7xaFm7XtBakr4O2 .rough-node .label text,#mermaid-svg-r7xaFm7XtBakr4O2 .node .label text,#mermaid-svg-r7xaFm7XtBakr4O2 .image-shape .label,#mermaid-svg-r7xaFm7XtBakr4O2 .icon-shape .label{text-anchor:middle;}#mermaid-svg-r7xaFm7XtBakr4O2 .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-r7xaFm7XtBakr4O2 .rough-node .label,#mermaid-svg-r7xaFm7XtBakr4O2 .node .label,#mermaid-svg-r7xaFm7XtBakr4O2 .image-shape .label,#mermaid-svg-r7xaFm7XtBakr4O2 .icon-shape .label{text-align:center;}#mermaid-svg-r7xaFm7XtBakr4O2 .node.clickable{cursor:pointer;}#mermaid-svg-r7xaFm7XtBakr4O2 .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-r7xaFm7XtBakr4O2 .arrowheadPath{fill:#333333;}#mermaid-svg-r7xaFm7XtBakr4O2 .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-r7xaFm7XtBakr4O2 .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-r7xaFm7XtBakr4O2 .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-r7xaFm7XtBakr4O2 .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-r7xaFm7XtBakr4O2 .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-r7xaFm7XtBakr4O2 .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster text{fill:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 .cluster span{color:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-r7xaFm7XtBakr4O2 .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-r7xaFm7XtBakr4O2 rect.text{fill:none;stroke-width:0;}#mermaid-svg-r7xaFm7XtBakr4O2 .icon-shape,#mermaid-svg-r7xaFm7XtBakr4O2 .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-r7xaFm7XtBakr4O2 .icon-shape p,#mermaid-svg-r7xaFm7XtBakr4O2 .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-r7xaFm7XtBakr4O2 .icon-shape .label rect,#mermaid-svg-r7xaFm7XtBakr4O2 .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-r7xaFm7XtBakr4O2 .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-r7xaFm7XtBakr4O2 .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-r7xaFm7XtBakr4O2 :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u8f93\u5165 Token<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Token Embedding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Position Encoding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u76f8\u52a0<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Transformer<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Multi-Head Attention<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>FFN<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u4e0b\u4e00\u5c42<\/p>\n<p><\/span><\/p>\n<h2>\u56db\u3001\u4e3a\u4ec0\u4e48\u4e0d\u540c\u7ef4\u5ea6\u8981\u4f7f\u7528\u4e0d\u540c\u9891\u7387&#xff1f;<\/h2>\n<p>\u73b0\u5728\u6211\u4eec\u518d\u56de\u5934\u770b\u516c\u5f0f&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i)}&#061; \\\\sin \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           &#043;<\/p>\n<p>           1<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i&#043;1)}&#061; \\\\cos \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6700\u503c\u5f97\u6ce8\u610f\u7684\u5c31\u662f&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>          10000<\/p>\n<p>            2<\/p>\n<p>            i<\/p>\n<p>            d<\/p>\n<p>            model<\/p>\n<p>         10000^{\\\\frac{2i}{d_{\\\\text{model}}}} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1219em\"><\/span><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5b83\u4f1a\u968f\u7740\u7ef4\u5ea6 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        i<\/p>\n<p>       i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6595em\"><\/span><span class=\"mord mathnormal\">i<\/span><\/span><\/span><\/span><\/span> \u7684\u53d8\u5316\u800c\u53d8\u5316\u3002<\/p>\n<p>\u4e8e\u662f\u4e0d\u540c\u7ef4\u5ea6\u5bf9\u5e94\u4e0d\u540c\u7684\u53d8\u5316\u9891\u7387\u3002<\/p>\n<p>\u53ef\u4ee5\u7c97\u7565\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>\u4f4e\u7ef4&#xff1a;<\/p>\n<p>\u53d8\u5316\u5feb<br \/>\n~~~~~~~<\/p>\n<p>\u9ad8\u7ef4&#xff1a;<\/p>\n<p>\u53d8\u5316\u6162<br \/>\n~~~~~~~~~~~~~~<\/p>\n<p>\u8fd9\u6837\u505a\u6709\u4ec0\u4e48\u597d\u5904&#xff1f;<\/p>\n<p>\u5047\u8bbe\u6240\u6709\u7ef4\u5ea6\u90fd\u4f7f\u7528\u5b8c\u5168\u76f8\u540c\u7684\u9891\u7387&#xff1a;<\/p>\n<p>\u7ef4\u5ea6 0 \u2192 \u5feb<br \/>\n\u7ef4\u5ea6 1 \u2192 \u5feb<br \/>\n\u7ef4\u5ea6 2 \u2192 \u5feb<br \/>\n\u7ef4\u5ea6 3 \u2192 \u5feb<br \/>\n&#8230;<\/p>\n<p>\u90a3\u4e48\u5f88\u591a\u7ef4\u5ea6\u5b9e\u9645\u4e0a\u662f\u5728\u91cd\u590d\u8868\u8fbe\u7c7b\u4f3c\u7684\u4fe1\u606f\u3002<\/p>\n<p>\u800c\u4f7f\u7528\u4e0d\u540c\u9891\u7387\u4e4b\u540e&#xff1a;<\/p>\n<p>\u7ef4\u5ea6 0 \u2192 \u9ad8\u9891<br \/>\n\u7ef4\u5ea6 1 \u2192 \u9ad8\u9891<\/p>\n<p>\u7ef4\u5ea6 2 \u2192 \u4e2d\u9ad8\u9891<br \/>\n\u7ef4\u5ea6 3 \u2192 \u4e2d\u9ad8\u9891<\/p>\n<p>\u7ef4\u5ea6 4 \u2192 \u4e2d\u4f4e\u9891<br \/>\n\u7ef4\u5ea6 5 \u2192 \u4e2d\u4f4e\u9891<\/p>\n<p>\u7ef4\u5ea6 6 \u2192 \u4f4e\u9891<br \/>\n\u7ef4\u5ea6 7 \u2192 \u4f4e\u9891<\/p>\n<p>\u4e0d\u540c\u7ef4\u5ea6\u5c31\u80fd\u591f\u4ece\u4e0d\u540c\u5c3a\u5ea6\u63cf\u8ff0\u4f4d\u7f6e\u4fe1\u606f\u3002<\/p>\n<p>\u53ef\u4ee5\u628a\u5b83\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>\u6709\u4e9b\u7ef4\u5ea6\u8d1f\u8d23\u89c2\u5bdf\u4f4d\u7f6e\u7684\u7ec6\u5fae\u53d8\u5316&#xff0c;\u6709\u4e9b\u7ef4\u5ea6\u8d1f\u8d23\u89c2\u5bdf\u66f4\u5927\u8303\u56f4\u7684\u4f4d\u7f6e\u53d8\u5316\u3002<\/p>\n<p>\u591a\u4e2a\u4e0d\u540c\u9891\u7387\u7ec4\u5408\u8d77\u6765&#xff0c;\u5c31\u5f62\u6210\u4e86\u4e00\u4e2a\u66f4\u52a0\u4e30\u5bcc\u7684\u4f4d\u7f6e\u8868\u793a\u3002<\/p>\n<hr \/>\n<h2>\u4e94\u3001\u6b63\u5f26\u4f4d\u7f6e\u7f16\u7801\u4e3a\u4ec0\u4e48\u80fd\u591f\u5e2e\u52a9\u6a21\u578b\u7406\u89e3\u76f8\u5bf9\u4f4d\u7f6e&#xff1f;<\/h2>\n<p>\u8fd9\u662f\u7406\u89e3 Sinusoidal Positional Encoding \u65f6\u975e\u5e38\u503c\u5f97\u6ce8\u610f\u7684\u5730\u65b9\u3002<\/p>\n<p>\u5047\u8bbe\u6211\u4eec\u6709&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>         E<\/p>\n<p>         (<\/p>\n<p>         p<\/p>\n<p>         o<\/p>\n<p>         s<\/p>\n<p>         )<\/p>\n<p>         PE(pos) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u548c&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>         E<\/p>\n<p>         (<\/p>\n<p>         p<\/p>\n<p>         o<\/p>\n<p>         s<\/p>\n<p>         &#043;<\/p>\n<p>         k<\/p>\n<p>         )<\/p>\n<p>         PE(pos&#043;k) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0315em\">k<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6211\u4eec\u5e0c\u671b\u6a21\u578b\u4e0d\u4ec5\u80fd\u591f\u77e5\u9053&#xff1a;<\/p>\n<p>\u8fd9\u662f\u7b2c 10 \u4e2a\u4f4d\u7f6e<br \/>\n\u8fd9\u662f\u7b2c 20 \u4e2a\u4f4d\u7f6e<\/p>\n<p>\u8fd8\u80fd\u591f\u5728\u67d0\u79cd\u7a0b\u5ea6\u4e0a\u5b66\u4e60&#xff1a;<\/p>\n<p>\u4e24\u4e2a\u4f4d\u7f6e\u4e4b\u95f4\u76f8\u5dee 10<\/p>\n<p>\u6b63\u5f26\u548c\u4f59\u5f26\u51fd\u6570\u5177\u6709\u975e\u5e38\u597d\u7684\u6570\u5b66\u6027\u8d28\u3002<\/p>\n<p>\u5229\u7528\u4e09\u89d2\u51fd\u6570\u7684\u548c\u89d2\u516c\u5f0f&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>         (<\/p>\n<p>         a<\/p>\n<p>         &#043;<\/p>\n<p>         b<\/p>\n<p>         )<\/p>\n<p>         &#061;<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>         a<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>         b<\/p>\n<p>         &#043;<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>         a<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>         b<\/p>\n<p>         \\\\sin(a&#043;b)&#061; \\\\sin a\\\\cos b &#043; \\\\cos a\\\\sin b <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7778em;vertical-align: -0.0833em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">b<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u4ee5\u53ca&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>         (<\/p>\n<p>         a<\/p>\n<p>         &#043;<\/p>\n<p>         b<\/p>\n<p>         )<\/p>\n<p>         &#061;<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>         a<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>         b<\/p>\n<p>         \u2212<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>         a<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>         b<\/p>\n<p>         \\\\cos(a&#043;b)&#061; \\\\cos a\\\\cos b- \\\\sin a\\\\sin b <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mopen\">(<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mclose\">)<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7778em;vertical-align: -0.0833em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">b<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u2212<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">b<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u53ef\u4ee5\u53d1\u73b0&#xff1a;<\/p>\n<p>\u5982\u679c\u77e5\u9053\u67d0\u4e2a\u4f4d\u7f6e\u7684\u6b63\u5f26\u548c\u4f59\u5f26\u8868\u793a&#xff0c;\u90a3\u4e48\u79fb\u52a8\u56fa\u5b9a\u8ddd\u79bb <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        k<\/p>\n<p>       k<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0315em\">k<\/span><\/span><\/span><\/span><\/span> \u540e\u7684\u4f4d\u7f6e&#xff0c;\u53ef\u4ee5\u901a\u8fc7\u56fa\u5b9a\u7684\u53d8\u6362\u5173\u7cfb\u5f97\u5230\u3002<\/p>\n<p>\u8fd9\u610f\u5473\u7740&#xff1a;<\/p>\n<p>\u4f4d\u7f6e\u7f16\u7801\u4e0d\u4ec5\u53ef\u4ee5\u8868\u793a\u7edd\u5bf9\u4f4d\u7f6e&#xff0c;\u8fd8\u80fd\u591f\u8ba9\u6a21\u578b\u6bd4\u8f83\u5bb9\u6613\u5730\u5b66\u4e60\u4f4d\u7f6e\u4e4b\u95f4\u7684\u76f8\u5bf9\u5173\u7cfb\u3002<\/p>\n<p>\u8fd9\u91cc\u4e0d\u9700\u8981\u628a\u6570\u5b66\u63a8\u5bfc\u5168\u90e8\u5c55\u5f00\u3002<\/p>\n<p>\u771f\u6b63\u9700\u8981\u7406\u89e3\u7684\u662f&#xff1a;<\/p>\n<p>\u4f4d\u7f6e 10<br \/>\n   \u2193<br \/>\n\u4e00\u7ec4\u6ce2\u5f62\u503c<\/p>\n<p>\u4f4d\u7f6e 11<br \/>\n   \u2193<br \/>\n\u53e6\u4e00\u7ec4\u6ce2\u5f62\u503c<\/p>\n<p>\u4f4d\u7f6e 12<br \/>\n   \u2193<br \/>\n\u7ee7\u7eed\u53d8\u5316<\/p>\n<p>\u7531\u4e8e\u8fd9\u4e9b\u53d8\u5316\u662f\u8fde\u7eed\u7684&#xff0c;\u6240\u4ee5\u76f8\u90bb\u4f4d\u7f6e\u7684\u8868\u793a\u4e5f\u5177\u6709\u8fde\u7eed\u53d8\u5316\u7684\u7279\u5f81\u3002<\/p>\n<hr \/>\n<h2>\u516d\u3001\u7edd\u5bf9\u4f4d\u7f6e\u7f16\u7801\u548c\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801<\/h2>\n<p>\u8bb2\u5230\u8fd9\u91cc&#xff0c;\u6211\u4eec\u9700\u8981\u533a\u5206\u4e24\u4e2a\u6982\u5ff5\u3002<\/p>\n<p>\u4e00\u79cd\u662f&#xff1a;<\/p>\n<p>\u56fa\u5b9a\u7684\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<p>\u53e6\u4e00\u79cd\u662f&#xff1a;<\/p>\n<p>\u53ef\u5b66\u4e60\u7684\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<h3>1. Sinusoidal Encoding&#xff1a;\u56fa\u5b9a\u4f4d\u7f6e\u7f16\u7801<\/h3>\n<p>Transformer \u539f\u8bba\u6587\u4f7f\u7528\u7684\u5c31\u662f\u524d\u9762\u4ecb\u7ecd\u7684\u6b63\u5f26\u4f59\u5f26\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<p>\u5b83\u7684\u7279\u70b9\u662f&#xff1a;<\/p>\n<p>\u4f4d\u7f6e<br \/>\n \u2193<br \/>\n\u6570\u5b66\u516c\u5f0f\u8ba1\u7b97<br \/>\n \u2193<br \/>\n\u5f97\u5230\u4f4d\u7f6e\u5411\u91cf<\/p>\n<p>\u8fd9\u4e9b\u4f4d\u7f6e\u5411\u91cf\u5e76\u4e0d\u662f\u901a\u8fc7\u8bad\u7ec3\u5f97\u5230\u7684\u3002<\/p>\n<p>\u4e5f\u5c31\u662f\u8bf4&#xff0c;\u6a21\u578b\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u4e0d\u4f1a\u53bb\u66f4\u65b0\u8fd9\u4e9b\u4f4d\u7f6e\u7f16\u7801\u672c\u8eab\u3002<\/p>\n<p>\u5b83\u4eec\u7531\u516c\u5f0f\u76f4\u63a5\u8ba1\u7b97\u51fa\u6765\u3002<\/p>\n<hr \/>\n<h3>2. Learned Positional Embedding&#xff1a;\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801<\/h3>\n<p>\u53e6\u4e00\u79cd\u65b9\u5f0f\u5219\u66f4\u52a0\u76f4\u63a5\u3002<\/p>\n<p>\u6211\u4eec\u53ef\u4ee5\u50cf Token Embedding \u4e00\u6837&#xff0c;\u521b\u5efa\u4e00\u4e2a\u4f4d\u7f6e Embedding \u8868&#xff1a;<\/p>\n<p>Position 0 \u2192 \u53ef\u5b66\u4e60\u5411\u91cf<br \/>\nPosition 1 \u2192 \u53ef\u5b66\u4e60\u5411\u91cf<br \/>\nPosition 2 \u2192 \u53ef\u5b66\u4e60\u5411\u91cf<br \/>\nPosition 3 \u2192 \u53ef\u5b66\u4e60\u5411\u91cf<br \/>\n&#8230;<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u4f4d\u7f6e 0 \u2192 P0<br \/>\n\u4f4d\u7f6e 1 \u2192 P1<br \/>\n\u4f4d\u7f6e 2 \u2192 P2<br \/>\n&#8230;<\/p>\n<p>\u8fd9\u4e9b <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>         i<\/p>\n<p>       P_i<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.8333em;vertical-align: -0.15em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3117em\"><span class=\"\" style=\"top: -2.55em;margin-left: -0.1389em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u672c\u8eab\u5c31\u662f\u6a21\u578b\u53c2\u6570\u3002<\/p>\n<p>\u8bad\u7ec3\u8fc7\u7a0b\u4e2d&#xff0c;\u5b83\u4eec\u4f1a\u968f\u7740\u68af\u5ea6\u4e0b\u964d\u4e0d\u65ad\u66f4\u65b0\u3002<\/p>\n<p>\u4e8e\u662f&#xff1a;<\/p>\n<p>Token Embedding<br \/>\n       &#043;<br \/>\n\u53ef\u5b66\u4e60 Position Embedding<br \/>\n       \u2193<br \/>\nTransformer<\/p>\n<p>\u8fd9\u79cd\u65b9\u5f0f\u975e\u5e38\u76f4\u89c2\u3002<\/p>\n<p>\u6a21\u578b\u53ef\u4ee5\u81ea\u5df1\u5b66\u4e60\u4ec0\u4e48\u6837\u7684\u4f4d\u7f6e\u8868\u793a\u5bf9\u4e8e\u5f53\u524d\u4efb\u52a1\u6700\u6709\u7528\u3002<\/p>\n<hr \/>\n<h3>3. \u4e24\u79cd\u65b9\u6cd5\u6709\u4ec0\u4e48\u533a\u522b&#xff1f;<\/h3>\n<p>\u53ef\u4ee5\u7b80\u5355\u6574\u7406\u6210&#xff1a;<\/p>\n<table>\n<tr>\u65b9\u5f0f\u4f4d\u7f6e\u5411\u91cf\u600e\u4e48\u5f97\u5230\u662f\u5426\u8bad\u7ec3\u7279\u70b9<\/tr>\n<tbody>\n<tr>\n<td>Sinusoidal<\/td>\n<td>\u6570\u5b66\u516c\u5f0f\u8ba1\u7b97<\/td>\n<td>\u5426<\/td>\n<td>\u56fa\u5b9a\u3001\u7b80\u5355\u3001\u5177\u6709\u8fde\u7eed\u53d8\u5316\u7279\u5f81<\/td>\n<\/tr>\n<tr>\n<td>Learned<\/td>\n<td>\u53c2\u6570\u8868<\/td>\n<td>\u662f<\/td>\n<td>\u7075\u6d3b&#xff0c;\u7531\u6a21\u578b\u81ea\u5df1\u5b66\u4e60<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8fd9\u91cc\u4e0d\u80fd\u7b80\u5355\u8bf4\u54ea\u4e00\u79cd\u201c\u7edd\u5bf9\u66f4\u597d\u201d\u3002<\/p>\n<p>\u56e0\u4e3a\u4e0d\u540c\u6a21\u578b\u3001\u4e0d\u540c\u67b6\u6784\u4f1a\u9009\u62e9\u4e0d\u540c\u7684\u4f4d\u7f6e\u8868\u793a\u65b9\u5f0f\u3002<\/p>\n<p>\u800c\u4e14\u968f\u7740 Transformer \u4e0d\u65ad\u53d1\u5c55&#xff0c;\u4f4d\u7f6e\u7f16\u7801\u672c\u8eab\u4e5f\u51fa\u73b0\u4e86\u5f88\u591a\u65b0\u7684\u65b9\u6848\u3002<\/p>\n<p>\u4f8b\u5982\u73b0\u4ee3\u5927\u8bed\u8a00\u6a21\u578b\u4e2d\u7ecf\u5e38\u51fa\u73b0\u7684&#xff1a;<\/p>\n<p>RoPE<br \/>\nRelative Position<br \/>\nALiBi<br \/>\n&#8230;<\/p>\n<p>\u5b83\u4eec\u90fd\u5728\u5c1d\u8bd5\u89e3\u51b3\u4e00\u4e2a\u66f4\u6df1\u5c42\u7684\u95ee\u9898&#xff1a;<\/p>\n<p>\u6a21\u578b\u5230\u5e95\u5e94\u8be5\u5982\u4f55\u8868\u793a Token \u4e4b\u95f4\u7684\u4f4d\u7f6e\u5173\u7cfb&#xff1f;<\/p>\n<hr \/>\n<h2>\u4e03\u3001\u4ece\u7edd\u5bf9\u4f4d\u7f6e\u5230\u76f8\u5bf9\u4f4d\u7f6e<\/h2>\n<p>\u6211\u4eec\u524d\u9762\u8ba8\u8bba\u7684\u7ecf\u5178\u4f4d\u7f6e\u7f16\u7801&#xff0c;\u672c\u8d28\u4e0a\u662f\u5728\u544a\u8bc9\u6a21\u578b&#xff1a;<\/p>\n<p>Token A&#xff1a;<\/p>\n<p>\u6211\u4f4d\u4e8e\u7b2c 10 \u4e2a\u4f4d\u7f6e<\/p>\n<p>\u8fd9\u5c5e\u4e8e\u7edd\u5bf9\u4f4d\u7f6e\u3002<\/p>\n<p>\u4f46\u662f\u5bf9\u4e8e\u8bed\u8a00\u6a21\u578b\u6765\u8bf4&#xff0c;\u5f88\u591a\u65f6\u5019\u771f\u6b63\u91cd\u8981\u7684\u5e76\u4e0d\u662f&#xff1a;<\/p>\n<p>\u4f60\u662f\u7b2c 10 \u4e2a Token<\/p>\n<p>\u800c\u662f&#xff1a;<\/p>\n<p>\u4f60\u548c\u6211\u4e4b\u95f4\u8ddd\u79bb\u662f\u591a\u5c11&#xff1f;<\/p>\n<p>\u4f8b\u5982&#xff1a;<\/p>\n<p>\u5c0f\u660e \u559c\u6b22 \u5403 \u82f9\u679c<\/p>\n<p>\u5982\u679c\u5f53\u524d Token \u662f&#xff1a;<\/p>\n<p>\u82f9\u679c<\/p>\n<p>\u90a3\u4e48&#xff1a;<\/p>\n<p>\u5403<\/p>\n<p>\u5c31\u5728\u5b83\u524d\u9762\u4e00\u4e2a\u4f4d\u7f6e\u3002<\/p>\n<p>\u800c&#xff1a;<\/p>\n<p>\u559c\u6b22<\/p>\n<p>\u5c31\u5728\u5b83\u524d\u9762\u4e24\u4e2a\u4f4d\u7f6e\u3002<\/p>\n<p>\u5bf9\u4e8e\u5f88\u591a\u8bed\u8a00\u5173\u7cfb\u6765\u8bf4&#xff1a;<\/p>\n<p>\u76f8\u5bf9\u8ddd\u79bb<\/p>\n<p>\u53ef\u80fd\u6bd4&#xff1a;<\/p>\n<p>\u7edd\u5bf9\u7f16\u53f7<\/p>\n<p>\u66f4\u52a0\u91cd\u8981\u3002<\/p>\n<p>\u56e0\u6b64\u540e\u6765\u7684\u5f88\u591a Transformer \u7ed3\u6784\u5f00\u59cb\u7814\u7a76&#xff1a;<\/p>\n<p>\u80fd\u4e0d\u80fd\u76f4\u63a5\u628a\u76f8\u5bf9\u4f4d\u7f6e\u4fe1\u606f\u878d\u5165 Attention&#xff1f;<\/p>\n<p>\u8fd9\u5c31\u5f62\u6210\u4e86\u76f8\u5bf9\u4f4d\u7f6e\u7f16\u7801\u7b49\u4e00\u7cfb\u5217\u65b9\u6cd5\u3002<\/p>\n<hr \/>\n<h2>\u516b\u3001RoPE&#xff1a;\u73b0\u4ee3\u5927\u8bed\u8a00\u6a21\u578b\u4e2d\u7684\u91cd\u8981\u4f4d\u7f6e\u8868\u793a\u65b9\u6cd5<\/h2>\n<p>\u5982\u679c\u6211\u4eec\u7ee7\u7eed\u5b66\u4e60 GPT \u4e00\u7c7b\u7684\u5927\u8bed\u8a00\u6a21\u578b&#xff0c;\u5c31\u4f1a\u7ecf\u5e38\u9047\u5230\u4e00\u4e2a\u540d\u5b57&#xff1a;<\/p>\n<p>RoPE&#xff0c;Rotary Positional Embedding&#xff0c;\u65cb\u8f6c\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n<p>\u8fd9\u91cc\u5148\u4e0d\u6025\u7740\u6df1\u5165\u6570\u5b66\u63a8\u5bfc&#xff0c;\u56e0\u4e3a\u5b83\u6d89\u53ca\u4e8c\u7ef4\u65cb\u8f6c\u4ee5\u53ca\u590d\u6570\u8868\u793a\u7b49\u5185\u5bb9\u3002<\/p>\n<p>\u6211\u4eec\u73b0\u5728\u53ea\u9700\u8981\u5efa\u7acb\u4e00\u4e2a\u57fa\u672c\u8ba4\u8bc6&#xff1a;<\/p>\n<p>\u4f20\u7edf\u4f4d\u7f6e\u7f16\u7801\u66f4\u50cf\u662f&#xff1a;<\/p>\n<p>Token Embedding<br \/>\n       &#043;<br \/>\nPosition Encoding<br \/>\n       \u2193<br \/>\n\u8f93\u5165 Transformer<\/p>\n<p>\u800c RoPE \u7684\u601d\u8def\u66f4\u52a0\u7279\u522b\u3002<\/p>\n<p>\u5b83\u4e0d\u662f\u7b80\u5355\u5730\u628a\u4e00\u4e2a\u4f4d\u7f6e\u5411\u91cf\u52a0\u5230 Token Embedding \u4e0a&#xff0c;\u800c\u662f&#xff1a;<\/p>\n<p>\u6839\u636e Token \u6240\u5904\u7684\u4f4d\u7f6e&#xff0c;\u5bf9 Query \u548c Key \u7684\u8868\u793a\u8fdb\u884c\u65cb\u8f6c\u3002<\/p>\n<p>\u53ef\u4ee5\u7b80\u5355\u7406\u89e3\u4e3a&#xff1a;<\/p>\n<p>Query \/ Key<br \/>\n     \u2193<br \/>\n\u6839\u636e\u4f4d\u7f6e\u8fdb\u884c\u65cb\u8f6c<br \/>\n     \u2193<br \/>\n\u5e26\u6709\u4f4d\u7f6e\u4fe1\u606f\u7684 Q \/ K<br \/>\n     \u2193<br \/>\n\u8ba1\u7b97 Attention<\/p>\n<p>\u8fd9\u6837\u505a\u7684\u4e00\u4e2a\u91cd\u8981\u76ee\u7684&#xff0c;\u5c31\u662f\u8ba9 Query \u548c Key \u7684\u70b9\u79ef\u5173\u7cfb\u80fd\u591f\u81ea\u7136\u5730\u5305\u542b\u76f8\u5bf9\u4f4d\u7f6e\u4fe1\u606f\u3002<\/p>\n<p>\u8fd9\u4e5f\u662f\u4e3a\u4ec0\u4e48\u5f53\u6211\u4eec\u540e\u9762\u5b66\u4e60\u73b0\u4ee3 GPT \u7c7b\u6a21\u578b\u65f6&#xff0c;RoPE \u4f1a\u6210\u4e3a\u4e00\u4e2a\u975e\u5e38\u91cd\u8981\u7684\u6982\u5ff5\u3002<\/p>\n<p>\u73b0\u5728\u6682\u65f6\u8bb0\u4f4f&#xff1a;<\/p>\n<p>\u4f4d\u7f6e\u7f16\u7801\u7684\u53d1\u5c55\u8def\u7ebf&#xff0c;\u5e76\u4e0d\u662f\u7b80\u5355\u5730\u4ece\u201c\u6ca1\u6709\u4f4d\u7f6e\u201d\u5230\u201c\u52a0\u4e00\u4e2a\u4f4d\u7f6e\u5411\u91cf\u201d\u5c31\u7ed3\u675f\u4e86&#xff0c;\u800c\u662f\u5728\u4e0d\u65ad\u63a2\u7d22\u5982\u4f55\u8ba9 Attention \u66f4\u81ea\u7136\u5730\u7406\u89e3\u76f8\u5bf9\u4f4d\u7f6e\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u9762\u8fd9\u5f20\u56fe\u5bf9\u6bd4\u4e86\u4f20\u7edf\u4f4d\u7f6e\u7f16\u7801\u4e0e RoPE \u7684\u6838\u5fc3\u533a\u522b&#xff1a;<\/p>\n<p>  #mermaid-svg-ks5hGsIYEUbpXr8K{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-ks5hGsIYEUbpXr8K .error-icon{fill:#552222;}#mermaid-svg-ks5hGsIYEUbpXr8K .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-ks5hGsIYEUbpXr8K .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-ks5hGsIYEUbpXr8K .marker{fill:#333333;stroke:#333333;}#mermaid-svg-ks5hGsIYEUbpXr8K .marker.cross{stroke:#333333;}#mermaid-svg-ks5hGsIYEUbpXr8K svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-ks5hGsIYEUbpXr8K p{margin:0;}#mermaid-svg-ks5hGsIYEUbpXr8K .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster-label text{fill:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster-label span{color:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster-label span p{background-color:transparent;}#mermaid-svg-ks5hGsIYEUbpXr8K .label text,#mermaid-svg-ks5hGsIYEUbpXr8K span{fill:#333;color:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K .node rect,#mermaid-svg-ks5hGsIYEUbpXr8K .node circle,#mermaid-svg-ks5hGsIYEUbpXr8K .node ellipse,#mermaid-svg-ks5hGsIYEUbpXr8K .node polygon,#mermaid-svg-ks5hGsIYEUbpXr8K .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-ks5hGsIYEUbpXr8K .rough-node .label text,#mermaid-svg-ks5hGsIYEUbpXr8K .node .label text,#mermaid-svg-ks5hGsIYEUbpXr8K .image-shape .label,#mermaid-svg-ks5hGsIYEUbpXr8K .icon-shape .label{text-anchor:middle;}#mermaid-svg-ks5hGsIYEUbpXr8K .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-ks5hGsIYEUbpXr8K .rough-node .label,#mermaid-svg-ks5hGsIYEUbpXr8K .node .label,#mermaid-svg-ks5hGsIYEUbpXr8K .image-shape .label,#mermaid-svg-ks5hGsIYEUbpXr8K .icon-shape .label{text-align:center;}#mermaid-svg-ks5hGsIYEUbpXr8K .node.clickable{cursor:pointer;}#mermaid-svg-ks5hGsIYEUbpXr8K .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-ks5hGsIYEUbpXr8K .arrowheadPath{fill:#333333;}#mermaid-svg-ks5hGsIYEUbpXr8K .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-ks5hGsIYEUbpXr8K .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-ks5hGsIYEUbpXr8K .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-ks5hGsIYEUbpXr8K .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-ks5hGsIYEUbpXr8K .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-ks5hGsIYEUbpXr8K .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster text{fill:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K .cluster span{color:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-ks5hGsIYEUbpXr8K .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-ks5hGsIYEUbpXr8K rect.text{fill:none;stroke-width:0;}#mermaid-svg-ks5hGsIYEUbpXr8K .icon-shape,#mermaid-svg-ks5hGsIYEUbpXr8K .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-ks5hGsIYEUbpXr8K .icon-shape p,#mermaid-svg-ks5hGsIYEUbpXr8K .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-ks5hGsIYEUbpXr8K .icon-shape .label rect,#mermaid-svg-ks5hGsIYEUbpXr8K .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-ks5hGsIYEUbpXr8K .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-ks5hGsIYEUbpXr8K .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-ks5hGsIYEUbpXr8K :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>RoPE \u65cb\u8f6c\u4f4d\u7f6e\u7f16\u7801<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Query \/ Key<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u6839\u636e\u4f4d\u7f6e\u8fdb\u884c\u65cb\u8f6c<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u5e26\u6709\u4f4d\u7f6e\u4fe1\u606f\u7684 Q \/ K<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u8ba1\u7b97 Attention<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u4f20\u7edf\u4f4d\u7f6e\u7f16\u7801<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"edgeLabel\"><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>Token Embedding<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>&#043; Position Encoding<\/p>\n<p><\/span><\/p>\n<p>           <span class=\"nodeLabel\"><\/p>\n<p>\u8f93\u5165 Transformer<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u6838\u5fc3\u533a\u522b<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>RoPE \u8ba9\u70b9\u79ef\u5173\u7cfb\u81ea\u7136\u5305\u542b\u76f8\u5bf9\u4f4d\u7f6e\u4fe1\u606f<\/p>\n<p><\/span><\/p>\n<h2>\u4e5d\u3001\u4f4d\u7f6e\u7f16\u7801\u653e\u5728\u54ea\u91cc&#xff1f;<\/h2>\n<p>\u6211\u4eec\u73b0\u5728\u77e5\u9053\u4f4d\u7f6e\u7f16\u7801\u8981\u52a0\u5165 Transformer\u3002<\/p>\n<p>\u4f46\u662f\u5b83\u5230\u5e95\u662f\u5728\u4ec0\u4e48\u65f6\u5019\u52a0\u5165&#xff1f;<\/p>\n<p>\u6700\u7ecf\u5178\u7684\u505a\u6cd5\u53ef\u4ee5\u7406\u89e3\u6210&#xff1a;<\/p>\n<p>\u8f93\u5165 Token<br \/>\n    \u2193<br \/>\nToken Embedding<br \/>\n    \u2193<br \/>\nPosition Encoding<br \/>\n    \u2193<br \/>\n\u76f8\u52a0<br \/>\n    \u2193<br \/>\nTransformer<br \/>\n    \u2193<br \/>\nMulti-Head Attention<br \/>\n    \u2193<br \/>\nFFN<br \/>\n    \u2193<br \/>\n\u4e0b\u4e00\u5c42<\/p>\n<p>\u6570\u5b66\u4e0a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         X<\/p>\n<p>         &#061;<\/p>\n<p>         E<\/p>\n<p>         &#043;<\/p>\n<p>         P<\/p>\n<p>         X&#061;E&#043;P <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7667em;vertical-align: -0.0833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         E<\/p>\n<p>         \u2208<\/p>\n<p>          R<\/p>\n<p>           n<\/p>\n<p>           \u00d7<\/p>\n<p>            d<\/p>\n<p>            model<\/p>\n<p>         E\\\\in\\\\mathbb{R}^{n\\\\times d_{\\\\text{model}}} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8991em\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8868\u793a Token Embedding\u3002<\/p>\n<p>\u4f4d\u7f6e\u7f16\u7801\u5219\u53ef\u4ee5\u8868\u793a\u4e3a&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>         \u2208<\/p>\n<p>          R<\/p>\n<p>           n<\/p>\n<p>           \u00d7<\/p>\n<p>            d<\/p>\n<p>            model<\/p>\n<p>         P\\\\in\\\\mathbb{R}^{n\\\\times d_{\\\\text{model}}} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8991em\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u6240\u4ee5&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         X<\/p>\n<p>         \u2208<\/p>\n<p>          R<\/p>\n<p>           n<\/p>\n<p>           \u00d7<\/p>\n<p>            d<\/p>\n<p>            model<\/p>\n<p>         X\\\\in\\\\mathbb{R}^{n\\\\times d_{\\\\text{model}}} <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.7224em;vertical-align: -0.0391em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2208<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.8991em\"><\/span><span class=\"mord\"><span class=\"mord mathbb\">R<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8991em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">n<\/span><span class=\"mbin mtight\">\u00d7<\/span><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3488em;margin-left: 0em;margin-right: 0.0714em\"><span class=\"pstrut\" style=\"height: 2.5em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1512em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u8fd9\u6837\u5c31\u53ef\u4ee5\u76f4\u63a5\u628a <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        X<\/p>\n<p>       X<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><\/span><\/span><\/span><\/span> \u9001\u5165 Transformer\u3002<\/p>\n<hr \/>\n<h2>\u5341\u3001\u628a\u524d\u9762\u4e94\u7bc7\u6587\u7ae0\u4e32\u8d77\u6765<\/h2>\n<p>\u5230\u8fd9\u91cc&#xff0c;\u6211\u4eec\u5df2\u7ecf\u53ef\u4ee5\u628a Transformer \u524d\u9762\u7684\u51e0\u4e2a\u6838\u5fc3\u6982\u5ff5\u771f\u6b63\u4e32\u8d77\u6765\u4e86\u3002<\/p>\n<p>\u6700\u5f00\u59cb&#xff0c;\u6211\u4eec\u9047\u5230\u4e86 RNN \u7684\u95ee\u9898&#xff1a;<\/p>\n<p>\u5e8f\u5217\u8ba1\u7b97<br \/>\n   \u2193<br \/>\n\u96be\u4ee5\u5b8c\u5168\u5e76\u884c<br \/>\n   \u2193<br \/>\n\u957f\u8ddd\u79bb\u4f9d\u8d56\u56f0\u96be<\/p>\n<p>\u4e8e\u662f Transformer \u4f7f\u7528 Attention&#xff1a;<\/p>\n<p>Token<br \/>\n \u2193<br \/>\nSelf-Attention<br \/>\n \u2193<br \/>\n\u76f4\u63a5\u5efa\u7acb Token \u4e4b\u95f4\u7684\u5173\u7cfb<\/p>\n<p>\u4f46\u662f\u4e00\u4e2a Attention \u53ef\u80fd\u4e0d\u8db3\u4ee5\u4ece\u591a\u4e2a\u89d2\u5ea6\u8868\u8fbe\u590d\u6742\u5173\u7cfb\u3002<\/p>\n<p>\u4e8e\u662f&#xff1a;<\/p>\n<p>Self-Attention<br \/>\n       \u2193<br \/>\nMulti-Head Attention<br \/>\n       \u2193<br \/>\n\u591a\u4e2a\u8868\u793a\u7a7a\u95f4<br \/>\n       \u2193<br \/>\n\u4ece\u591a\u4e2a\u89d2\u5ea6\u5efa\u7acb\u5173\u7cfb<\/p>\n<p>\u53ef\u662f\u6211\u4eec\u53c8\u53d1\u73b0&#xff1a;<\/p>\n<p>Attention \u672c\u8eab\u4e0d\u77e5\u9053\u987a\u5e8f<\/p>\n<p>\u4e8e\u662f&#xff1a;<\/p>\n<p>Token Embedding<br \/>\n       &#043;<br \/>\nPosition Encoding<br \/>\n       \u2193<br \/>\n\u5305\u542b Token \u4fe1\u606f\u548c\u4f4d\u7f6e\u4fe1\u606f<br \/>\n       \u2193<br \/>\nMulti-Head Attention<\/p>\n<p>\u6240\u4ee5\u5230\u8fd9\u91cc&#xff0c;\u4e00\u4e2a\u975e\u5e38\u91cd\u8981\u7684 Transformer \u8f93\u5165\u8fc7\u7a0b\u5c31\u5df2\u7ecf\u51fa\u73b0\u4e86&#xff1a;<\/p>\n<p>                 Token<br \/>\n                   \u2193<br \/>\n              Token Embedding<br \/>\n                   \u2193<br \/>\n          \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2534\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510<br \/>\n          \u2502                 \u2502<br \/>\n          \u2502 Position        \u2502<br \/>\n          \u2502 Encoding        \u2502<br \/>\n          \u2502                 \u2502<br \/>\n          \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518<br \/>\n                   \u2193<br \/>\n                   &#043;<br \/>\n                   \u2193<br \/>\n              Transformer<br \/>\n                   \u2193<br \/>\n          Multi-Head Attention<\/p>\n<p>\u73b0\u5728\u6211\u4eec\u7ec8\u4e8e\u53ef\u4ee5\u56de\u7b54\u6700\u5f00\u59cb\u7684\u95ee\u9898&#xff1a;<\/p>\n<p>Transformer \u662f\u5982\u4f55\u7406\u89e3\u987a\u5e8f\u7684&#xff1f;<\/p>\n<p>\u7b54\u6848\u662f&#xff1a;<\/p>\n<p>Self-Attention \u672c\u8eab\u6ca1\u6709\u5929\u7136\u7684\u987a\u5e8f\u6982\u5ff5&#xff0c;\u56e0\u6b64 Transformer \u9700\u8981\u989d\u5916\u5f15\u5165\u4f4d\u7f6e\u4fe1\u606f\u3002\u7ecf\u5178 Transformer \u4f7f\u7528\u6b63\u5f26\u548c\u4f59\u5f26\u51fd\u6570\u6784\u9020\u4f4d\u7f6e\u7f16\u7801&#xff0c;\u5e76\u5c06\u5b83\u4e0e Token Embedding \u878d\u5408&#xff0c;\u8ba9\u540e\u7eed\u7684 Attention \u5728\u5904\u7406 Token \u65f6\u80fd\u591f\u540c\u65f6\u5229\u7528\u5185\u5bb9\u4fe1\u606f\u548c\u4f4d\u7f6e\u4fe1\u606f\u3002<\/p>\n<hr \/>\n<p>\u4e0b\u9762\u8fd9\u5f20\u56fe\u628a\u524d\u9762\u51e0\u7bc7\u6587\u7ae0\u7684\u6838\u5fc3\u6982\u5ff5\u5b8c\u6574\u4e32\u8054\u8d77\u6765&#xff1a;<\/p>\n<p>  #mermaid-svg-rWM86DAdmzZCOogn{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-rWM86DAdmzZCOogn .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-rWM86DAdmzZCOogn .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-rWM86DAdmzZCOogn .error-icon{fill:#552222;}#mermaid-svg-rWM86DAdmzZCOogn .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-rWM86DAdmzZCOogn .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-rWM86DAdmzZCOogn .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-rWM86DAdmzZCOogn .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-rWM86DAdmzZCOogn .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-rWM86DAdmzZCOogn .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-rWM86DAdmzZCOogn .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-rWM86DAdmzZCOogn .marker{fill:#333333;stroke:#333333;}#mermaid-svg-rWM86DAdmzZCOogn .marker.cross{stroke:#333333;}#mermaid-svg-rWM86DAdmzZCOogn svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-rWM86DAdmzZCOogn p{margin:0;}#mermaid-svg-rWM86DAdmzZCOogn .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-rWM86DAdmzZCOogn .cluster-label text{fill:#333;}#mermaid-svg-rWM86DAdmzZCOogn .cluster-label span{color:#333;}#mermaid-svg-rWM86DAdmzZCOogn .cluster-label span p{background-color:transparent;}#mermaid-svg-rWM86DAdmzZCOogn .label text,#mermaid-svg-rWM86DAdmzZCOogn span{fill:#333;color:#333;}#mermaid-svg-rWM86DAdmzZCOogn .node rect,#mermaid-svg-rWM86DAdmzZCOogn .node circle,#mermaid-svg-rWM86DAdmzZCOogn .node ellipse,#mermaid-svg-rWM86DAdmzZCOogn .node polygon,#mermaid-svg-rWM86DAdmzZCOogn .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-rWM86DAdmzZCOogn .rough-node .label text,#mermaid-svg-rWM86DAdmzZCOogn .node .label text,#mermaid-svg-rWM86DAdmzZCOogn .image-shape .label,#mermaid-svg-rWM86DAdmzZCOogn .icon-shape .label{text-anchor:middle;}#mermaid-svg-rWM86DAdmzZCOogn .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-rWM86DAdmzZCOogn .rough-node .label,#mermaid-svg-rWM86DAdmzZCOogn .node .label,#mermaid-svg-rWM86DAdmzZCOogn .image-shape .label,#mermaid-svg-rWM86DAdmzZCOogn .icon-shape .label{text-align:center;}#mermaid-svg-rWM86DAdmzZCOogn .node.clickable{cursor:pointer;}#mermaid-svg-rWM86DAdmzZCOogn .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-rWM86DAdmzZCOogn .arrowheadPath{fill:#333333;}#mermaid-svg-rWM86DAdmzZCOogn .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-rWM86DAdmzZCOogn .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-rWM86DAdmzZCOogn .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-rWM86DAdmzZCOogn .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-rWM86DAdmzZCOogn .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-rWM86DAdmzZCOogn .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-rWM86DAdmzZCOogn .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-rWM86DAdmzZCOogn .cluster text{fill:#333;}#mermaid-svg-rWM86DAdmzZCOogn .cluster span{color:#333;}#mermaid-svg-rWM86DAdmzZCOogn div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-rWM86DAdmzZCOogn .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-rWM86DAdmzZCOogn rect.text{fill:none;stroke-width:0;}#mermaid-svg-rWM86DAdmzZCOogn .icon-shape,#mermaid-svg-rWM86DAdmzZCOogn .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-rWM86DAdmzZCOogn .icon-shape p,#mermaid-svg-rWM86DAdmzZCOogn .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-rWM86DAdmzZCOogn .icon-shape .label rect,#mermaid-svg-rWM86DAdmzZCOogn .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-rWM86DAdmzZCOogn .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-rWM86DAdmzZCOogn .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-rWM86DAdmzZCOogn :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"edgeLabel\"><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>RNN \u7684\u5c40\u9650<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u96be\u4ee5\u5b8c\u5168\u5e76\u884c<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u957f\u8ddd\u79bb\u4f9d\u8d56\u56f0\u96be<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Transformer \u4f7f\u7528 Attention<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Self-Attention<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Multi-Head Attention<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Attention \u4e0d\u77e5\u9053\u987a\u5e8f<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Token Embedding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Position Encoding<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u76f8\u52a0<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>\u5305\u542b Token \u4fe1\u606f\u548c\u4f4d\u7f6e\u4fe1\u606f<\/p>\n<p><\/span><\/p>\n<p>         <span class=\"nodeLabel\"><\/p>\n<p>Multi-Head Attention<\/p>\n<p><\/span><\/p>\n<h2>\u5341\u4e00\u3001\u8fd9\u4e00\u7bc7\u6700\u5e94\u8be5\u8bb0\u4f4f\u7684\u5185\u5bb9<\/h2>\n<p>\u5982\u679c\u8fd9\u4e00\u7bc7\u53ea\u8bb0\u4f4f\u51e0\u4e2a\u6838\u5fc3\u77e5\u8bc6\u70b9&#xff0c;\u53ef\u4ee5\u8bb0\u4f4f\u4e0b\u9762\u8fd9\u4e9b\u3002<\/p>\n<p>\u7b2c\u4e00&#xff0c;Self-Attention \u672c\u8eab\u5e76\u4e0d\u5929\u7136\u5305\u542b Token \u7684\u987a\u5e8f\u4fe1\u606f\u3002<\/p>\n<p>\u5b83\u64c5\u957f\u56de\u7b54&#xff1a;<\/p>\n<p>\u8c01\u5e94\u8be5\u5173\u6ce8\u8c01&#xff1f;<\/p>\n<p>\u4f46\u5e76\u4e0d\u4f1a\u81ea\u52a8\u77e5\u9053&#xff1a;<\/p>\n<p>\u8c01\u5728\u7b2c\u51e0\u4e2a\u4f4d\u7f6e&#xff1f;<\/p>\n<p>\u7b2c\u4e8c&#xff0c;\u56e0\u6b64 Transformer \u9700\u8981\u989d\u5916\u52a0\u5165\u4f4d\u7f6e\u8868\u793a\u3002<\/p>\n<p>\u6700\u7ecf\u5178\u7684\u8f93\u5165\u5f62\u5f0f\u662f&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         X<\/p>\n<p>         &#061;<\/p>\n<p>         E<\/p>\n<p>         &#043;<\/p>\n<p>         P<\/p>\n<p>         X&#061;E&#043;P <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0785em\">X<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.7667em;vertical-align: -0.0833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">&#043;<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5176\u4e2d <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        E<\/p>\n<p>       E<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><\/span><\/span><\/span><\/span> \u662f Token Embedding&#xff0c;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>        P<\/p>\n<p>       P<\/p>\n<p>    <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6833em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><\/span><\/span><\/span><\/span> \u662f Position Encoding\u3002<\/p>\n<p>\u7b2c\u4e09&#xff0c;Transformer \u539f\u8bba\u6587\u4f7f\u7528\u6b63\u5f26\u548c\u4f59\u5f26\u6784\u9020\u4f4d\u7f6e\u7f16\u7801&#xff1a;<\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         sin<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i)}&#061; \\\\sin \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">sin<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\"><\/p>\n<p>         P<\/p>\n<p>          E<\/p>\n<p>           (<\/p>\n<p>           p<\/p>\n<p>           o<\/p>\n<p>           s<\/p>\n<p>           ,<\/p>\n<p>           2<\/p>\n<p>           i<\/p>\n<p>           &#043;<\/p>\n<p>           1<\/p>\n<p>           )<\/p>\n<p>         &#061;<\/p>\n<p>         cos<\/p>\n<p>         \u2061<\/p>\n<p>          (<\/p>\n<p>            p<\/p>\n<p>            o<\/p>\n<p>            s<\/p>\n<p>            10000<\/p>\n<p>              2<\/p>\n<p>              i<\/p>\n<p>              d<\/p>\n<p>              model<\/p>\n<p>          )<\/p>\n<p>         PE_{(pos,2i&#043;1)}&#061; \\\\cos \\\\left( \\\\frac{pos}{10000^{\\\\frac{2i}{d_{\\\\text{model}}}}} \\\\right) <\/p>\n<p>     <\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0385em;vertical-align: -0.3552em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.1389em\">P<\/span><span class=\"mord\"><span class=\"mord mathnormal\" style=\"margin-right: 0.0576em\">E<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.5198em;margin-left: -0.0576em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mopen mtight\">(<\/span><span class=\"mord mathnormal mtight\">p<\/span><span class=\"mord mathnormal mtight\">os<\/span><span class=\"mpunct mtight\">,<\/span><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><span class=\"mbin mtight\">&#043;<\/span><span class=\"mord mtight\">1<\/span><span class=\"mclose mtight\">)<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3552em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 2.4619em;vertical-align: -1.0119em\"><\/span><span class=\"mop\">cos<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">(<\/span><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1076em\"><span class=\"\" style=\"top: -2.11em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord\">1000<\/span><span class=\"mord\"><span class=\"mord\">0<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.1219em\"><span class=\"\" style=\"top: -3.5234em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mopen nulldelimiter sizing reset-size3 size6\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8551em\"><span class=\"\" style=\"top: -2.656em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3448em\"><span class=\"\" style=\"top: -2.3448em;margin-left: 0em;margin-right: 0.1em\"><span class=\"pstrut\" style=\"height: 2.6944em\"><\/span><span class=\"mord mtight\"><span class=\"mord text mtight\"><span class=\"mord mtight\">model<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3496em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.2255em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line mtight\" style=\"border-bottom-width: 0.049em\"><\/span><\/span><span class=\"\" style=\"top: -3.384em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size3 size1 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><span class=\"mord mathnormal mtight\">i<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.5937em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter sizing reset-size3 size6\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.3519em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.7989em\"><span class=\"pstrut\" style=\"height: 3.1219em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"mord mathnormal\">os<\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 1.0119em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size3\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u5b83\u901a\u8fc7\u4e0d\u540c\u9891\u7387\u7684\u5468\u671f\u51fd\u6570&#xff0c;\u4e3a\u4e0d\u540c\u4f4d\u7f6e\u6784\u9020\u4e0d\u540c\u7684\u5411\u91cf\u8868\u793a\u3002<\/p>\n<p>\u7b2c\u56db&#xff0c;\u4f4d\u7f6e\u7f16\u7801\u4e0d\u4ec5\u4ec5\u662f\u4e3a\u4e86\u544a\u8bc9\u6a21\u578b&#xff1a;<\/p>\n<p>\u201c\u6211\u662f\u7b2c\u51e0\u4e2a Token\u201d<\/p>\n<p>\u8fd8\u5e0c\u671b\u8ba9\u6a21\u578b\u80fd\u591f\u7406\u89e3&#xff1a;<\/p>\n<p>\u201c\u4e24\u4e2a Token \u4e4b\u95f4\u76f8\u9694\u591a\u8fdc\u201d<\/p>\n<p>\u56e0\u6b64\u540e\u6765\u51fa\u73b0\u4e86\u8d8a\u6765\u8d8a\u591a\u5173\u6ce8\u76f8\u5bf9\u4f4d\u7f6e\u4fe1\u606f\u7684\u65b9\u6cd5\u3002<\/p>\n<p>\u7b2c\u4e94&#xff0c;\u73b0\u4ee3 Transformer \u548c\u5927\u8bed\u8a00\u6a21\u578b\u5df2\u7ecf\u4e0d\u5c40\u9650\u4e8e\u6700\u521d\u7684 Sinusoidal Positional Encoding&#xff0c;RoPE \u7b49\u4f4d\u7f6e\u8868\u793a\u65b9\u6cd5\u4e5f\u6210\u4e3a\u73b0\u4ee3\u67b6\u6784\u4e2d\u7684\u91cd\u8981\u7ec4\u6210\u90e8\u5206\u3002<\/p>\n<hr \/>\n<h2>\u7ed3\u8bed&#xff1a;Attention \u6709\u4e86&#xff0c;\u987a\u5e8f\u4e5f\u6709\u4e86&#xff0c;\u90a3\u4e48\u4e00\u4e2a Transformer Block \u8fd8\u7f3a\u4ec0\u4e48&#xff1f;<\/h2>\n<p>\u5230\u73b0\u5728&#xff0c;\u6211\u4eec\u5df2\u7ecf\u628a Transformer \u4e2d\u6700\u6838\u5fc3\u7684\u4e00\u6761\u4e3b\u7ebf\u5efa\u7acb\u8d77\u6765\u4e86&#xff1a;<\/p>\n<p>\u4e3a\u4ec0\u4e48\u9700\u8981 Transformer&#xff1f;<br \/>\n        \u2193<br \/>\nSelf-Attention<br \/>\n        \u2193<br \/>\nMulti-Head Attention<br \/>\n        \u2193<br \/>\nPosition Encoding<\/p>\n<p>\u73b0\u5728 Transformer \u5df2\u7ecf\u53ef\u4ee5&#xff1a;<\/p>\n<p>\u770b\u5230 Token<br \/>\n   \u2193<br \/>\n\u7406\u89e3 Token \u4e4b\u95f4\u7684\u5173\u7cfb<br \/>\n   \u2193<br \/>\n\u4ece\u591a\u4e2a\u89d2\u5ea6\u63d0\u53d6\u4fe1\u606f<br \/>\n   \u2193<br \/>\n\u540c\u65f6\u77e5\u9053 Token \u7684\u4f4d\u7f6e\u4fe1\u606f<\/p>\n<p>\u4f46\u662f\u5982\u679c\u4f60\u56de\u5934\u770b\u770b Transformer \u7684\u5b8c\u6574\u7ed3\u6784&#xff0c;\u5c31\u4f1a\u53d1\u73b0\u8fd8\u6709\u51e0\u4e2a\u975e\u5e38\u91cd\u8981\u7684\u6a21\u5757\u6ca1\u6709\u771f\u6b63\u5c55\u5f00\u3002<\/p>\n<p>\u5c24\u5176\u662f&#xff1a;<\/p>\n<p>Multi-Head Attention<br \/>\n        \u2193<br \/>\n        ?<br \/>\n        \u2193<br \/>\n\u4e0b\u4e00\u5c42 Transformer<\/p>\n<p>Attention \u5f97\u5230\u7684\u4fe1\u606f&#xff0c;\u7a76\u7adf\u8fd8\u8981\u7ecf\u8fc7\u4ec0\u4e48\u5904\u7406&#xff1f;<\/p>\n<p>\u4e3a\u4ec0\u4e48 Transformer \u4e2d\u4e00\u5b9a\u8fd8\u6709\u4e00\u4e2a Feed Forward Network&#xff0c;\u524d\u9988\u795e\u7ecf\u7f51\u7edc&#xff1f;<\/p>\n<p>\u4e3a\u4ec0\u4e48 Attention \u540e\u9762\u603b\u662f\u80fd\u591f\u770b\u5230&#xff1a;<\/p>\n<p>Residual Connection<br \/>\nLayer Normalization<br \/>\nFeed Forward Network<\/p>\n<p>\u8fd9\u4e9b\u6a21\u5757\u5230\u5e95\u5206\u522b\u5728\u505a\u4ec0\u4e48&#xff1f;<\/p>\n<p>\u5982\u679c\u8bf4 Attention \u8d1f\u8d23\u7684\u662f&#xff1a;<\/p>\n<p>\u8ba9\u4e0d\u540c Token \u4e4b\u95f4\u8fdb\u884c\u4fe1\u606f\u4ea4\u6d41\u3002<\/p>\n<p>\u90a3\u4e48\u4e0b\u4e00\u7bc7\u6587\u7ae0&#xff0c;\u6211\u4eec\u5c31\u8981\u7ee7\u7eed\u7814\u7a76&#xff1a;<\/p>\n<p>Attention \u5f97\u5230\u4fe1\u606f\u4e4b\u540e&#xff0c;Transformer \u53c8\u662f\u5982\u4f55\u5bf9\u8fd9\u4e9b\u4fe1\u606f\u8fdb\u884c\u8fdb\u4e00\u6b65\u52a0\u5de5\u7684&#xff1f;<\/p>\n<p>\u4e0b\u4e00\u7bc7&#xff1a;<\/p>\n<p>\u7b2c\u516d\u7bc7&#xff1a;FFN\u3001\u6b8b\u5dee\u8fde\u63a5\u4e0e LayerNorm\u2014\u2014Attention \u4e4b\u540e\u53d1\u751f\u4e86\u4ec0\u4e48&#xff1f;<\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00<br \/>\n\u5728\u524d\u9762\u7684\u6587\u7ae0\u4e2d&#xff0c;\u6211\u4eec\u5df2\u7ecf\u9010\u6b65\u89e3\u51b3\u4e86 Transformer \u4e2d\u4e00\u4e2a\u975e\u5e38\u6838\u5fc3\u7684\u95ee\u9898\u3002<br \/>\n\u9996\u5148&#xff0c;\u6211\u4eec\u5728\u7b2c\u4e00\u7bc7\u4e2d\u8ba8\u8bba\u4e86\u4e3a\u4ec0\u4e48\u4f20\u7edf\u7684 RNN\u3001LSTM \u5728\u5904\u7406\u957f\u8ddd\u79bb\u4f9d\u8d56\u548c\u5e76\u884c\u8ba1\u7b97\u65f6\u5b58\u5728\u5c40\u9650&#xff0c;\u56e0\u6b64\u9700\u8981\u4e00\u79cd\u80fd\u591f\u76f4\u63a5\u5efa\u7acb\u4efb\u610f Token \u4e4b\u95f4\u8054\u7cfb\u7684\u673a\u5236\u3002<br \/>\n\u968f\u540e&#xff0c;\u6211\u4eec\u8fdb\u5165 Transformer \u7684\u6838\u5fc3\u3002<br \/>\n\u6211\u4eec\u5148\u7406\u89e3\u4e86 Self-Attention&#xff0c;\u77e5\u9053\u4e86 Query\u3001Key\u3001Value \u662f\u5982\u4f55\u8ba1\u7b97\u7684&amp;#xff<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[841,50,207,86,523],"topic":[],"class_list":["post-107134","post","type-post","status-publish","format-standard","hentry","category-server","tag-transformer","tag-50","tag-207","tag-86","tag-523"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/107134.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u524d\u8a00 \u5728\u524d\u9762\u7684\u6587\u7ae0\u4e2d&#xff0c;\u6211\u4eec\u5df2\u7ecf\u9010\u6b65\u89e3\u51b3\u4e86 Transformer \u4e2d\u4e00\u4e2a\u975e\u5e38\u6838\u5fc3\u7684\u95ee\u9898\u3002 \u9996\u5148&#xff0c;\u6211\u4eec\u5728\u7b2c\u4e00\u7bc7\u4e2d\u8ba8\u8bba\u4e86\u4e3a\u4ec0\u4e48\u4f20\u7edf\u7684 RNN\u3001LSTM \u5728\u5904\u7406\u957f\u8ddd\u79bb\u4f9d\u8d56\u548c\u5e76\u884c\u8ba1\u7b97\u65f6\u5b58\u5728\u5c40\u9650&#xff0c;\u56e0\u6b64\u9700\u8981\u4e00\u79cd\u80fd\u591f\u76f4\u63a5\u5efa\u7acb\u4efb\u610f Token \u4e4b\u95f4\u8054\u7cfb\u7684\u673a\u5236\u3002 \u968f\u540e&#xff0c;\u6211\u4eec\u8fdb\u5165 Transformer \u7684\u6838\u5fc3\u3002 \u6211\u4eec\u5148\u7406\u89e3\u4e86 Self-Attention&#xff0c;\u77e5\u9053\u4e86 Query\u3001Key\u3001Value \u662f\u5982\u4f55\u8ba1\u7b97\u7684&amp;#xff\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/107134.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-09-18T20:19:50+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"19 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/107134.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/107134.html\",\"name\":\"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-09-18T20:19:50+00:00\",\"dateModified\":\"2026-09-18T20:19:50+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/107134.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/107134.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/107134.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/107134.html","og_locale":"zh_CN","og_type":"article","og_title":"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u524d\u8a00 \u5728\u524d\u9762\u7684\u6587\u7ae0\u4e2d&#xff0c;\u6211\u4eec\u5df2\u7ecf\u9010\u6b65\u89e3\u51b3\u4e86 Transformer \u4e2d\u4e00\u4e2a\u975e\u5e38\u6838\u5fc3\u7684\u95ee\u9898\u3002 \u9996\u5148&#xff0c;\u6211\u4eec\u5728\u7b2c\u4e00\u7bc7\u4e2d\u8ba8\u8bba\u4e86\u4e3a\u4ec0\u4e48\u4f20\u7edf\u7684 RNN\u3001LSTM \u5728\u5904\u7406\u957f\u8ddd\u79bb\u4f9d\u8d56\u548c\u5e76\u884c\u8ba1\u7b97\u65f6\u5b58\u5728\u5c40\u9650&#xff0c;\u56e0\u6b64\u9700\u8981\u4e00\u79cd\u80fd\u591f\u76f4\u63a5\u5efa\u7acb\u4efb\u610f Token \u4e4b\u95f4\u8054\u7cfb\u7684\u673a\u5236\u3002 \u968f\u540e&#xff0c;\u6211\u4eec\u8fdb\u5165 Transformer \u7684\u6838\u5fc3\u3002 \u6211\u4eec\u5148\u7406\u89e3\u4e86 Self-Attention&#xff0c;\u77e5\u9053\u4e86 Query\u3001Key\u3001Value \u662f\u5982\u4f55\u8ba1\u7b97\u7684&amp;#xff","og_url":"https:\/\/www.wsisp.com\/helps\/107134.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-09-18T20:19:50+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"19 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/107134.html","url":"https:\/\/www.wsisp.com\/helps\/107134.html","name":"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-09-18T20:19:50+00:00","dateModified":"2026-09-18T20:19:50+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/107134.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/107134.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/107134.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u6df1\u5165\u7406\u89e3 Transformer\uff1a\u7406\u89e3\u987a\u5e8fPositional Encoding"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/107134","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=107134"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/107134\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=107134"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=107134"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=107134"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=107134"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}