{"id":120636,"date":"2026-10-31T23:47:56","date_gmt":"2026-10-31T15:47:56","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/120636.html"},"modified":"2026-10-31T23:47:56","modified_gmt":"2026-10-31T15:47:56","slug":"%e7%ac%ac20%e7%ab%a0-%e4%bb%8e%e9%9b%b6%e5%ae%9e%e7%8e%b0-transformer%ef%bc%88pytorch%ef%bc%89","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/120636.html","title":{"rendered":"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09"},"content":{"rendered":"<h2>\u7b2c\u516d\u7bc7 \u5b9e\u6218<\/h2>\n<h2>\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer&#xff08;PyTorch&#xff09;<\/h2>\n<p>\u672c\u7ae0\u76ee\u6807&#xff1a;<\/p>\n<p>\u4e0d\u8c03\u7528 nn.Transformer \/ nn.MultiheadAttention&#xff0c;\u7528 PyTorch \u4ece\u96f6\u5199\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684 Encoder-Decoder Transformer&#xff0c;\u5728 RTX 4090 \u4e0a 17 \u79d2\u8bad\u7ec3\u5230 99.6% \u7684\u6574\u53e5\u51c6\u786e\u7387&#xff0c;\u5e76\u4eb2\u773c\u770b\u5230 Cross-Attention \u5b66\u51fa\u6765\u7684\u5bf9\u9f50\u77e9\u9635\u3002<\/p>\n<hr \/>\n<h3>\u672c\u7ae0\u76ee\u5f55<\/h3>\n<ul>\n<li>20.1 \u672c\u7ae0\u505a\u4ec0\u4e48 \u2014\u2014 \u4efb\u52a1\u9009\u62e9\u3001\u6574\u4f53\u67b6\u6784\u3001\u9879\u76ee\u7ed3\u6784\u4e0e\u73af\u5883<\/li>\n<li>20.2 \u8f93\u5165\u5c42&#xff1a;Embedding &#043; \u4f4d\u7f6e\u7f16\u7801 \u2014\u2014 \u5bf9\u5e94\u7b2c3\u7ae0<\/li>\n<li>20.3 Multi-Head Attention \u2014\u2014 \u5bf9\u5e94\u7b2c6\u30017\u7ae0&#xff0c;\u9010\u884c\u6807\u6ce8 Shape<\/li>\n<li>20.4 Feed Forward Network \u2014\u2014 \u5bf9\u5e94\u7b2c8\u7ae0<\/li>\n<li>20.5 Encoder \u2014\u2014 Pre-Norm \u4e0e\u6700\u540e\u90a3\u4e00\u5c42 LayerNorm<\/li>\n<li>20.6 Decoder \u2014\u2014 Masked Self-Attention &#043; Cross-Attention<\/li>\n<li>20.7 \u4e09\u79cd Mask \u4e0e\u5b8c\u6574\u6a21\u578b \u2014\u2014 \u5bf9\u5e94\u7b2c13\u7ae0&#xff0c;\u4e00\u6b21\u7406\u6e05<\/li>\n<li>20.8 \u5148\u6d4b\u518d\u8bad&#xff1a;\u4e24\u4e2a\u5355\u5143\u6d4b\u8bd5<\/li>\n<li>20.9 \u6570\u636e&#xff1a;\u5e8f\u5217\u7ffb\u8f6c\u4efb\u52a1<\/li>\n<li>20.10 \u8bad\u7ec3 \u2014\u2014 Teacher Forcing\u3001Warmup\u3001bf16 \u6df7\u5408\u7cbe\u5ea6<\/li>\n<li>20.11 \u63a8\u7406&#xff1a;\u8d2a\u5fc3\u89e3\u7801<\/li>\n<li>20.12 \u8fd0\u884c\u7ed3\u679c&#xff1a;\u4e00\u6b21&#034;\u4ee3\u7801\u5168\u5bf9\u3001\u6a21\u578b\u5374\u5b66\u4e0d\u4f1a&#034;\u7684\u6392\u67e5<\/li>\n<li>20.13 \u5b9e\u9a8c&#xff1a;\u53ef\u89c6\u5316 Cross-Attention<\/li>\n<li>20.14 \u5b9e\u9a8c&#xff1a;\u624b\u5199 Attention vs SDPA&#xff08;FlashAttention&#xff09;<\/li>\n<li>20.15 Hugging Face \u6e90\u7801\u5b9a\u4f4d<\/li>\n<li>20.16 \u5e38\u89c1\u5751<\/li>\n<\/ul>\n<hr \/>\n<h3>20.1 \u672c\u7ae0\u505a\u4ec0\u4e48<\/h3>\n<h4>20.1.1 \u4e3a\u4ec0\u4e48\u7528&#034;\u5e8f\u5217\u7ffb\u8f6c&#034;\u505a\u4efb\u52a1&#xff1f;<\/h4>\n<p>\u771f\u5b9e\u7684\u673a\u5668\u7ffb\u8bd1\u9700\u8981\u4e0b\u8f7d\u8bed\u6599\u3001\u8bad\u7ec3\u5206\u8bcd\u5668\u3001\u8dd1\u51e0\u4e2a\u5c0f\u65f6&#xff0c;\u5927\u90e8\u5206\u7cbe\u529b\u4f1a\u82b1\u5728\u6570\u636e\u5de5\u7a0b\u4e0a\u3002\u672c\u7ae0\u9009\u4e00\u4e2a\u5408\u6210\u4efb\u52a1&#xff1a;\u628a\u4e00\u4e32\u6570\u5b57\u5012\u8fc7\u6765\u3002<\/p>\n<p>\u8f93\u5165:  7 3 0 9 5 &lt;eos&gt;<br \/>\n\u8f93\u51fa:  &lt;bos&gt; 5 9 0 3 7 &lt;eos&gt;<\/p>\n<p>\u8fd9\u4e2a\u4efb\u52a1\u770b\u4f3c\u7b80\u5355&#xff0c;\u5374\u6070\u597d\u80fd\u68c0\u9a8c Transformer \u7684\u6bcf\u4e2a\u96f6\u4ef6&#xff1a;<\/p>\n<table>\n<tr>\u8981\u6c42\u7528\u5230\u7684\u7ec4\u4ef6<\/tr>\n<tbody>\n<tr>\n<td>\u8f93\u51fa\u7b2c 1 \u4e2a\u6570\u5b57\u65f6\u5fc5\u987b&#034;\u770b&#034;\u8f93\u5165\u7684\u6700\u540e\u4e00\u4e2a\u6570\u5b57<\/td>\n<td>Cross-Attention \u5fc5\u987b\u5b66\u4f1a\u5bf9\u9f50<\/td>\n<\/tr>\n<tr>\n<td>\u8981\u77e5\u9053\u6bcf\u4e2a\u6570\u5b57\u5728\u7b2c\u51e0\u4f4d<\/td>\n<td>\u4f4d\u7f6e\u7f16\u7801\u4e0d\u80fd\u5199\u9519&#xff0c;\u4e5f\u4e0d\u80fd\u88ab\u6df9\u6ca1<\/td>\n<\/tr>\n<tr>\n<td>\u8f93\u5165\u957f\u5ea6 3~12 \u4e0d\u7b49<\/td>\n<td>Padding Mask \u5fc5\u987b\u6b63\u786e<\/td>\n<\/tr>\n<tr>\n<td>\u751f\u6210\u65f6\u4e0d\u80fd\u5077\u770b\u540e\u9762\u7684\u7b54\u6848<\/td>\n<td>Causal Mask \u5fc5\u987b\u6b63\u786e<\/td>\n<\/tr>\n<tr>\n<td>\u6709\u660e\u786e\u7684\u6807\u51c6\u7b54\u6848<\/td>\n<td>\u53ef\u4ee5\u7528&#034;\u6574\u53e5\u5b8c\u5168\u6b63\u786e\u7387&#034;\u91cf\u5316\u6548\u679c<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4efb\u4f55\u4e00\u4e2a\u7ec4\u4ef6\u51fa\u95ee\u9898&#xff0c;\u6a21\u578b\u90fd\u5b66\u4e0d\u597d\u2014\u201420.12 \u8282\u5c31\u4f1a\u770b\u5230\u4e00\u4e2a\u771f\u5b9e\u7684\u4f8b\u5b50\u3002<\/p>\n<h4>20.1.2 \u6574\u4f53\u67b6\u6784<\/h4>\n<p>#mermaid-svg-QjkyVt3CTxqgwa7R{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-QjkyVt3CTxqgwa7R .error-icon{fill:#552222;}#mermaid-svg-QjkyVt3CTxqgwa7R .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-QjkyVt3CTxqgwa7R .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-QjkyVt3CTxqgwa7R .marker{fill:#333333;stroke:#333333;}#mermaid-svg-QjkyVt3CTxqgwa7R .marker.cross{stroke:#333333;}#mermaid-svg-QjkyVt3CTxqgwa7R svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-QjkyVt3CTxqgwa7R p{margin:0;}#mermaid-svg-QjkyVt3CTxqgwa7R .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster-label text{fill:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster-label span{color:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster-label span p{background-color:transparent;}#mermaid-svg-QjkyVt3CTxqgwa7R .label text,#mermaid-svg-QjkyVt3CTxqgwa7R span{fill:#333;color:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R .node rect,#mermaid-svg-QjkyVt3CTxqgwa7R .node circle,#mermaid-svg-QjkyVt3CTxqgwa7R .node ellipse,#mermaid-svg-QjkyVt3CTxqgwa7R .node polygon,#mermaid-svg-QjkyVt3CTxqgwa7R .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-QjkyVt3CTxqgwa7R .rough-node .label text,#mermaid-svg-QjkyVt3CTxqgwa7R .node .label text,#mermaid-svg-QjkyVt3CTxqgwa7R .image-shape .label,#mermaid-svg-QjkyVt3CTxqgwa7R .icon-shape .label{text-anchor:middle;}#mermaid-svg-QjkyVt3CTxqgwa7R .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-QjkyVt3CTxqgwa7R .rough-node .label,#mermaid-svg-QjkyVt3CTxqgwa7R .node .label,#mermaid-svg-QjkyVt3CTxqgwa7R .image-shape .label,#mermaid-svg-QjkyVt3CTxqgwa7R .icon-shape .label{text-align:center;}#mermaid-svg-QjkyVt3CTxqgwa7R .node.clickable{cursor:pointer;}#mermaid-svg-QjkyVt3CTxqgwa7R .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-QjkyVt3CTxqgwa7R .arrowheadPath{fill:#333333;}#mermaid-svg-QjkyVt3CTxqgwa7R .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-QjkyVt3CTxqgwa7R .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-QjkyVt3CTxqgwa7R .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-QjkyVt3CTxqgwa7R .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-QjkyVt3CTxqgwa7R .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-QjkyVt3CTxqgwa7R .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster text{fill:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R .cluster span{color:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-QjkyVt3CTxqgwa7R .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-QjkyVt3CTxqgwa7R rect.text{fill:none;stroke-width:0;}#mermaid-svg-QjkyVt3CTxqgwa7R .icon-shape,#mermaid-svg-QjkyVt3CTxqgwa7R .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-QjkyVt3CTxqgwa7R .icon-shape p,#mermaid-svg-QjkyVt3CTxqgwa7R .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-QjkyVt3CTxqgwa7R .icon-shape .label rect,#mermaid-svg-QjkyVt3CTxqgwa7R .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-QjkyVt3CTxqgwa7R .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-QjkyVt3CTxqgwa7R .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-QjkyVt3CTxqgwa7R :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u6e90\u5e8f\u5217 src[B, Ls]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Token Embedding \u00d7 \u221ad&#043; \u4f4d\u7f6e\u7f16\u7801<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Encoder Layer \u00d7 NSelf-Attn &#043; FFN<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>memory[B, Ls, d_model]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u76ee\u6807\u8f93\u5165 tgt_in[B, Lt]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Token Embedding \u00d7 \u221ad&#043; \u4f4d\u7f6e\u7f16\u7801<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Decoder Layer \u00d7 NMasked Self-AttnCross-AttnFFN<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Linear[B, Lt, vocab]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>logits<\/p>\n<p><\/span><\/p>\n<h4>20.1.3 \u9879\u76ee\u7ed3\u6784\u4e0e\u73af\u5883<\/h4>\n<p>ch20_transformer\/<br \/>\n\u251c\u2500\u2500 model.py         # \u5168\u90e8\u6a21\u578b\u7ec4\u4ef6&#xff08;20.2 ~ 20.7&#xff09;<br \/>\n\u251c\u2500\u2500 test_model.py    # Mask \u5355\u5143\u6d4b\u8bd5&#xff08;20.8&#xff09;<br \/>\n\u251c\u2500\u2500 train.py         # \u6570\u636e\u3001\u8bad\u7ec3\u3001\u63a8\u7406&#xff08;20.9 ~ 20.11&#xff09;<br \/>\n\u251c\u2500\u2500 viz.py           # Cross-Attention \u53ef\u89c6\u5316&#xff08;20.13&#xff09;<br \/>\n\u2514\u2500\u2500 mha_vs_sdpa.py   # \u624b\u5199 Attention vs SDPA&#xff08;20.14&#xff09;<\/p>\n<p>\u672c\u7ae0&#xff08;\u4ee5\u53ca\u7b2c21\u300122\u7ae0&#xff09;\u7684\u5b9e\u6d4b\u73af\u5883&#xff1a;<\/p>\n<table>\n<tr>\u9879\u76ee\u7248\u672c<\/tr>\n<tbody>\n<tr>\n<td>Python<\/td>\n<td>3.12<\/td>\n<\/tr>\n<tr>\n<td>PyTorch<\/td>\n<td>2.7.0&#043;cu126&#xff08;CUDA 12.6&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>GPU<\/td>\n<td>NVIDIA RTX 4090 24GB<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>pip <span class=\"token function\">install<\/span> <span class=\"token assign-left variable\">numpy<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">2.2<\/span>.6 <span class=\"token assign-left variable\">matplotlib<\/span><span class=\"token operator\">&#061;&#061;<\/span><span class=\"token number\">3.10<\/span>.3   <span class=\"token comment\"># PyTorch \u6309\u5b98\u7f51\u547d\u4ee4\u5355\u72ec\u5b89\u88c5\u5bf9\u5e94 CUDA \u7248\u672c<\/span><\/p>\n<p>\u4ee3\u7801\u4f1a\u81ea\u52a8\u68c0\u6d4b\u8bbe\u5907&#xff1a;\u6ca1\u6709 GPU \u65f6\u9000\u56de CPU&#xff0c;\u672c\u7ae0\u7684\u5c0f\u6a21\u578b\u5728 CPU \u4e0a\u4e5f\u80fd\u5728\u51e0\u5206\u949f\u5185\u8bad\u5b8c\u3002\u4e0b\u6587\u6240\u6709\u8f93\u51fa\u90fd\u6765\u81ea\u771f\u5b9e\u8fd0\u884c\u3002<\/p>\n<hr \/>\n<h3>20.2 \u8f93\u5165\u5c42&#xff1a;Embedding &#043; \u4f4d\u7f6e\u7f16\u7801<\/h3>\n<p>\u7b2c3\u7ae0\u8bb2\u8fc7&#xff1a;Transformer \u81ea\u8eab\u4e0d\u8ba4\u8bc6\u987a\u5e8f&#xff0c;\u5fc5\u987b\u628a\u4f4d\u7f6e\u4fe1\u606f\u52a0\u5230 Token \u5411\u91cf\u4e0a\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py<\/span><br \/>\n<span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer&#xff08;PyTorch \u7248&#xff09;\u2014\u2014 \u6a21\u578b<\/p>\n<p>Mask \u7ea6\u5b9a&#xff08;\u5168\u6587\u4ef6\u7edf\u4e00&#xff09;&#xff1a;bool \u5f20\u91cf&#xff0c;True &#061; \u5c4f\u853d&#xff08;\u4e0d\u8bb8\u770b&#xff09;&#xff0c;False &#061; \u53ef\u89c1\u3002<br \/>\n\u6ce8\u610f&#xff1a;F.scaled_dot_product_attention \u7684 bool mask \u8bed\u4e49\u6b63\u597d\u76f8\u53cd&#xff08;True &#061; \u53ef\u89c1&#xff09;&#xff0c;<br \/>\n\u672c\u7ae0\u4e3a\u4e86\u62ff\u5230\u6ce8\u610f\u529b\u6743\u91cd\u505a\u53ef\u89c6\u5316&#xff0c;\u624b\u5199 Attention&#xff0c;\u4e0d\u8c03\u7528 SDPA\u3002<br \/>\n&#034;&#034;&#034;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> math<\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> torch<span class=\"token punctuation\">.<\/span>nn <span class=\"token keyword\">as<\/span> nn<br \/>\n<span class=\"token keyword\">import<\/span> torch<span class=\"token punctuation\">.<\/span>nn<span class=\"token punctuation\">.<\/span>functional <span class=\"token keyword\">as<\/span> F<\/p>\n<p>PAD_ID <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">sinusoidal_position_encoding<\/span><span class=\"token punctuation\">(<\/span>max_len<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">:<\/span> <span class=\"token builtin\">int<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token operator\">&gt;<\/span> torch<span class=\"token punctuation\">.<\/span>Tensor<span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u6b63\u5f26\u4f4d\u7f6e\u7f16\u7801&#xff0c;\u8fd4\u56de [max_len, d_model]&#034;&#034;&#034;<\/span><br \/>\n    pos <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>arange<span class=\"token punctuation\">(<\/span>max_len<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>float32<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>unsqueeze<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>            <span class=\"token comment\"># [L, 1]<\/span><br \/>\n    div <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>exp<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>arange<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>float32<span class=\"token punctuation\">)<\/span><br \/>\n                    <span class=\"token operator\">*<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span>math<span class=\"token punctuation\">.<\/span>log<span class=\"token punctuation\">(<\/span><span class=\"token number\">10000.0<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span>                         <span class=\"token comment\"># [d\/2]<\/span><br \/>\n    pe <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>max_len<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><br \/>\n    pe<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">:<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>sin<span class=\"token punctuation\">(<\/span>pos <span class=\"token operator\">*<\/span> div<span class=\"token punctuation\">)<\/span>                                        <span class=\"token comment\"># \u5076\u6570\u7ef4 sin<\/span><br \/>\n    pe<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">:<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>cos<span class=\"token punctuation\">(<\/span>pos <span class=\"token operator\">*<\/span> div<span class=\"token punctuation\">)<\/span>                                        <span class=\"token comment\"># \u5947\u6570\u7ef4 cos<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> pe<\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">TokenAndPosition<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;Token Embedding \u00d7 sqrt(d_model) &#043; \u4f4d\u7f6e\u7f16\u7801&#034;&#034;&#034;<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>d_model <span class=\"token operator\">&#061;<\/span> d_model<br \/>\n        self<span class=\"token punctuation\">.<\/span>emb <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Embedding<span class=\"token punctuation\">(<\/span>vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> padding_idx<span class=\"token operator\">&#061;<\/span>PAD_ID<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># PyTorch \u7684 Embedding \u9ed8\u8ba4\u6309 N(0, 1) \u521d\u59cb\u5316&#xff0c;\u4e58 sqrt(d_model) \u540e\u6807\u51c6\u5dee\u224811&#xff0c;<\/span><br \/>\n        <span class=\"token comment\"># \u800c\u4f4d\u7f6e\u7f16\u7801\u53ea\u5728 [-1, 1]&#xff0c;\u4f4d\u7f6e\u4fe1\u53f7\u4f1a\u88ab\u6df9\u6ca1\u3002\u6309 N(0, d^-0.5) \u521d\u59cb\u5316&#xff0c;\u4e58\u5b8c\u540e\u6807\u51c6\u5dee\u22481\u3002<\/span><br \/>\n        nn<span class=\"token punctuation\">.<\/span>init<span class=\"token punctuation\">.<\/span>normal_<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>emb<span class=\"token punctuation\">.<\/span>weight<span class=\"token punctuation\">,<\/span> mean<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">,<\/span> std<span class=\"token operator\">&#061;<\/span>d_model <span class=\"token operator\">**<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">0.5<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            self<span class=\"token punctuation\">.<\/span>emb<span class=\"token punctuation\">.<\/span>weight<span class=\"token punctuation\">[<\/span>PAD_ID<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>zero_<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># register_buffer&#xff1a;\u968f\u6a21\u578b .to(device)&#xff0c;\u4f46\u4e0d\u662f\u53ef\u8bad\u7ec3\u53c2\u6570&#xff0c;\u4e5f\u4f1a\u5b58\u8fdb state_dict<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>register_buffer<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;pe&#034;<\/span><span class=\"token punctuation\">,<\/span> sinusoidal_position_encoding<span class=\"token punctuation\">(<\/span>max_len<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> persistent<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>drop <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> ids<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>                                                   <span class=\"token comment\"># ids: [B, L]<\/span><br \/>\n        L <span class=\"token operator\">&#061;<\/span> ids<span class=\"token punctuation\">.<\/span>size<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>emb<span class=\"token punctuation\">(<\/span>ids<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">*<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>d_model<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>pe<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span>L<span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>drop<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4e09\u4e2a\u503c\u5f97\u6ce8\u610f\u7684\u7ec6\u8282&#xff1a;<\/p>\n<ul>\n<li>register_buffer&#xff1a;\u4f4d\u7f6e\u7f16\u7801\u4e0d\u662f\u53ef\u8bad\u7ec3\u53c2\u6570&#xff0c;\u4f46\u8981\u8ddf\u7740 model.to(&#034;cuda&#034;) \u4e00\u8d77\u642c\u5230 GPU\u3002\u6ce8\u518c\u6210 buffer \u5c31\u80fd\u505a\u5230&#xff1b;persistent&#061;False \u8868\u793a\u4e0d\u5b58\u8fdb state_dict&#xff08;\u5b83\u53ef\u4ee5\u968f\u65f6\u91cd\u65b0\u7b97\u51fa\u6765&#xff09;\u3002<\/li>\n<li>\u4e3a\u4ec0\u4e48 Embedding \u8981\u4e58 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">dmodel\\\\sqrt{d_{model}}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1828em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8572em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.3361em\"><span class=\"\" style=\"top: -2.55em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"mord mathnormal mtight\">o<\/span><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em\">l<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.15em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -2.8172em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1828em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff1a;\u539f\u8bba\u6587 3.4 \u8282\u7684\u505a\u6cd5&#xff0c;\u76ee\u7684\u662f\u8ba9 Token \u4fe1\u53f7\u548c\u4f4d\u7f6e\u7f16\u7801\u5904\u5728\u76f8\u8fd1\u7684\u6570\u503c\u8303\u56f4\u3002<\/li>\n<li>\u4e3a\u4ec0\u4e48\u8981\u624b\u52a8\u521d\u59cb\u5316 Embedding&#xff1a;\u8fd9\u662f\u4ece\u5176\u4ed6\u6846\u67b6\u79fb\u690d\u65f6\u6700\u5bb9\u6613\u8e29\u7684\u5751\u3002PyTorch \u7684 nn.Embedding \u9ed8\u8ba4\u6309 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(0,1)\\\\mathcal{N}(0, 1)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u521d\u59cb\u5316&#xff0c;\u4e58\u4ee5 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">128\u224811.3\\\\sqrt{128} \u2248 11.3<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\">128<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">\u2248<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6444em\"><\/span><span class=\"mord\">11.3<\/span><\/span><\/span><\/span><\/span> \u4e4b\u540e\u6807\u51c6\u5dee\u7ea6\u4e3a 11&#xff0c;\u800c\u4f4d\u7f6e\u7f16\u7801\u53ea\u5728 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">[\u22121,1][-1, 1]<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mopen\">[<\/span><span class=\"mord\">\u2212<\/span><span class=\"mord\">1<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">]<\/span><\/span><\/span><\/span><\/span> \u4e4b\u95f4\u2014\u2014\u4f4d\u7f6e\u4fe1\u53f7\u53ea\u5269\u4e0d\u5230\u5341\u5206\u4e4b\u4e00&#xff0c;\u88ab\u6df9\u6ca1\u4e86\u3002\u6309 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(0,d\u22120.5)\\\\mathcal{N}(0, d^{-0.5})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">0.5<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u521d\u59cb\u5316\u540e&#xff0c;\u4e58\u5b8c\u6b63\u597d\u662f\u6807\u51c6\u5dee 1\u300220.12 \u8282\u4f1a\u770b\u5230\u4e0d\u6539\u8fd9\u4e00\u884c\u65f6&#xff0c;\u6a21\u578b\u5b66\u5f97\u6709\u591a\u6162\u3002<\/li>\n<\/ul>\n<hr \/>\n<h3>20.3 Multi-Head Attention<\/h3>\n<p>\u8fd9\u662f\u7b2c6\u30017\u7ae0\u7684\u4ee3\u7801\u5316\u3002forward \u540c\u65f6\u8fd4\u56de\u4e86\u6ce8\u610f\u529b\u6743\u91cd&#xff0c;20.13 \u8282\u8981\u7528\u5b83\u753b\u56fe\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">MultiHeadAttention<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">assert<\/span> d_model <span class=\"token operator\">%<\/span> num_heads <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;d_model \u5fc5\u987b\u80fd\u88ab num_heads \u6574\u9664&#034;<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>h<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>d_k <span class=\"token operator\">&#061;<\/span> num_heads<span class=\"token punctuation\">,<\/span> d_model <span class=\"token operator\">\/\/<\/span> num_heads<br \/>\n        self<span class=\"token punctuation\">.<\/span>W_q <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>W_k <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>W_v <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>W_o <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>attn_drop <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_split<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>                       <span class=\"token comment\"># [B, L, d_model] -&gt; [B, h, L, d_k]<\/span><br \/>\n        B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> x<span class=\"token punctuation\">.<\/span>shape<br \/>\n        <span class=\"token keyword\">return<\/span> x<span class=\"token punctuation\">.<\/span>view<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>h<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>d_k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">_merge<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>                       <span class=\"token comment\"># [B, h, L, d_k] -&gt; [B, L, d_model]<\/span><br \/>\n        B<span class=\"token punctuation\">,<\/span> _<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> x<span class=\"token punctuation\">.<\/span>shape<br \/>\n        <span class=\"token keyword\">return<\/span> x<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>h <span class=\"token operator\">*<\/span> self<span class=\"token punctuation\">.<\/span>d_k<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> q<span class=\"token punctuation\">,<\/span> k<span class=\"token punctuation\">,<\/span> v<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        Q<span class=\"token punctuation\">,<\/span> K<span class=\"token punctuation\">,<\/span> V <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>_split<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>W_q<span class=\"token punctuation\">(<\/span>q<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>_split<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>W_k<span class=\"token punctuation\">(<\/span>k<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>_split<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>W_v<span class=\"token punctuation\">(<\/span>v<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        scores <span class=\"token operator\">&#061;<\/span> Q &#064; K<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> math<span class=\"token punctuation\">.<\/span>sqrt<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>d_k<span class=\"token punctuation\">)<\/span>                <span class=\"token comment\"># [B, h, Lq, Lk]<\/span><br \/>\n        <span class=\"token keyword\">if<\/span> attn_mask <span class=\"token keyword\">is<\/span> <span class=\"token keyword\">not<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">:<\/span>                                             <span class=\"token comment\"># True &#061; \u5c4f\u853d<\/span><br \/>\n            <span class=\"token comment\"># \u7528 dtype \u7684\u6700\u5c0f\u503c\u800c\u4e0d\u662f -1e9&#xff1a;bf16\/fp16 \u4e0b -1e9 \u4f1a\u6ea2\u51fa\u6210 -inf \u6216\u5931\u771f<\/span><br \/>\n            scores <span class=\"token operator\">&#061;<\/span> scores<span class=\"token punctuation\">.<\/span>masked_fill<span class=\"token punctuation\">(<\/span>attn_mask<span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>finfo<span class=\"token punctuation\">(<\/span>scores<span class=\"token punctuation\">.<\/span>dtype<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">min<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        weights <span class=\"token operator\">&#061;<\/span> scores<span class=\"token punctuation\">.<\/span>softmax<span class=\"token punctuation\">(<\/span>dim<span class=\"token operator\">&#061;<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span>                                      <span class=\"token comment\"># [B, h, Lq, Lk]<\/span><br \/>\n        out <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>attn_drop<span class=\"token punctuation\">(<\/span>weights<span class=\"token punctuation\">)<\/span> &#064; V                                     <span class=\"token comment\"># [B, h, Lq, d_k]<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>W_o<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>_merge<span class=\"token punctuation\">(<\/span>out<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> weights<\/p>\n<p>Shape \u53d8\u5316\u4e00\u89c8&#xff08;\u4ee5\u672c\u7ae0\u914d\u7f6e B&#061;64, L&#061;14, d_model&#061;128, h&#061;4 \u4e3a\u4f8b&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u6b65\u9aa4\u5f20\u91cfShape<\/tr>\n<tbody>\n<tr>\n<td>\u8f93\u5165<\/td>\n<td>q<\/td>\n<td>[64, 14, 128]<\/td>\n<\/tr>\n<tr>\n<td>\u7ebf\u6027\u6295\u5f71<\/td>\n<td>W_q(q)<\/td>\n<td>[64, 14, 128]<\/td>\n<\/tr>\n<tr>\n<td>\u62c6\u5934 _split<\/td>\n<td>Q<\/td>\n<td>[64, 4, 14, 32]<\/td>\n<\/tr>\n<tr>\n<td>\u6253\u5206<\/td>\n<td>Q &#064; K.transpose(-2, -1)<\/td>\n<td>[64, 4, 14, 14]<\/td>\n<\/tr>\n<tr>\n<td>\u52a0\u6743\u6c42\u548c<\/td>\n<td>weights &#064; V<\/td>\n<td>[64, 4, 14, 32]<\/td>\n<\/tr>\n<tr>\n<td>\u5408\u5e76 _merge<\/td>\n<td>\u2014<\/td>\n<td>[64, 14, 128]<\/td>\n<\/tr>\n<tr>\n<td>\u8f93\u51fa\u6295\u5f71<\/td>\n<td>W_o(&#8230;)<\/td>\n<td>[64, 14, 128]<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u4e24\u5904 PyTorch \u7279\u6709\u7684\u5199\u6cd5&#xff1a;<\/p>\n<ul>\n<li>view vs reshape&#xff1a;_split \u91cc\u7528 view&#xff0c;\u56e0\u4e3a\u7ebf\u6027\u5c42\u7684\u8f93\u51fa\u662f\u8fde\u7eed\u5185\u5b58&#xff1b;_merge \u91cc\u5fc5\u987b\u7528 reshape&#xff0c;\u56e0\u4e3a transpose \u4e4b\u540e\u5185\u5b58\u4e0d\u518d\u8fde\u7eed&#xff0c;view \u4f1a\u62a5\u9519\u3002<\/li>\n<li>\u5c4f\u853d\u503c\u7528 torch.finfo(dtype).min \u800c\u4e0d\u662f -1e9&#xff1a;\u672c\u7ae0\u8bad\u7ec3\u7528 bf16 \u6df7\u5408\u7cbe\u5ea6&#xff08;20.10 \u8282&#xff09;&#xff0c;\u4e0d\u540c\u7cbe\u5ea6\u4e0b&#034;\u8db3\u591f\u5c0f\u7684\u6570&#034;\u4e0d\u4e00\u6837&#xff0c;\u76f4\u63a5\u53d6\u5f53\u524d dtype \u80fd\u8868\u793a\u7684\u6700\u5c0f\u503c\u6700\u7a33\u59a5\u3002<\/li>\n<\/ul>\n<p>Mask \u7ea6\u5b9a&#xff1a;\u672c\u7ae0\u6240\u6709 Mask \u90fd\u662f bool \u5f20\u91cf&#xff0c;True &#061; \u5c4f\u853d&#xff0c;\u4e0e masked_fill \u7684\u8bed\u4e49\u4e00\u81f4\u3002\u8981\u7279\u522b\u5c0f\u5fc3\u7684\u662f&#xff0c;PyTorch \u5185\u7f6e\u7684 F.scaled_dot_product_attention \u7684 bool mask \u6070\u597d\u76f8\u53cd&#xff08;True &#061; \u53ef\u4ee5\u770b&#xff09;&#xff0c;Hugging Face \u7684 attention_mask \u5219\u662f\u6574\u6570 1 &#061; \u53ef\u4ee5\u770b\u3002\u4e09\u79cd\u7ea6\u5b9a\u6df7\u7528\u662f\u6700\u5e38\u89c1\u7684 bug \u6765\u6e90\u3002<\/p>\n<hr \/>\n<h3>20.4 Feed Forward Network<\/h3>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">FeedForward<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>net <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Sequential<span class=\"token punctuation\">(<\/span><br \/>\n            nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>GELU<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n            nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>dff<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        <span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>net<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><\/p>\n<p>dff &#061; 4 \u00d7 d_model&#xff08;\u7b2c8\u7ae0&#xff09;&#xff0c;\u6fc0\u6d3b\u51fd\u6570\u7528 GELU\u3002\u60f3\u6362\u6210 SwiGLU \u53ef\u4ee5\u4f5c\u4e3a\u7ec3\u4e60&#xff08;\u89c1\u7ae0\u672b&#xff09;\u3002<\/p>\n<hr \/>\n<h3>20.5 Encoder<\/h3>\n<p>\u672c\u7ae0\u91c7\u7528\u73b0\u4ee3 LLM \u901a\u7528\u7684 Pre-Norm&#xff1a;x &#043; Sublayer(LayerNorm(x))&#xff08;\u7b2c9\u300110\u7ae0&#xff09;\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">EncoderLayer<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;Pre-Norm&#xff1a;x &#043; Sublayer(LN(x))&#034;&#034;&#034;<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ln1<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>ln2 <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>mha <span class=\"token operator\">&#061;<\/span> MultiHeadAttention<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ffn <span class=\"token operator\">&#061;<\/span> FeedForward<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>drop <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> pad_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        h <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>ln1<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><br \/>\n        a<span class=\"token punctuation\">,<\/span> w <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>mha<span class=\"token punctuation\">(<\/span>h<span class=\"token punctuation\">,<\/span> h<span class=\"token punctuation\">,<\/span> h<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span>pad_mask<span class=\"token punctuation\">)<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> x <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>drop<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">)<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> x <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>ffn<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>ln2<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> x<span class=\"token punctuation\">,<\/span> w<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">Encoder<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> num_layers<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>embed <span class=\"token operator\">&#061;<\/span> TokenAndPosition<span class=\"token punctuation\">(<\/span>vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>layers <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>ModuleList<span class=\"token punctuation\">(<\/span>EncoderLayer<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> _ <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>num_layers<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ln_final <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span>          <span class=\"token comment\"># Pre-Norm \u9700\u8981\u5728\u51fa\u53e3\u8865\u4e00\u4e2a LN<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">,<\/span> pad_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>embed<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> layer <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>layers<span class=\"token punctuation\">:<\/span><br \/>\n            x<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> layer<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> pad_mask<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>ln_final<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4e3a\u4ec0\u4e48 Pre-Norm \u6700\u540e\u8981\u591a\u4e00\u4e2a ln_final&#xff1f; Pre-Norm \u4e2d&#xff0c;\u4e3b\u5e72 x \u4e00\u8def\u53ea\u505a\u52a0\u6cd5&#xff0c;\u4ece\u672a\u88ab\u5f52\u4e00\u5316\u8fc7&#xff0c;\u5c42\u6570\u8d8a\u591a\u6570\u503c\u8d8a\u5927\u3002\u6240\u4ee5\u8981\u5728\u6574\u4e2a Stack \u7684\u51fa\u53e3\u8865\u4e00\u4e2a LayerNorm&#xff0c;GPT-2&#xff08;ln_f&#xff09;\u3001Llama&#xff08;norm&#xff09;\u90fd\u6709\u8fd9\u4e00\u5c42\u3002Post-Norm \u6bcf\u5c42\u51fa\u53e3\u5df2\u7ecf\u5f52\u4e00\u5316\u8fc7&#xff0c;\u4e0d\u9700\u8981\u3002<\/p>\n<p>nn.ModuleList \u4e0d\u80fd\u6362\u6210\u666e\u901a\u7684 Python list&#xff1a;\u53ea\u6709\u6ce8\u518c\u8fdb ModuleList \u7684\u5b50\u6a21\u5757&#xff0c;\u53c2\u6570\u624d\u4f1a\u51fa\u73b0\u5728 model.parameters() \u91cc\u3001\u624d\u4f1a\u88ab\u4f18\u5316\u5668\u66f4\u65b0\u3001\u624d\u4f1a\u8ddf\u7740 .to(device) \u642c\u5230 GPU\u3002<\/p>\n<hr \/>\n<h3>20.6 Decoder<\/h3>\n<p>Decoder Layer \u6bd4 Encoder Layer \u591a\u4e00\u4e2a Cross-Attention&#xff1a;Q \u6765\u81ea Decoder \u81ea\u5df1&#xff0c;K\/V \u6765\u81ea Encoder \u7684\u8f93\u51fa memory&#xff08;\u7b2c13\u7ae0&#xff09;\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">DecoderLayer<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ln1<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>ln2<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>ln3 <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>self_mha <span class=\"token operator\">&#061;<\/span> MultiHeadAttention<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>cross_mha <span class=\"token operator\">&#061;<\/span> MultiHeadAttention<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ffn <span class=\"token operator\">&#061;<\/span> FeedForward<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>drop1<span class=\"token punctuation\">,<\/span> self<span class=\"token punctuation\">.<\/span>drop2 <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>Dropout<span class=\"token punctuation\">(<\/span>dropout<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> self_mask<span class=\"token punctuation\">,<\/span> cross_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token comment\"># \u2460 Masked Self-Attention&#xff1a;\u53ea\u80fd\u770b\u81ea\u5df1\u548c\u4e4b\u524d\u7684 Token<\/span><br \/>\n        h <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>ln1<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><br \/>\n        a<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>self_mha<span class=\"token punctuation\">(<\/span>h<span class=\"token punctuation\">,<\/span> h<span class=\"token punctuation\">,<\/span> h<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span>self_mask<span class=\"token punctuation\">)<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> x <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>drop1<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># \u2461 Cross-Attention&#xff1a;Q \u6765\u81ea Decoder&#xff0c;K\/V \u6765\u81ea Encoder \u8f93\u51fa<\/span><br \/>\n        h <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>ln2<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><br \/>\n        a<span class=\"token punctuation\">,<\/span> cross_w <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>cross_mha<span class=\"token punctuation\">(<\/span>h<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span>cross_mask<span class=\"token punctuation\">)<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> x <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>drop2<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># \u2462 FFN<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> x <span class=\"token operator\">&#043;<\/span> self<span class=\"token punctuation\">.<\/span>ffn<span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">.<\/span>ln3<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> x<span class=\"token punctuation\">,<\/span> cross_w<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">class<\/span> <span class=\"token class-name\">Decoder<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> num_layers<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>embed <span class=\"token operator\">&#061;<\/span> TokenAndPosition<span class=\"token punctuation\">(<\/span>vocab_size<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>layers <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>ModuleList<span class=\"token punctuation\">(<\/span>DecoderLayer<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> _ <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>num_layers<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>ln_final <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>LayerNorm<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> self_mask<span class=\"token punctuation\">,<\/span> cross_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        x <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>embed<span class=\"token punctuation\">(<\/span>tgt<span class=\"token punctuation\">)<\/span><br \/>\n        cross_weights <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">]<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> layer <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>layers<span class=\"token punctuation\">:<\/span><br \/>\n            x<span class=\"token punctuation\">,<\/span> w <span class=\"token operator\">&#061;<\/span> layer<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> self_mask<span class=\"token punctuation\">,<\/span> cross_mask<span class=\"token punctuation\">)<\/span><br \/>\n            cross_weights<span class=\"token punctuation\">.<\/span>append<span class=\"token punctuation\">(<\/span>w<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>ln_final<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> cross_weights<\/p>\n<p>Decoder \u6536\u96c6\u4e86\u6bcf\u4e00\u5c42\u7684 Cross-Attention \u6743\u91cd&#xff0c;\u540e\u9762\u53ef\u89c6\u5316\u65f6\u76f4\u63a5\u53d6\u7528\u3002<\/p>\n<hr \/>\n<h3>20.7 \u4e09\u79cd Mask \u4e0e\u5b8c\u6574\u6a21\u578b<\/h3>\n<p>Encoder-Decoder \u91cc\u4e00\u5171\u6709\u4e09\u5904 Attention&#xff0c;\u5404\u81ea\u9700\u8981\u7684 Mask \u4e0d\u4e00\u6837&#xff1a;<\/p>\n<table>\n<tr>\u4f4d\u7f6eQ \u6765\u81eaK\/V \u6765\u81ea\u9700\u8981\u5c4f\u853dMask Shape<\/tr>\n<tbody>\n<tr>\n<td>Encoder Self-Attention<\/td>\n<td>src<\/td>\n<td>src<\/td>\n<td>\u6e90\u7aef PAD<\/td>\n<td>[B, 1, 1, Ls]<\/td>\n<\/tr>\n<tr>\n<td>Decoder Self-Attention<\/td>\n<td>tgt<\/td>\n<td>tgt<\/td>\n<td>\u672a\u6765\u4f4d\u7f6e &#043; \u76ee\u6807\u7aef PAD<\/td>\n<td>[B, 1, Lt, Lt]<\/td>\n<\/tr>\n<tr>\n<td>Decoder Cross-Attention<\/td>\n<td>tgt<\/td>\n<td>memory<\/td>\n<td>\u6e90\u7aef PAD<\/td>\n<td>[B, 1, 1, Ls]<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>#mermaid-svg-1DQiVraax83oQ4oV{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-1DQiVraax83oQ4oV .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-1DQiVraax83oQ4oV .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-1DQiVraax83oQ4oV .error-icon{fill:#552222;}#mermaid-svg-1DQiVraax83oQ4oV .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-1DQiVraax83oQ4oV .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-1DQiVraax83oQ4oV .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-1DQiVraax83oQ4oV .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-1DQiVraax83oQ4oV .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-1DQiVraax83oQ4oV .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-1DQiVraax83oQ4oV .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-1DQiVraax83oQ4oV .marker{fill:#333333;stroke:#333333;}#mermaid-svg-1DQiVraax83oQ4oV .marker.cross{stroke:#333333;}#mermaid-svg-1DQiVraax83oQ4oV svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-1DQiVraax83oQ4oV p{margin:0;}#mermaid-svg-1DQiVraax83oQ4oV .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-1DQiVraax83oQ4oV .cluster-label text{fill:#333;}#mermaid-svg-1DQiVraax83oQ4oV .cluster-label span{color:#333;}#mermaid-svg-1DQiVraax83oQ4oV .cluster-label span p{background-color:transparent;}#mermaid-svg-1DQiVraax83oQ4oV .label text,#mermaid-svg-1DQiVraax83oQ4oV span{fill:#333;color:#333;}#mermaid-svg-1DQiVraax83oQ4oV .node rect,#mermaid-svg-1DQiVraax83oQ4oV .node circle,#mermaid-svg-1DQiVraax83oQ4oV .node ellipse,#mermaid-svg-1DQiVraax83oQ4oV .node polygon,#mermaid-svg-1DQiVraax83oQ4oV .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-1DQiVraax83oQ4oV .rough-node .label text,#mermaid-svg-1DQiVraax83oQ4oV .node .label text,#mermaid-svg-1DQiVraax83oQ4oV .image-shape .label,#mermaid-svg-1DQiVraax83oQ4oV .icon-shape .label{text-anchor:middle;}#mermaid-svg-1DQiVraax83oQ4oV .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-1DQiVraax83oQ4oV .rough-node .label,#mermaid-svg-1DQiVraax83oQ4oV .node .label,#mermaid-svg-1DQiVraax83oQ4oV .image-shape .label,#mermaid-svg-1DQiVraax83oQ4oV .icon-shape .label{text-align:center;}#mermaid-svg-1DQiVraax83oQ4oV .node.clickable{cursor:pointer;}#mermaid-svg-1DQiVraax83oQ4oV .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-1DQiVraax83oQ4oV .arrowheadPath{fill:#333333;}#mermaid-svg-1DQiVraax83oQ4oV .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-1DQiVraax83oQ4oV .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-1DQiVraax83oQ4oV .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-1DQiVraax83oQ4oV .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-1DQiVraax83oQ4oV .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-1DQiVraax83oQ4oV .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-1DQiVraax83oQ4oV .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-1DQiVraax83oQ4oV .cluster text{fill:#333;}#mermaid-svg-1DQiVraax83oQ4oV .cluster span{color:#333;}#mermaid-svg-1DQiVraax83oQ4oV div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-1DQiVraax83oQ4oV .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-1DQiVraax83oQ4oV rect.text{fill:none;stroke-width:0;}#mermaid-svg-1DQiVraax83oQ4oV .icon-shape,#mermaid-svg-1DQiVraax83oQ4oV .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-1DQiVraax83oQ4oV .icon-shape p,#mermaid-svg-1DQiVraax83oQ4oV .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-1DQiVraax83oQ4oV .icon-shape .label rect,#mermaid-svg-1DQiVraax83oQ4oV .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-1DQiVraax83oQ4oV .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-1DQiVraax83oQ4oV .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-1DQiVraax83oQ4oV :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"nodeLabel\"><\/p>\n<p>Cross-Attn<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>padding_mask(src)\u548c Encoder \u590d\u7528<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Decoder Self-Attn<\/p>\n<p><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"nodeLabel\"><\/p>\n<p>causal_mask(Lt)<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u6309\u4f4d\u6216 OR<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>padding_mask(tgt_in)<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Encoder Self-Attn<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>padding_mask(src)<\/p>\n<p><\/span><\/p>\n<p>Shape \u91cc\u7684 1 \u90fd\u9760\u5e7f\u64ad\u5c55\u5f00&#xff1a;[B, 1, 1, Ls] \u5e7f\u64ad\u5230 [B, h, Lq, Ls]&#xff0c;\u610f\u601d\u662f&#034;\u6240\u6709 Head\u3001\u6240\u6709 Query \u90fd\u4e0d\u8bb8\u770b\u8fd9\u51e0\u4e2a PAD Key&#034;\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/model.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">make_padding_mask<\/span><span class=\"token punctuation\">(<\/span>ids<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;[B, L] -&gt; [B, 1, 1, L]&#xff0c;PAD \u4f4d\u7f6e\u4e3a True&#034;&#034;&#034;<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> <span class=\"token punctuation\">(<\/span>ids <span class=\"token operator\">&#061;&#061;<\/span> PAD_ID<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">make_causal_mask<\/span><span class=\"token punctuation\">(<\/span>L<span class=\"token punctuation\">,<\/span> device<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;[1, 1, L, L]&#xff0c;\u4e0a\u4e09\u89d2&#xff08;\u672a\u6765\u4f4d\u7f6e&#xff09;\u4e3a True&#034;&#034;&#034;<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> torch<span class=\"token punctuation\">.<\/span>triu<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>ones<span class=\"token punctuation\">(<\/span>L<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">bool<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>device<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> diagonal<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token boolean\">None<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><\/p>\n<p><span class=\"token keyword\">class<\/span> <span class=\"token class-name\">Transformer<\/span><span class=\"token punctuation\">(<\/span>nn<span class=\"token punctuation\">.<\/span>Module<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">__init__<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> src_vocab<span class=\"token punctuation\">,<\/span> tgt_vocab<span class=\"token punctuation\">,<\/span> d_model<span class=\"token operator\">&#061;<\/span><span class=\"token number\">128<\/span><span class=\"token punctuation\">,<\/span> num_heads<span class=\"token operator\">&#061;<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                 dff<span class=\"token operator\">&#061;<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span> num_layers<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> max_len<span class=\"token operator\">&#061;<\/span><span class=\"token number\">64<\/span><span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token builtin\">super<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>__init__<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>encoder <span class=\"token operator\">&#061;<\/span> Encoder<span class=\"token punctuation\">(<\/span>src_vocab<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> num_layers<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>decoder <span class=\"token operator\">&#061;<\/span> Decoder<span class=\"token punctuation\">(<\/span>tgt_vocab<span class=\"token punctuation\">,<\/span> d_model<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token punctuation\">,<\/span> dff<span class=\"token punctuation\">,<\/span> num_layers<span class=\"token punctuation\">,<\/span> max_len<span class=\"token punctuation\">,<\/span> dropout<span class=\"token punctuation\">)<\/span><br \/>\n        self<span class=\"token punctuation\">.<\/span>out_proj <span class=\"token operator\">&#061;<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> tgt_vocab<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># \u7ebf\u6027\u5c42\u7528 Xavier \u521d\u59cb\u5316&#xff08;nn.Transformer\u3001The Annotated Transformer\u3001Keras Dense \u9ed8\u8ba4\u90fd\u662f Glorot\/Xavier&#xff09;&#xff1b;<\/span><br \/>\n        <span class=\"token comment\"># PyTorch nn.Linear \u9ed8\u8ba4\u7684 Kaiming-uniform(a&#061;\u221a5) \u65b9\u5dee\u66f4\u5c0f<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> m <span class=\"token keyword\">in<\/span> self<span class=\"token punctuation\">.<\/span>modules<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">if<\/span> <span class=\"token builtin\">isinstance<\/span><span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">,<\/span> nn<span class=\"token punctuation\">.<\/span>Linear<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                nn<span class=\"token punctuation\">.<\/span>init<span class=\"token punctuation\">.<\/span>xavier_uniform_<span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">.<\/span>weight<span class=\"token punctuation\">)<\/span><br \/>\n                nn<span class=\"token punctuation\">.<\/span>init<span class=\"token punctuation\">.<\/span>zeros_<span class=\"token punctuation\">(<\/span>m<span class=\"token punctuation\">.<\/span>bias<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">encode<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        src_mask <span class=\"token operator\">&#061;<\/span> make_padding_mask<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>encoder<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> src_mask<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">decode<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> tgt_in<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        L <span class=\"token operator\">&#061;<\/span> tgt_in<span class=\"token punctuation\">.<\/span>size<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token comment\"># Decoder \u81ea\u6ce8\u610f\u529b&#xff1a;\u672a\u6765\u4f4d\u7f6e \u6216 \u76ee\u6807\u7aef PAD&#xff0c;\u4efb\u4e00\u6210\u7acb\u5c31\u5c4f\u853d<\/span><br \/>\n        self_mask <span class=\"token operator\">&#061;<\/span> make_causal_mask<span class=\"token punctuation\">(<\/span>L<span class=\"token punctuation\">,<\/span> tgt_in<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">|<\/span> make_padding_mask<span class=\"token punctuation\">(<\/span>tgt_in<span class=\"token punctuation\">)<\/span><br \/>\n        h<span class=\"token punctuation\">,<\/span> cross_w <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>decoder<span class=\"token punctuation\">(<\/span>tgt_in<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> self_mask<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> self<span class=\"token punctuation\">.<\/span>out_proj<span class=\"token punctuation\">(<\/span>h<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> cross_w<\/p>\n<p>    <span class=\"token keyword\">def<\/span> <span class=\"token function\">forward<\/span><span class=\"token punctuation\">(<\/span>self<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">,<\/span> tgt_in<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        memory<span class=\"token punctuation\">,<\/span> src_mask <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><br \/>\n        logits<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> self<span class=\"token punctuation\">.<\/span>decode<span class=\"token punctuation\">(<\/span>tgt_in<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> logits                                                         <span class=\"token comment\"># [B, Lt, tgt_vocab]<\/span><\/p>\n<p>\u51e0\u70b9\u8bf4\u660e&#xff1a;<\/p>\n<ul>\n<li>\u4e24\u4e2a bool Mask \u7528 | \u5408\u5e76&#xff1a;\u4efb\u4e00\u6761\u4ef6\u6210\u7acb\u5c31\u5c4f\u853d\u3002<\/li>\n<li>make_causal_mask \u8981\u4f20 device&#xff1a;Mask \u5728 forward \u91cc\u4e34\u65f6\u521b\u5efa&#xff0c;\u5fc5\u987b\u548c\u6570\u636e\u5728\u540c\u4e00\u5757 GPU \u4e0a&#xff0c;\u5426\u5219\u4f1a\u62a5 Expected all tensors to be on the same device\u3002<\/li>\n<li>Xavier \u521d\u59cb\u5316&#xff1a;PyTorch nn.Linear \u9ed8\u8ba4\u662f Kaiming-uniform&#xff08;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a&#061;5a&#061;\\\\sqrt5<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\">5<\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff09;&#xff0c;\u65b9\u5dee\u6bd4 Xavier \u5c0f\u3002nn.Transformer \u5185\u90e8\u3001The Annotated Transformer \u90fd\u4f1a\u6539\u7528 Xavier&#xff0c;\u672c\u7ae0\u4e5f\u4e00\u6837\u3002<\/li>\n<li>encode \u548c decode \u62c6\u5f00&#xff1a;\u63a8\u7406\u65f6 Encoder \u53ea\u9700\u8981\u8dd1\u4e00\u6b21&#xff0c;Decoder \u8981\u5faa\u73af\u8dd1\u5f88\u591a\u6b21&#xff08;20.11 \u8282&#xff09;&#xff0c;\u62c6\u5f00\u624d\u80fd\u590d\u7528 memory\u3002<\/li>\n<\/ul>\n<hr \/>\n<h3>20.8 \u5148\u6d4b\u518d\u8bad&#xff1a;\u4e24\u4e2a\u5355\u5143\u6d4b\u8bd5<\/h3>\n<p>Mask \u5199\u9519\u65f6\u6a21\u578b\u7167\u6837\u80fd\u8dd1\u3001loss \u7167\u6837\u4e0b\u964d&#xff0c;\u53ea\u662f\u5b66\u5230\u4e86&#034;\u4f5c\u5f0a&#034;\u6216&#034;\u566a\u58f0&#034;\u3002\u6240\u4ee5\u8bad\u7ec3\u524d\u5148\u7528\u4e24\u4e2a\u6027\u8d28\u6d4b\u4e00\u4e0b&#xff1a;<\/p>\n<li>\u56e0\u679c\u6027&#xff1a;\u6539\u52a8 Decoder \u8f93\u5165\u4e2d\u7b2c 3 \u4e2a\u4f4d\u7f6e\u4e4b\u540e\u7684 Token&#xff0c;\u524d 3 \u4e2a\u4f4d\u7f6e\u7684\u8f93\u51fa\u5fc5\u987b\u5b8c\u5168\u4e0d\u53d8\u3002<\/li>\n<li>Padding \u4e0d\u53d8\u6027&#xff1a;\u6e90\u5e8f\u5217\u672b\u5c3e\u591a\u8865\u51e0\u4e2a PAD&#xff0c;\u8f93\u51fa\u5fc5\u987b\u5b8c\u5168\u4e0d\u53d8\u3002<\/li>\n<p><span class=\"token comment\"># ch20_transformer\/test_model.py<\/span><br \/>\n<span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u8bad\u7ec3\u524d\u5148\u9a8c\u8bc1 Mask&#xff1a;\u56e0\u679c\u6027 &#043; Padding \u4e0d\u53d8\u6027&#034;&#034;&#034;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> torch<\/p>\n<p><span class=\"token keyword\">from<\/span> model <span class=\"token keyword\">import<\/span> Transformer<\/p>\n<p>torch<span class=\"token punctuation\">.<\/span>manual_seed<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\nm <span class=\"token operator\">&#061;<\/span> Transformer<span class=\"token punctuation\">(<\/span><span class=\"token number\">13<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">13<\/span><span class=\"token punctuation\">,<\/span> max_len<span class=\"token operator\">&#061;<\/span><span class=\"token number\">14<\/span><span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>      <span class=\"token comment\"># \u5173\u6389 Dropout&#xff0c;\u4e24\u6b21\u524d\u5411\u624d\u53ef\u6bd4<\/span><\/p>\n<p>src <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\ntgt <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    a <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\"># 1) \u56e0\u679c\u6027&#xff1a;\u6539\u52a8\u7b2c 3 \u4e2a\u4f4d\u7f6e\u4e4b\u540e\u7684 Token&#xff0c;\u524d 3 \u4e2a\u4f4d\u7f6e\u8f93\u51fa\u5fc5\u987b\u4e0d\u53d8<\/span><br \/>\n    b <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">9<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">9<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;causal ok:&#034;<\/span><span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>allclose<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> b<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> atol<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1e-5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n          <span class=\"token string\">&#034;\u540e\u9762\u53d8\u4e86:&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token keyword\">not<\/span> torch<span class=\"token punctuation\">.<\/span>allclose<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> b<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\"># 2) Padding&#xff1a;\u6e90\u7aef\u591a\u8865\u51e0\u4e2a PAD&#xff0c;\u8f93\u51fa\u5fc5\u987b\u4e0d\u53d8<\/span><br \/>\n    c <span class=\"token operator\">&#061;<\/span> m<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;padding ok:&#034;<\/span><span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>allclose<span class=\"token punctuation\">(<\/span>a<span class=\"token punctuation\">,<\/span> c<span class=\"token punctuation\">,<\/span> atol<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1e-5<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>causal ok: True \u540e\u9762\u53d8\u4e86: True<br \/>\npadding ok: True<\/p>\n<p>\u6ce8\u610f .eval()&#xff1a;\u6d4b\u8bd5\u65f6\u8981\u5173\u6389 Dropout&#xff0c;\u5426\u5219\u4e24\u6b21\u524d\u5411\u672c\u6765\u5c31\u4e0d\u4e00\u6837\u3002\u628a make_causal_mask \u91cc\u7684 diagonal&#061;1 \u6545\u610f\u6539\u6210 diagonal&#061;0&#xff08;\u8fde\u81ea\u5df1\u90fd\u770b\u4e0d\u5230&#xff09;\u6216\u53bb\u6389 triu&#xff0c;\u7b2c\u4e00\u4e2a\u6d4b\u8bd5\u4f1a\u7acb\u523b\u5931\u8d25\u2014\u2014\u5efa\u8bae\u4eb2\u624b\u8bd5\u4e00\u6b21\u3002<\/p>\n<hr \/>\n<h3>20.9 \u6570\u636e&#xff1a;\u5e8f\u5217\u7ffb\u8f6c\u4efb\u52a1<\/h3>\n<p>\u8bcd\u8868\u53ea\u6709 13 \u4e2a Token&#xff1a;PAD&#061;0, BOS&#061;1, EOS&#061;2&#xff0c;\u6570\u5b57 0~9 \u6620\u5c04\u5230 3~12\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/train.py<\/span><br \/>\n<span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u7b2c20\u7ae0&#xff1a;\u5e8f\u5217\u7ffb\u8f6c\u4efb\u52a1 \u2014\u2014 \u6570\u636e\u3001\u8bad\u7ec3\u3001\u8d2a\u5fc3\u89e3\u7801<\/p>\n<p>\u8fd0\u884c&#xff1a;python train.py<br \/>\n&#034;&#034;&#034;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> time<\/p>\n<p><span class=\"token keyword\">import<\/span> numpy <span class=\"token keyword\">as<\/span> np<br \/>\n<span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> torch<span class=\"token punctuation\">.<\/span>nn<span class=\"token punctuation\">.<\/span>functional <span class=\"token keyword\">as<\/span> F<\/p>\n<p><span class=\"token keyword\">from<\/span> model <span class=\"token keyword\">import<\/span> PAD_ID<span class=\"token punctuation\">,<\/span> Transformer<\/p>\n<p><span class=\"token comment\"># &#8212;&#8212;&#8212;&#8212;&#8212;- \u8bbe\u5907\u4e0e\u7cbe\u5ea6 &#8212;&#8212;&#8212;&#8212;&#8212;-<\/span><br \/>\nDEVICE <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;cuda&#034;<\/span> <span class=\"token keyword\">if<\/span> torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>is_available<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token string\">&#034;cpu&#034;<\/span><br \/>\nUSE_BF16 <span class=\"token operator\">&#061;<\/span> DEVICE <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;cuda&#034;<\/span> <span class=\"token keyword\">and<\/span> torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>is_bf16_supported<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>     <span class=\"token comment\"># RTX 4090 \u652f\u6301 bf16<\/span><br \/>\ntorch<span class=\"token punctuation\">.<\/span>backends<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">.<\/span>allow_tf32 <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">True<\/span>                       <span class=\"token comment\"># Ampere \u53ca\u4ee5\u4e0a&#xff1a;FP32 \u77e9\u9635\u4e58\u8d70 TF32<\/span><br \/>\ntorch<span class=\"token punctuation\">.<\/span>backends<span class=\"token punctuation\">.<\/span>cudnn<span class=\"token punctuation\">.<\/span>allow_tf32 <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">True<\/span><\/p>\n<p><span class=\"token comment\"># &#8212;&#8212;&#8212;&#8212;&#8212;- \u6570\u636e&#xff1a;\u5e8f\u5217\u7ffb\u8f6c\u4efb\u52a1 &#8212;&#8212;&#8212;&#8212;&#8212;-<\/span><br \/>\nBOS_ID<span class=\"token punctuation\">,<\/span> EOS_ID <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><br \/>\nNUM_OFFSET <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">3<\/span>                 <span class=\"token comment\"># \u6570\u5b57 0~9 \u6620\u5c04\u4e3a id 3~12<\/span><br \/>\nVOCAB <span class=\"token operator\">&#061;<\/span> NUM_OFFSET <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">10<\/span>        <span class=\"token comment\"># 13<\/span><br \/>\nMIN_LEN<span class=\"token punctuation\">,<\/span> MAX_LEN <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">12<\/span><br \/>\nSEQ_LEN <span class=\"token operator\">&#061;<\/span> MAX_LEN <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">2<\/span>          <span class=\"token comment\"># \u52a0\u4e0a BOS \/ EOS<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">make_dataset<\/span><span class=\"token punctuation\">(<\/span>n<span class=\"token punctuation\">,<\/span> seed<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    rng <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>random<span class=\"token punctuation\">.<\/span>default_rng<span class=\"token punctuation\">(<\/span>seed<span class=\"token punctuation\">)<\/span><br \/>\n    src <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>n<span class=\"token punctuation\">,<\/span> SEQ_LEN<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>np<span class=\"token punctuation\">.<\/span>int64<span class=\"token punctuation\">)<\/span><br \/>\n    tgt <span class=\"token operator\">&#061;<\/span> np<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>n<span class=\"token punctuation\">,<\/span> SEQ_LEN<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>np<span class=\"token punctuation\">.<\/span>int64<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>n<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        L <span class=\"token operator\">&#061;<\/span> rng<span class=\"token punctuation\">.<\/span>integers<span class=\"token punctuation\">(<\/span>MIN_LEN<span class=\"token punctuation\">,<\/span> MAX_LEN <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        digits <span class=\"token operator\">&#061;<\/span> rng<span class=\"token punctuation\">.<\/span>integers<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">10<\/span><span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> NUM_OFFSET<br \/>\n        src<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span>L<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> digits<br \/>\n        src<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> EOS_ID<br \/>\n        tgt<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> BOS_ID<br \/>\n        tgt<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span>L <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> digits<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">:<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span>               <span class=\"token comment\"># \u76ee\u6807 &#061; \u6e90\u5e8f\u5217\u7ffb\u8f6c<\/span><br \/>\n        tgt<span class=\"token punctuation\">[<\/span>i<span class=\"token punctuation\">,<\/span> L <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> EOS_ID<br \/>\n    <span class=\"token keyword\">return<\/span> torch<span class=\"token punctuation\">.<\/span>from_numpy<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>from_numpy<span class=\"token punctuation\">(<\/span>tgt<span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4e00\u6761\u6837\u672c&#xff08;\u6570\u5b57 7 3 0 9 5&#xff09;\u5728\u5185\u5b58\u91cc\u7684\u6837\u5b50&#xff1a;<\/p>\n<p>src : [10  6  3 12  8  2  0  0  0  0  0  0  0  0]   \u2190 7 3 0 9 5 &lt;eos&gt; &lt;pad&gt;&#8230;<br \/>\ntgt : [ 1  8 12  3  6 10  2  0  0  0  0  0  0  0]   \u2190 &lt;bos&gt; 5 9 0 3 7 &lt;eos&gt; &lt;pad&gt;&#8230;<\/p>\n<p>\u6587\u4ef6\u5f00\u5934\u7684\u4e24\u884c allow_tf32 &#061; True&#xff1a;Ampere \u53ca\u4e4b\u540e\u7684 GPU&#xff08;\u5305\u62ec 4090&#xff09;\u53ef\u4ee5\u7528 TF32 \u683c\u5f0f\u8ba1\u7b97 FP32 \u77e9\u9635\u4e58\u6cd5&#xff0c;\u7cbe\u5ea6\u635f\u5931\u5f88\u5c0f\u3001\u901f\u5ea6\u5feb\u6570\u500d\u3002<\/p>\n<hr \/>\n<h3>20.10 \u8bad\u7ec3<\/h3>\n<h4>20.10.1 Teacher Forcing&#xff1a;\u8f93\u5165\u548c\u6807\u7b7e\u9519\u5f00\u4e00\u4f4d<\/h4>\n<p>\u8bad\u7ec3\u65f6 Decoder \u7684\u8f93\u5165\u662f\u6b63\u786e\u7b54\u6848\u53bb\u6389\u6700\u540e\u4e00\u4f4d&#xff0c;\u6807\u7b7e\u662f\u6b63\u786e\u7b54\u6848\u53bb\u6389\u7b2c\u4e00\u4f4d&#xff1a;<\/p>\n<p>tgt     : &lt;bos&gt;  5   9   0   3   7  &lt;eos&gt;<br \/>\ntgt_in  : &lt;bos&gt;  5   9   0   3   7          \u2190 \u5582\u7ed9 Decoder<br \/>\ntgt_out :   5    9   0   3   7 &lt;eos&gt;        \u2190 \u6bcf\u4e2a\u4f4d\u7f6e\u8981\u9884\u6d4b\u7684\u4e0b\u4e00\u4e2a Token<\/p>\n<p>\u6bcf\u4e2a\u4f4d\u7f6e\u90fd&#034;\u770b\u7740\u6b63\u786e\u7684\u524d\u6587\u3001\u9884\u6d4b\u4e0b\u4e00\u4e2a&#034;&#xff0c;\u518d\u914d\u5408 Causal Mask&#xff0c;\u4e00\u6b21\u524d\u5411\u5c31\u80fd\u5e76\u884c\u7b97\u51fa\u6240\u6709\u4f4d\u7f6e\u7684 loss&#xff08;\u7b2c14\u7ae0 14.4.2&#xff09;\u3002<\/p>\n<h4>20.10.2 Warmup \u5b66\u4e60\u7387<\/h4>\n<p><span class=\"katex--display\"><span class=\"katex-display\"><span class=\"katex\"><span class=\"katex-mathml\">lr&#061;dmodel\u22120.5\u22c5min\u2061(step\u22120.5,\u00a0step\u22c5warmup\u22121.5)<br \/>\nlr &#061; d_{model}^{-0.5} \\\\cdot \\\\min\\\\left(step^{-0.5},\\\\ step \\\\cdot warmup^{-1.5}\\\\right)<br \/>\n<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0197em\">l<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.1555em;vertical-align: -0.2914em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -2.4086em;margin-left: 0em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mathnormal mtight\">m<\/span><span class=\"mord mathnormal mtight\">o<\/span><span class=\"mord mathnormal mtight\">d<\/span><span class=\"mord mathnormal mtight\">e<\/span><span class=\"mord mathnormal mtight\" style=\"margin-right: 0.0197em\">l<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">0.5<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2914em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.2141em;vertical-align: -0.35em\"><\/span><span class=\"mop\">min<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"minner\"><span class=\"mopen delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size1\">(<\/span><\/span><span class=\"mord mathnormal\">s<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">0.5<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\">\u00a0<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord mathnormal\">s<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">p<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u22c5<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0269em\">w<\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mord mathnormal\" style=\"margin-right: 0.0278em\">r<\/span><span class=\"mord mathnormal\">m<\/span><span class=\"mord mathnormal\">u<\/span><span class=\"mord\"><span class=\"mord mathnormal\">p<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8641em\"><span class=\"\" style=\"top: -3.113em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">1.5<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose delimcenter\" style=\"top: 0em\"><span class=\"delimsizing size1\">)<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/p>\n<p>\u524d warmup \u6b65\u7ebf\u6027\u5347\u9ad8&#xff0c;\u4e4b\u540e\u6309 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">1\/step1\/\\\\sqrt{step}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0453em;vertical-align: -0.25em\"><\/span><span class=\"mord\">1\/<\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.7953em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\">s<\/span><span class=\"mord mathnormal\">t<\/span><span class=\"mord mathnormal\">e<\/span><span class=\"mord mathnormal\">p<\/span><\/span><\/span><span class=\"\" style=\"top: -2.7553em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.2447em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u8870\u51cf\u3002\u8bad\u7ec3\u521d\u671f\u53c2\u6570\u662f\u968f\u673a\u7684&#xff0c;\u68af\u5ea6\u65b9\u5411\u4e0d\u53ef\u9760&#xff0c;Adam \u7684\u4e8c\u9636\u77e9\u4f30\u8ba1\u4e5f\u8fd8\u4e0d\u51c6&#xff0c;\u5c0f\u5b66\u4e60\u7387\u8d77\u6b65\u66f4\u7a33\u3002<\/p>\n<p>#mermaid-svg-yVVHZkAsMRxEq70K{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-yVVHZkAsMRxEq70K .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-yVVHZkAsMRxEq70K .error-icon{fill:#552222;}#mermaid-svg-yVVHZkAsMRxEq70K .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-yVVHZkAsMRxEq70K .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-yVVHZkAsMRxEq70K .marker{fill:#333333;stroke:#333333;}#mermaid-svg-yVVHZkAsMRxEq70K .marker.cross{stroke:#333333;}#mermaid-svg-yVVHZkAsMRxEq70K svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-yVVHZkAsMRxEq70K p{margin:0;}#mermaid-svg-yVVHZkAsMRxEq70K .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-yVVHZkAsMRxEq70K .cluster-label text{fill:#333;}#mermaid-svg-yVVHZkAsMRxEq70K .cluster-label span{color:#333;}#mermaid-svg-yVVHZkAsMRxEq70K .cluster-label span p{background-color:transparent;}#mermaid-svg-yVVHZkAsMRxEq70K .label text,#mermaid-svg-yVVHZkAsMRxEq70K span{fill:#333;color:#333;}#mermaid-svg-yVVHZkAsMRxEq70K .node rect,#mermaid-svg-yVVHZkAsMRxEq70K .node circle,#mermaid-svg-yVVHZkAsMRxEq70K .node ellipse,#mermaid-svg-yVVHZkAsMRxEq70K .node polygon,#mermaid-svg-yVVHZkAsMRxEq70K .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-yVVHZkAsMRxEq70K .rough-node .label text,#mermaid-svg-yVVHZkAsMRxEq70K .node .label text,#mermaid-svg-yVVHZkAsMRxEq70K .image-shape .label,#mermaid-svg-yVVHZkAsMRxEq70K .icon-shape .label{text-anchor:middle;}#mermaid-svg-yVVHZkAsMRxEq70K .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-yVVHZkAsMRxEq70K .rough-node .label,#mermaid-svg-yVVHZkAsMRxEq70K .node .label,#mermaid-svg-yVVHZkAsMRxEq70K .image-shape .label,#mermaid-svg-yVVHZkAsMRxEq70K .icon-shape .label{text-align:center;}#mermaid-svg-yVVHZkAsMRxEq70K .node.clickable{cursor:pointer;}#mermaid-svg-yVVHZkAsMRxEq70K .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-yVVHZkAsMRxEq70K .arrowheadPath{fill:#333333;}#mermaid-svg-yVVHZkAsMRxEq70K .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-yVVHZkAsMRxEq70K .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-yVVHZkAsMRxEq70K .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-yVVHZkAsMRxEq70K .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-yVVHZkAsMRxEq70K .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-yVVHZkAsMRxEq70K .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-yVVHZkAsMRxEq70K .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-yVVHZkAsMRxEq70K .cluster text{fill:#333;}#mermaid-svg-yVVHZkAsMRxEq70K .cluster span{color:#333;}#mermaid-svg-yVVHZkAsMRxEq70K div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-yVVHZkAsMRxEq70K .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-yVVHZkAsMRxEq70K rect.text{fill:none;stroke-width:0;}#mermaid-svg-yVVHZkAsMRxEq70K .icon-shape,#mermaid-svg-yVVHZkAsMRxEq70K .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-yVVHZkAsMRxEq70K .icon-shape p,#mermaid-svg-yVVHZkAsMRxEq70K .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-yVVHZkAsMRxEq70K .icon-shape .label rect,#mermaid-svg-yVVHZkAsMRxEq70K .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-yVVHZkAsMRxEq70K .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-yVVHZkAsMRxEq70K .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-yVVHZkAsMRxEq70K :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"nodeLabel\"><\/p>\n<p>step 0lr\u22480<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Warmup\u7ebf\u6027\u5347\u9ad8<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>step&#061;warmuplr \u5cf0\u503c<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u4e4b\u540e\u221d 1\/\u221astep \u8870\u51cf<\/p>\n<p><\/span><\/p>\n<p>PyTorch \u91cc\u7528 LambdaLR \u5b9e\u73b0&#xff1a;\u5b83\u628a\u5b66\u4e60\u7387\u8bbe\u4e3a base_lr \u00d7 f(step)&#xff0c;\u6240\u4ee5\u4f18\u5316\u5668\u7684 lr \u8bbe\u6210 1.0&#xff0c;f \u76f4\u63a5\u8fd4\u56de\u4e0a\u9762\u516c\u5f0f\u7684\u503c\u3002<\/p>\n<h4>20.10.3 \u8bad\u7ec3\u4ee3\u7801<\/h4>\n<p><span class=\"token comment\"># ch20_transformer\/train.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">noam_lambda<\/span><span class=\"token punctuation\">(<\/span>d_model<span class=\"token punctuation\">,<\/span> warmup<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;lr &#061; d^-0.5 \u00b7 min(step^-0.5, step \u00b7 warmup^-1.5)&#xff0c;\u914d\u5408 base lr &#061; 1.0 \u4f7f\u7528&#034;&#034;&#034;<\/span><br \/>\n    <span class=\"token keyword\">def<\/span> <span class=\"token function\">f<\/span><span class=\"token punctuation\">(<\/span>step<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        step <span class=\"token operator\">&#061;<\/span> step <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span>                                  <span class=\"token comment\"># LambdaLR \u4ece step&#061;0 \u5f00\u59cb&#xff0c;\u907f\u514d\u9664 0<\/span><br \/>\n        <span class=\"token keyword\">return<\/span> d_model <span class=\"token operator\">**<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">0.5<\/span> <span class=\"token operator\">*<\/span> <span class=\"token builtin\">min<\/span><span class=\"token punctuation\">(<\/span>step <span class=\"token operator\">**<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">0.5<\/span><span class=\"token punctuation\">,<\/span> step <span class=\"token operator\">*<\/span> warmup <span class=\"token operator\">**<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1.5<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> f<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">main<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    torch<span class=\"token punctuation\">.<\/span>manual_seed<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    src_train<span class=\"token punctuation\">,<\/span> tgt_train <span class=\"token operator\">&#061;<\/span> make_dataset<span class=\"token punctuation\">(<\/span><span class=\"token number\">20000<\/span><span class=\"token punctuation\">,<\/span> seed<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    src_val<span class=\"token punctuation\">,<\/span> tgt_val <span class=\"token operator\">&#061;<\/span> make_dataset<span class=\"token punctuation\">(<\/span><span class=\"token number\">1000<\/span><span class=\"token punctuation\">,<\/span> seed<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    loader <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>utils<span class=\"token punctuation\">.<\/span>data<span class=\"token punctuation\">.<\/span>DataLoader<span class=\"token punctuation\">(<\/span><br \/>\n        torch<span class=\"token punctuation\">.<\/span>utils<span class=\"token punctuation\">.<\/span>data<span class=\"token punctuation\">.<\/span>TensorDataset<span class=\"token punctuation\">(<\/span>src_train<span class=\"token punctuation\">,<\/span> tgt_train<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n        batch_size<span class=\"token operator\">&#061;<\/span><span class=\"token number\">64<\/span><span class=\"token punctuation\">,<\/span> shuffle<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">,<\/span> drop_last<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    D_MODEL<span class=\"token punctuation\">,<\/span> EPOCHS <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">128<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">10<\/span><br \/>\n    model <span class=\"token operator\">&#061;<\/span> Transformer<span class=\"token punctuation\">(<\/span>VOCAB<span class=\"token punctuation\">,<\/span> VOCAB<span class=\"token punctuation\">,<\/span> d_model<span class=\"token operator\">&#061;<\/span>D_MODEL<span class=\"token punctuation\">,<\/span> num_heads<span class=\"token operator\">&#061;<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                        dff<span class=\"token operator\">&#061;<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span> num_layers<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> max_len<span class=\"token operator\">&#061;<\/span>SEQ_LEN<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;device&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>DEVICE<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> bf16&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>USE_BF16<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> \u53c2\u6570\u91cf&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span>p<span class=\"token punctuation\">.<\/span>numel<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> p <span class=\"token keyword\">in<\/span> model<span class=\"token punctuation\">.<\/span>parameters<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">,<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    optimizer <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>optim<span class=\"token punctuation\">.<\/span>Adam<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">.<\/span>parameters<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> lr<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1.0<\/span><span class=\"token punctuation\">,<\/span> betas<span class=\"token operator\">&#061;<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">0.9<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0.98<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> eps<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1e-9<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    scheduler <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>optim<span class=\"token punctuation\">.<\/span>lr_scheduler<span class=\"token punctuation\">.<\/span>LambdaLR<span class=\"token punctuation\">(<\/span>optimizer<span class=\"token punctuation\">,<\/span> noam_lambda<span class=\"token punctuation\">(<\/span>D_MODEL<span class=\"token punctuation\">,<\/span> warmup<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1000<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    t0 <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> epoch <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> EPOCHS <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        model<span class=\"token punctuation\">.<\/span>train<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> src<span class=\"token punctuation\">,<\/span> tgt <span class=\"token keyword\">in<\/span> loader<span class=\"token punctuation\">:<\/span><br \/>\n            src<span class=\"token punctuation\">,<\/span> tgt <span class=\"token operator\">&#061;<\/span> src<span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">,<\/span> non_blocking<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">,<\/span> non_blocking<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            tgt_in<span class=\"token punctuation\">,<\/span> tgt_out <span class=\"token operator\">&#061;<\/span> tgt<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span>    <span class=\"token comment\"># Teacher Forcing&#xff1a;\u53f3\u79fb\u4e00\u4f4d<\/span><br \/>\n            <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>autocast<span class=\"token punctuation\">(<\/span>device_type<span class=\"token operator\">&#061;<\/span>DEVICE<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>bfloat16<span class=\"token punctuation\">,<\/span> enabled<span class=\"token operator\">&#061;<\/span>USE_BF16<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                logits <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">,<\/span> tgt_in<span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token comment\"># ignore_index&#061;PAD&#xff1a;PAD \u4f4d\u7f6e\u4e0d\u8ba1\u5165 loss&#xff08;\u7b49\u4ef7\u4e8e TF \u7248\u7684 masked_loss&#xff09;<\/span><br \/>\n                loss <span class=\"token operator\">&#061;<\/span> F<span class=\"token punctuation\">.<\/span>cross_entropy<span class=\"token punctuation\">(<\/span>logits<span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> VOCAB<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                                       tgt_out<span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> ignore_index<span class=\"token operator\">&#061;<\/span>PAD_ID<span class=\"token punctuation\">)<\/span><br \/>\n            optimizer<span class=\"token punctuation\">.<\/span>zero_grad<span class=\"token punctuation\">(<\/span>set_to_none<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            loss<span class=\"token punctuation\">.<\/span>backward<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            optimizer<span class=\"token punctuation\">.<\/span>step<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            scheduler<span class=\"token punctuation\">.<\/span>step<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>        <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span>                           <span class=\"token comment\"># \u6700\u540e\u4e00\u4e2a batch \u7684\u9010 Token \u51c6\u786e\u7387<\/span><br \/>\n            mask <span class=\"token operator\">&#061;<\/span> tgt_out <span class=\"token operator\">!&#061;<\/span> PAD_ID<br \/>\n            acc <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>logits<span class=\"token punctuation\">.<\/span>argmax<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#061;&#061;<\/span> tgt_out<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&amp;<\/span> mask<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> mask<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">sum<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        em <span class=\"token operator\">&#061;<\/span> exact_match<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> src_val<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token number\">500<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> tgt_val<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token number\">500<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;epoch <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>epoch<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">2d<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">  loss&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>loss<span class=\"token punctuation\">.<\/span>item<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.4f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">  token_acc&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>acc<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.4f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">  &#034;<\/span><\/span><br \/>\n              <span class=\"token string-interpolation\"><span class=\"token string\">f&#034;val_exact_match&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>em<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.3f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">  lr&#061;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>scheduler<span class=\"token punctuation\">.<\/span>get_last_lr<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2e<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">  <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>time<span class=\"token punctuation\">.<\/span>time<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> t0<span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.0f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\">s&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token comment\"># \u5355\u6761\u63a8\u7406\u6f14\u793a<\/span><br \/>\n    demo <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">7<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">9<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">]<\/span><br \/>\n    s <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> SEQ_LEN<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">long<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    s<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> NUM_OFFSET<br \/>\n    s<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> EOS_ID<br \/>\n    p <span class=\"token operator\">&#061;<\/span> greedy_decode<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> s<span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>tolist<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u8f93\u5165:&#034;<\/span><span class=\"token punctuation\">,<\/span> demo<span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;-&gt; \u8f93\u51fa:&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">[<\/span>i <span class=\"token operator\">&#8211;<\/span> NUM_OFFSET <span class=\"token keyword\">for<\/span> i <span class=\"token keyword\">in<\/span> p<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token keyword\">if<\/span> i <span class=\"token operator\">&gt;&#061;<\/span> NUM_OFFSET<span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    torch<span class=\"token punctuation\">.<\/span>save<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">.<\/span>state_dict<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token string\">&#034;reverse_transformer.pt&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">if<\/span> __name__ <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;__main__&#034;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    main<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u4e0e TensorFlow \u5199\u6cd5\u76f8\u6bd4\u7684\u51e0\u4e2a\u8981\u70b9&#xff1a;<\/p>\n<table>\n<tr>\u8981\u70b9\u505a\u6cd5\u539f\u56e0<\/tr>\n<tbody>\n<tr>\n<td>\u5ffd\u7565 PAD<\/td>\n<td>F.cross_entropy(&#8230;, ignore_index&#061;PAD_ID)<\/td>\n<td>\u4e00\u884c\u4ee3\u66ff\u624b\u5199 Mask \u6c42\u5e73\u5747&#xff1b;\u5426\u5219\u6a21\u578b\u53ea\u8981\u5b66\u4f1a\u8f93\u51fa PAD \u5c31\u80fd\u62ff\u5230\u5f88\u4f4e\u7684 loss<\/td>\n<\/tr>\n<tr>\n<td>bf16 \u6df7\u5408\u7cbe\u5ea6<\/td>\n<td>torch.autocast(dtype&#061;torch.bfloat16)<\/td>\n<td>4090 \u7684 bf16 Tensor Core \u541e\u5410\u8fdc\u9ad8\u4e8e FP32&#xff1b;bf16 \u7684\u6307\u6570\u4f4d\u548c FP32 \u4e00\u6837&#xff0c;\u4e0d\u9700\u8981 fp16 \u7684 GradScaler<\/td>\n<\/tr>\n<tr>\n<td>loss \u7528 FP32 \u7b97<\/td>\n<td>logits.float()<\/td>\n<td>Softmax &#043; log \u5bf9\u7cbe\u5ea6\u654f\u611f&#xff0c;\u4ea4\u53c9\u71b5\u653e\u5728 FP32 \u4e0b\u8ba1\u7b97\u66f4\u7a33<\/td>\n<\/tr>\n<tr>\n<td>\u6bcf\u6b65\u4e09\u4ef6\u5957<\/td>\n<td>zero_grad \u2192 backward \u2192 step<\/td>\n<td>PyTorch \u7684\u68af\u5ea6\u9ed8\u8ba4\u7d2f\u52a0&#xff0c;\u4e0d\u6e05\u96f6\u5c31\u4f1a\u548c\u4e0a\u4e00\u6b65\u7684\u68af\u5ea6\u6df7\u5728\u4e00\u8d77<\/td>\n<\/tr>\n<tr>\n<td>set_to_none&#061;True<\/td>\n<td>\u628a\u68af\u5ea6\u8bbe\u4e3a None \u800c\u4e0d\u662f\u586b 0<\/td>\n<td>\u5c11\u4e00\u6b21\u663e\u5b58\u5199\u5165&#xff0c;PyTorch 2.x \u7684\u9ed8\u8ba4\u884c\u4e3a<\/td>\n<\/tr>\n<tr>\n<td>\u8bad\u7ec3\/\u8bc4\u4f30\u6a21\u5f0f<\/td>\n<td>model.train() \/ model.eval()<\/td>\n<td>\u63a7\u5236 Dropout \u5f00\u5173&#xff1b;greedy_decode \u91cc\u5207\u5230 eval&#xff0c;\u56de\u5230\u8bad\u7ec3\u5faa\u73af\u65f6\u8981\u5207\u56de train<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>20.11 \u63a8\u7406&#xff1a;\u8d2a\u5fc3\u89e3\u7801<\/h3>\n<p>\u63a8\u7406\u65f6\u6ca1\u6709\u6b63\u786e\u7b54\u6848\u53ef\u4ee5&#034;\u5582&#034;&#xff0c;\u53ea\u80fd\u81ea\u5df1\u751f\u6210\u4e00\u4e2a\u3001\u62fc\u56de\u53bb\u3001\u518d\u751f\u6210\u4e0b\u4e00\u4e2a&#xff1a;<\/p>\n<p>#mermaid-svg-Q29LWbEJJsraQtIO{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-Q29LWbEJJsraQtIO .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-Q29LWbEJJsraQtIO .error-icon{fill:#552222;}#mermaid-svg-Q29LWbEJJsraQtIO .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-Q29LWbEJJsraQtIO .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-Q29LWbEJJsraQtIO .marker{fill:#333333;stroke:#333333;}#mermaid-svg-Q29LWbEJJsraQtIO .marker.cross{stroke:#333333;}#mermaid-svg-Q29LWbEJJsraQtIO svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-Q29LWbEJJsraQtIO p{margin:0;}#mermaid-svg-Q29LWbEJJsraQtIO .label{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;color:#333;}#mermaid-svg-Q29LWbEJJsraQtIO .cluster-label text{fill:#333;}#mermaid-svg-Q29LWbEJJsraQtIO .cluster-label span{color:#333;}#mermaid-svg-Q29LWbEJJsraQtIO .cluster-label span p{background-color:transparent;}#mermaid-svg-Q29LWbEJJsraQtIO .label text,#mermaid-svg-Q29LWbEJJsraQtIO span{fill:#333;color:#333;}#mermaid-svg-Q29LWbEJJsraQtIO .node rect,#mermaid-svg-Q29LWbEJJsraQtIO .node circle,#mermaid-svg-Q29LWbEJJsraQtIO .node ellipse,#mermaid-svg-Q29LWbEJJsraQtIO .node polygon,#mermaid-svg-Q29LWbEJJsraQtIO .node path{fill:#ECECFF;stroke:#9370DB;stroke-width:1px;}#mermaid-svg-Q29LWbEJJsraQtIO .rough-node .label text,#mermaid-svg-Q29LWbEJJsraQtIO .node .label text,#mermaid-svg-Q29LWbEJJsraQtIO .image-shape .label,#mermaid-svg-Q29LWbEJJsraQtIO .icon-shape .label{text-anchor:middle;}#mermaid-svg-Q29LWbEJJsraQtIO .node .katex path{fill:#000;stroke:#000;stroke-width:1px;}#mermaid-svg-Q29LWbEJJsraQtIO .rough-node .label,#mermaid-svg-Q29LWbEJJsraQtIO .node .label,#mermaid-svg-Q29LWbEJJsraQtIO .image-shape .label,#mermaid-svg-Q29LWbEJJsraQtIO .icon-shape .label{text-align:center;}#mermaid-svg-Q29LWbEJJsraQtIO .node.clickable{cursor:pointer;}#mermaid-svg-Q29LWbEJJsraQtIO .root .anchor path{fill:#333333!important;stroke-width:0;stroke:#333333;}#mermaid-svg-Q29LWbEJJsraQtIO .arrowheadPath{fill:#333333;}#mermaid-svg-Q29LWbEJJsraQtIO .edgePath .path{stroke:#333333;stroke-width:2.0px;}#mermaid-svg-Q29LWbEJJsraQtIO .flowchart-link{stroke:#333333;fill:none;}#mermaid-svg-Q29LWbEJJsraQtIO .edgeLabel{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-Q29LWbEJJsraQtIO .edgeLabel p{background-color:rgba(232,232,232, 0.8);}#mermaid-svg-Q29LWbEJJsraQtIO .edgeLabel rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-Q29LWbEJJsraQtIO .labelBkg{background-color:rgba(232, 232, 232, 0.5);}#mermaid-svg-Q29LWbEJJsraQtIO .cluster rect{fill:#ffffde;stroke:#aaaa33;stroke-width:1px;}#mermaid-svg-Q29LWbEJJsraQtIO .cluster text{fill:#333;}#mermaid-svg-Q29LWbEJJsraQtIO .cluster span{color:#333;}#mermaid-svg-Q29LWbEJJsraQtIO div.mermaidTooltip{position:absolute;text-align:center;max-width:200px;padding:2px;font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:12px;background:hsl(80, 100%, 96.2745098039%);border:1px solid #aaaa33;border-radius:2px;pointer-events:none;z-index:100;}#mermaid-svg-Q29LWbEJJsraQtIO .flowchartTitleText{text-anchor:middle;font-size:18px;fill:#333;}#mermaid-svg-Q29LWbEJJsraQtIO rect.text{fill:none;stroke-width:0;}#mermaid-svg-Q29LWbEJJsraQtIO .icon-shape,#mermaid-svg-Q29LWbEJJsraQtIO .image-shape{background-color:rgba(232,232,232, 0.8);text-align:center;}#mermaid-svg-Q29LWbEJJsraQtIO .icon-shape p,#mermaid-svg-Q29LWbEJJsraQtIO .image-shape p{background-color:rgba(232,232,232, 0.8);padding:2px;}#mermaid-svg-Q29LWbEJJsraQtIO .icon-shape .label rect,#mermaid-svg-Q29LWbEJJsraQtIO .image-shape .label rect{opacity:0.5;background-color:rgba(232,232,232, 0.8);fill:rgba(232,232,232, 0.8);}#mermaid-svg-Q29LWbEJJsraQtIO .label-icon{display:inline-block;height:1em;overflow:visible;vertical-align:-0.125em;}#mermaid-svg-Q29LWbEJJsraQtIO .node .label-icon path{fill:currentColor;stroke:revert;stroke-width:revert;}#mermaid-svg-Q29LWbEJJsraQtIO :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/p>\n<p>No<\/p>\n<p><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/p>\n<p>Yes<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>src<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Encoder \u53ea\u8dd1\u4e00\u6b21\u5f97\u5230 memory<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>out &#061; [BOS]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Decoder(out, memory)<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u53d6\u6700\u540e\u4e00\u4e2a\u4f4d\u7f6e\u7684 argmax<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u5168\u90e8\u6837\u672c\u90fd\u8f93\u51fa\u4e86 EOS?<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>out &#061; out &#043; [next]<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u7ed3\u675f<\/p>\n<p><\/span><br \/>\n<span class=\"token comment\"># ch20_transformer\/train.py&#xff08;\u7eed&#xff09;<\/span><br \/>\n<span class=\"token comment\"># &#8212;&#8212;&#8212;&#8212;&#8212;- \u63a8\u7406&#xff1a;\u8d2a\u5fc3\u89e3\u7801 &#8212;&#8212;&#8212;&#8212;&#8212;-<\/span><br \/>\n<span class=\"token decorator annotation punctuation\">&#064;torch<span class=\"token punctuation\">.<\/span>no_grad<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">def<\/span> <span class=\"token function\">greedy_decode<\/span><span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">,<\/span> max_len<span class=\"token operator\">&#061;<\/span>SEQ_LEN<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    <span class=\"token triple-quoted-string string\">&#034;&#034;&#034;src: [B, Ls] -&gt; [B, &lt;&#061;max_len]&#034;&#034;&#034;<\/span><br \/>\n    model<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    memory<span class=\"token punctuation\">,<\/span> src_mask <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><br \/>\n    B <span class=\"token operator\">&#061;<\/span> src<span class=\"token punctuation\">.<\/span>size<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    out <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>full<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> BOS_ID<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">long<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>src<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n    finished <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">bool<\/span><span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>src<span class=\"token punctuation\">.<\/span>device<span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> _ <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span>max_len <span class=\"token operator\">&#8211;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        logits<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">.<\/span>decode<span class=\"token punctuation\">(<\/span>out<span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><br \/>\n        nxt <span class=\"token operator\">&#061;<\/span> logits<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>argmax<span class=\"token punctuation\">(<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        nxt <span class=\"token operator\">&#061;<\/span> nxt<span class=\"token punctuation\">.<\/span>masked_fill<span class=\"token punctuation\">(<\/span>finished<span class=\"token punctuation\">,<\/span> PAD_ID<span class=\"token punctuation\">)<\/span>          <span class=\"token comment\"># \u5df2\u7ed3\u675f\u7684\u6837\u672c\u8865 PAD<\/span><br \/>\n        out <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>cat<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">[<\/span>out<span class=\"token punctuation\">,<\/span> nxt<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token boolean\">None<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> dim<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">)<\/span><br \/>\n        finished <span class=\"token operator\">|<\/span><span class=\"token operator\">&#061;<\/span> nxt <span class=\"token operator\">&#061;&#061;<\/span> EOS_ID<br \/>\n        <span class=\"token keyword\">if<\/span> finished<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">all<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">break<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> out<\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">exact_match<\/span><span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    pred <span class=\"token operator\">&#061;<\/span> greedy_decode<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>cpu<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>tolist<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    ok <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">0<\/span><br \/>\n    <span class=\"token keyword\">for<\/span> p<span class=\"token punctuation\">,<\/span> t <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">zip<\/span><span class=\"token punctuation\">(<\/span>pred<span class=\"token punctuation\">,<\/span> tgt<span class=\"token punctuation\">.<\/span>tolist<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        t <span class=\"token operator\">&#061;<\/span> t<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span>t<span class=\"token punctuation\">.<\/span>index<span class=\"token punctuation\">(<\/span>EOS_ID<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span>                     <span class=\"token comment\"># \u53bb\u6389 BOS&#xff0c;\u4fdd\u7559\u5230 EOS<\/span><br \/>\n        ok <span class=\"token operator\">&#043;&#061;<\/span> p<span class=\"token punctuation\">[<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">:<\/span><span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>t<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;&#061;<\/span> t<br \/>\n    <span class=\"token keyword\">return<\/span> ok <span class=\"token operator\">\/<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><\/p>\n<p>&#064;torch.no_grad() \u8ba9\u8fd9\u6bb5\u4ee3\u7801\u4e0d\u8bb0\u5f55\u8ba1\u7b97\u56fe&#xff1a;\u63a8\u7406\u4e0d\u9700\u8981\u53cd\u5411\u4f20\u64ad&#xff0c;\u7701\u663e\u5b58\u4e5f\u66f4\u5feb\u3002<\/p>\n<p>\u8fd9\u91cc\u6bcf\u751f\u6210\u4e00\u4e2a Token \u90fd\u628a\u6574\u4e2a out \u91cd\u65b0\u9001\u8fdb Decoder&#xff0c;\u524d\u9762\u4f4d\u7f6e\u7684 K\/V \u88ab\u53cd\u590d\u8ba1\u7b97\u3002\u8fd9\u6b63\u662f\u7b2c15\u7ae0 KV Cache \u8981\u89e3\u51b3\u7684\u95ee\u9898&#xff0c;\u4e0b\u4e00\u7ae0\u4f1a\u5728 GPT \u4e0a\u4eb2\u624b\u5b9e\u73b0\u5b83\u3002<\/p>\n<hr \/>\n<h3>20.12 \u8fd0\u884c\u7ed3\u679c&#xff1a;\u4e00\u6b21&#034;\u4ee3\u7801\u5168\u5bf9\u3001\u6a21\u578b\u5374\u5b66\u4e0d\u4f1a&#034;\u7684\u6392\u67e5<\/h3>\n<p>python test_model.py   <span class=\"token comment\"># \u5148\u8fc7\u5355\u5143\u6d4b\u8bd5<\/span><br \/>\npython train.py<\/p>\n<h4>20.12.1 \u7b2c\u4e00\u6b21\u8fd0\u884c&#xff1a;\u6162\u5f97\u53cd\u5e38<\/h4>\n<p>\u7b14\u8005\u7b2c\u4e00\u7248\u4ee3\u7801\u6ca1\u6709 20.2 \u8282\u90a3\u51e0\u884c Embedding \u521d\u59cb\u5316&#xff0c;\u4e5f\u6ca1\u6709 20.7 \u8282\u7684 Xavier \u521d\u59cb\u5316&#xff08;\u7528\u7684\u662f PyTorch \u9ed8\u8ba4\u503c&#xff09;\u3002\u5355\u5143\u6d4b\u8bd5\u5168\u90e8\u901a\u8fc7&#xff0c;\u8bad\u7ec3\u7ed3\u679c\u5374\u662f\u8fd9\u6837&#xff1a;<\/p>\n<p>device&#061;cuda bf16&#061;True \u53c2\u6570\u91cf&#061;931,213<br \/>\nepoch  1  loss&#061;1.6167  token_acc&#061;0.3487  val_exact_match&#061;0.020  lr&#061;8.75e-04  2s<br \/>\nepoch  2  loss&#061;1.3009  token_acc&#061;0.4825  val_exact_match&#061;0.046  lr&#061;1.75e-03  4s<br \/>\nepoch  3  loss&#061;1.2627  token_acc&#061;0.4906  val_exact_match&#061;0.040  lr&#061;2.62e-03  5s<br \/>\nepoch  4  loss&#061;1.0233  token_acc&#061;0.5657  val_exact_match&#061;0.118  lr&#061;2.50e-03  7s<br \/>\nepoch  5  loss&#061;0.9464  token_acc&#061;0.5835  val_exact_match&#061;0.166  lr&#061;2.24e-03  9s<br \/>\nepoch  6  loss&#061;0.8655  token_acc&#061;0.6466  val_exact_match&#061;0.186  lr&#061;2.04e-03  10s<br \/>\nepoch  7  loss&#061;0.6120  token_acc&#061;0.7418  val_exact_match&#061;0.370  lr&#061;1.89e-03  12s<br \/>\nepoch  8  loss&#061;0.5351  token_acc&#061;0.8015  val_exact_match&#061;0.482  lr&#061;1.77e-03  14s<br \/>\nepoch  9  loss&#061;0.4389  token_acc&#061;0.8445  val_exact_match&#061;0.660  lr&#061;1.67e-03  15s<br \/>\nepoch 10  loss&#061;0.3849  token_acc&#061;0.8509  val_exact_match&#061;0.604  lr&#061;1.58e-03  17s<\/p>\n<p>\u6a21\u578b\u5728\u5b66&#xff0c;\u4f46\u5b66\u5f97\u5f88\u6162&#xff1a;10 \u4e2a epoch \u540e\u6574\u53e5\u6b63\u786e\u7387\u53ea\u6709 0.60\u3002\u540c\u6837\u7684\u7ed3\u6784\u548c\u8d85\u53c2\u6570&#xff0c;\u7528 TensorFlow \u5b9e\u73b0\u65f6 5 \u4e2a epoch \u5c31\u5230\u4e86 0.97\u3002<\/p>\n<p>\u6392\u67e5\u601d\u8def&#xff1a;<\/p>\n<li>\u5355\u5143\u6d4b\u8bd5\u901a\u8fc7 \u2192 Mask \u6ca1\u95ee\u9898&#xff0c;\u6392\u9664&#034;\u5077\u770b\u7b54\u6848&#034;\u6216&#034;PAD \u6ca1\u5c4f\u853d&#034;\u3002<\/li>\n<li>loss \u5728\u7a33\u5b9a\u4e0b\u964d \u2192 \u5b66\u4e60\u7387\u3001\u68af\u5ea6\u65b9\u5411\u6ca1\u95ee\u9898\u3002<\/li>\n<li>\u5269\u4e0b\u7684\u5dee\u5f02\u53ea\u53ef\u80fd\u6765\u81ea\u6846\u67b6\u9ed8\u8ba4\u503c \u2192 \u9010\u4e2a\u5bf9\u6bd4 TensorFlow \u548c PyTorch \u5404\u5c42\u7684\u9ed8\u8ba4\u521d\u59cb\u5316&#xff1a;<\/li>\n<table>\n<tr>\u5c42Keras \u9ed8\u8ba4PyTorch \u9ed8\u8ba4\u672c\u7ae0\u7684\u95ee\u9898<\/tr>\n<tbody>\n<tr>\n<td>Embedding<\/td>\n<td>\u5747\u5300\u5206\u5e03 \u00b10.05<\/td>\n<td><span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(0,1)\\\\mathcal{N}(0, 1)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span><\/td>\n<td>\u4e58 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">128\\\\sqrt{128}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord\">128<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u540e\u6807\u51c6\u5dee \u2248 11&#xff0c;\u4f4d\u7f6e\u7f16\u7801\u88ab\u6df9\u6ca1<\/td>\n<\/tr>\n<tr>\n<td>Dense \/ Linear<\/td>\n<td>Glorot&#xff08;Xavier&#xff09;\u5747\u5300<\/td>\n<td>Kaiming \u5747\u5300&#xff08;<span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">a&#061;5a&#061;\\\\sqrt5<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 0.4306em\"><\/span><span class=\"mord mathnormal\">a<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><span class=\"mrel\">&#061;<\/span><span class=\"mspace\" style=\"margin-right: 0.2778em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1328em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9072em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\">5<\/span><\/span><span class=\"\" style=\"top: -2.8672em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1328em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span>&#xff09;<\/td>\n<td>\u65b9\u5dee\u504f\u5c0f&#xff0c;\u5f71\u54cd\u8f83\u8f7b<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7ffb\u8f6c\u4efb\u52a1\u5b8c\u5168\u4f9d\u8d56\u4f4d\u7f6e\u4fe1\u606f&#xff0c;Token \u4fe1\u53f7\u6bd4\u4f4d\u7f6e\u4fe1\u53f7\u5927 10 \u500d&#xff0c;\u6a21\u578b\u5c31\u5f88\u96be\u5206\u6e05&#034;\u8fd9\u4e2a 7 \u5728\u7b2c\u51e0\u4f4d&#034;\u300220.13 \u8282\u7684\u6ce8\u610f\u529b\u77e9\u9635\u4e5f\u5370\u8bc1\u4e86\u8fd9\u4e00\u70b9&#xff1a;\u7b2c\u4e00\u7248\u6a21\u578b\u7684 Cross-Attention \u6bcf\u884c\u90fd\u5206\u6563\u5728 2~3 \u4e2a\u4f4d\u7f6e\u4e0a\u3002<\/p>\n<h4>20.12.2 \u4fee\u590d\u540e<\/h4>\n<p>\u52a0\u4e0a 20.2 \u8282\u548c 20.7 \u8282\u7684\u521d\u59cb\u5316&#xff0c;\u5176\u4ed6\u4e00\u884c\u4e0d\u6539&#xff1a;<\/p>\n<p>device&#061;cuda bf16&#061;True \u53c2\u6570\u91cf&#061;931,213<br \/>\nepoch  1  loss&#061;0.5494  token_acc&#061;0.7948  val_exact_match&#061;0.386  lr&#061;8.75e-04  2s<br \/>\nepoch  2  loss&#061;0.1301  token_acc&#061;0.9545  val_exact_match&#061;0.876  lr&#061;1.75e-03  4s<br \/>\nepoch  3  loss&#061;0.1097  token_acc&#061;0.9639  val_exact_match&#061;0.872  lr&#061;2.62e-03  5s<br \/>\nepoch  4  loss&#061;0.0552  token_acc&#061;0.9841  val_exact_match&#061;0.946  lr&#061;2.50e-03  7s<br \/>\nepoch  5  loss&#061;0.0449  token_acc&#061;0.9819  val_exact_match&#061;0.964  lr&#061;2.24e-03  9s<br \/>\nepoch  6  loss&#061;0.0376  token_acc&#061;0.9893  val_exact_match&#061;0.992  lr&#061;2.04e-03  10s<br \/>\nepoch  7  loss&#061;0.0299  token_acc&#061;0.9926  val_exact_match&#061;0.988  lr&#061;1.89e-03  12s<br \/>\nepoch  8  loss&#061;0.0185  token_acc&#061;0.9904  val_exact_match&#061;0.996  lr&#061;1.77e-03  14s<br \/>\nepoch  9  loss&#061;0.0300  token_acc&#061;0.9910  val_exact_match&#061;0.996  lr&#061;1.67e-03  15s<br \/>\nepoch 10  loss&#061;0.0211  token_acc&#061;0.9907  val_exact_match&#061;0.996  lr&#061;1.58e-03  17s<br \/>\n\u8f93\u5165: [7, 3, 0, 9, 5] -&gt; \u8f93\u51fa: [5, 9, 0, 3, 7]<\/p>\n<table>\n<tr>\u6307\u6807\u4fee\u590d\u524d\u4fee\u590d\u540e<\/tr>\n<tbody>\n<tr>\n<td>epoch 1 token_acc<\/td>\n<td>0.349<\/td>\n<td>0.795<\/td>\n<\/tr>\n<tr>\n<td>epoch 5 \u6574\u53e5\u6b63\u786e\u7387<\/td>\n<td>0.166<\/td>\n<td>0.964<\/td>\n<\/tr>\n<tr>\n<td>epoch 10 \u6574\u53e5\u6b63\u786e\u7387<\/td>\n<td>0.604<\/td>\n<td>0.996<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u8bfb\u6570\u8981\u70b9&#xff1a;<\/p>\n<ul>\n<li>token_acc \u662f Teacher Forcing \u4e0b\u7684\u9010 Token \u51c6\u786e\u7387&#xff08;\u6700\u540e\u4e00\u4e2a batch&#xff09;&#xff0c;val_exact_match \u662f\u81ea\u56de\u5f52\u751f\u6210\u65f6\u6574\u53e5\u5b8c\u5168\u6b63\u786e\u7684\u6bd4\u4f8b&#xff0c;\u540e\u8005\u66f4\u4e25\u683c&#xff1a;\u4e00\u53e5 12 \u4e2a\u6570\u5b57\u9519 1 \u4e2a\u5c31\u7b97\u9519\u3002<\/li>\n<li>epoch 2\u21923 \u7684\u6574\u53e5\u6b63\u786e\u7387\u7565\u964d&#xff08;0.876 \u2192 0.872&#xff09;&#xff1a;1 \u4e2a epoch \u2248 312 \u6b65&#xff0c;warmup&#061;1000&#xff0c;\u7b2c 3 \u4e2a epoch \u7ed3\u675f\u65f6\u5b66\u4e60\u7387\u624d\u5347\u5230\u5cf0\u503c 2.6e-3&#xff0c;\u8fd9\u65f6\u51fa\u73b0\u5c0f\u5e45\u6ce2\u52a8\u662f\u6b63\u5e38\u7684\u3002<\/li>\n<li>\u6a21\u578b\u7ea6 93 \u4e07\u53c2\u6570&#xff0c;4090 \u4e0a 10 \u4e2a epoch \u5171 17 \u79d2\u3002<\/li>\n<\/ul>\n<p>\u6559\u8bad&#xff1a;\u8de8\u6846\u67b6\u79fb\u690d\u6a21\u578b\u65f6&#xff0c;\u201c\u4ee3\u7801\u903b\u8f91\u4e00\u6837&#034;\u4e0d\u7b49\u4e8e&#034;\u6a21\u578b\u4e00\u6837\u201d\u2014\u2014\u9ed8\u8ba4\u521d\u59cb\u5316\u3001\u9ed8\u8ba4 epsilon\u3001\u9ed8\u8ba4\u6fc0\u6d3b\u51fd\u6570\u8fd1\u4f3c\u65b9\u5f0f\u90fd\u53ef\u80fd\u4e0d\u540c\u3002\u5355\u5143\u6d4b\u8bd5\u80fd\u4fdd\u8bc1\u903b\u8f91\u6b63\u786e&#xff0c;\u4f46\u4fdd\u8bc1\u4e0d\u4e86\u8bad\u7ec3\u52a8\u6001\u4e00\u81f4\u3002<\/p>\n<hr \/>\n<h3>20.13 \u5b9e\u9a8c&#xff1a;\u53ef\u89c6\u5316 Cross-Attention<\/h3>\n<p>\u6a21\u578b\u771f\u7684\u5b66\u4f1a\u4e86&#034;\u8f93\u51fa\u7b2c i \u4e2a\u6570\u65f6\u53bb\u770b\u8f93\u5165\u7684\u5012\u6570\u7b2c i \u4e2a&#034;\u5417&#xff1f;\u76f4\u63a5\u770b Cross-Attention \u6743\u91cd\u3002<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/viz.py<\/span><br \/>\n<span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u53ef\u89c6\u5316 Cross-Attention&#xff1a;\u6a21\u578b\u662f\u5426\u5b66\u4f1a\u4e86&#034;\u5012\u7740\u5bf9\u9f50&#034;&#xff1f;<\/p>\n<p>\u5148\u8fd0\u884c python train.py \u751f\u6210 reverse_transformer.pt&#xff0c;\u518d\u8fd0\u884c python viz.py<br \/>\n&#034;&#034;&#034;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> matplotlib<br \/>\nmatplotlib<span class=\"token punctuation\">.<\/span>use<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Agg&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">import<\/span> matplotlib<span class=\"token punctuation\">.<\/span>pyplot <span class=\"token keyword\">as<\/span> plt<br \/>\n<span class=\"token keyword\">import<\/span> torch<\/p>\n<p><span class=\"token keyword\">from<\/span> model <span class=\"token keyword\">import<\/span> Transformer<br \/>\n<span class=\"token keyword\">from<\/span> train <span class=\"token keyword\">import<\/span> DEVICE<span class=\"token punctuation\">,<\/span> EOS_ID<span class=\"token punctuation\">,<\/span> NUM_OFFSET<span class=\"token punctuation\">,<\/span> SEQ_LEN<span class=\"token punctuation\">,<\/span> VOCAB<span class=\"token punctuation\">,<\/span> greedy_decode<\/p>\n<p>model <span class=\"token operator\">&#061;<\/span> Transformer<span class=\"token punctuation\">(<\/span>VOCAB<span class=\"token punctuation\">,<\/span> VOCAB<span class=\"token punctuation\">,<\/span> d_model<span class=\"token operator\">&#061;<\/span><span class=\"token number\">128<\/span><span class=\"token punctuation\">,<\/span> num_heads<span class=\"token operator\">&#061;<\/span><span class=\"token number\">4<\/span><span class=\"token punctuation\">,<\/span> dff<span class=\"token operator\">&#061;<\/span><span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                    num_layers<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> max_len<span class=\"token operator\">&#061;<\/span>SEQ_LEN<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><br \/>\nmodel<span class=\"token punctuation\">.<\/span>load_state_dict<span class=\"token punctuation\">(<\/span>torch<span class=\"token punctuation\">.<\/span>load<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;reverse_transformer.pt&#034;<\/span><span class=\"token punctuation\">,<\/span> map_location<span class=\"token operator\">&#061;<\/span>DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\nmodel<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>demo <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token number\">7<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">3<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">9<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">5<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">8<\/span><span class=\"token punctuation\">]<\/span><br \/>\nsrc <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>zeros<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> SEQ_LEN<span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span><span class=\"token builtin\">long<\/span><span class=\"token punctuation\">)<\/span><br \/>\nsrc<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>tensor<span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> NUM_OFFSET<br \/>\nsrc<span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#061;<\/span> EOS_ID<br \/>\nsrc <span class=\"token operator\">&#061;<\/span> src<span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    pred <span class=\"token operator\">&#061;<\/span> greedy_decode<span class=\"token punctuation\">(<\/span>model<span class=\"token punctuation\">,<\/span> src<span class=\"token punctuation\">)<\/span><br \/>\n    memory<span class=\"token punctuation\">,<\/span> src_mask <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">.<\/span>encode<span class=\"token punctuation\">(<\/span>src<span class=\"token punctuation\">)<\/span><br \/>\n    _<span class=\"token punctuation\">,<\/span> cross_w <span class=\"token operator\">&#061;<\/span> model<span class=\"token punctuation\">.<\/span>decode<span class=\"token punctuation\">(<\/span>pred<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">,<\/span> memory<span class=\"token punctuation\">,<\/span> src_mask<span class=\"token punctuation\">)<\/span><\/p>\n<p>w <span class=\"token operator\">&#061;<\/span> cross_w<span class=\"token punctuation\">[<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">[<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">.<\/span>mean<span class=\"token punctuation\">(<\/span>dim<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">float<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>cpu<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span>       <span class=\"token comment\"># \u6700\u540e\u4e00\u5c42&#xff0c;\u6240\u6709 Head \u5e73\u5747 [Lt, Ls]<\/span><br \/>\nn <span class=\"token operator\">&#061;<\/span> <span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>demo<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token number\">1<\/span><br \/>\nw <span class=\"token operator\">&#061;<\/span> w<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span>n<span class=\"token punctuation\">,<\/span> <span class=\"token punctuation\">:<\/span>n<span class=\"token punctuation\">]<\/span><br \/>\ntorch<span class=\"token punctuation\">.<\/span>set_printoptions<span class=\"token punctuation\">(<\/span>precision<span class=\"token operator\">&#061;<\/span><span class=\"token number\">2<\/span><span class=\"token punctuation\">,<\/span> sci_mode<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">False<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span>w<span class=\"token punctuation\">)<\/span><\/p>\n<p>xl <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token builtin\">str<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> d <span class=\"token keyword\">in<\/span> demo<span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;&lt;eos&gt;&#034;<\/span><span class=\"token punctuation\">]<\/span><br \/>\nyl <span class=\"token operator\">&#061;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token string\">&#034;&lt;bos&gt;&#034;<\/span><span class=\"token punctuation\">]<\/span> <span class=\"token operator\">&#043;<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token builtin\">str<\/span><span class=\"token punctuation\">(<\/span>d<span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">for<\/span> d <span class=\"token keyword\">in<\/span> demo<span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">:<\/span><span class=\"token punctuation\">:<\/span><span class=\"token operator\">&#8211;<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">]<\/span><br \/>\nplt<span class=\"token punctuation\">.<\/span>imshow<span class=\"token punctuation\">(<\/span>w<span class=\"token punctuation\">.<\/span>numpy<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> cmap<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;Blues&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nplt<span class=\"token punctuation\">.<\/span>xticks<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>xl<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> xl<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span> plt<span class=\"token punctuation\">.<\/span>yticks<span class=\"token punctuation\">(<\/span><span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token builtin\">len<\/span><span class=\"token punctuation\">(<\/span>yl<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> yl<span class=\"token punctuation\">)<\/span><br \/>\nplt<span class=\"token punctuation\">.<\/span>xlabel<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Source (Encoder)&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span> plt<span class=\"token punctuation\">.<\/span>ylabel<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Decoder input&#034;<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span> plt<span class=\"token punctuation\">.<\/span>colorbar<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nplt<span class=\"token punctuation\">.<\/span>title<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;Cross-Attention (last layer, mean of heads)&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\nplt<span class=\"token punctuation\">.<\/span>savefig<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;cross_attn.png&#034;<\/span><span class=\"token punctuation\">,<\/span> dpi<span class=\"token operator\">&#061;<\/span><span class=\"token number\">120<\/span><span class=\"token punctuation\">,<\/span> bbox_inches<span class=\"token operator\">&#061;<\/span><span class=\"token string\">&#034;tight&#034;<\/span><span class=\"token punctuation\">)<\/span><br \/>\n<span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u5df2\u4fdd\u5b58 cross_attn.png&#034;<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5b9e\u6d4b\u8f93\u51fa&#xff08;\u884c &#061; Decoder \u5f53\u524d\u8f93\u5165&#xff0c;\u5217 &#061; \u6e90\u5e8f\u5217 7 3 0 9 5 1 8 &lt;eos&gt;&#xff1b;\u884c\u5217\u6807\u7b7e\u4e3a\u4fbf\u4e8e\u9605\u8bfb\u624b\u5de5\u6dfb\u52a0&#xff09;&#xff1a;<\/p>\n<p>              7     3     0     9     5     1     8   &lt;eos&gt;<br \/>\n&lt;bos&gt;    [[0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 1.00, 0.00],<br \/>\n  8       [0.01, 0.01, 0.00, 0.00, 0.00, 0.97, 0.00, 0.00],<br \/>\n  1       [0.00, 0.00, 0.00, 0.00, 1.00, 0.00, 0.00, 0.00],<br \/>\n  5       [0.00, 0.00, 0.00, 0.96, 0.04, 0.00, 0.00, 0.00],<br \/>\n  9       [0.00, 0.00, 0.99, 0.01, 0.00, 0.00, 0.00, 0.00],<br \/>\n  0       [0.00, 0.94, 0.02, 0.00, 0.00, 0.00, 0.04, 0.00],<br \/>\n  3       [0.99, 0.01, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00],<br \/>\n  7       [0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 1.00]]<\/p>\n<p>\u4e00\u6761\u6e05\u6670\u7684\u53cd\u5bf9\u89d2\u7ebf&#xff1a;<\/p>\n<ul>\n<li>\u7b2c 1 \u884c&#xff0c;Decoder \u8f93\u5165\u662f &lt;bos&gt;&#xff0c;\u8981\u9884\u6d4b\u7b2c\u4e00\u4e2a\u8f93\u51fa 8&#xff0c;\u6ce8\u610f\u529b 100% \u843d\u5728\u6e90\u5e8f\u5217\u7684 8 \u4e0a&#xff1b;<\/li>\n<li>\u7b2c 2 \u884c&#xff0c;\u8f93\u5165\u662f 8&#xff0c;\u8981\u9884\u6d4b 1&#xff0c;\u6ce8\u610f\u529b\u843d\u5728\u6e90\u5e8f\u5217\u7684 1 \u4e0a&#xff1b;<\/li>\n<li>\u2026\u2026\u4f9d\u6b21\u5f80\u5de6\u79fb&#xff0c;\u6bcf\u884c\u7684\u6700\u5927\u6743\u91cd\u90fd\u5728 0.94 \u4ee5\u4e0a\u3002<\/li>\n<li>\u6700\u540e\u4e00\u884c\u8981\u9884\u6d4b &lt;eos&gt;&#xff0c;\u6ce8\u610f\u529b 100% \u843d\u5728\u6e90\u5e8f\u5217\u7684 &lt;eos&gt; \u4e0a\u2014\u2014\u6a21\u578b\u5b66\u5230\u4e86&#034;\u6e90\u5e8f\u5217\u8bfb\u5b8c\u4e86&#xff0c;\u8be5\u7ed3\u675f\u4e86&#034;\u3002<\/li>\n<\/ul>\n<p>\u6ca1\u6709\u4efb\u4f55\u4eba\u544a\u8bc9\u6a21\u578b&#034;\u8981\u5012\u7740\u5bf9\u9f50&#034;&#xff0c;\u5b83\u662f\u4ece loss \u91cc\u81ea\u5df1\u5b66\u51fa\u6765\u7684\u3002\u8fd9\u5c31\u662f\u7b2c5\u7ae0\u8bf4\u7684&#034;Attention &#061; \u8f6f\u5bf9\u9f50&#034;\u3002<\/p>\n<p>\u4f5c\u4e3a\u5bf9\u7167&#xff0c;20.12.1 \u8282\u521d\u59cb\u5316\u6709\u95ee\u9898\u7684\u6a21\u578b&#xff0c;\u540c\u4e00\u4e2a\u8f93\u5165\u7684\u6ce8\u610f\u529b\u77e9\u9635\u662f\u8fd9\u6837\u7684&#xff1a;<\/p>\n<p>&lt;bos&gt;    [[0.00, 0.00, 0.00, 0.00, 0.00, 0.12, 0.88, 0.00],<br \/>\n  8       [0.00, 0.00, 0.00, 0.00, 0.18, 0.66, 0.15, 0.00],<br \/>\n  1       [0.00, 0.00, 0.03, 0.09, 0.77, 0.06, 0.05, 0.00],<br \/>\n  5       [0.00, 0.01, 0.29, 0.63, 0.06, 0.02, 0.01, 0.00],<br \/>\n  9       [0.02, 0.28, 0.57, 0.08, 0.01, 0.00, 0.00, 0.03],<br \/>\n  0       [0.21, 0.51, 0.03, 0.01, 0.00, 0.00, 0.00, 0.24],<br \/>\n  3       [0.55, 0.04, 0.00, 0.00, 0.00, 0.00, 0.00, 0.40],<br \/>\n  7       [0.36, 0.11, 0.00, 0.00, 0.00, 0.00, 0.00, 0.53]]<\/p>\n<p>\u65b9\u5411\u662f\u5bf9\u7684&#xff0c;\u4f46\u6bcf\u884c\u90fd&#034;\u7cca&#034;\u5728\u76f8\u90bb\u7684 2~3 \u4e2a\u4f4d\u7f6e\u4e0a\u2014\u2014\u6a21\u578b\u5927\u81f4\u77e5\u9053\u8be5\u5f80\u54ea\u770b&#xff0c;\u5374\u5206\u4e0d\u6e05\u76f8\u90bb\u7684\u4e24\u4e2a\u4f4d\u7f6e\u3002\u8fd9\u6b63\u662f\u4f4d\u7f6e\u4fe1\u53f7\u88ab\u6df9\u6ca1\u7684\u8868\u73b0\u3002<\/p>\n<hr \/>\n<h3>20.14 \u5b9e\u9a8c&#xff1a;\u624b\u5199 Attention vs SDPA&#xff08;FlashAttention&#xff09;<\/h3>\n<p>\u672c\u7ae0\u624b\u5199 Attention \u662f\u4e3a\u4e86\u62ff\u5230\u6ce8\u610f\u529b\u6743\u91cd\u3002\u5b9e\u9645\u5de5\u7a0b\u4e2d&#xff0c;PyTorch 2.x \u63d0\u4f9b\u4e86 F.scaled_dot_product_attention&#xff08;SDPA&#xff09;&#xff0c;\u4f1a\u6839\u636e\u786c\u4ef6\u81ea\u52a8\u9009\u62e9 FlashAttention\u3001Memory-Efficient Attention \u7b49\u9ad8\u6548\u5185\u6838&#xff08;\u7b2c15\u7ae0 15.6 \u8282&#xff09;\u3002\u4e24\u8005\u5728\u6570\u5b66\u4e0a\u5b8c\u5168\u7b49\u4ef7&#xff0c;\u4e0b\u9762\u9a8c\u8bc1\u8fd9\u4e00\u70b9&#xff0c;\u5e76\u6bd4\u8f83\u901f\u5ea6\u548c\u663e\u5b58&#xff1a;<\/p>\n<p><span class=\"token comment\"># ch20_transformer\/mha_vs_sdpa.py<\/span><br \/>\n<span class=\"token triple-quoted-string string\">&#034;&#034;&#034;\u5bf9\u7167&#xff1a;\u540c\u4e00\u4e2a Multi-Head Attention&#xff0c;\u624b\u5199\u7248 vs PyTorch \u5185\u7f6e SDPA<\/p>\n<p>\u8fd0\u884c&#xff1a;python mha_vs_sdpa.py<br \/>\n&#xff08;\u7b2c20\u7ae0\u7528\u624b\u5199\u7248\u662f\u4e3a\u4e86\u62ff\u5230\u6ce8\u610f\u529b\u6743\u91cd&#xff1b;\u7b2c21\u7ae0\u6362\u6210 SDPA \u662f\u4e3a\u4e86\u901f\u5ea6&#xff09;<br \/>\n&#034;&#034;&#034;<\/span><br \/>\n<span class=\"token keyword\">import<\/span> time<\/p>\n<p><span class=\"token keyword\">import<\/span> torch<br \/>\n<span class=\"token keyword\">import<\/span> torch<span class=\"token punctuation\">.<\/span>nn<span class=\"token punctuation\">.<\/span>functional <span class=\"token keyword\">as<\/span> F<\/p>\n<p><span class=\"token keyword\">from<\/span> model <span class=\"token keyword\">import<\/span> MultiHeadAttention<span class=\"token punctuation\">,<\/span> make_causal_mask<\/p>\n<p>DEVICE <span class=\"token operator\">&#061;<\/span> <span class=\"token string\">&#034;cuda&#034;<\/span> <span class=\"token keyword\">if<\/span> torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>is_available<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token keyword\">else<\/span> <span class=\"token string\">&#034;cpu&#034;<\/span><br \/>\ntorch<span class=\"token punctuation\">.<\/span>manual_seed<span class=\"token punctuation\">(<\/span><span class=\"token number\">0<\/span><span class=\"token punctuation\">)<\/span><br \/>\nB<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> D<span class=\"token punctuation\">,<\/span> H <span class=\"token operator\">&#061;<\/span> <span class=\"token number\">8<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">1024<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">512<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">8<\/span><br \/>\nmha <span class=\"token operator\">&#061;<\/span> MultiHeadAttention<span class=\"token punctuation\">(<\/span>D<span class=\"token punctuation\">,<\/span> H<span class=\"token punctuation\">,<\/span> dropout<span class=\"token operator\">&#061;<\/span><span class=\"token number\">0.0<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>to<span class=\"token punctuation\">(<\/span>DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span><span class=\"token builtin\">eval<\/span><span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\nx <span class=\"token operator\">&#061;<\/span> torch<span class=\"token punctuation\">.<\/span>randn<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> D<span class=\"token punctuation\">,<\/span> device<span class=\"token operator\">&#061;<\/span>DEVICE<span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">def<\/span> <span class=\"token function\">sdpa_forward<\/span><span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    sp <span class=\"token operator\">&#061;<\/span> <span class=\"token keyword\">lambda<\/span> t<span class=\"token punctuation\">:<\/span> t<span class=\"token punctuation\">.<\/span>view<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> H<span class=\"token punctuation\">,<\/span> D <span class=\"token operator\">\/\/<\/span> H<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    y <span class=\"token operator\">&#061;<\/span> F<span class=\"token punctuation\">.<\/span>scaled_dot_product_attention<span class=\"token punctuation\">(<\/span>sp<span class=\"token punctuation\">(<\/span>mha<span class=\"token punctuation\">.<\/span>W_q<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> sp<span class=\"token punctuation\">(<\/span>mha<span class=\"token punctuation\">.<\/span>W_k<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> sp<span class=\"token punctuation\">(<\/span>mha<span class=\"token punctuation\">.<\/span>W_v<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> is_causal<span class=\"token operator\">&#061;<\/span><span class=\"token boolean\">True<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">return<\/span> mha<span class=\"token punctuation\">.<\/span>W_o<span class=\"token punctuation\">(<\/span>y<span class=\"token punctuation\">.<\/span>transpose<span class=\"token punctuation\">(<\/span><span class=\"token number\">1<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token number\">2<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">.<\/span>reshape<span class=\"token punctuation\">(<\/span>B<span class=\"token punctuation\">,<\/span> L<span class=\"token punctuation\">,<\/span> D<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p><span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>no_grad<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n    torch<span class=\"token punctuation\">.<\/span>backends<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>matmul<span class=\"token punctuation\">.<\/span>allow_tf32 <span class=\"token operator\">&#061;<\/span> <span class=\"token boolean\">False<\/span><br \/>\n    ours<span class=\"token punctuation\">,<\/span> _ <span class=\"token operator\">&#061;<\/span> mha<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span>make_causal_mask<span class=\"token punctuation\">(<\/span>L<span class=\"token punctuation\">,<\/span> DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><br \/>\n    <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u624b\u5199\u7248\u4e0e SDPA \u4e00\u81f4:&#034;<\/span><span class=\"token punctuation\">,<\/span> torch<span class=\"token punctuation\">.<\/span>allclose<span class=\"token punctuation\">(<\/span>ours<span class=\"token punctuation\">,<\/span> sdpa_forward<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span> atol<span class=\"token operator\">&#061;<\/span><span class=\"token number\">1e-4<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>    <span class=\"token keyword\">if<\/span> DEVICE <span class=\"token operator\">&#061;&#061;<\/span> <span class=\"token string\">&#034;cuda&#034;<\/span><span class=\"token punctuation\">:<\/span><br \/>\n        <span class=\"token keyword\">for<\/span> name<span class=\"token punctuation\">,<\/span> fn <span class=\"token keyword\">in<\/span> <span class=\"token punctuation\">[<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;\u624b\u5199\u7248&#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token keyword\">lambda<\/span><span class=\"token punctuation\">:<\/span> mha<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> x<span class=\"token punctuation\">,<\/span> attn_mask<span class=\"token operator\">&#061;<\/span>make_causal_mask<span class=\"token punctuation\">(<\/span>L<span class=\"token punctuation\">,<\/span> DEVICE<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">,<\/span><br \/>\n                         <span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;SDPA  &#034;<\/span><span class=\"token punctuation\">,<\/span> <span class=\"token keyword\">lambda<\/span><span class=\"token punctuation\">:<\/span> sdpa_forward<span class=\"token punctuation\">(<\/span>x<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">]<\/span><span class=\"token punctuation\">:<\/span><br \/>\n            <span class=\"token keyword\">with<\/span> torch<span class=\"token punctuation\">.<\/span>autocast<span class=\"token punctuation\">(<\/span><span class=\"token string\">&#034;cuda&#034;<\/span><span class=\"token punctuation\">,<\/span> dtype<span class=\"token operator\">&#061;<\/span>torch<span class=\"token punctuation\">.<\/span>bfloat16<span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                fn<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">;<\/span> torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>synchronize<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>reset_peak_memory_stats<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                t <span class=\"token operator\">&#061;<\/span> time<span class=\"token punctuation\">.<\/span>perf_counter<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                <span class=\"token keyword\">for<\/span> _ <span class=\"token keyword\">in<\/span> <span class=\"token builtin\">range<\/span><span class=\"token punctuation\">(<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">)<\/span><span class=\"token punctuation\">:<\/span><br \/>\n                    fn<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n                torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>synchronize<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span><br \/>\n            <span class=\"token keyword\">print<\/span><span class=\"token punctuation\">(<\/span><span class=\"token string-interpolation\"><span class=\"token string\">f&#034;<\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>name<span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> bf16: <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span><span class=\"token punctuation\">(<\/span>time<span class=\"token punctuation\">.<\/span>perf_counter<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">&#8211;<\/span> t<span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> <span class=\"token number\">20<\/span> <span class=\"token operator\">*<\/span> <span class=\"token number\">1000<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.2f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> ms\/\u6b21, &#034;<\/span><\/span><br \/>\n                  <span class=\"token string-interpolation\"><span class=\"token string\">f&#034;\u5cf0\u503c\u663e\u5b58 <\/span><span class=\"token interpolation\"><span class=\"token punctuation\">{<\/span>torch<span class=\"token punctuation\">.<\/span>cuda<span class=\"token punctuation\">.<\/span>max_memory_allocated<span class=\"token punctuation\">(<\/span><span class=\"token punctuation\">)<\/span> <span class=\"token operator\">\/<\/span> <span class=\"token number\">2<\/span><span class=\"token operator\">**<\/span><span class=\"token number\">20<\/span><span class=\"token punctuation\">:<\/span><span class=\"token format-spec\">.0f<\/span><span class=\"token punctuation\">}<\/span><\/span><span class=\"token string\"> MB&#034;<\/span><\/span><span class=\"token punctuation\">)<\/span><\/p>\n<p>\u5b9e\u6d4b&#xff08;RTX 4090&#xff0c;B&#061;8, L&#061;1024, d_model&#061;512, h&#061;8&#xff0c;bf16&#xff09;&#xff1a;<\/p>\n<p>\u624b\u5199\u7248\u4e0e SDPA \u4e00\u81f4: True<br \/>\n\u624b\u5199\u7248 bf16: 2.97 ms\/\u6b21, \u5cf0\u503c\u663e\u5b58 712 MB<br \/>\nSDPA   bf16: 0.25 ms\/\u6b21, \u5cf0\u503c\u663e\u5b58 79 MB<\/p>\n<table>\n<tr>\u5b9e\u73b0\u8017\u65f6\u5cf0\u503c\u663e\u5b58<\/tr>\n<tbody>\n<tr>\n<td>\u624b\u5199 Attention<\/td>\n<td>2.97 ms<\/td>\n<td>712 MB<\/td>\n<\/tr>\n<tr>\n<td>SDPA&#xff08;FlashAttention&#xff09;<\/td>\n<td>0.25 ms<\/td>\n<td>79 MB<\/td>\n<\/tr>\n<tr>\n<td>\u5dee\u8ddd<\/td>\n<td>\u5feb\u7ea6 12 \u500d<\/td>\n<td>\u7ea6 1\/9<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u5dee\u8ddd\u4ece\u54ea\u6765&#xff1f;\u624b\u5199\u7248\u5fc5\u987b\u628a [8, 8, 1024, 1024] \u7684\u6ce8\u610f\u529b\u77e9\u9635&#xff08;6700 \u4e07\u4e2a\u5143\u7d20&#xff09;\u5b8c\u6574\u5199\u8fdb\u663e\u5b58&#xff0c;\u518d\u8bfb\u51fa\u6765\u505a Softmax\u3001\u518d\u8bfb\u51fa\u6765\u4e58 V\u3002FlashAttention \u628a\u8fd9\u4e2a\u77e9\u9635\u5207\u6210\u5c0f\u5757\u653e\u5728 GPU \u7247\u4e0a\u7684 SRAM \u91cc\u7b97\u5b8c&#xff0c;\u4ece\u4e0d\u628a\u5b8c\u6574\u77e9\u9635\u5199\u56de\u663e\u5b58&#xff08;\u9644\u5f55A \u63a8\u5bfc\u4e86\u5206\u5757 Softmax \u4e3a\u4ec0\u4e48\u7cbe\u786e\u7b49\u4ef7&#xff09;\u3002\u5e8f\u5217\u8d8a\u957f&#xff0c;\u5dee\u8ddd\u8d8a\u5927\u3002<\/p>\n<p>\u6240\u4ee5\u7b2c21\u7ae0\u7684 GPT \u4f1a\u76f4\u63a5\u7528 SDPA\u3002<\/p>\n<hr \/>\n<h3>20.15 Hugging Face \u6e90\u7801\u5b9a\u4f4d<\/h3>\n<p>\u8bfb\u5b8c\u672c\u7ae0\u4ee3\u7801&#xff0c;\u53ef\u4ee5\u76f4\u63a5\u5bf9\u7167 transformers \u5e93\u7684\u5b9e\u73b0&#xff08;\u8def\u5f84\u76f8\u5bf9\u4e8e src\/transformers\/&#xff09;&#xff1a;<\/p>\n<table>\n<tr>\u672c\u7ae0\u7ec4\u4ef6Hugging Face \u5bf9\u5e94\u4f4d\u7f6e\u5dee\u5f02<\/tr>\n<tbody>\n<tr>\n<td>MultiHeadAttention<\/td>\n<td>models\/bart\/modeling_bart.py \u2192 BartAttention<\/td>\n<td>\u652f\u6301 KV Cache&#xff0c;\u53ef\u5207\u6362 eager \/ SDPA \/ FlashAttention-2 \u540e\u7aef<\/td>\n<\/tr>\n<tr>\n<td>EncoderLayer<\/td>\n<td>models\/bart\/modeling_bart.py \u2192 BartEncoderLayer<\/td>\n<td>BART \u662f Post-Norm<\/td>\n<\/tr>\n<tr>\n<td>DecoderLayer<\/td>\n<td>models\/bart\/modeling_bart.py \u2192 BartDecoderLayer<\/td>\n<td>\u540c\u4e0a&#xff0c;\u542b encoder_attn&#xff08;Cross-Attention&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>\u6b63\u5f26\u4f4d\u7f6e\u7f16\u7801<\/td>\n<td>models\/marian\/modeling_marian.py \u2192 MarianSinusoidalPositionalEmbedding<\/td>\n<td>BART \u672c\u8eab\u7528\u53ef\u5b66\u4e60\u4f4d\u7f6e\u7f16\u7801<\/td>\n<\/tr>\n<tr>\n<td>greedy_decode<\/td>\n<td>generation\/utils.py \u2192 GenerationMixin.generate<\/td>\n<td>\u7edf\u4e00\u652f\u6301 Greedy \/ Beam \/ Sampling<\/td>\n<\/tr>\n<tr>\n<td>PyTorch \u5b98\u65b9\u5b9e\u73b0<\/td>\n<td>torch\/nn\/modules\/transformer.py \u2192 nn.Transformer<\/td>\n<td>norm_first&#061;True \u5373 Pre-Norm&#xff1b;_reset_parameters \u91cc\u7528 Xavier \u521d\u59cb\u5316<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>20.16 \u5e38\u89c1\u5751<\/h3>\n<table>\n<tr>\u73b0\u8c61\u539f\u56e0\u89e3\u51b3<\/tr>\n<tbody>\n<tr>\n<td>\u5355\u5143\u6d4b\u8bd5\u5168\u8fc7&#xff0c;\u8bad\u7ec3\u5374\u6162\u5f97\u53cd\u5e38<\/td>\n<td>nn.Embedding \u9ed8\u8ba4 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(0,1)\\\\mathcal{N}(0,1)<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\">1<\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span>&#xff0c;\u4e58 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">d\\\\sqrt{d}<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.04em;vertical-align: -0.1078em\"><\/span><span class=\"mord sqrt\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.9322em\"><span class=\"svg-align\" style=\"top: -3em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"mord\" style=\"padding-left: 0.833em\"><span class=\"mord mathnormal\">d<\/span><\/span><\/span><span class=\"\" style=\"top: -2.8922em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"hide-tail\" style=\"min-width: 0.853em;height: 1.08em\"><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.1078em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span> \u540e\u6df9\u6ca1\u4f4d\u7f6e\u7f16\u7801<\/td>\n<td>\u6309 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">N(0,d\u22120.5)\\\\mathcal{N}(0, d^{-0.5})<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.0641em;vertical-align: -0.25em\"><\/span><span class=\"mord mathcal\" style=\"margin-right: 0.1474em\">N<\/span><span class=\"mopen\">(<\/span><span class=\"mord\">0<\/span><span class=\"mpunct\">,<\/span><span class=\"mspace\" style=\"margin-right: 0.1667em\"><\/span><span class=\"mord\"><span class=\"mord mathnormal\">d<\/span><span class=\"msupsub\"><span class=\"vlist-t\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8141em\"><span class=\"\" style=\"top: -3.063em;margin-right: 0.05em\"><span class=\"pstrut\" style=\"height: 2.7em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">\u2212<\/span><span class=\"mord mtight\">0.5<\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><\/span><span class=\"mclose\">)<\/span><\/span><\/span><\/span><\/span> \u521d\u59cb\u5316&#xff08;20.12&#xff09;<\/td>\n<\/tr>\n<tr>\n<td>RuntimeError: view size is not compatible&#8230;<\/td>\n<td>transpose \u540e\u5185\u5b58\u4e0d\u8fde\u7eed&#xff0c;\u4e0d\u80fd\u7528 view<\/td>\n<td>\u6539\u7528 reshape&#xff0c;\u6216\u5148 .contiguous()<\/td>\n<\/tr>\n<tr>\n<td>Expected all tensors to be on the same device<\/td>\n<td>forward \u91cc\u65b0\u5efa\u7684 Mask \u9ed8\u8ba4\u5728 CPU<\/td>\n<td>\u521b\u5efa\u65f6\u4f20 device&#061;x.device<\/td>\n<\/tr>\n<tr>\n<td>\u4f4d\u7f6e\u7f16\u7801\u6ca1\u8ddf\u7740\u6a21\u578b\u4e0a GPU<\/td>\n<td>\u7528\u666e\u901a\u5c5e\u6027 self.pe &#061; &#8230; \u4fdd\u5b58<\/td>\n<td>\u7528 register_buffer<\/td>\n<\/tr>\n<tr>\n<td>\u5b50\u5c42\u53c2\u6570\u6ca1\u88ab\u8bad\u7ec3<\/td>\n<td>\u7528\u666e\u901a list \u88c5\u5b50\u6a21\u5757<\/td>\n<td>\u7528 nn.ModuleList<\/td>\n<\/tr>\n<tr>\n<td>\u6539\u6210 SDPA \u540e\u7ed3\u679c\u5168\u9519<\/td>\n<td>SDPA \u7684 bool mask \u662f True &#061; \u53ef\u89c1&#xff0c;\u4e0e masked_fill \u76f8\u53cd<\/td>\n<td>\u4f20 ~mask&#xff0c;\u6216\u7528 is_causal&#061;True<\/td>\n<\/tr>\n<tr>\n<td>loss \u964d\u5f97\u5f88\u5feb&#xff0c;\u751f\u6210\u65f6\u5374\u5168\u662f\u4e71\u7801<\/td>\n<td>Causal Mask \u65b9\u5411\u5199\u53cd\u6216\u6ca1\u4f20&#xff0c;\u8bad\u7ec3\u65f6\u5077\u770b\u4e86\u7b54\u6848<\/td>\n<td>\u8dd1 20.8 \u7684\u56e0\u679c\u6027\u6d4b\u8bd5<\/td>\n<\/tr>\n<tr>\n<td>\u77ed\u53e5\u6548\u679c\u597d\u3001\u957f\u53e5\u6548\u679c\u5dee<\/td>\n<td>\u53ea\u5c4f\u853d\u4e86\u6e90\u7aef PAD&#xff0c;\u6ca1\u5c4f\u853d\u76ee\u6807\u7aef PAD<\/td>\n<td>Decoder \u81ea\u6ce8\u610f\u529b\u7528 causal | padding<\/td>\n<\/tr>\n<tr>\n<td>\u8f93\u51fa\u6574\u4f53\u9519\u4e00\u4f4d<\/td>\n<td>tgt_in \/ tgt_out \u5207\u7247\u5199\u53cd<\/td>\n<td>tgt_in &#061; tgt[:, :-1]&#xff0c;tgt_out &#061; tgt[:, 1:]<\/td>\n<\/tr>\n<tr>\n<td>bf16\/fp16 \u4e0b\u51fa\u73b0 NaN<\/td>\n<td>\u5c4f\u853d\u503c -1e9 \u8d85\u51fa fp16 \u8303\u56f4<\/td>\n<td>\u7528 torch.finfo(scores.dtype).min<\/td>\n<\/tr>\n<tr>\n<td>\u63a8\u7406\u7ed3\u679c\u6bcf\u6b21\u90fd\u4e0d\u4e00\u6837<\/td>\n<td>\u5fd8\u4e86 model.eval()&#xff0c;Dropout \u8fd8\u5f00\u7740<\/td>\n<td>\u63a8\u7406\u524d model.eval()<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<hr \/>\n<h3>\u672c\u7ae0\u7ec3\u4e60\u4e0e\u5b9e\u9a8c<\/h3>\n<li>\u590d\u73b0\u521d\u59cb\u5316\u95ee\u9898&#xff1a;\u5220\u6389 TokenAndPosition \u91cc\u7684 nn.init.normal_(&#8230;) \u90a3\u51e0\u884c&#xff0c;\u91cd\u65b0\u8bad\u7ec3&#xff0c;\u590d\u73b0 20.12.1 \u7684\u6162\u6536\u655b\u3002\u518d\u8bd5\u8bd5\u4fdd\u7559\u9ed8\u8ba4\u521d\u59cb\u5316\u3001\u4f46\u53bb\u6389 * math.sqrt(self.d_model)&#xff0c;\u6548\u679c\u5982\u4f55&#xff1f;<\/li>\n<li>\u53bb\u6389\u4f4d\u7f6e\u7f16\u7801&#xff1a;\u628a &#043; self.pe[:L] \u5220\u6389\u91cd\u65b0\u8bad\u7ec3&#xff0c;exact_match \u4f1a\u600e\u6837&#xff1f;\u4e3a\u4ec0\u4e48\u7ffb\u8f6c\u4efb\u52a1\u79bb\u4e0d\u5f00\u4f4d\u7f6e\u4fe1\u606f&#xff1f;<\/li>\n<li>\u53ea\u7528 1 \u4e2a Head&#xff1a;num_heads&#061;1&#xff0c;\u5bf9\u6bd4\u6536\u655b\u901f\u5ea6\u548c\u6700\u7ec8\u51c6\u786e\u7387\u3002<\/li>\n<li>Post-Norm \u7248\u672c&#xff1a;\u628a Encoder\/Decoder Layer \u6539\u6210 LN(x &#043; Sublayer(x))&#xff0c;\u53bb\u6389 ln_final&#xff0c;\u5728 num_layers&#061;6 \u65f6\u4e0e Pre-Norm \u5bf9\u6bd4&#xff1b;\u518d\u628a warmup \u6539\u6210 1 \u770b\u770b\u4e24\u8005\u8c01\u66f4\u5bb9\u6613\u5d29\u3002<\/li>\n<li>\u6362\u4efb\u52a1&#xff1a;\u628a\u76ee\u6807\u6539\u6210&#034;\u6392\u5e8f&#034;&#xff08;np.sort(digits)&#xff09;\u3002Cross-Attention \u77e9\u9635\u8fd8\u4f1a\u662f\u53cd\u5bf9\u89d2\u7ebf\u5417&#xff1f;<\/li>\n<li>SwiGLU&#xff1a;\u53c2\u7167\u7b2c8\u7ae0&#xff0c;\u628a FeedForward \u6362\u6210 SwiGLU&#xff0c;\u5e76\u4fdd\u6301\u53c2\u6570\u91cf\u5927\u81f4\u76f8\u540c&#xff08;\u9690\u85cf\u7ef4\u5ea6\u53d6 <span class=\"katex--inline\"><span class=\"katex\"><span class=\"katex-mathml\">23\u00d74d\\\\frac{2}{3} \\\\times 4d<\/span><span class=\"katex-html\"><span class=\"base\"><span class=\"strut\" style=\"height: 1.1901em;vertical-align: -0.345em\"><\/span><span class=\"mord\"><span class=\"mopen nulldelimiter\"><\/span><span class=\"mfrac\"><span class=\"vlist-t vlist-t2\"><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.8451em\"><span class=\"\" style=\"top: -2.655em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">3<\/span><\/span><\/span><\/span><span class=\"\" style=\"top: -3.23em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"frac-line\" style=\"border-bottom-width: 0.04em\"><\/span><\/span><span class=\"\" style=\"top: -3.394em\"><span class=\"pstrut\" style=\"height: 3em\"><\/span><span class=\"sizing reset-size6 size3 mtight\"><span class=\"mord mtight\"><span class=\"mord mtight\">2<\/span><\/span><\/span><\/span><\/span><span class=\"vlist-s\">\u200b<\/span><\/span><span class=\"vlist-r\"><span class=\"vlist\" style=\"height: 0.345em\"><span class=\"\"><\/span><\/span><\/span><\/span><\/span><span class=\"mclose nulldelimiter\"><\/span><\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><span class=\"mbin\">\u00d7<\/span><span class=\"mspace\" style=\"margin-right: 0.2222em\"><\/span><\/span><span class=\"base\"><span class=\"strut\" style=\"height: 0.6944em\"><\/span><span class=\"mord\">4<\/span><span class=\"mord mathnormal\">d<\/span><\/span><\/span><\/span><\/span>&#xff09;\u3002<\/li>\n<li>\u4e0e nn.Transformer \u5bf9\u6bd4&#xff1a;\u7528 nn.Transformer(d_model&#061;128, nhead&#061;4, num_encoder_layers&#061;2, num_decoder_layers&#061;2, dim_feedforward&#061;512, batch_first&#061;True, norm_first&#061;True) \u66ff\u6362\u672c\u7ae0\u7684 Encoder\/Decoder&#xff0c;\u8bad\u7ec3\u7ed3\u679c\u662f\u5426\u4e00\u81f4&#xff1f;<\/li>\n<hr \/>\n<h3>\u672c\u7ae0\u603b\u7ed3<\/h3>\n<p>#mermaid-svg-utoh6k5YdpNDfHrG{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;fill:#333;}@keyframes edge-animation-frame{from{stroke-dashoffset:0;}}@keyframes dash{to{stroke-dashoffset:0;}}#mermaid-svg-utoh6k5YdpNDfHrG .edge-animation-slow{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 50s linear infinite;stroke-linecap:round;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-animation-fast{stroke-dasharray:9,5!important;stroke-dashoffset:900;animation:dash 20s linear infinite;stroke-linecap:round;}#mermaid-svg-utoh6k5YdpNDfHrG .error-icon{fill:#552222;}#mermaid-svg-utoh6k5YdpNDfHrG .error-text{fill:#552222;stroke:#552222;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-thickness-normal{stroke-width:1px;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-thickness-thick{stroke-width:3.5px;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-pattern-solid{stroke-dasharray:0;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-thickness-invisible{stroke-width:0;fill:none;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-pattern-dashed{stroke-dasharray:3;}#mermaid-svg-utoh6k5YdpNDfHrG .edge-pattern-dotted{stroke-dasharray:2;}#mermaid-svg-utoh6k5YdpNDfHrG .marker{fill:#333333;stroke:#333333;}#mermaid-svg-utoh6k5YdpNDfHrG .marker.cross{stroke:#333333;}#mermaid-svg-utoh6k5YdpNDfHrG svg{font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;font-size:16px;}#mermaid-svg-utoh6k5YdpNDfHrG p{margin:0;}#mermaid-svg-utoh6k5YdpNDfHrG .edge{stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 path,#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 path{fill:hsl(240, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 text{fill:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon&#8211;1{font-size:40px;color:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge&#8211;1{stroke:hsl(240, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth&#8211;1{stroke-width:17;}#mermaid-svg-utoh6k5YdpNDfHrG .section&#8211;1 line{stroke:hsl(60, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-0 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-0 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-0 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-0 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-0 path{fill:hsl(60, 100%, 73.5294117647%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-0 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-0{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-0{stroke:hsl(60, 100%, 73.5294117647%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-0{stroke-width:14;}#mermaid-svg-utoh6k5YdpNDfHrG .section-0 line{stroke:hsl(240, 100%, 83.5294117647%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-1 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-1 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-1 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-1 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-1 path{fill:hsl(80, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-1 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-1{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-1{stroke:hsl(80, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-1{stroke-width:11;}#mermaid-svg-utoh6k5YdpNDfHrG .section-1 line{stroke:hsl(260, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-2 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-2 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-2 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-2 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-2 path{fill:hsl(270, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-2 text{fill:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-2{font-size:40px;color:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-2{stroke:hsl(270, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-2{stroke-width:8;}#mermaid-svg-utoh6k5YdpNDfHrG .section-2 line{stroke:hsl(90, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-3 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-3 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-3 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-3 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-3 path{fill:hsl(300, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-3 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-3{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-3{stroke:hsl(300, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-3{stroke-width:5;}#mermaid-svg-utoh6k5YdpNDfHrG .section-3 line{stroke:hsl(120, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-4 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-4 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-4 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-4 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-4 path{fill:hsl(330, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-4 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-4{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-4{stroke:hsl(330, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-4{stroke-width:2;}#mermaid-svg-utoh6k5YdpNDfHrG .section-4 line{stroke:hsl(150, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-5 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-5 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-5 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-5 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-5 path{fill:hsl(0, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-5 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-5{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-5{stroke:hsl(0, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-5{stroke-width:-1;}#mermaid-svg-utoh6k5YdpNDfHrG .section-5 line{stroke:hsl(180, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-6 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-6 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-6 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-6 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-6 path{fill:hsl(30, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-6 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-6{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-6{stroke:hsl(30, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-6{stroke-width:-4;}#mermaid-svg-utoh6k5YdpNDfHrG .section-6 line{stroke:hsl(210, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-7 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-7 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-7 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-7 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-7 path{fill:hsl(90, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-7 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-7{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-7{stroke:hsl(90, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-7{stroke-width:-7;}#mermaid-svg-utoh6k5YdpNDfHrG .section-7 line{stroke:hsl(270, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-8 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-8 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-8 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-8 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-8 path{fill:hsl(150, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-8 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-8{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-8{stroke:hsl(150, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-8{stroke-width:-10;}#mermaid-svg-utoh6k5YdpNDfHrG .section-8 line{stroke:hsl(330, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-9 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-9 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-9 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-9 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-9 path{fill:hsl(180, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-9 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-9{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-9{stroke:hsl(180, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-9{stroke-width:-13;}#mermaid-svg-utoh6k5YdpNDfHrG .section-9 line{stroke:hsl(0, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-10 rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-10 path,#mermaid-svg-utoh6k5YdpNDfHrG .section-10 circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-10 polygon,#mermaid-svg-utoh6k5YdpNDfHrG .section-10 path{fill:hsl(210, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-10 text{fill:black;}#mermaid-svg-utoh6k5YdpNDfHrG .node-icon-10{font-size:40px;color:black;}#mermaid-svg-utoh6k5YdpNDfHrG .section-edge-10{stroke:hsl(210, 100%, 76.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .edge-depth-10{stroke-width:-16;}#mermaid-svg-utoh6k5YdpNDfHrG .section-10 line{stroke:hsl(30, 100%, 86.2745098039%);stroke-width:3;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled,#mermaid-svg-utoh6k5YdpNDfHrG .disabled circle,#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:lightgray;}#mermaid-svg-utoh6k5YdpNDfHrG .disabled text{fill:#efefef;}#mermaid-svg-utoh6k5YdpNDfHrG .section-root rect,#mermaid-svg-utoh6k5YdpNDfHrG .section-root path,#mermaid-svg-utoh6k5YdpNDfHrG .section-root circle,#mermaid-svg-utoh6k5YdpNDfHrG .section-root polygon{fill:hsl(240, 100%, 46.2745098039%);}#mermaid-svg-utoh6k5YdpNDfHrG .section-root text{fill:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .section-root span{color:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .section-2 span{color:#ffffff;}#mermaid-svg-utoh6k5YdpNDfHrG .icon-container{height:100%;display:flex;justify-content:center;align-items:center;}#mermaid-svg-utoh6k5YdpNDfHrG .edge{fill:none;}#mermaid-svg-utoh6k5YdpNDfHrG .mindmap-node-label{dy:1em;alignment-baseline:middle;text-anchor:middle;dominant-baseline:middle;text-align:center;}#mermaid-svg-utoh6k5YdpNDfHrG :root{&#8211;mermaid-font-family:\\&#8221;trebuchet ms\\&#8221;,verdana,arial,sans-serif;}<span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"edgeLabel\"><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u4ece\u96f6\u5b9e\u73b0 Transformer<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u7ec4\u4ef6<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u4f4d\u7f6e\u7f16\u7801 register_buffer<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Multi-Head Attention<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>FFN<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Pre-Norm \u4e0e ln_final<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Mask<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Encoder \u6e90\u7aef Padding<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Decoder \u56e0\u679c \u6216 \u76ee\u6807\u7aef Padding<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Cross \u6e90\u7aef Padding<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u8bad\u7ec3<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Teacher Forcing<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Warmup LambdaLR<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>ignore_index \u5ffd\u7565 PAD<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>bf16 autocast<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u63a8\u7406<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Encoder \u8dd1\u4e00\u6b21<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Decoder \u5faa\u73af\u751f\u6210<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u9a8c\u8bc1<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u5355\u5143\u6d4b\u8bd5 \u56e0\u679c\u6027 \u4e0e Padding<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>Cross-Attention \u53cd\u5bf9\u89d2\u7ebf<\/p>\n<p><\/span><span class=\"nodeLabel\"><\/p>\n<p>\u521d\u59cb\u5316\u51b3\u5b9a\u6536\u655b\u901f\u5ea6<\/p>\n<p><\/span><\/p>\n<p>\u4e00\u53e5\u8bdd\u603b\u7ed3&#xff1a;Transformer \u7684\u4ee3\u7801\u91cf\u4e0d\u5927&#xff0c;\u771f\u6b63\u5bb9\u6613\u51fa\u9519\u7684\u662f Mask\u3001\u6570\u636e\u9519\u4f4d\u548c\u521d\u59cb\u5316&#xff1b;\u5148\u7528\u5355\u5143\u6d4b\u8bd5\u9a8c\u8bc1\u6027\u8d28&#xff0c;\u518d\u7528\u53ef\u89e3\u91ca\u7684\u5c0f\u4efb\u52a1&#xff08;\u770b\u6ce8\u610f\u529b\u77e9\u9635&#xff09;\u9a8c\u8bc1\u6548\u679c&#xff0c;\u6bd4\u76ef\u7740 loss \u66f2\u7ebf\u53ef\u9760\u5f97\u591a\u3002<\/p>\n<hr \/>\n<h3>\u672c\u7ae0\u601d\u8003\u9898<\/h3>\n<li>\u63a8\u7406\u65f6 Encoder \u53ea\u8dd1\u4e00\u6b21&#xff0c;Decoder \u8981\u8dd1\u591a\u6b21\u3002\u5982\u679c\u6e90\u5e8f\u5217\u5f88\u957f\u3001\u76ee\u6807\u5e8f\u5217\u5f88\u77ed&#xff08;\u5982\u6587\u672c\u6458\u8981&#xff09;&#xff0c;\u8ba1\u7b97\u74f6\u9888\u5728\u54ea\u4e00\u4fa7&#xff1f;<\/li>\n<li>Cross-Attention \u7684 Mask \u4e3a\u4ec0\u4e48\u53ea\u9700\u8981\u6e90\u7aef Padding&#xff0c;\u4e0d\u9700\u8981 Causal Mask&#xff1f;<\/li>\n<li>\u4e3a\u4ec0\u4e48 val_exact_match \u4f1a\u6bd4 token_acc \u4f4e&#xff1f;\u4e24\u8005\u7684\u5dee\u8ddd\u6765\u81ea\u54ea\u91cc&#xff1f;&#xff08;\u63d0\u793a&#xff1a;Exposure Bias&#xff09;<\/li>\n<li>20.12 \u8282\u4e2d&#xff0c;Embedding \u7684\u6570\u503c\u6bd4\u4f4d\u7f6e\u7f16\u7801\u5927 10 \u500d\u65f6&#xff0c;\u4e3a\u4ec0\u4e48\u6a21\u578b&#034;\u65b9\u5411\u5bf9\u3001\u4f46\u5206\u4e0d\u6e05\u76f8\u90bb\u4f4d\u7f6e&#034;&#xff1f;\u4ece\u70b9\u79ef\u6ce8\u610f\u529b\u7684\u89d2\u5ea6\u89e3\u91ca\u3002<\/li>\n<li>\u672c\u7ae0\u7684\u4f4d\u7f6e\u7f16\u7801\u662f\u56fa\u5b9a\u7684\u6b63\u5f26\u51fd\u6570\u3002\u5982\u679c\u6d4b\u8bd5\u65f6\u8f93\u5165\u957f\u5ea6\u8d85\u8fc7\u8bad\u7ec3\u65f6\u7684\u6700\u5927\u957f\u5ea6 12&#xff0c;\u6a21\u578b\u8fd8\u80fd\u6b63\u786e\u7ffb\u8f6c\u5417&#xff1f;\u8bd5\u7740\u505a\u5b9e\u9a8c\u9a8c\u8bc1&#xff08;\u6ce8\u610f max_len \u4e5f\u8981\u8c03\u5927&#xff09;\u3002<\/li>\n<hr \/>\n<h3>\u4e0b\u4e00\u7ae0\u9884\u544a<\/h3>\n<p>\u4e0b\u4e00\u7ae0\u53bb\u6389 Encoder \u548c Cross-Attention&#xff0c;\u53ea\u4fdd\u7559 Decoder&#xff0c;\u8bad\u7ec3\u4e00\u4e2a\u5b57\u7b26\u7ea7 GPT&#xff1a;<\/p>\n<ul>\n<li>\u5728\u838e\u58eb\u6bd4\u4e9a\u5168\u96c6\u4e0a\u505a Next Token Prediction&#xff0c;\u89c2\u5bdf\u8fc7\u62df\u5408\u5e76\u7528\u6700\u4f73 checkpoint \u65e9\u505c<\/li>\n<li>\u7528 SDPA&#xff08;FlashAttention&#xff09;&#043; bf16 &#043; torch.compile \u5728 4090 \u4e0a 90 \u79d2\u8bad\u5b8c 1000 \u4e07\u53c2\u6570\u7684\u6a21\u578b<\/li>\n<li>\u5b9e\u73b0 Temperature \/ Top-k \u91c7\u6837<\/li>\n<li>\u4eb2\u624b\u5b9e\u73b0 KV Cache&#xff0c;\u9a8c\u8bc1\u5b83\u4e0e\u6734\u7d20\u751f\u6210\u9010\u5b57\u4e00\u81f4&#xff0c;\u5e76\u627e\u51fa\u5b83\u771f\u6b63\u53d1\u6325\u4f5c\u7528\u7684\u6761\u4ef6<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"<p>\u7b2c\u516d\u7bc7 \u5b9e\u6218<br \/>\n\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer&#xff08;PyTorch&#xff09;\u672c\u7ae0\u76ee\u6807&#xff1a;<br \/>\n\u4e0d\u8c03\u7528 nn.Transformer \/ nn.MultiheadAttention&#xff0c;\u7528 PyTorch \u4ece\u96f6\u5199\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684 Encoder-Decoder Transformer&#xff0c;\u5728 RTX 4090 \u4e0a 17 \u79d2\u8bad\u7ec3\u5230 99.6% \u7684\u6574\u53e5\u51c6\u786e\u7387&#xff0c;\u5e76\u4eb2\u773c\u770b\u5230 Cross-Attention \u5b66\u51fa\u6765\u7684\u5bf9\u9f50\u77e9\u9635<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[152,841,86],"topic":[],"class_list":["post-120636","post","type-post","status-publish","format-standard","hentry","category-server","tag-pytorch","tag-transformer","tag-86"],"yoast_head":"<!-- This site is optimized with the Yoast SEO plugin v20.3 - https:\/\/yoast.com\/wordpress\/plugins\/seo\/ -->\n<title>\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/120636.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u7b2c\u516d\u7bc7 \u5b9e\u6218 \u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer&#xff08;PyTorch&#xff09;\u672c\u7ae0\u76ee\u6807&#xff1a; \u4e0d\u8c03\u7528 nn.Transformer \/ nn.MultiheadAttention&#xff0c;\u7528 PyTorch \u4ece\u96f6\u5199\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684 Encoder-Decoder Transformer&#xff0c;\u5728 RTX 4090 \u4e0a 17 \u79d2\u8bad\u7ec3\u5230 99.6% \u7684\u6574\u53e5\u51c6\u786e\u7387&#xff0c;\u5e76\u4eb2\u773c\u770b\u5230 Cross-Attention \u5b66\u51fa\u6765\u7684\u5bf9\u9f50\u77e9\u9635\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/120636.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-10-31T15:47:56+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"36 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/120636.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/120636.html\",\"name\":\"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-10-31T15:47:56+00:00\",\"dateModified\":\"2026-10-31T15:47:56+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/120636.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/120636.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/120636.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/120636.html","og_locale":"zh_CN","og_type":"article","og_title":"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u7b2c\u516d\u7bc7 \u5b9e\u6218 \u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer&#xff08;PyTorch&#xff09;\u672c\u7ae0\u76ee\u6807&#xff1a; \u4e0d\u8c03\u7528 nn.Transformer \/ nn.MultiheadAttention&#xff0c;\u7528 PyTorch \u4ece\u96f6\u5199\u51fa\u4e00\u4e2a\u5b8c\u6574\u7684 Encoder-Decoder Transformer&#xff0c;\u5728 RTX 4090 \u4e0a 17 \u79d2\u8bad\u7ec3\u5230 99.6% \u7684\u6574\u53e5\u51c6\u786e\u7387&#xff0c;\u5e76\u4eb2\u773c\u770b\u5230 Cross-Attention \u5b66\u51fa\u6765\u7684\u5bf9\u9f50\u77e9\u9635","og_url":"https:\/\/www.wsisp.com\/helps\/120636.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-10-31T15:47:56+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"36 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/120636.html","url":"https:\/\/www.wsisp.com\/helps\/120636.html","name":"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-10-31T15:47:56+00:00","dateModified":"2026-10-31T15:47:56+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/120636.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/120636.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/120636.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u7b2c20\u7ae0 \u4ece\u96f6\u5b9e\u73b0 Transformer\uff08PyTorch\uff09"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/120636","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=120636"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/120636\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=120636"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=120636"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=120636"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=120636"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}