{"id":72570,"date":"2026-02-06T00:32:50","date_gmt":"2026-02-05T16:32:50","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/72570.html"},"modified":"2026-02-06T00:32:50","modified_gmt":"2026-02-05T16:32:50","slug":"%e6%89%8b%e6%8e%a8transformer%e5%bc%a0%e9%87%8f%e4%bc%a0%e9%80%92%e8%bf%87%e7%a8%8b","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/72570.html","title":{"rendered":"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b"},"content":{"rendered":"

\"\"<\/p>\n

\u6b65\u9aa41：\u4f4d\u7f6e\u5d4c\u5165\u4e0e\u8bcd\u6620\u5c04<\/h3>\n

\"\"<\/p>\n

import torch
\nimport torch.nn as nn
\nimport math<\/p>\n

class TransformerInput(nn.Module):
\n def __init__(self, vocab_size, d_model, max_len=5000):
\n super().__init__()
\n # 1. Embedding \u5c42：\u5c06\u5355\u8bcd\u7d22\u5f15\u8f6c\u4e3a 512 \u7ef4\u5411\u91cf
\n self.embedding = nn.Embedding(vocab_size, d_model)<\/p>\n

# 2. Positional Encoding：\u751f\u6210\u4f4d\u7f6e\u504f\u7f6e
\n pe = torch.zeros(max_len, d_model)
\n position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
\n # \u4f7f\u7528\u6b63\u4f59\u5f26\u516c\u5f0f：PE(pos, 2i) = sin(pos\/10000^(2i\/d_model))
\n div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) \/ d_model))
\n pe[:, 0::2] = torch.sin(position * div_term)
\n pe[:, 1::2] = torch.cos(position * div_term)
\n self.register_buffer('pe', pe.unsqueeze(0)) # \u56fa\u5b9a\u6743\u91cd，\u4e0d\u53c2\u4e0e\u8bad\u7ec3<\/p>\n

def forward(self, x):
\n # x \u5f62\u72b6: [batch_size, seq_len] -> \u8bcd\u5411\u91cf: [batch_size, seq_len, 512]
\n x = self.embedding(x) * math.sqrt(x.size(-1)) # \u7f29\u653e\u4ee5\u5339\u914d PE \u5f3a\u5ea6
\n # \u76f4\u63a5\u76f8\u52a0：\u56fe\u4e2d\u90a3\u4e2a\u5708\u5708\u91cc\u7684 "+" \u53f7
\n x = x + self.pe[:, :x.size(1)]
\n return x<\/p>\n

\u76f8\u6bd4\u4f20\u7edf RNN \u7684\u9010\u8bcd\u8f93\u5165，Transformer \u7684\u8f93\u5165\u662f\u4e00\u6574\u53e5\u8bdd\u3002\u4e3a\u4e86\u533a\u5206\u6bcf\u4e2a\u8bcd\u7684\u4f4d\u7f6e\u4fe1\u606f，\u9700\u8981\u7ed9\u5b83\u52a0\u4e0a\u4f4d\u7f6e\u7f16\u7801\u3002<\/p>\n

\u6ce8\u610f：vocab_size \u6d89\u53ca\u5230\u53e5\u5b50\u89e3\u7801\u6240\u6d89\u53ca\u7684 token \u91cf，\u5373 BPE \u7b97\u6cd5\u6700\u7ec8\u505c\u6b65\u7684\u9650\u5236\u503c\u3002\u4f46\u662f Embedding \u8fd9\u4e2a\u4ece token \u6620\u5c04\u5230\u5411\u91cf\u7684\u6b65\u9aa4\u662f\u9700\u8981\u8bad\u7ec3\u7684\u3002<\/p>\n\n\n\n\n\n\n\n\n
\u7ec4\u4ef6\u540d\u79f0<\/td>\n\u6765\u6e90<\/td>\n\u662f\u5426\u6d89\u53ca Transformer \u8bad\u7ec3<\/td>\n\u6bd4\u55bb<\/td>\n<\/tr>\n
\u8bcd\u8868 (Vocabulary)<\/td>\n\u7edf\u8ba1\u7b97\u6cd5 (Tokenizer) \u626b\u63cf\u8bed\u6599\u5e93<\/td>\n\u5426 (\u9884\u5904\u7406\u9636\u6bb5\u5b8c\u6210)<\/td>\n\u5b57\u5178\u7684\u76ee\u5f55（\u51b3\u5b9a\u4e86\u4e66\u91cc\u6536\u5f55\u54ea\u4e9b\u5b57）<\/td>\n<\/tr>\n
vocab_size<\/td>\n\u4eba\u4e3a\u8bbe\u5b9a\u7684\u9608\u503c<\/td>\n\u5426 (\u914d\u7f6e\u53c2\u6570)<\/td>\n\u5b57\u5178\u7684\u539a\u5ea6（\u51b3\u5b9a\u4e86\u5b57\u5178\u80fd\u67e5\u591a\u5c11\u4e2a\u5b57）<\/td>\n<\/tr>\n
Embedding \u77e9\u9635<\/td>\nnn.Embedding<\/td>\n\u662f (\u6838\u5fc3\u8bad\u7ec3\u5bf9\u8c61)<\/td>\n\u5bf9\u5b57\u7684\u7406\u89e3（\u4e00\u5f00\u59cb\u778e\u731c，\u8d8a\u5b66\u8d8a\u61c2）<\/td>\n<\/tr>\n
d_model<\/td>\n\u4eba\u4e3a\u8bbe\u5b9a\u7684\u5bbd\u5ea6<\/td>\n\u5426 (\u914d\u7f6e\u53c2\u6570)<\/td>\n\u8111\u5bb9\u91cf（\u7528\u591a\u5c11\u4e2a\u8111\u7ec6\u80de\u53bb\u8bb0\u4e00\u4e2a\u8bcd）<\/td>\n<\/tr>\n
\u4f4d\u7f6e\u7f16\u7801 (PE)<\/td>\nsin cos \u6570\u5b66\u516c\u5f0f<\/td>\n\u5426 (\u56fa\u5b9a\u53c2\u6570)<\/td>\n\u9875\u7801（\u65e0\u8bba\u4e66\u5199\u4ec0\u4e48\u5185\u5bb9，\u9875\u7801\u6c38\u8fdc\u662f 1, 2, 3…）<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n
\n

\u6b65\u9aa42：\u7f16\u7801\u5668\u90e8\u5206（\u65c1\u8fb9\u7684 Nx \u8868\u793a\u53ef\u4ee5\u7ec4\u88c5 N \u4e2a\u7f16\u7801\u5668）<\/h3>\n

\"\"<\/p>\n

\u5bf9\u4e8e\u8fd9\u4e2a\u5c0f\u7070\u6846，\u6211\u4eec\u8ba4\u4e3a\u662f\u4e00\u5c42\u7684 Encoder，\u79f0\u4e3a EncoderLayer，\u5305\u542b\u4e24\u4e2a\u5b50\u5c42（\u591a\u5934\u6ce8\u610f\u529b + \u524d\u9988 FFN）\u3002\u6bcf\u4e2a\u5b50\u5c42\u540e\u90fd\u5305\u542b\u4e86\u5c42\u5f52\u4e00\u4e0e\u6b8b\u5dee\u94fe\u63a5\u3002<\/p>\n

import torch
\nimport torch.nn as nn
\nimport copy<\/p>\n

# \u8f85\u52a9\u51fd\u6570：\u7528\u4e8e\u514b\u9686 N \u4e2a\u76f8\u540c\u7684\u5c42（\u540e\u9762\u6784\u5efa Encoder \u65f6\u7528\u5230）
\ndef clones(module, N):
\n return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])<\/p>\n

class EncoderLayer(nn.Module):
\n def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
\n """
\n \u521d\u59cb\u5316\u5355\u5c42\u7f16\u7801\u5668
\n :param d_model: \u8bcd\u5411\u91cf\u7ef4\u5ea6 (\u4f8b\u5982 512)
\n :param num_heads: \u591a\u5934\u6ce8\u610f\u529b\u7684\u5934\u6570 (\u4f8b\u5982 8)
\n :param d_ff: \u524d\u9988\u795e\u7ecf\u7f51\u7edc\u7684\u4e2d\u95f4\u5c42\u7ef4\u5ea6 (\u4f8b\u5982 2048)
\n :param dropout: \u4e22\u5f03\u7387，\u9632\u6b62\u8fc7\u62df\u5408
\n """
\n super().__init__()<\/p>\n

# — \u5b50\u5c42 1: \u591a\u5934\u81ea\u6ce8\u610f\u529b —
\n # \u5bf9\u5e94\u56fe\u4e2d: Multi-Head Attention
\n self.self_attn = MultiHeadAttention(d_model, num_heads)<\/p>\n

# — \u5b50\u5c42 2: \u524d\u9988\u795e\u7ecf\u7f51\u7edc —
\n # \u5bf9\u5e94\u56fe\u4e2d: Feed Forward
\n self.feed_forward = PositionwiseFeedForward(d_model, d_ff)<\/p>\n

# — \u8fde\u63a5\u7ec4\u4ef6 —
\n # \u5bf9\u5e94\u56fe\u4e2d: Add & Norm (\u6709\u4e24\u4e2a，\u5206\u522b\u8ddf\u5728\u4e24\u4e2a\u5b50\u5c42\u540e\u9762)
\n self.norm1 = nn.LayerNorm(d_model)
\n self.norm2 = nn.LayerNorm(d_model)<\/p>\n

# Dropout (\u653e\u5728\u6b8b\u5dee\u8fde\u63a5\u76f8\u52a0\u4e4b\u524d)
\n self.dropout = nn.Dropout(dropout)<\/p>\n

def forward(self, x, mask):
\n """
\n \u524d\u5411\u4f20\u64ad\u903b\u8f91
\n :param x: \u8f93\u5165\u6570\u636e [batch_size, seq_len, d_model]
\n :param mask: \u63a9\u7801\u77e9\u9635 (\u7528\u4e8e\u5c4f\u853d PAD token)
\n """<\/p>\n

# — \u6b65\u9aa4 1: Self-Attention + Add + Norm —
\n # 1.1 \u4fdd\u7559\u539f\u59cb\u8f93\u5165\u7528\u4e8e\u6b8b\u5dee\u8fde\u63a5 (Residue)
\n residual = x <\/p>\n

# 1.2 \u6267\u884c\u591a\u5934\u6ce8\u610f\u529b
\n # \u6ce8\u610f：Encoder \u4e2d Q, K, V \u5168\u90e8\u6765\u6e90\u4e8e\u540c\u4e00\u4e2a x (\u6240\u4ee5\u53eb"\u81ea"\u6ce8\u610f\u529b)
\n x_attn = self.self_attn(x, x, x, mask)<\/p>\n

# 1.3 Dropout + Add (\u6b8b\u5dee\u8fde\u63a5) + Norm
\n # \u516c\u5f0f: LayerNorm(x + Sublayer(x))
\n x = self.norm1(residual + self.dropout(x_attn))<\/p>\n

# — \u6b65\u9aa4 2: Feed Forward + Add + Norm —
\n # 2.1 \u518d\u6b21\u4fdd\u7559\u5f53\u524d\u72b6\u6001\u7528\u4e8e\u6b8b\u5dee\u8fde\u63a5
\n residual = x<\/p>\n

# 2.2 \u6267\u884c\u524d\u9988\u7f51\u7edc
\n x_ff = self.feed_forward(x)<\/p>\n

# 2.3 Dropout + Add + Norm
\n x = self.norm2(residual + self.dropout(x_ff))<\/p>\n

return x<\/p>\n

\u591a\u5934\u6ce8\u610f\u529b\u5b9e\u9645\u4e0a\u5c31\u662f\u4f7f\u7528\u4e09\u4e2a\u77e9\u9635\u5bf9\u8f93\u5165\u8fdb\u884c\u7ebf\u6027\u53d8\u6362（\u77e9\u9635\u4e58\u6cd5）\u3002<\/p>\n

import torch
\nimport torch.nn as nn
\nimport math<\/p>\n

class MultiHeadAttention(nn.Module):
\n def __init__(self, d_model, num_heads):
\n super().__init__()
\n assert d_model % num_heads == 0<\/p>\n

self.d_k = d_model \/\/ num_heads # 64
\n self.num_heads = num_heads # 8
\n self.d_model = d_model # 512<\/p>\n

# \u3010\u5173\u952e\u70b91\u3011\u5b9a\u4e49\u53c2\u6570\u77e9\u9635
\n # \u6ce8\u610f：\u8fd9\u91cc\u5b9a\u4e49\u7684\u662f"\u6240\u6709\u5934\u52a0\u8d77\u6765"\u7684\u5927\u77e9\u9635
\n # W_q, W_k, W_v \u7684\u5f62\u72b6\u90fd\u662f [512, 512]
\n self.linear_q = nn.Linear(d_model, d_model)
\n self.linear_k = nn.Linear(d_model, d_model)
\n self.linear_v = nn.Linear(d_model, d_model)<\/p>\n

# \u6700\u540e\u8f93\u51fa\u7684\u7ebf\u6027\u5c42 W_o
\n self.linear_out = nn.Linear(d_model, d_model)<\/p>\n

def forward(self, query, key, value, mask=None):
\n batch_size = query.size(0) # 32<\/p>\n

# — \u7b2c\u4e00\u6b65：\u7ebf\u6027\u6295\u5f71 + \u5207\u5206\u5934 —
\n # 1. \u7ebf\u6027\u53d8\u6362: [32, 10, 512] * [512, 512] -> [32, 10, 512]
\n # 2. view\u91cd\u5851: -> [32, 10, 8, 64] (\u628a 512 \u62c6\u6210 8 * 64)
\n # 3. transpose\u8f6c\u7f6e: -> [32, 8, 10, 64] (\u4ea4\u6362 seq_len \u548c num_heads \u7ef4\u5ea6)
\n # \u4e3a\u4ec0\u4e48\u8981\u8f6c\u7f6e？\u56e0\u4e3a\u6211\u4eec\u8981\u8ba9 Attention \u53d1\u751f\u5728 seq_len \u7ef4\u5ea6\u4e0a，
\n # \u628a num_heads \u653e\u5230\u524d\u9762，PyTorch \u5c31\u4f1a\u628a\u5b83\u4eec\u5f53\u6210\u72ec\u7acb\u7684"\u6279\u6b21"\u5e76\u884c\u8ba1\u7b97\u3002<\/p>\n

Q = self.linear_q(query).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
\n K = self.linear_k(key).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)
\n V = self.linear_v(value).view(batch_size, -1, self.num_heads, self.d_k).transpose(1, 2)<\/p>\n

# — \u7b2c\u4e8c\u6b65：\u7f29\u653e\u70b9\u79ef\u6ce8\u610f\u529b (Scaled Dot-Product Attention) —
\n # Q: [32, 8, 10, 64]
\n # K.T: [32, 8, 64, 10] (\u6700\u540e\u4e24\u4e2a\u7ef4\u5ea6\u8f6c\u7f6e)
\n # Matmul: [32, 8, 10, 10] -> \u5f97\u5230\u6bcf\u4e2a\u5934\u5185\u90e8\u7684\u8bcd\u4e0e\u8bcd\u7684\u5173\u7cfb\u5206\u6570
\n scores = torch.matmul(Q, K.transpose(-2, -1)) \/ math.sqrt(self.d_k)<\/p>\n

if mask is not None:
\n scores = scores.masked_fill(mask == 0, -1e9)<\/p>\n

attn_weights = torch.softmax(scores, dim=-1) # [32, 8, 10, 10]<\/p>\n

# \u52a0\u6743\u6c42\u548c: [32, 8, 10, 10] * [32, 8, 10, 64] -> [32, 8, 10, 64]
\n context = torch.matmul(attn_weights, V)<\/p>\n

# — \u7b2c\u4e09\u6b65：\u62fc\u63a5\u5934 (Concat) —
\n # transpose: -> [32, 10, 8, 64] (\u628a seq_len \u6362\u56de\u6765)
\n # contiguous().view: -> [32, 10, 512] (\u628a 8 \u548c 64 \u91cd\u65b0\u7c98\u5408\u5728\u4e00\u8d77)
\n context = context.transpose(1, 2).contiguous().view(batch_size, -1, self.d_model)<\/p>\n

# — \u7b2c\u56db\u6b65：\u6700\u7ec8\u7ebf\u6027\u53d8\u6362 —
\n return self.linear_out(context)<\/p>\n

\u8981\u70b9：\u81ea\u6ce8\u610f\u529b\u7684\u201c\u81ea\u201c\u7684\u5355\u4f4d\u662f\u53e5\u5b50，\u56e0\u6b64\u7f16\u7801\u5668 layer \u4e2d x_attn = self.self_attn(x, x, x, mask)，\u5b9e\u9645\u4e0a\u8f93\u51fa\u4e86\u53e5\u5b50\u4e2d\u6bcf\u4e2a token \u5bf9\u5176\u5b83 token \u7684\u6ce8\u610f\u529b\u54cd\u5e94\u3002<\/p>\n

\u8f93\u5165\u5f20\u91cf，\u5f62\u72b6：[32, 8, 10, 64]<\/p>\n\n\n\n\n\n\n\n
\u7ef4\u5ea6\u7d22\u5f15<\/td>\n\u7ef4\u5ea6\u540d\u79f0<\/td>\n\u793a\u4f8b\u503c<\/td>\n\u89e3\u91ca\u4e0e\u4ea4\u4e92\u60c5\u51b5<\/td>\n<\/tr>\n
Dim 0<\/td>\nbatch_size<\/td>\n32<\/td>\n\u72ec\u7acb\u7684\u6837\u672c\u7d22\u5f15\u3002 \u4ee3\u8868 32 \u4e2a\u6beb\u65e0\u5173\u7cfb\u7684\u53e5\u5b50\u540c\u65f6\u5904\u7406（\u4f8b\u5982\u53e5\u5b50 A \u548c\u53e5\u5b50 B）\u3002<\/td>\n<\/tr>\n
Dim 1<\/td>\nnum_heads<\/td>\n8<\/td>\n\u8bed\u4e49\u5b50\u7a7a\u95f4\u3002 \u5c06 512 \u7ef4\u62c6\u5206\u6210 8 \u7ec4，\u6bcf\u7ec4\u5173\u6ce8\u4e0d\u540c\u7279\u5f81（\u5982 Head 1 \u770b\u8bed\u6cd5，Head 2 \u770b\u6307\u4ee3）\u3002<\/td>\n<\/tr>\n
Dim 2<\/td>\nseq_len<\/td>\n10<\/td>\n\u8bcd\u7684\u4f4d\u7f6e\u3002 \u4ee3\u8868\u53e5\u5b50\u91cc\u7684\u5177\u4f53\u54ea\u4e2a\u8bcd（\u5982\u7b2c 3 \u4e2a\u8bcd\u201c\u7231\u201d）\u3002<\/td>\n<\/tr>\n
Dim 3<\/td>\nd_k<\/td>\n64<\/td>\n\u7279\u5f81\u5c5e\u6027\u5411\u91cf\u3002 \u63cf\u8ff0\u8be5\u8bcd\u5728\u5f53\u524d\u89c6\u89d2\u4e0b\u7684\u5177\u4f53\u5c5e\u6027（\u5982\u201c\u6211\u662f\u7ea2\u8272\u7684\u201d\u6216\u201c\u6211\u662f\u540d\u8bcd\u201d）\u3002Q \u548c K \u5728\u6b64\u7ef4\u5ea6\u8fdb\u884c\u70b9\u79ef，\u7b97\u51fa\u5339\u914d\u5206\u6570\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

d_k \u5b9e\u9645\u4e0a\u5c31\u662f\u5bf9\u8bcd\u5411\u91cf\u7684\u5206\u5272\u3002\u56e0\u4e3a\u5e0c\u671b\u8ba9\u6ce8\u610f\u529b\u7684\u6bcf\u4e2a\u5934\u80fd\u9488\u5bf9\u4e0d\u540c\u7684\u8bed\u4e49\u505a\u89c2\u5bdf，\u5c06\u8bcd\u5411\u91cf d_model \u5206\u4e3a\u201c\u6ce8\u610f\u529b\u5934\u201d\u6570\u4e2a\u5b50\u5411\u91cf d_k = d_model \/\/ num_heads\u3002<\/p>\n

\u62c6\u89e3\u89c6\u89d2：\u5c06\u5f85\u8bad\u7ec3\u7684\u53e5\u5b50\u7528\u4e00\u4e2a 32 * 10 \u7ef4\u7684\u8868\u683c，\u5176\u4e2d，\u53e5\u5b50\u603b\u6570\u662f 32（\u4e00\u4e2a batch），\u6bcf\u4e2a\u53e5\u5b50\u957f\u5ea6\u56fa\u5b9a\u4e3a 10（seq_len）\u3002\u8868\u683c\u4e2d\u7684\u6bcf\u4e2a\u5404\u81ea\u586b\u5199\u4e86\u4e00\u4e2a 8 * 64 \u7ef4\u7684\u8868\u683c\u7528\u4e8e\u8868\u793a\u4e00\u4e2a\u8bcd\u3002<\/p>\n

QKV \u7684\u5f62\u72b6\u5206\u6790：$W_{Q,K,V} = [512 * 512]$, \u53d8\u6362\u540e\u7684 $Q,K,V = [32, 8, 10, 64]$\u3002<\/p>\n

$W_{Q,K,V} = [512 * 512]$ \u662f\u5bf9\u8f93\u5165\u7684\u7ebf\u6027\u53d8\u5316，\u4e0d\u8003\u8651\u610f\u4e49\u7684\u60c5\u51b5\u4e0b，\u5c31\u662f\u77e9\u9635\u4e58\u6cd5\u3002\u4e3a\u4e86\u5339\u914d\u6bcf\u4e2a\u8bcd\u7684\u7279\u5f81\u5411\u91cf\u957f\u5ea6\u4e14\u8f6c\u6362\u4e0d\u4e22\u5931\u8bed\u4e49，\u56e0\u6b64\u53d8\u5316\u77e9\u9635\u5927\u5c0f\u540c\u8bcd\u5411\u91cf\u957f\u5ea6\u5339\u914d\u3002<\/p>\n

$Q,K,V = [32, 8, 10, 64]$ \u662f\u7ecf\u8fc7\u4e0a\u8ff0\u53d8\u5316\u540e\u7684\u7ed3\u679c\u300232 \u4e3a batch \u4e2d\u5305\u542b\u7684\u53e5\u5b50\u6570\u91cf，10 \u4e3a\u786e\u5b9a\u7684\u53e5\u5b50\u957f\u5ea6\u3002\u6709\u533a\u522b\u7684\u662f 8 * 64 \u7684\u5411\u91cf\u542b\u4e49\u3002QKV \u5206\u522b\u5bf9\u5e94\u7684\u662f Q: \u4e00\u4e2a\u8bcd\u5411\u5916\u67e5\u8be2\u7684\u5185\u5bb9, K: \u4e00\u4e2a\u8bcd\u5305\u542b\u7684\u8bed\u4e49，V: QK \u4f1a\u5bfc\u81f4\u7684\u4fe1\u606f\u6536\u76ca\u3002（\u4ee5\u4e0a\u4e3a\u4e2a\u4eba\u7406\u89e3）<\/p>\n

\u6ce8\u610f，\u4e58\u79ef\u7ed3\u679c\u7684\u539f\u59cb QKV \u662f $Q,K,V = [32, 10, 512]$，\u63a5\u4e0b\u6765\u7b80\u5355\u5206\u5272\u7ef4\u5ea6\u5c31\u53ef\u4ee5\u5f97\u5230 $Q,K,V = [32, 8, 10, 64]$\u3002<\/p>\n

QK \u78b0\u649e\u7684\u7ed3\u679c<\/p>\n\n\n\n\n\n\n
\u64cd\u4f5c\u6b65\u9aa4<\/td>\n\u8f93\u5165\u5f62\u72b6<\/td>\n\u53d8\u6362\u540e\u5f62\u72b6<\/td>\n\u7269\u7406\u610f\u4e49\u53d8\u5316<\/td>\n<\/tr>\n
1. \u51c6\u5907<\/td>\n\n

Q: [32, 8, 10, 64]<\/p>\n

\u00a0<\/p>\n

K: [32, 8, 10, 64]<\/p>\n<\/td>\n

\n

K \u8f6c\u7f6e\u4e3a:<\/p>\n

\u00a0<\/p>\n

[32, 8, 64, 10]<\/p>\n<\/td>\n

\u4e3a\u4e86\u8ba9 K \u7684\u7279\u5f81\u7ef4\u5ea6 (64) \u5bf9\u9f50 Q \u7684\u7279\u5f81\u7ef4\u5ea6，\u51c6\u5907\u8fdb\u884c\u70b9\u79ef\u3002<\/td>\n<\/tr>\n
2. \u78b0\u649e (Matmul)<\/td>\nQ $\\\\times$ K\u1d40<\/td>\n[32, 8, 10, 10]<\/td>\n\u7ef4\u5ea6 64 \u88ab\u6d88\u8017 (\u6c42\u548c) \u6389\u4e86\u3002\u5269\u4e0b\u7684 [10, 10] \u662f Attention Map (\u6ce8\u610f\u529b\u56fe)\u3002<\/td>\n<\/tr>\n
3. \u7ed3\u679c\u89e3\u8bfb<\/td>\nScore[b, h, i, j]<\/td>\n(\u6807\u91cf\u6570\u503c)<\/td>\n\u5173\u6ce8\u5ea6 \/ \u6743\u91cd\u3002 \u4ee3\u8868\u7b2c b \u4e2a\u53e5\u5b50\u7684\u7b2c h \u4e2a\u89c6\u89d2\u4e0b，\u7b2c i \u4e2a\u8bcd\u5bf9\u7b2c j \u4e2a\u8bcd\u7684\u5173\u6ce8\u7a0b\u5ea6\u3002\u4f8b\u5982：Score("it", "animal") = 0.9 (\u9ad8\u5ea6\u5173\u6ce8)\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n

\u5c06 QK \u8fdb\u884c\u78b0\u649e（\u77e9\u9635\u4e58\u6cd5），\u5f97\u5230\u4e86\u4e00\u4e2a [32, 8, 10, 10] \u7684\u77e9\u9635，\u8fd9\u4e2a\u77e9\u9635\u524d\u4e24\u4e2a\u6807\u8bc6\u662f\u54ea\u4e00\u4e2a\u53e5\u5b50\u7684\u90a3\u4e00\u4e2a\u5173\u6ce8\u70b9，\u540e\u4e24\u4e2a 10 \u8868\u793a\u5355\u4e2a\u53e5\u5b50\u7684\u4efb\u610f\u4e24\u4e2a\u8bcd\u4e4b\u95f4\u7684\u5173\u7cfb\u5f3a\u5f31\u3002<\/p>\n

\u4e3a\u4e86\u8ba9\u8fd9\u4e2a\u5173\u7cfb\u8868\u73b0\u4e3a\u6982\u7387，\u6ce8\u610f\u5bf9\u5176\u8fdb\u884c Softmax\u3002<\/p>\n

\u57fa\u4e8e\u8fd9\u4e2a\u5173\u7cfb\u5f3a\u5f31\u7684\u5173\u7cfb：score = [32, 8, 10, 10], \u6211\u4eec\u9700\u8981\u57fa\u4e8e Value \u83b7\u5f97\u771f\u6b63\u7684\u8bed\u4e49\u4fe1\u606f，\u5373\u6709：<\/p>\n

# [10, 10] * [10, 64] -> [10, 64]
\ncontext = torch.matmul(attn_weights, V)<\/p>\n

\u524d\u4e24\u4e2a\u7ef4\u5ea6\u7684 batch \u4e2d\u53e5\u5b50\u603b\u6570\u548c\u6ce8\u610f\u529b\u5934\u6570\u4e0d\u53d8，\u540e\u9762\u53d8\u6210\u4e86\u53e5\u5b50\u4e2d\u6bcf\u4e2a\u8bcd\u6240\u643a\u5e26\u7684\u5176\u4ed6\u8bcd\u8d4b\u4e88\u7684\u8bed\u4e49\u91cf（\u4f8b\u5982\u4ee3\u8bcd "it" \u539f\u672c\u7684\u8f93\u5165\u7279\u5f81\u53ea\u6709\u5176\u672c\u8eab\u7684\u201c\u5b83\u201d，\u73b0\u5728\u548c\u53e5\u5b50\u4e2d\u5176\u5b83\u8bcd\u4f8b\u5982\u201cdog\u201d\u7684 value \u4e58\u4e86\u4e0b，\u73b0\u5728 "it" \u4e5f\u5305\u542b "dog" \u7684\u8bed\u4e49\u4fe1\u606f\u4e86）\u3002<\/p>\n

\u6700\u540e\u63d0\u4f9b\u7ed9\u4e0b\u4e00\u5c42 FFN \u7684\u8bed\u4e49\u4fe1\u606f\u4e3a：<\/p>\n

# [32, 8, 10, 64] -> \u8f6c\u7f6e -> [32, 10, 8, 64]
\n# view -> [32, 10, 512] (8 * 64 = 512)
\ncontext = context.transpose(1, 2).contiguous().view(batch_size, -1, self.d_model)<\/p>\n

# — \u7b2c\u56db\u6b65：\u6700\u7ec8\u7ebf\u6027\u53d8\u6362 —
\nreturn self.linear_out(context)<\/p>\n

\u9996\u5148\u8fdb\u884c\u76f4\u63a5\u7684\u591a\u5934\u62fc\u63a5，\u4f46\u662f\u8fd9\u79cd\u62fc\u63a5\u592a\u786c\u4e86，\u56e0\u6b64\u4f7f\u7528\u4e00\u4e2a\u7ebf\u6027\u5c42（\u77e9\u9635\u4e58\u6cd5）\u505a\u5f62\u72b6\u4e0d\u53d8\u7684\u4eff\u5c04\u53d8\u6362（\u4e58\u4ee5\u4e00\u4e2a [512, 512] \u7684\u77e9\u9635，\u8bed\u4e49\u77e9\u9635\u7684\u5f62\u72b6\u4e0d\u53d8，\u4f46\u662f\u5177\u4f53\u8bed\u4e49\u4fe1\u606f\u6709\u4e00\u5b9a\u7684\u878d\u5408）\u3002<\/p>\n

\u6c47\u603b\u4e0b\u6570\u636e\u5f62\u72b6\u7684\u53d8\u5316：<\/p>\n

    \n
  • \n

    \u8f93\u5165\u4e00\u4e2a batch \u7684\u53e5\u5b50：input[32, 10, 512] (32 \u53e5\u8bdd， \u6bcf\u4e2a\u53e5\u5b50\u957f\u5ea6 10 token，\u6bcf\u4e2a token \u7684\u8bed\u4e49\u7528 512 \u7ef4\u7684\u8bed\u4e49\u5411\u91cf\u8868\u793a)\u3002<\/p>\n<\/li>\n

  • \n

    \u8f93\u5165\u5230\u591a\u5934（\u5047\u8bbe 8 \u4e2a\u5934）\u6ce8\u610f\u529b\u5c42\u3002<\/p>\n<\/li>\n

  • \n

    \u901a\u8fc7\u4e09\u4e2a\u4eff\u5c04\u53d8\u6362\u83b7\u5f97\u4e0d\u540c\u7528\u9014\u7684\u8bcd\u8bed\u8bed\u4e49 Q, K, V \u4e09\u4e2a\u8bed\u4e49\u77e9\u9635\u90fd\u662f\u7531 [512 * 512] \u5927\u5c0f\u7684\u4e09\u4e2a\u53d8\u6362\u9635\u4e58\u51fa\u6765\u7684\u3002\u7136\u540e\u5bf9\u4e8e\u8f93\u51fa\u7684 QKV \u62c6\u5206\u5230\u5c06\u8bed\u4e49\u5207\u5206\u4e3a input_multi[32, 8, 10, 64] (32 \u53e5\u8bdd，\u9700\u8981\u5173\u6ce8 8 \u4e2a\u7ef4\u5ea6\u7684\u8bed\u4e49， \u6bcf\u4e2a\u53e5\u5b50\u957f\u5ea6 10 token，\u6bcf\u4e2a token \u7684\u5728\u67d0\u4e2a\u7ef4\u5ea6\u7684\u8bed\u4e49\u7528 64 \u7ef4\u7684\u8bed\u4e49\u5411\u91cf\u8868\u793a)\u3002<\/p>\n<\/li>\n

  • \n

    \u4e4b\u540e\u901a\u8fc7 Q, K \u78b0\u649e\u5f97\u5230\u8bcd\u4e0e\u8bcd\u4e4b\u95f4\u7684\u5173\u7cfb\u5206\u6570 Score[32, 8, 10, 10]，\u8fd9\u4e2a Score \u4f1a\u518d\u548c V \u505a\u4e58\u79ef\u4ee5\u901a\u8fc7\u5176\u5b83\u8bcd\u8bed\u7684\u8bed\u4e49\u6765\u5f97\u5230\u67d0\u4e2a\u8bcd\u7684\u8868\u8fbe context[32, 8, 10, 64]\u3002<\/p>\n<\/li>\n

  • \n

    \u6700\u540e\u5c06 context \u5404\u7ef4\u5ea6\u62fc\u63a5，\u8f6c\u4e3a [32, 10, 512]，\u63a5\u7740\u4e3a\u4e86\u9632\u6b62\u62fc\u63a5\u8fc7\u4e8e\u751f\u786c，\u901a\u8fc7\u4e00\u5c42\u53cd\u5c04\u53d8\u6362（\u4e0e\u4e00\u4e2a [512, 512] \u7684\u77e9\u9635\u505a\u4e58\u79ef）\u5f97\u5230\u6a59\u8272\u90e8\u5206\u7684\u8f93\u51fa Atten_out[32, 10, 512]\u3002<\/p>\n<\/li>\n

  • \n

    \u5c06\u8f93\u5165 input[32, 10, 512] \u548c\u8f93\u51fa Atten_out \u76f8\u52a0\u7136\u540e\u8fdb\u884c\u6b63\u6001\u5206\u5e03\u5f52\u4e00\u5316，\u6700\u540e\u5728\u8fdb\u884c\u4e00\u6b21\u7f29\u653e\u5e73\u79fb\u4ee5\u5f25\u8865\u5f52\u4e00\u5316\u5bfc\u81f4\u7684\u8bed\u4e49\u635f\u5931\u3002（\u7f29\u653e\u5e73\u79fb\u662f\u5728\u505a\u9010\u5143\u7d20\u4e58\u6cd5，\u4e0d\u662f\u5f88\u6e05\u695a\u5f25\u8865\u7684\u80fd\u529b，\u5e94\u8be5\u662f\u6d88\u878d\u5b9e\u9a8c\u7ed3\u679c）\u8f93\u51fa\u5230 FFN \u7684\u8f93\u5165\u5f62\u72b6\u4fdd\u6301\u4e3a [32, 10, 512]\u3002\u8fd9\u90e8\u5206\u8017\u65f6\u57fa\u672c\u53ef\u5ffd\u7565\u3002<\/p>\n<\/li>\n

  • \n

    FFN \u5c31\u662f\u4e24\u4e2a\u57fa\u672c\u7ebf\u6027\u5c42，\u539f\u59cb\u64cd\u4f5c\u662f\u5148\u5347\u4e3a 2048 \u7ef4，\u7136\u540e\u964d\u56de 512 \u7ef4：[32, 10, 512] \u2192 [32, 10, 2048] \u2192 [32, 10, 512]\u3002<\/p>\n<\/li>\n

  • \n

    \u4e4b\u540e\u5bf9\u4e8e\u7ebf\u6027\u5c42\u8f93\u51fa\u5185\u5bb9，\u91cd\u65b0\u8f93\u56de\u7b2c\u4e00\u6b65，\u91cd\u590d N \u6b21\u540e\u8fdb\u5165\u89e3\u7801\u5668\u3002<\/p>\n<\/li>\n<\/ul>\n

    \n

    \u6b65\u9aa43：\u89e3\u7801\u5668\u90e8\u5206<\/h3>\n

    \"\"<\/p>\n

    import torch
    \nimport torch.nn as nn
    \nfrom copy import deepcopy<\/p>\n

    class Decoder(nn.Module):
    \n def __init__(self, layer, N):
    \n """
    \n \u6574\u4f53\u89e3\u7801\u5668 (\u7531 N \u4e2a DecoderLayer \u5806\u53e0\u800c\u6210)
    \n """
    \n super().__init__()
    \n # \u514b\u9686 N \u4e2a DecoderLayer
    \n self.layers = nn.ModuleList([deepcopy(layer) for _ in range(N)])
    \n self.norm = nn.LayerNorm(layer.norm1.normalized_shape[0])<\/p>\n

    def forward(self, x, enc_output, src_mask, trg_mask):
    \n """
    \n :param x: \u7ecf\u8fc7 Embedding + Positional Encoding \u540e\u7684\u76ee\u6807\u5e8f\u5217\u8f93\u5165
    \n :param enc_output: Encoder \u7684\u8f93\u51fa\u7ed3\u679c (Memory)
    \n """
    \n # \u4e32\u884c\u6d41\u7ecf\u6bcf\u4e00\u5c42
    \n for layer in self.layers:
    \n x = layer(x, enc_output, src_mask, trg_mask)<\/p>\n

    return self.norm(x)<\/p>\n

    \u7b2c\u4e00\u90e8\u5206：\u5e95\u90e8 outputs \u8f93\u5165\u6a21\u578b<\/h4>\n

    \u8fd9\u4e2a\u5730\u65b9\u548c RNN \u4e00\u6837，\u9996\u5148\u6211\u4eec\u83b7\u5f97\u4e00\u4e2a\u7a7a\u5e8f\u5217 [<SOS>] \u8868\u793a\u53e5\u5b50\u5f00\u59cb（Start of the Sentence）\u3002<\/p>\n

    \u7136\u540e\u6bcf\u4e00\u6b21\u89e3\u7801\u8fc7\u7a0b\u540e，\u9876\u90e8\u7684 Output \u8f93\u51fa\u9700\u8981\u52a0\u5165\u8fd9\u4e2a\u5e8f\u5217\u3002<\/p>\n

    \u4e00\u76f4\u8fd9\u6837\u91cd\u590d，\u76f4\u5230\u9876\u90e8 Output \u8f93\u51fa\u4e86 <EOS> (End of Sentence)\u3002<\/p>\n

    \u5177\u4f53\u7684\u5904\u7406\u65b9\u5f0f\u8fd8\u662f\u548c\u4e4b\u524d\u7684\u7f16\u7801\u5668\u90e8\u5206\u4e00\u6837，\u52a0\u4e0a\u4f4d\u7f6e\u7f16\u7801，\u7136\u540e\u8f93\u5165\u5230\u591a\u5934\u4e2d\u3002<\/p>\n

    \u9700\u8981\u6ce8\u610f\u7684\u662f，\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u8fd9\u4e2a outputs \u4e0d\u662f\u52a8\u6001\u751f\u6210\u7684，\u800c\u662f\u4e00\u4e2a\u5b57\u4e00\u4e2a\u5b57\u6f0f\u7ed9\u6a21\u578b\u770b，\u5e76\u884c\u8bad\u7ec3（\u5e94\u8be5\u662f\u5e76\u884c，\u6bcf\u4e2a\u6279\u6b21\u66f4\u65b0\u4e00\u6b21\u53c2\u6570）\u3002<\/p>\n

    \u7b2c\u4e8c\u90e8\u5206：\u63a9\u7801\u6ce8\u610f\u529b<\/h4>\n

    \u5728\u7f16\u7801\u5668\u4e2d\u7684\u53e5\u5b50\u5185\u5bb9\u662f\u6211\u4eec\u7684\u8f93\u5165\u7684 prompt，\u8fd9\u4e2a\u89e3\u7801\u5668\u90e8\u5206\u7684\u53e5\u5b50\u5185\u5bb9\u662f\u8f93\u51fa\u53e5\u5b50（LLM \u7684\u56de\u590d）\u3002\u6a21\u578b\u7684\u5176\u5b83\u8d85\u53c2\u6570\u90fd\u662f\u4e00\u81f4\u7684，\u53ea\u6709 Seq_len \u4e0d\u540c（\u5c31\u662f\u4e4b\u524d\u662f 10 \u7684\u90a3\u4e2a\u53c2\u6570）\u3002<\/p>\n

    \u5047\u8bbe\u5b83\u73b0\u5728\u662f 4\u3002<\/p>\n

    \u91cd\u65b0\u5217\u4e00\u904d\u53c2\u6570：<\/p>\n

      \n
    • \n

      Batch Size (B): 32 (\u4e00\u6b21\u5e76\u884c\u5904\u7406 32 \u53e5\u8bdd)<\/p>\n<\/li>\n

    • \n

      Seq Len (L): 4 (\u5047\u8bbe\u8fd9\u53e5\u8bdd\u662f [<SOS>, A, B, C])<\/p>\n<\/li>\n

    • \n

      d_model (D): 512<\/p>\n<\/li>\n

    • \n

      Heads (H): 8<\/p>\n<\/li>\n

    • \n

      d_k: 64<\/p>\n<\/li>\n<\/ul>\n

      \u524d\u9762 QKV \u751f\u6210\u8fc7\u7a0b\u8fd8\u662f\u4e00\u6837\u7684，\u5177\u4f53\u7684\u5f62\u72b6\u53d8\u6362\u8fc7\u7a0b\u5982\u4e0b：<\/p>\n

      Input: [32, 4, 512] \u2192 W_{Q,K,V}: [512, 512] \u21d2 Q,K,V: [32, 4, 512] \u2192 \u591a\u5934\u5212\u5206：[32, 8, 4, 64]<\/p>\n

      \u7136\u540e\u8ba1\u7b97 Score = Q \u00d7 K^t : [32, 8, 4, 4]\u3002<\/p>\n

      \u8fd9\u4e00\u6b65\u91cc\u9762\u9700\u8981\u4f7f\u7528\u63a9\u7801\u3002<\/p>\n

      \u8fd9\u4e2a Score \u8868\u793a\u7684\u662f\u5f53\u524d\u884c\u7d22\u5f15\u7684\u5bf9\u5e94\u8bcd\u6c47\u5bf9\u5217\u7d22\u5f15\u7684\u8bcd\u6c47\u7684\u76f8\u5173\u5ea6\u3002<\/p>\n

      \u4f46\u662f\u73b0\u5728\u7684\u8bcd\u6c47\u4e0d\u80fd\u770b\u5230\u540e\u9762\u8fd8\u672a\u751f\u6210\u7684\u8bcd\u6c47\u3002\u63a9\u7801\u903b\u8f91\u53c2\u8003\u4ee5\u4e0b\u5185\u5bb9：<\/p>\n

      Mask \u77e9\u9635<\/p>\n

      SOS A B C
      \nSOS [[ 0, 1, 1, 1 ], <- \u906e\u6321 A, B, C
      \n A [ 0, 0, 1, 1 ], <- \u906e\u6321 B, C
      \n B [ 0, 0, 0, 1 ], <- \u906e\u6321 C
      \n C [ 0, 0, 0, 0 ]] <- \u5168\u90fd\u80fd\u770b<\/p>\n

      Score \u77e9\u9635\u4e2d\u63a9\u7801\u4e3a 1 \u7684\u5730\u65b9\u8bbe\u7f6e\u4e3a -1e9 (\u6781\u5c0f\u503c)\u3002<\/p>\n

      \u56e0\u6b64\u53d8\u6362\u540e\u7684 Score \u77e9\u9635\u5927\u6982\u5982\u4e0b：<\/p>\n

      SOS A B C
      \nSOS [[ 2.5, -1e9, -1e9, -1e9 ], <- \u540e\u9762\u5168\u5e9f\u4e86
      \n A [ 1.8, 3.2, -1e9, -1e9 ],
      \n B [ 0.5, 1.5, 2.8, -1e9 ],
      \n C [ 0.2, 0.4, 1.2, 3.0 ]]<\/p>\n

      \u5bf9\u6bcf\u4e00\u884c Softmax \u540e, \u6781\u5c0f\u503c\u5c31\u8f6c\u4e3a\u6982\u7387 0 \u4e86\u3002<\/p>\n

      \u7136\u540e\u548c\u4e4b\u524d\u4e00\u6837，Score \u00d7 V = context[32, 4, 512]\u3002<\/p>\n

      \u7b2c\u4e09\u90e8\u5206：\u7f16\u7801\u5668\u4ea4\u4e92\u7684\u591a\u5934\u6ce8\u610f\u529b (\u4ea4\u53c9\u6ce8\u610f\u529b，\u533a\u522b\u4e8e\u4e4b\u524d\u7684\u81ea\u6ce8\u610f\u529b)<\/h4>\n

      \u53c2\u8003\u539f\u59cb\u7684\u6ce8\u610f\u529b\u4e09\u4e2a\u77e9\u9635：<\/p>\n

        \n
      • \n

        Q：\u76ee\u524d\u7684\u95ee\u9898<\/p>\n<\/li>\n

      • \n

        K：\u76ee\u524d\u53ef\u4ee5\u53c2\u8003\u7684\u4fe1\u606f<\/p>\n<\/li>\n

      • \n

        V：\u53e5\u5b50\u4e2d\u5404\u4e2a\u8bcd\u5bf9\u5e94\u7684\u542b\u4e49<\/p>\n<\/li>\n<\/ul>\n

        # x: \u89e3\u7801\u5668\u76ee\u524d\u7684\u8fdb\u5ea6 (\u6bd4\u5982\u8bf4\u5df2\u7ecf\u751f\u6210\u4e86\u524d3\u4e2a\u8bcd)
        \n# enc_output: \u7f16\u7801\u5668\u8f93\u51fa\u7684\u8bb0\u5fc6 (\u4e5f\u5c31\u662f\u539f\u6587\u7684\u5b8c\u6574\u7406\u89e3)<\/p>\n

        x = self.cross_attn(q=x, k=enc_output, v=enc_output, mask=src_mask)<\/p>\n

        \u5047\u8bbe\u6700\u7ec8\u7684 Ground Truth \u662f\u201c\u6211\u7231\u4f60！\u201d，\u7136\u540e LLM \u5df2\u7ecf\u8f93\u51fa\u4e86\u201c\u6211\u201d\u3002<\/p>\n

        \u73b0\u5728\u9700\u8981\u89e3\u7b54\u7684\u95ee\u9898\u662f：\u4e0b\u4e00\u4e2a\u5b57\u7b26\u5e94\u8be5\u8f93\u51fa\u4ec0\u4e48，\u56e0\u6b64\u9700\u8981\u8f93\u5165\u5df2\u6709\u7684\u8f93\u51fa\u4f5c\u4e3a\u67e5\u8be2\u4e5f\u5c31\u662f\u76ee\u524d\u8f93\u51fa\u7684 \u201c<SOS> \u6211\u201d\u3002<\/p>\n

        \u90a3\u4e48\u4f5c\u4e3a\u53c2\u8003\u7684\u5e94\u8be5\u662f\u4f60\u8f93\u5165\u7684 prompt（\u8981\u8f93\u51fa\u201c\u6211\u7231\u4f60\u201d\u5927\u7ea6\u662f\u8f93\u5165\u4e86\u201c\u4f60\u7231\u6211\u5417？\u201d\u8fd9\u79cd input）\u3002<\/p>\n

        \u7136\u540e\u6211\u9700\u8981\u7684\u8bed\u4e49\u4e5f\u662f\u4ece\u539f\u59cb\u8f93\u5165\u4e2d\u62ff\u7684\u3002<\/p>\n

        \u6240\u4ee5 Q=x（\u5f53\u524d\u8f93\u51fa\u4e00\u534a\u7684\u5185\u5bb9），KV \u90fd\u662f encoder \u5728\u7ecf\u8fc7 N \u5c42\u540e\u7684\u539f\u59cb\u8f93\u5165\u3002<\/p>\n

        \u6765\u770b\u770b\u5f20\u91cf\u5f62\u72b6：<\/p>\n

        \u4e4b\u524d\u8bf4\u5230 Encoder \u7684\u8f93\u51fa\u6309\u7167\u8bbe\u5b9a\u662f [32, 10, 512]\u3002<\/p>\n

        \u8fd9\u91cc\u7684 x \u867d\u7136\u6ca1\u751f\u6210\u5b8c，\u4f46\u662f\u4e5f\u4f1a\u88ab\u586b\u5145\u5230\u9884\u671f\u7684\u5927\u5c0f，\u662f [32, 4, 512]\u3002<\/p>\n

        \u7136\u540e\u6211\u4eec QKV \u8fd8\u662f\u548c\u4e4b\u524d\u4e00\u6837\u4f7f\u7528 [512, 512] \u5927\u5c0f\u7684\u8f6c\u6362\u77e9\u9635\u3002\u90a3\u4e48\u5f97\u5230：<\/p>\n

          \n
        • \n

          Q：[32, 4, 512]<\/p>\n<\/li>\n

        • \n

          K：[32, 10, 512]<\/p>\n<\/li>\n

        • \n

          V：[32, 10, 512]<\/p>\n<\/li>\n<\/ul>\n

          \u7136\u540e\u6211\u4eec $QK^T$ \u5f97\u5230 score [32, 4, 10]\u3002<\/p>\n

          \u5bf9\u4e8e prompt \u4e2d\u7684\u586b\u5145\u90e8\u5206（\u4e00\u53e5\u8bdd\u957f\u5ea6\u8fd8\u4e0d\u5230 10，\u5269\u4e0b\u7684\u7528 <PAD> \u586b\u5145）\u7528\u6781\u5c0f\u503c\u8fdb\u884c\u586b\u5145\u3002<\/p>\n

          \u5bf9\u4e8e decoder \u7684\u8f93\u51fa，\u7531\u4e8e\u4e0a\u4e00\u6b65\u7684\u63a9\u7801\u64cd\u4f5c，\u8fd9\u4e00\u6b65 decoder \u5185\u5bb9\u4e0d\u9700\u8981\u91cd\u65b0\u63a9\u7801，\u76f4\u63a5\u6cbf\u7528\u4e0a\u4e00\u8f6e\u7684\u4e0a\u4e0b\u6587\u7ed3\u679c\u5373\u53ef\u3002<\/p>\n

          \u8fd9\u6837\u6211\u4eec\u5c31\u5f97\u5230\u4e86 Q \u4e2d\u6bcf\u4e2a\u5b57\u7b26\u5bf9\u5e94 K \u4e2d\u6bcf\u4e2a\u5b57\u7b26\u7684\u76f8\u5173\u6027\u4e86\u3002<\/p>\n

          Transformer \u5f3a\u5927\u7684\u5730\u65b9\u5728\u4e8e，\u53ea\u8981\u8bcd\u7684\u5411\u91cf\u8868\u793a\u7ef4\u5ea6\u76f8\u540c，\u90a3\u4e48\u4ea4\u53c9\u6ce8\u610f\u529b\u53ef\u4ee5\u6355\u83b7\u4e24\u4e2a\u4efb\u610f\u957f\u5e8f\u5217\u7684\u6bcf\u4e2a\u8bcd\u4e4b\u95f4\u7684\u76f8\u5173\u6027\u3002<\/p>\n

          \u6700\u7ec8\u6211\u4eec\u4e58\u4e0a V \u77e9\u9635，\u5f97\u5230\u4e86 context[32, 4, 512], \u4e5f\u5c31\u662f\u8fd9\u53e5\u8bdd\u4e2d\u7684\u6bcf\u4e2a\u8bcd\u6240\u5305\u542b\u7684\u5185\u5bb9\u542b\u4e49\u3002<\/p>\n

          \u7b2c\u56db\u90e8\u5206：FFN<\/h4>\n

          \u548c\u7f16\u7801\u5668\u90e8\u5206 FFN \u7684\u903b\u8f91\u5b8c\u5168\u4e00\u81f4，\u9996\u5148\u8fdb\u884c\u76f4\u63a5\u7684\u591a\u5934\u62fc\u63a5，\u4f46\u662f\u8fd9\u79cd\u62fc\u63a5\u592a\u786c\u4e86，\u56e0\u6b64\u4f7f\u7528\u4e00\u4e2a\u7ebf\u6027\u5c42（\u77e9\u9635\u4e58\u6cd5）\u505a\u5f62\u72b6\u4e0d\u53d8\u7684\u4eff\u5c04\u53d8\u6362（\u4e58\u4ee5\u4e00\u4e2a [512, 512] \u7684\u77e9\u9635，\u8bed\u4e49\u77e9\u9635\u7684\u5f62\u72b6\u4e0d\u53d8，\u4f46\u662f\u5177\u4f53\u8bed\u4e49\u4fe1\u606f\u6709\u4e00\u5b9a\u7684\u878d\u5408）\u3002<\/p>\n

          \u6700\u540e\u8f93\u51fa\u4e86\u4e00\u4e2a\u5e73\u6ed1\u539a\u7684 context[32, 4, 512]\u3002<\/p>\n

          \n

          \u6b65\u9aa44：\u6700\u7ec8\u7ebf\u6027\u5c42\u4e0e\u5206\u7c7b\u8f93\u51fa<\/h3>\n

          \u6700\u540e\u4e00\u5c42\u7684\u4ea4\u53c9\u6ce8\u610f\u529b\u6211\u4eec\u83b7\u5f97\u4e86 context[32, 4, 512]，\u5176\u5177\u4f53\u542b\u4e49\u662f：<\/p>\n

            \n
          • \n

            \u8fd9\u4e2a batch \u4e2d\u7684\u53e5\u5b50\u4e2a\u6570：32<\/p>\n<\/li>\n

          • \n

            \u6bcf\u4e2a\u53e5\u5b50\u5728\u586b\u5145\u540e\u7684 token \u5b9a\u957f：4<\/p>\n<\/li>\n

          • \n

            \u6bcf\u4e2a token \u7684\u8bed\u4e49\u5411\u91cf：512（\u878d\u5408\u4e86\u8fd9\u4e2a token \u5bf9\u5176\u5b83 token \u7684\u8bed\u4e49\u6444\u5165，\u8fd9\u91cc\u7684 "it" \u6216 "\u4f60\u6211\u4ed6" \u4e0d\u662f\u7b80\u5355\u4ee3\u8bcd\u4e86，\u5df2\u7ecf\u878d\u5408\u4e86\u5b83\u5177\u4f53\u6307\u5e26\u7684\u5bf9\u8c61\u7684\u8bed\u4e49）\u3002<\/p>\n<\/li>\n<\/ul>\n

            \u6b64\u65f6\u6211\u4eec\u505a\u4e00\u6b21\u7ebf\u6027\u6620\u5c04，\u628a\u8fd9\u4e2a\u53e5\u5b50 4 \u4e2a token \u7684\u8bed\u4e49\u5411\u91cf\u6620\u5c04\u5230\u8bcd\u8868\u5927\u5c0f：<\/p>\n

              \n
            • \n

              pred_result: [32, 4, 30000] \u8868\u793a\u6bcf\u4e2a\u4f4d\u7f6e\u7684\u8bcd\u4e3a\u8bcd\u8868\u4e2d\u67d0\u4e2a\u8bcd\u7684\u6982\u7387，\u6982\u7387\u6700\u9ad8\u7684\u90a3\u4e2a\u8bcd\u5373\u4e3a\u7ed3\u679c\u3002<\/p>\n<\/li>\n

            • \n

              ground_truth: [32, 4, 1] \u2192 \u63a8\u5e7f\u5230 30000 \u7ef4，\u5373\u53ea\u6709\u6b63\u786e\u7684\u8bcd\u4e3a 1，\u5176\u4f59\u4e3a 0 \u2192 [32, 4, 30000]\u3002<\/p>\n<\/li>\n<\/ul>\n","protected":false},"excerpt":{"rendered":"

              \u6b65\u9aa41：\u4f4d\u7f6e\u5d4c\u5165\u4e0e\u8bcd\u6620\u5c04import torch
              \nimport torch.nn as nn
              \nimport mathclass TransformerInput(nn.Module):def __init__(self, vocab_size, d_model, max_len5000):super().__init__()# 1. Embedding \u5c42：\u5c06\u5355\u8bcd\u7d22\u5f15\u8f6c\u4e3a 512 \u7ef4\u5411\u91cfself.embedding nn.Embedding(vo<\/p>\n","protected":false},"author":2,"featured_media":72566,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[841,50,86],"topic":[],"class_list":["post-72570","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-server","tag-transformer","tag-50","tag-86"],"yoast_head":"\n\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/72570.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u6b65\u9aa41：\u4f4d\u7f6e\u5d4c\u5165\u4e0e\u8bcd\u6620\u5c04import torch import torch.nn as nn import mathclass TransformerInput(nn.Module):def __init__(self, vocab_size, d_model, max_len5000):super().__init__()# 1. Embedding \u5c42：\u5c06\u5355\u8bcd\u7d22\u5f15\u8f6c\u4e3a 512 \u7ef4\u5411\u91cfself.embedding nn.Embedding(vo\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/72570.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-02-05T16:32:50+00:00\" \/>\n<meta property=\"og:image\" content=\"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/02\/20260205163249-6984c63177b9b.png\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"7 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/72570.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/72570.html\",\"name\":\"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-02-05T16:32:50+00:00\",\"dateModified\":\"2026-02-05T16:32:50+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/72570.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/72570.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/72570.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/72570.html","og_locale":"zh_CN","og_type":"article","og_title":"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u6b65\u9aa41：\u4f4d\u7f6e\u5d4c\u5165\u4e0e\u8bcd\u6620\u5c04import torch import torch.nn as nn import mathclass TransformerInput(nn.Module):def __init__(self, vocab_size, d_model, max_len5000):super().__init__()# 1. Embedding \u5c42：\u5c06\u5355\u8bcd\u7d22\u5f15\u8f6c\u4e3a 512 \u7ef4\u5411\u91cfself.embedding nn.Embedding(vo","og_url":"https:\/\/www.wsisp.com\/helps\/72570.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-02-05T16:32:50+00:00","og_image":[{"url":"https:\/\/www.wsisp.com\/helps\/wp-content\/uploads\/2026\/02\/20260205163249-6984c63177b9b.png"}],"author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"7 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/72570.html","url":"https:\/\/www.wsisp.com\/helps\/72570.html","name":"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-02-05T16:32:50+00:00","dateModified":"2026-02-05T16:32:50+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/72570.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/72570.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/72570.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"\u624b\u63a8Transformer\u5f20\u91cf\u4f20\u9012\u8fc7\u7a0b"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/72570","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=72570"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/72570\/revisions"}],"wp:featuredmedia":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media\/72566"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=72570"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=72570"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=72570"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=72570"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}