{"id":97858,"date":"2026-08-30T14:24:52","date_gmt":"2026-08-30T06:24:52","guid":{"rendered":"https:\/\/www.wsisp.com\/helps\/97858.html"},"modified":"2026-08-30T14:24:52","modified_gmt":"2026-08-30T06:24:52","slug":"nanogpt-%e9%80%90%e8%a1%8c%e8%ae%b2%e8%a7%a3","status":"publish","type":"post","link":"https:\/\/www.wsisp.com\/helps\/97858.html","title":{"rendered":"nanoGPT \u9010\u884c\u8bb2\u89e3"},"content":{"rendered":"
\u4e00\u3001model.py \u5b8c\u6574\u89e3\u6790<\/h5>\n

model.py \u662f\u6574\u4e2a\u9879\u76ee\u7684\u6838\u5fc3，\u53ea\u6709 330 \u884c\u4ee3\u7801，\u5374\u5b9e\u73b0\u4e86\u5b8c\u6574\u7684 GPT \u6a21\u578b\u3002<\/p>\n

1. LayerNorm（\u7b2c18-27\u884c）<\/h6>\n

\nclass LayerNorm(nn.Module):
\n\u00a0 \u00a0 def __init__(self, ndim, bias):
\n\u00a0 \u00a0 \u00a0 \u00a0 super().__init__()
\n\u00a0 \u00a0 \u00a0 \u00a0 self.weight = nn.Parameter(torch.ones(ndim))
\n\u00a0 \u00a0 \u00a0 \u00a0 self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None<\/p>\n

\u00a0 \u00a0 def forward(self, input):
\n\u00a0 \u00a0 \u00a0 \u00a0 return F.layer_norm(input, self.weight.shape, self.weight, self.bias, 1e-5)
\n\u00a0<\/p>\n

\u4f5c\u7528：\u5c42\u5f52\u4e00\u5316，\u8ba9\u6bcf\u5c42\u7684\u8f93\u51fa\u5206\u5e03\u66f4\u7a33\u5b9a<\/p>\n

\u5173\u952e\u70b9：
\n– `weight` \u548c `bias` \u662f\u53ef\u5b66\u4e60\u53c2\u6570
\n– PyTorch \u7684 `F.layer_norm` \u9ed8\u8ba4\u4e0d\u652f\u6301 bias=False，\u8fd9\u91cc\u505a\u4e86\u5c01\u88c5
\n– `1e-5` \u662f epsilon，\u9632\u6b62\u9664\u96f6<\/p>\n

\u00a02. CausalSelfAttention（\u7b2c29-76\u884c）\u2014\u2014 \u6700\u91cd\u8981！<\/h6>\n

\nclass CausalSelfAttention(nn.Module):
\n\u00a0 \u00a0 def __init__(self, config):
\n\u00a0 \u00a0 \u00a0 \u00a0 super().__init__()
\n\u00a0 \u00a0 \u00a0 \u00a0 assert config.n_embd % config.n_head == 0
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # QKV \u6295\u5f71：\u4e00\u4e2a\u7ebf\u6027\u5c42\u540c\u65f6\u8ba1\u7b97 Q, K, V
\n\u00a0 \u00a0 \u00a0 \u00a0 self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u8f93\u51fa\u6295\u5f71
\n\u00a0 \u00a0 \u00a0 \u00a0 self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u6b63\u5219\u5316
\n\u00a0 \u00a0 \u00a0 \u00a0 self.attn_dropout = nn.Dropout(config.dropout)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.resid_dropout = nn.Dropout(config.dropout)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 self.n_head = config.n_head
\n\u00a0 \u00a0 \u00a0 \u00a0 self.n_embd = config.n_embd
\n\u00a0 \u00a0 \u00a0 \u00a0 self.dropout = config.dropout
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # Flash Attention \u68c0\u67e5（PyTorch >= 2.0）
\n\u00a0 \u00a0 \u00a0 \u00a0 self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention')
\n\u00a0 \u00a0 \u00a0 \u00a0 if not self.flash:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u56e0\u679c\u63a9\u7801：\u786e\u4fdd\u53ea\u80fd\u770b\u5230\u5de6\u8fb9\u7684token
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size))
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 .view(1, 1, config.block_size, config.block_size))<\/p>\n

\u00a0 \u00a0 def forward(self, x):
\n\u00a0 \u00a0 \u00a0 \u00a0 B, T, C = x.size() \u00a0# Batch, Time, Channel
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 1. \u6295\u5f71\u5f97\u5230 Q, K, V
\n\u00a0 \u00a0 \u00a0 \u00a0 q, k, v = self.c_attn(x).split(self.n_embd, dim=2)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 2. \u591a\u5934\u91cd\u5851：[B, T, C] \u2192 [B, heads, T, head_dim]
\n\u00a0 \u00a0 \u00a0 \u00a0 k = k.view(B, T, self.n_head, C \/\/ self.n_head).transpose(1, 2)
\n\u00a0 \u00a0 \u00a0 \u00a0 q = q.view(B, T, self.n_head, C \/\/ self.n_head).transpose(1, 2)
\n\u00a0 \u00a0 \u00a0 \u00a0 v = v.view(B, T, self.n_head, C \/\/ self.n_head).transpose(1, 2)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 3. \u8ba1\u7b97\u6ce8\u610f\u529b
\n\u00a0 \u00a0 \u00a0 \u00a0 if self.flash:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u9ad8\u6548\u5b9e\u73b0（PyTorch 2.0+）
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 y = torch.nn.functional.scaled_dot_product_attention(
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 q, k, v, attn_mask=None,\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 dropout_p=self.dropout if self.training else 0,\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 is_causal=True
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 )
\n\u00a0 \u00a0 \u00a0 \u00a0 else:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u624b\u52a8\u5b9e\u73b0
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 att = (q @ k.transpose(-2, -1)) * (1.0 \/ math.sqrt(k.size(-1)))
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 att = att.masked_fill(self.bias[:,:,:T,:T] == 0, float('-inf'))
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 att = F.softmax(att, dim=-1)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 att = self.attn_dropout(att)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 y = att @ v
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 4. \u5408\u5e76\u591a\u5934\u8f93\u51fa
\n\u00a0 \u00a0 \u00a0 \u00a0 y = y.transpose(1, 2).contiguous().view(B, T, C)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 5. \u8f93\u51fa\u6295\u5f71
\n\u00a0 \u00a0 \u00a0 \u00a0 y = self.resid_dropout(self.c_proj(y))
\n\u00a0 \u00a0 \u00a0 \u00a0 return y<\/p>\n

\u6570\u636e\u6d41\u8be6\u89e3：
\n\u8f93\u5165: [B, T, C] \u00a0(Batch, Sequence, Embedding Dim)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u00a0c_attn: Linear(C \u2192 3*C) \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u2502 \u00a0\u4e00\u6b21\u6295\u5f71\u540c\u65f6\u5f97\u5230 Q, K, V \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u00a0\u91cd\u5851\u4e3a\u591a\u5934 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u2502 \u00a0[B, T, 3*C] \u2192 [B, heads, T, hs] \u00a0 \u2502
\n\u2502 \u00a0\u5176\u4e2d hs = C \/ heads \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u00a0Attention(Q, K, V) \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u2502 \u00a0= softmax(QK^T \/ \u221ahs) @ V \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u2502 \u00a0\u5408\u5e76\u591a\u5934 + \u8f93\u51fa\u6295\u5f71 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u2502 \u00a0[B, heads, T, hs] \u2192 [B, T, C] \u00a0 \u00a0 \u00a0\u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0<\/p>\n

\u4e3a\u4ec0\u4e48\u53eb"Causal"（\u56e0\u679c）？<\/p>\n

\u56e0\u4e3a\u63a9\u7801\u77e9\u9635 `self.bias` \u662f\u4e0b\u4e09\u89d2\u77e9\u9635：<\/p>\n

[[1, 0, 0, 0],
\n\u00a0[1, 1, 0, 0],
\n\u00a0[1, 1, 1, 0],
\n\u00a0[1, 1, 1, 1]]<\/p>\n

\u8fd9\u786e\u4fdd\u4f4d\u7f6e i \u53ea\u80fd\u770b\u5230\u4f4d\u7f6e 0~i，\u770b\u4e0d\u5230\u672a\u6765\u3002<\/p>\n<\/p>\n

\u00a03. MLP \/ Feed-Forward Network（\u7b2c78-92\u884c）<\/h6>\n

\nclass MLP(nn.Module):
\n\u00a0 \u00a0 def __init__(self, config):
\n\u00a0 \u00a0 \u00a0 \u00a0 super().__init__()
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u8f93\u5165\u6295\u5f71：C \u2192 4C
\n\u00a0 \u00a0 \u00a0 \u00a0 self.c_fc \u00a0 \u00a0= nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.gelu \u00a0 \u00a0= nn.GELU() \u00a0# \u6fc0\u6d3b\u51fd\u6570
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u8f93\u51fa\u6295\u5f71：4C \u2192 C
\n\u00a0 \u00a0 \u00a0 \u00a0 self.c_proj \u00a0= nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.dropout = nn.Dropout(config.dropout)<\/p>\n

\u00a0 \u00a0 def forward(self, x):
\n\u00a0 \u00a0 \u00a0 \u00a0 x = self.c_fc(x) \u00a0 \u00a0 \u00a0# [B, T, C] \u2192 [B, T, 4C]
\n\u00a0 \u00a0 \u00a0 \u00a0 x = self.gelu(x) \u00a0 \u00a0 \u00a0# GELU \u6fc0\u6d3b
\n\u00a0 \u00a0 \u00a0 \u00a0 x = self.c_proj(x) \u00a0 \u00a0# [B, T, 4C] \u2192 [B, T, C]
\n\u00a0 \u00a0 \u00a0 \u00a0 x = self.dropout(x)
\n\u00a0 \u00a0 \u00a0 \u00a0 return x<\/p>\n

\u4f5c\u7528：\u9010\u4f4d\u7f6e\u5904\u7406，\u6bcf\u4e2a token \u72ec\u7acb\u53d8\u6362<\/p>\n

\u4e3a\u4ec0\u4e48\u662f 4 \u500d？
\n– GPT-2 \u8bba\u6587\u4e2d\u7684\u6807\u51c6\u8bbe\u7f6e
\n– \u6269\u5927\u5bb9\u91cf\u540e\u518d\u538b\u7f29，\u5b66\u4e60\u66f4\u590d\u6742\u7684\u7279\u5f81<\/p>\n<\/p>\n

4. Block（\u7b2c94-106\u884c）\u2014\u2014 Transformer \u5c42<\/h6>\n

\nclass Block(nn.Module):
\n\u00a0 \u00a0 def __init__(self, config):
\n\u00a0 \u00a0 \u00a0 \u00a0 super().__init__()
\n\u00a0 \u00a0 \u00a0 \u00a0 self.ln_1 = LayerNorm(config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.attn = CausalSelfAttention(config)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.ln_2 = LayerNorm(config.n_embd, bias=config.bias)
\n\u00a0 \u00a0 \u00a0 \u00a0 self.mlp = MLP(config)<\/p>\n

\u00a0 \u00a0 def forward(self, x):
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u9884\u5f52\u4e00\u5316 + \u6b8b\u5dee\u8fde\u63a5
\n\u00a0 \u00a0 \u00a0 \u00a0 x = x + self.attn(self.ln_1(x)) \u00a0 # \u6ce8\u610f\u529b\u5b50\u5c42
\n\u00a0 \u00a0 \u00a0 \u00a0 x = x + self.mlp(self.ln_2(x)) \u00a0 \u00a0# FFN \u5b50\u5c42
\n\u00a0 \u00a0 \u00a0 \u00a0 return x<\/p>\n

\u7ed3\u6784\u56fe\u793a：
\n\u00a0 \u00a0 \u00a0 \u00a0 x
\n\u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u25bc\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u25bc
\n\u00a0 \u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u00a0 \u2502 Layer\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502 Layer\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u2502 Norm\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u2502 Norm\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u25bc\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u25bc
\n\u00a0 \u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u00a0 \u2502 Attention\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502 \u00a0MLP\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u2502 (Multi-\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502(FFN)\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u2502 \u00a0Head)\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0 x (\u8f93\u51fa)
\n\u00a0<\/p>\n

\u6b8b\u5dee\u8fde\u63a5\u7684\u4f5c\u7528：
\n– \u68af\u5ea6\u53ef\u4ee5\u76f4\u901a，\u7f13\u89e3\u6d88\u5931\u68af\u5ea6\u95ee\u9898
\n– \u4fdd\u7559\u539f\u59cb\u4fe1\u606f，\u795e\u7ecf\u7f51\u7edc\u53ea\u5b66"\u589e\u91cf"<\/p>\n

5. GPTConfig（\u7b2c108-116\u884c）<\/h6>\n

\n@dataclass
\nclass GPTConfig:
\n\u00a0 \u00a0 block_size: int = 1024 \u00a0 \u00a0 \u00a0# \u6700\u5927\u5e8f\u5217\u957f\u5ea6
\n\u00a0 \u00a0 vocab_size: int = 50304 \u00a0 \u00a0 # GPT-2 \u8bcd\u8868\u5927\u5c0f (50257 \u8865\u9f50\u5230 64 \u7684\u500d\u6570)
\n\u00a0 \u00a0 n_layer: int = 12 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # Transformer \u5c42\u6570
\n\u00a0 \u00a0 n_head: int = 12 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0# \u6ce8\u610f\u529b\u5934\u6570
\n\u00a0 \u00a0 n_embd: int = 768 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u5d4c\u5165\u7ef4\u5ea6
\n\u00a0 \u00a0 dropout: float = 0.0 \u00a0 \u00a0 \u00a0 \u00a0# \u00a0dropout \u7387
\n\u00a0 \u00a0 bias: bool = True \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u662f\u5426\u4f7f\u7528 bias<\/p>\n

6. GPT \u6a21\u578b（\u7b2c118-330\u884c）<\/h6>\n

\nclass GPT(nn.Module):
\n\u00a0 \u00a0 def __init__(self, config):
\n\u00a0 \u00a0 \u00a0 \u00a0 super().__init__()
\n\u00a0 \u00a0 \u00a0 \u00a0 self.config = config<\/p>\n

\u00a0 \u00a0 \u00a0 \u00a0 # Transformer \u4e3b\u4f53
\n\u00a0 \u00a0 \u00a0 \u00a0 self.transformer = nn.ModuleDict(dict(
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 wte = nn.Embedding(config.vocab_size, config.n_embd), \u00a0# \u8bcd\u5d4c\u5165
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 wpe = nn.Embedding(config.block_size, config.n_embd), \u00a0# \u4f4d\u7f6e\u5d4c\u5165
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 drop = nn.Dropout(config.dropout),
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]), \u00a0# \u591a\u5c42 Block
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 ln_f = LayerNorm(config.n_embd, bias=config.bias), \u00a0# \u6700\u7ec8\u5f52\u4e00\u5316
\n\u00a0 \u00a0 \u00a0 \u00a0 ))
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u8bed\u8a00\u6a21\u578b\u5934
\n\u00a0 \u00a0 \u00a0 \u00a0 self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # Weight Tying：\u8f93\u51fa\u6295\u5f71\u590d\u7528\u8bcd\u5d4c\u5165\u6743\u91cd
\n\u00a0 \u00a0 \u00a0 \u00a0 self.transformer.wte.weight = self.lm_head.weight<\/p>\n

\u00a0 \u00a0 \u00a0 \u00a0 # \u6743\u91cd\u521d\u59cb\u5316
\n\u00a0 \u00a0 \u00a0 \u00a0 self.apply(self._init_weights)
\n\u00a0 \u00a0 \u00a0 \u00a0 for pn, p in self.named_parameters():
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 if pn.endswith('c_proj.weight'):
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 torch.nn.init.normal_(p, mean=0.0, std=0.02\/math.sqrt(2 * config.n_layer))
\n```<\/p>\n

**\u524d\u5411\u4f20\u64ad（\u7b2c170-193\u884c）：**<\/p>\n

```python
\ndef forward(self, idx, targets=None):
\n\u00a0 \u00a0 device = idx.device
\n\u00a0 \u00a0 b, t = idx.size()
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # \u4f4d\u7f6e\u7f16\u7801
\n\u00a0 \u00a0 pos = torch.arange(0, t, dtype=torch.long, device=device)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 1. Token Embedding + Position Embedding
\n\u00a0 \u00a0 tok_emb = self.transformer.wte(idx) \u00a0 \u00a0 \u00a0# [B, T, C]
\n\u00a0 \u00a0 pos_emb = self.transformer.wpe(pos) \u00a0 \u00a0 \u00a0# [T, C]
\n\u00a0 \u00a0 x = self.transformer.drop(tok_emb + pos_emb)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 2. \u901a\u8fc7\u6240\u6709 Transformer \u5c42
\n\u00a0 \u00a0 for block in self.transformer.h:
\n\u00a0 \u00a0 \u00a0 \u00a0 x = block(x)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 3. \u6700\u7ec8\u5f52\u4e00\u5316
\n\u00a0 \u00a0 x = self.transformer.ln_f(x)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 4. \u8f93\u51fa logits
\n\u00a0 \u00a0 if targets is not None:
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u8bad\u7ec3\u6a21\u5f0f：\u8ba1\u7b97\u6240\u6709\u4f4d\u7f6e\u7684\u635f\u5931
\n\u00a0 \u00a0 \u00a0 \u00a0 logits = self.lm_head(x)
\n\u00a0 \u00a0 \u00a0 \u00a0 loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1))
\n\u00a0 \u00a0 else:
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u63a8\u7406\u6a21\u5f0f：\u53ea\u53d6\u6700\u540e\u4e00\u4e2a\u4f4d\u7f6e\u7684 logits（\u6548\u7387\u4f18\u5316）
\n\u00a0 \u00a0 \u00a0 \u00a0 logits = self.lm_head(x[:, [-1], :])
\n\u00a0 \u00a0 \u00a0 \u00a0 loss = None
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 return logits, loss<\/p>\n

\u5173\u952e\u4f18\u5316：\u63a8\u7406\u65f6\u53ea\u53d6\u6700\u540e\u4e00\u4e2a\u4f4d\u7f6e<\/p>\n

\nlogits = self.lm_head(x[:, [-1], :]) \u00a0# \u53ea\u53d6\u6700\u540e\u4e00\u6b65
\n\u00a0<\/p>\n

\u56e0\u4e3a GPT \u662f\u81ea\u56de\u5f52\u6a21\u578b，\u6211\u4eec\u53ea\u9700\u8981\u9884\u6d4b\u4e0b\u4e00\u4e2a token，\u524d\u9762\u7684 position \u7684 logits \u4e0d\u7528\u8ba1\u7b97\u3002<\/p>\n

7. \u751f\u6210\u51fd\u6570（\u7b2c305-330\u884c）<\/h6>\n

\n@torch.no_grad()
\ndef generate(self, idx, max_new_tokens, temperature=1.0, top_k=None):
\n\u00a0 \u00a0 for _ in range(max_new_tokens):
\n\u00a0 \u00a0 \u00a0 \u00a0 # 1. \u5982\u679c\u5e8f\u5217\u592a\u957f，\u53ea\u4fdd\u7559\u6700\u540e block_size \u4e2a token
\n\u00a0 \u00a0 \u00a0 \u00a0 idx_cond = idx if idx.size(1) <= self.config.block_size else idx[:, -self.config.block_size:]
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 2. \u524d\u5411\u4f20\u64ad
\n\u00a0 \u00a0 \u00a0 \u00a0 logits, _ = self(idx_cond)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 3. \u53d6\u6700\u540e\u4e00\u4e2a\u4f4d\u7f6e\u7684 logits，\u9664\u4ee5 temperature
\n\u00a0 \u00a0 \u00a0 \u00a0 logits = logits[:, -1, :] \/ temperature
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 4. Top-k \u88c1\u526a
\n\u00a0 \u00a0 \u00a0 \u00a0 if top_k is not None:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 v, _ = torch.topk(logits, min(top_k, logits.size(-1)))
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 logits[logits < v[:, [-1]]] = -float('Inf')
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 5. Softmax \u8f6c\u6982\u7387
\n\u00a0 \u00a0 \u00a0 \u00a0 probs = F.softmax(logits, dim=-1)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 6. \u91c7\u6837
\n\u00a0 \u00a0 \u00a0 \u00a0 idx_next = torch.multinomial(probs, num_samples=1)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # 7. \u8ffd\u52a0\u5230\u5e8f\u5217
\n\u00a0 \u00a0 \u00a0 \u00a0 idx = torch.cat((idx, idx_next), dim=1)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 return idx<\/p>\n

\u91c7\u6837\u7b56\u7565：<\/p>\n

| \u53c2\u6570\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 | \u4f5c\u7528\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 | \u63a8\u8350\u503c |
\n|——————–|————————————————-|———–|
\n| `temperature` | \u63a7\u5236\u968f\u673a\u6027，\u8d8a\u4f4e\u8d8a\u4fdd\u5b88\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 | 0.8-1.0 |
\n| `top_k`\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0| \u53ea\u4ece\u6982\u7387\u6700\u9ad8\u7684 k \u4e2a token \u4e2d\u91c7\u6837 | 50-200 |<\/p>\n<\/p>\n

\u4e8c\u3001train.py \u5b8c\u6574\u89e3\u6790<\/h5>\n

train.py \u662f\u8bad\u7ec3\u5faa\u73af，\u7ea6 330 \u884c\u3002<\/p>\n

1. \u914d\u7f6e\u53c2\u6570（\u7b2c32-78\u884c）<\/h6>\n

\n# \u6570\u636e
\ndataset = 'openwebtext'
\nbatch_size = 12 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 # \u6bcf\u4e2a GPU \u7684 batch size
\nblock_size = 1024 \u00a0 \u00a0 \u00a0 \u00a0 # \u5e8f\u5217\u957f\u5ea6
\ngradient_accumulation_steps = 5 * 8 \u00a0# \u68af\u5ea6\u7d2f\u79ef\u6b65\u6570<\/p>\n

# \u6a21\u578b
\nn_layer = 12
\nn_head = 12
\nn_embd = 768
\ndropout = 0.0<\/p>\n

# \u4f18\u5316\u5668
\nlearning_rate = 6e-4
\nmax_iters = 600000
\nweight_decay = 1e-1
\nbeta1 = 0.9
\nbeta2 = 0.95
\ngrad_clip = 1.0<\/p>\n

# \u5b66\u4e60\u7387\u8c03\u5ea6
\nwarmup_iters = 2000
\nlr_decay_iters = 600000
\nmin_lr = 6e-5<\/p>\n

2. \u6570\u636e\u52a0\u8f7d（\u7b2c114-131\u884c）<\/h6>\n

\ndef get_batch(split):
\n\u00a0 \u00a0 # \u4f7f\u7528 memmap \u907f\u514d\u5185\u5b58\u6cc4\u6f0f
\n\u00a0 \u00a0 if split == 'train':
\n\u00a0 \u00a0 \u00a0 \u00a0 data = np.memmap(os.path.join(data_dir, 'train.bin'), dtype=np.uint16, mode='r')
\n\u00a0 \u00a0 else:
\n\u00a0 \u00a0 \u00a0 \u00a0 data = np.memmap(os.path.join(data_dir, 'val.bin'), dtype=np.uint16, mode='r')
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # \u968f\u673a\u9009\u62e9 batch_size \u4e2a\u8d77\u59cb\u4f4d\u7f6e
\n\u00a0 \u00a0 ix = torch.randint(len(data) – block_size, (batch_size,))
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # \u63d0\u53d6\u8f93\u5165 x \u548c\u76ee\u6807 y
\n\u00a0 \u00a0 x = torch.stack([torch.from_numpy((data[i:i+block_size]).astype(np.int64)) for i in ix])
\n\u00a0 \u00a0 y = torch.stack([torch.from_numpy((data[i+1:i+1+block_size]).astype(np.int64)) for i in ix])
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # \u79fb\u5230 GPU
\n\u00a0 \u00a0 if device_type == 'cuda':
\n\u00a0 \u00a0 \u00a0 \u00a0 x, y = x.pin_memory().to(device, non_blocking=True), y.pin_memory().to(device, non_blocking=True)
\n\u00a0 \u00a0 else:
\n\u00a0 \u00a0 \u00a0 \u00a0 x, y = x.to(device), y.to(device)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 return x, y<\/p>\n

\u5173\u952e\u8bbe\u8ba1：
\n– `memmap`：\u5185\u5b58\u6620\u5c04\u6587\u4ef6，\u4e0d\u5360\u7528\u5b9e\u9645\u5185\u5b58
\n– `x` \u548c `y` \u7684\u5173\u7cfb：y \u662f x \u53f3\u79fb\u4e00\u4f4d（\u9884\u6d4b\u4e0b\u4e00\u4e2a token）
\n– `pin_memory()`：\u52a0\u901f CPU\u2192GPU \u6570\u636e\u4f20\u8f93<\/p>\n

3. \u6a21\u578b\u521d\u59cb\u5316（\u7b2c146-193\u884c）<\/h6>\n

\n# \u4e09\u79cd\u521d\u59cb\u5316\u65b9\u5f0f
\nif init_from == 'scratch':
\n\u00a0 \u00a0 # \u4ece\u96f6\u5f00\u59cb
\n\u00a0 \u00a0 gptconf = GPTConfig(**model_args)
\n\u00a0 \u00a0 model = GPT(gptconf)
\nelif init_from == 'resume':
\n\u00a0 \u00a0 # \u4ece\u68c0\u67e5\u70b9\u6062\u590d
\n\u00a0 \u00a0 checkpoint = torch.load(ckpt_path, map_location=device)
\n\u00a0 \u00a0 model.load_state_dict(checkpoint['model'])
\nelif init_from.startswith('gpt2'):
\n\u00a0 \u00a0 # \u4ece OpenAI \u9884\u8bad\u7ec3\u6743\u91cd\u52a0\u8f7d
\n\u00a0 \u00a0 model = GPT.from_pretrained(init_from, override_args)<\/p>\n<\/p>\n

\u00a04. \u635f\u5931\u4f30\u7b97（\u7b2c215-228\u884c）<\/h6>\n

\n@torch.no_grad()
\ndef estimate_loss():
\n\u00a0 \u00a0 out = {}
\n\u00a0 \u00a0 model.eval()
\n\u00a0 \u00a0 for split in ['train', 'val']:
\n\u00a0 \u00a0 \u00a0 \u00a0 losses = torch.zeros(eval_iters)
\n\u00a0 \u00a0 \u00a0 \u00a0 for k in range(eval_iters):
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 X, Y = get_batch(split)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 with ctx:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 logits, loss = model(X, Y)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 losses[k] = loss.item()
\n\u00a0 \u00a0 \u00a0 \u00a0 out[split] = losses.mean()
\n\u00a0 \u00a0 model.train()
\n\u00a0 \u00a0 return out<\/p>\n

\u4e3a\u4ec0\u4e48\u7528 `@torch.no_grad()`？
\n– \u8bc4\u4f30\u65f6\u4e0d\u9700\u8981\u8ba1\u7b97\u68af\u5ea6，\u8282\u7701\u5185\u5b58
\n– \u901f\u5ea6\u66f4\u5feb<\/p>\n

\u00a05. \u5b66\u4e60\u7387\u8c03\u5ea6（\u7b2c230-242\u884c）<\/h6>\n

\ndef get_lr(it):
\n\u00a0 \u00a0 # 1. \u7ebf\u6027 warmup
\n\u00a0 \u00a0 if it < warmup_iters:
\n\u00a0 \u00a0 \u00a0 \u00a0 return learning_rate * (it + 1) \/ (warmup_iters + 1)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 2. \u8d85\u8fc7 decay_iters \u540e\u4f7f\u7528\u6700\u5c0f\u5b66\u4e60\u7387
\n\u00a0 \u00a0 if it > lr_decay_iters:
\n\u00a0 \u00a0 \u00a0 \u00a0 return min_lr
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 3. Cosine \u8870\u51cf
\n\u00a0 \u00a0 decay_ratio = (it – warmup_iters) \/ (lr_decay_iters – warmup_iters)
\n\u00a0 \u00a0 coeff = 0.5 * (1.0 + math.cos(math.pi * decay_ratio))
\n\u00a0 \u00a0 return min_lr + coeff * (learning_rate – min_lr)<\/p>\n

\u5b66\u4e60\u7387\u66f2\u7ebf：<\/p>\n

lr
\n\u2502 \u00a0 \u00a0\u256d\u2500\u2500\u2500 warmup
\n\u2502 \u00a0 \u2571
\n\u2502 \u00a0\u2571
\n\u2502 \u2571
\n\u256d\u2500\u256f
\n\u2502 \u00a0 \u00a0 \u00a0 \u00a0\u2572_________ cosine decay
\n\u2502
\n\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500 iter
\n\u00a0 \u00a0 \u00a00 \u00a0 2000 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0600000<\/p>\n

\u00a06. \u8bad\u7ec3\u5faa\u73af（\u7b2c249-333\u884c）<\/p>\n

\n# \u83b7\u53d6\u7b2c\u4e00\u6279\u6570\u636e
\nX, Y = get_batch('train')
\nt0 = time.time()<\/p>\n

while True:
\n\u00a0 \u00a0 # 1. \u8bbe\u7f6e\u5b66\u4e60\u7387
\n\u00a0 \u00a0 lr = get_lr(iter_num)
\n\u00a0 \u00a0 for param_group in optimizer.param_groups:
\n\u00a0 \u00a0 \u00a0 \u00a0 param_group['lr'] = lr
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 2. \u5b9a\u671f\u8bc4\u4f30\u548c\u4fdd\u5b58
\n\u00a0 \u00a0 if iter_num % eval_interval == 0 and master_process:
\n\u00a0 \u00a0 \u00a0 \u00a0 losses = estimate_loss()
\n\u00a0 \u00a0 \u00a0 \u00a0 print(f"step {iter_num}: train loss {losses['train']:.4f}, val loss {losses['val']:.4f}")
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u4fdd\u5b58\u6700\u4f73\u6a21\u578b
\n\u00a0 \u00a0 \u00a0 \u00a0 if losses['val'] < best_val_loss:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 best_val_loss = losses['val']
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 torch.save(checkpoint, os.path.join(out_dir, 'ckpt.pt'))
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 3. \u524d\u5411 + \u53cd\u5411 + \u66f4\u65b0
\n\u00a0 \u00a0 for micro_step in range(gradient_accumulation_steps):
\n\u00a0 \u00a0 \u00a0 \u00a0 if ddp:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 model.require_backward_grad_sync = (micro_step == gradient_accumulation_steps – 1)
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 with ctx:
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 logits, loss = model(X, Y)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 loss = loss \/ gradient_accumulation_steps \u00a0# \u68af\u5ea6\u7d2f\u79ef
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u5f02\u6b65\u9884\u53d6\u4e0b\u4e00\u6279\u6570\u636e
\n\u00a0 \u00a0 \u00a0 \u00a0 X, Y = get_batch('train')
\n\u00a0 \u00a0 \u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 \u00a0 \u00a0 # \u53cd\u5411\u4f20\u64ad
\n\u00a0 \u00a0 \u00a0 \u00a0 scaler.scale(loss).backward()
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 4. \u68af\u5ea6\u88c1\u526a
\n\u00a0 \u00a0 if grad_clip != 0.0:
\n\u00a0 \u00a0 \u00a0 \u00a0 scaler.unscale_(optimizer)
\n\u00a0 \u00a0 \u00a0 \u00a0 torch.nn.utils.clip_grad_norm_(model.parameters(), grad_clip)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 5. \u4f18\u5316\u5668\u6b65\u8fdb
\n\u00a0 \u00a0 scaler.step(optimizer)
\n\u00a0 \u00a0 scaler.update()
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 6. \u6e05\u7a7a\u68af\u5ea6
\n\u00a0 \u00a0 optimizer.zero_grad(set_to_none=True)
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 # 7. \u8ba1\u65f6\u548c\u65e5\u5fd7
\n\u00a0 \u00a0 t1 = time.time()
\n\u00a0 \u00a0 dt = t1 – t0
\n\u00a0 \u00a0 # … \u6253\u5370 loss, time, mfu …
\n\u00a0 \u00a0\u00a0
\n\u00a0 \u00a0 iter_num += 1
\n\u00a0 \u00a0 if iter_num > max_iters:
\n\u00a0 \u00a0 \u00a0 \u00a0 break<\/p>\n<\/p>\n

\u00a0\u4e09\u3001\u6574\u4f53\u6570\u636e\u6d41\u56fe<\/h5>\n

\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u8bad\u7ec3\u6d41\u7a0b
\n\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550\u2550<\/p>\n

\u00a0 train.bin \/ val.bin (uint16 token IDs)
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u2502 \u00a0 get_batch() \u00a0 \u2502 \u00a0\u2190 \u968f\u673a\u91c7\u6837 block_size \u4e2a\u8fde\u7eed token
\n\u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc
\n\u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0 GPT Model\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u2502 \u00a0\u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\u00a0 \u00a0 \u00a0 \u00a0\u00a0\u00a0\u2502
\n\u00a0 \u2502 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 wte + wpe\u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502 \u00a0\u2190 Token + Position Embedding
\n\u00a0 \u2502 \u00a0\u251c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2524\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u2502 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 Block \u00d7 N\u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502 \u00a0\u2190 N \u5c42 Transformer
\n\u00a0 \u2502 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0– Attn\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u2502 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0– FFN \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502\u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u2502 \u00a0\u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\u00a0 \u00a0 \u00a0 \u00a0 \u2502
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u25bc\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 lm_head\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u2502 \u00a0\u2190 \u6295\u5f71\u5230\u8bcd\u8868\u5927\u5c0f
\n\u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u25bc
\n\u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u00a0CrossEntropy\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502 \u00a0\u2190 \u8ba1\u7b97 loss
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 Loss\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u25bc
\n\u00a0 \u250c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u00a0Backward +\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0\u2502 \u00a0\u2190 \u53cd\u5411\u4f20\u64ad
\n\u00a0 \u2502\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 Optimizer\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u252c\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u2502
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u25bc
\n\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u66f4\u65b0\u6743\u91cd<\/p>\n

\u00a0\u56db\u3001\u5173\u952e\u6570\u5b57\u603b\u7ed3<\/h5>\n

| \u53c2\u6570 | GPT-2 (124M) | \u8bf4\u660e |
\n|——————-|————-|————————-|
\n| `vocab_size` | 50304\u00a0 \u00a0| \u8bcd\u8868\u5927\u5c0f\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0 |
\n| `n_embd`\u00a0 \u00a0 \u00a0 \u00a0| 768\u00a0 \u00a0 \u00a0 \u00a0| \u5d4c\u5165\u7ef4\u5ea6\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0\u00a0 |
\n| `n_layer`\u00a0 \u00a0 \u00a0 \u00a0\u00a0 | 12\u00a0 \u00a0 \u00a0 \u00a0 \u00a0| Transformer \u5c42\u6570 |
\n| `n_head`\u00a0 \u00a0 \u00a0 \u00a0 \u00a0| 12\u00a0 \u00a0 \u00a0 \u00a0 | \u6ce8\u610f\u529b\u5934\u6570\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0|
\n| `block_size`\u00a0 \u00a0| 1024\u00a0 \u00a0 | \u6700\u5927\u5e8f\u5217\u957f\u5ea6\u00a0 \u00a0 \u00a0\u00a0 |
\n| \u53c2\u6570\u91cf\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 \u00a0 | ~124M\u00a0| 1.24 \u4ebf\u53c2\u6570\u00a0 \u00a0 \u00a0 \u00a0 \u00a0 |<\/p>\n

\u8ba1\u7b97\u91cf\u4f30\u7b97：<\/p>\n

\u6bcf token FLOPs \u2248 6 \u00d7 \u53c2\u6570\u91cf = 6 \u00d7 124M \u2248 744M FLOPs<\/p>\n

\u00a0\u4e94\u3001\u5b66\u4e60\u5efa\u8bae<\/h5>\n

1. \u5148\u8dd1\u901a：`python train.py config\/train_shakespeare_char.py`
\n2. \u8bfb\u4ee3\u7801：\u9010\u884c\u7406\u89e3 model.py \u7684\u6bcf\u4e2a\u7c7b
\n3. \u6539\u53c2\u6570：\u5c1d\u8bd5\u4fee\u6539 n_layer, n_head, n_embd \u89c2\u5bdf\u53d8\u5316
\n4. \u52a0\u529f\u80fd：\u5c1d\u8bd5\u52a0\u5165\u65b0\u7684\u5c42\u6216\u4fee\u6539 attention \u673a\u5236
\n\u00a0<\/p>\n","protected":false},"excerpt":{"rendered":"

\u4e00\u3001model.py \u5b8c\u6574\u89e3\u6790model.py \u662f\u6574\u4e2a\u9879\u76ee\u7684\u6838\u5fc3，\u53ea\u6709 330 \u884c\u4ee3\u7801，\u5374\u5b9e\u73b0\u4e86\u5b8c\u6574\u7684 GPT \u6a21\u578b\u30021. LayerNorm（\u7b2c18-27\u884c）class LayerNorm(nn.Module):def __init__(self, ndim, bias):super().__init__()self.weight nn.Parameter(torch.ones(ndim))self.bias nn<\/p>\n","protected":false},"author":2,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[1],"tags":[11215,11214,152,50,86],"topic":[],"class_list":["post-97858","post","type-post","status-publish","format-standard","hentry","category-server","tag-nanochat","tag-nanogtp","tag-pytorch","tag-50","tag-86"],"yoast_head":"\nnanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3<\/title>\n<meta name=\"robots\" content=\"index, follow, max-snippet:-1, max-image-preview:large, max-video-preview:-1\" \/>\n<link rel=\"canonical\" href=\"https:\/\/www.wsisp.com\/helps\/97858.html\" \/>\n<meta property=\"og:locale\" content=\"zh_CN\" \/>\n<meta property=\"og:type\" content=\"article\" \/>\n<meta property=\"og:title\" content=\"nanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"og:description\" content=\"\u4e00\u3001model.py \u5b8c\u6574\u89e3\u6790model.py \u662f\u6574\u4e2a\u9879\u76ee\u7684\u6838\u5fc3，\u53ea\u6709 330 \u884c\u4ee3\u7801，\u5374\u5b9e\u73b0\u4e86\u5b8c\u6574\u7684 GPT \u6a21\u578b\u30021. LayerNorm（\u7b2c18-27\u884c）class LayerNorm(nn.Module):def __init__(self, ndim, bias):super().__init__()self.weight nn.Parameter(torch.ones(ndim))self.bias nn\" \/>\n<meta property=\"og:url\" content=\"https:\/\/www.wsisp.com\/helps\/97858.html\" \/>\n<meta property=\"og:site_name\" content=\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\" \/>\n<meta property=\"article:published_time\" content=\"2026-08-30T06:24:52+00:00\" \/>\n<meta name=\"author\" content=\"admin\" \/>\n<meta name=\"twitter:card\" content=\"summary_large_image\" \/>\n<meta name=\"twitter:label1\" content=\"\u4f5c\u8005\" \/>\n\t<meta name=\"twitter:data1\" content=\"admin\" \/>\n\t<meta name=\"twitter:label2\" content=\"\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4\" \/>\n\t<meta name=\"twitter:data2\" content=\"8 \u5206\" \/>\n<script type=\"application\/ld+json\" class=\"yoast-schema-graph\">{\"@context\":\"https:\/\/schema.org\",\"@graph\":[{\"@type\":\"WebPage\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/97858.html\",\"url\":\"https:\/\/www.wsisp.com\/helps\/97858.html\",\"name\":\"nanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"isPartOf\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\"},\"datePublished\":\"2026-08-30T06:24:52+00:00\",\"dateModified\":\"2026-08-30T06:24:52+00:00\",\"author\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\"},\"breadcrumb\":{\"@id\":\"https:\/\/www.wsisp.com\/helps\/97858.html#breadcrumb\"},\"inLanguage\":\"zh-Hans\",\"potentialAction\":[{\"@type\":\"ReadAction\",\"target\":[\"https:\/\/www.wsisp.com\/helps\/97858.html\"]}]},{\"@type\":\"BreadcrumbList\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/97858.html#breadcrumb\",\"itemListElement\":[{\"@type\":\"ListItem\",\"position\":1,\"name\":\"\u9996\u9875\",\"item\":\"https:\/\/www.wsisp.com\/helps\"},{\"@type\":\"ListItem\",\"position\":2,\"name\":\"nanoGPT \u9010\u884c\u8bb2\u89e3\"}]},{\"@type\":\"WebSite\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#website\",\"url\":\"https:\/\/www.wsisp.com\/helps\/\",\"name\":\"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3\",\"description\":\"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b\",\"potentialAction\":[{\"@type\":\"SearchAction\",\"target\":{\"@type\":\"EntryPoint\",\"urlTemplate\":\"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}\"},\"query-input\":\"required name=search_term_string\"}],\"inLanguage\":\"zh-Hans\"},{\"@type\":\"Person\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41\",\"name\":\"admin\",\"image\":{\"@type\":\"ImageObject\",\"inLanguage\":\"zh-Hans\",\"@id\":\"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/\",\"url\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"contentUrl\":\"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery\",\"caption\":\"admin\"},\"sameAs\":[\"http:\/\/wp.wsisp.com\"],\"url\":\"https:\/\/www.wsisp.com\/helps\/author\/admin\"}]}<\/script>\n<!-- \/ Yoast SEO plugin. -->","yoast_head_json":{"title":"nanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","robots":{"index":"index","follow":"follow","max-snippet":"max-snippet:-1","max-image-preview":"max-image-preview:large","max-video-preview":"max-video-preview:-1"},"canonical":"https:\/\/www.wsisp.com\/helps\/97858.html","og_locale":"zh_CN","og_type":"article","og_title":"nanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","og_description":"\u4e00\u3001model.py \u5b8c\u6574\u89e3\u6790model.py \u662f\u6574\u4e2a\u9879\u76ee\u7684\u6838\u5fc3，\u53ea\u6709 330 \u884c\u4ee3\u7801，\u5374\u5b9e\u73b0\u4e86\u5b8c\u6574\u7684 GPT \u6a21\u578b\u30021. LayerNorm（\u7b2c18-27\u884c）class LayerNorm(nn.Module):def __init__(self, ndim, bias):super().__init__()self.weight nn.Parameter(torch.ones(ndim))self.bias nn","og_url":"https:\/\/www.wsisp.com\/helps\/97858.html","og_site_name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","article_published_time":"2026-08-30T06:24:52+00:00","author":"admin","twitter_card":"summary_large_image","twitter_misc":{"\u4f5c\u8005":"admin","\u9884\u8ba1\u9605\u8bfb\u65f6\u95f4":"8 \u5206"},"schema":{"@context":"https:\/\/schema.org","@graph":[{"@type":"WebPage","@id":"https:\/\/www.wsisp.com\/helps\/97858.html","url":"https:\/\/www.wsisp.com\/helps\/97858.html","name":"nanoGPT \u9010\u884c\u8bb2\u89e3 - \u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","isPartOf":{"@id":"https:\/\/www.wsisp.com\/helps\/#website"},"datePublished":"2026-08-30T06:24:52+00:00","dateModified":"2026-08-30T06:24:52+00:00","author":{"@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41"},"breadcrumb":{"@id":"https:\/\/www.wsisp.com\/helps\/97858.html#breadcrumb"},"inLanguage":"zh-Hans","potentialAction":[{"@type":"ReadAction","target":["https:\/\/www.wsisp.com\/helps\/97858.html"]}]},{"@type":"BreadcrumbList","@id":"https:\/\/www.wsisp.com\/helps\/97858.html#breadcrumb","itemListElement":[{"@type":"ListItem","position":1,"name":"\u9996\u9875","item":"https:\/\/www.wsisp.com\/helps"},{"@type":"ListItem","position":2,"name":"nanoGPT \u9010\u884c\u8bb2\u89e3"}]},{"@type":"WebSite","@id":"https:\/\/www.wsisp.com\/helps\/#website","url":"https:\/\/www.wsisp.com\/helps\/","name":"\u7f51\u7855\u4e92\u8054\u5e2e\u52a9\u4e2d\u5fc3","description":"\u9999\u6e2f\u670d\u52a1\u5668_\u9999\u6e2f\u4e91\u670d\u52a1\u5668\u8d44\u8baf_\u670d\u52a1\u5668\u5e2e\u52a9\u6587\u6863_\u670d\u52a1\u5668\u6559\u7a0b","potentialAction":[{"@type":"SearchAction","target":{"@type":"EntryPoint","urlTemplate":"https:\/\/www.wsisp.com\/helps\/?s={search_term_string}"},"query-input":"required name=search_term_string"}],"inLanguage":"zh-Hans"},{"@type":"Person","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/358e386c577a3ab51c4493330a20ad41","name":"admin","image":{"@type":"ImageObject","inLanguage":"zh-Hans","@id":"https:\/\/www.wsisp.com\/helps\/#\/schema\/person\/image\/","url":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","contentUrl":"https:\/\/gravatar.wp-china-yes.net\/avatar\/?s=96&d=mystery","caption":"admin"},"sameAs":["http:\/\/wp.wsisp.com"],"url":"https:\/\/www.wsisp.com\/helps\/author\/admin"}]}},"_links":{"self":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/97858","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/users\/2"}],"replies":[{"embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/comments?post=97858"}],"version-history":[{"count":0,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/posts\/97858\/revisions"}],"wp:attachment":[{"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/media?parent=97858"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/categories?post=97858"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/tags?post=97858"},{"taxonomy":"topic","embeddable":true,"href":"https:\/\/www.wsisp.com\/helps\/wp-json\/wp\/v2\/topic?post=97858"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}