The author provides a free 170-page PDF guide titled " Test Yourself On Build a Large Language Model (From Scratch) ." It contains quiz questions and solutions for each chapter and is available on the Manning website or via the official GitHub repository .
class TransformerBlock(nn.Module): def __init__(self, cfg): super().__init__() self.att = MultiHeadAttention( d_in=cfg['emb_dim'], d_out=cfg['emb_dim'], context_length=cfg['context_len'], num_heads=cfg['n_heads'] ) self.ffn = nn.Sequential( nn.Linear(cfg['emb_dim'], 4 * cfg['emb_dim']), nn.GELU(), nn.Linear(4 * cfg['emb_dim'], cfg['emb_dim']) ) self.norm1 = nn.LayerNorm(cfg['emb_dim']) self.norm2 = nn.LayerNorm(cfg['emb_dim']) def forward(self, x): # Residual connections around attention and feed-forward loops x = x + self.att(self.norm1(x)) x = x + self.ffn(self.norm2(x)) return x class ScratchLLM(nn.Module): def __init__(self, cfg): super().__init__() self.tok_emb = nn.Embedding(cfg['vocab_size'], cfg['emb_dim']) self.pos_emb = nn.Embedding(cfg['context_len'], cfg['emb_dim']) self.trf_blocks = nn.Sequential(*[ TransformerBlock(cfg) for _ in range(cfg['n_layers']) ]) self.final_norm = nn.LayerNorm(cfg['emb_dim']) self.out_head = nn.Linear(cfg['emb_dim'], cfg['vocab_size'], bias=False) def forward(self, in_idx): batch_size, seq_len = in_idx.shape tok_embeds = self.tok_emb(in_idx) pos_embeds = self.pos_emb(torch.arange(seq_len, device=in_idx.device)) x = tok_embeds + pos_embeds x = self.trf_blocks(x) x = self.final_norm(x) logits = self.out_head(x) return logits Use code with caution. 5. Step 4: The Training Loop and Pre-training build a large language model from scratch pdf