From 29f93927538e9b7a12cfd992212927993b4fd585 Mon Sep 17 00:00:00 2001 From: cat0825 <1759138827@qq.com> Date: Sun, 31 May 2026 13:22:28 +0800 Subject: [PATCH] Clarify attention output projection --- src/components/Attention.svelte | 4 ++-- src/components/article/Article.svelte | 11 ++++++----- src/utils/textbookPages.ts | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/components/Attention.svelte b/src/components/Attention.svelte index f87540f..11f4047 100644 --- a/src/components/Attention.svelte +++ b/src/components/Attention.svelte @@ -139,8 +139,8 @@ {/each} Attention Out, Head 1, vector({$modelMeta.dimension / - $modelMeta.attention_head_num})Attention Out before concatenation and projection, Head {$attentionHeadIdx + 1}, + vector({$modelMeta.dimension / $modelMeta.attention_head_num}) diff --git a/src/components/article/Article.svelte b/src/components/article/Article.svelte index 607b84c..3ef7a36 100644 --- a/src/components/article/Article.svelte +++ b/src/components/article/Article.svelte @@ -328,11 +328,12 @@

Step 4: Output and Concatenation

The model uses the masked self-attention scores and multiplies them with the - Value matrix to get the - final output - of the self-attention mechanism. GPT-2 has 12 self-attention heads, each capturing - different relationships between tokens. The outputs of these heads are concatenated and passed - through a linear projection. + Value matrix to get each head's attention output. GPT-2 has + 12 self-attention heads, each capturing different relationships between tokens. + The outputs of these heads are concatenated into one vector and then passed through the + attention block's learned output projection matrix (c_proj) before the result + flows to the next layer. The projection matrix is part of the model computation, but it is + not drawn as a separate matrix in this visualization.

diff --git a/src/utils/textbookPages.ts b/src/utils/textbookPages.ts index bc95163..a07e544 100644 --- a/src/utils/textbookPages.ts +++ b/src/utils/textbookPages.ts @@ -333,7 +333,7 @@ export const textPages: TextbookPage[] = [ id: 'output-concatenation', title: 'Attention Output & Concatenation', content: - '

Each head multiplies its attention scores with the Value embeddings to produce its attention output—a refined representation of each token after considering context.

GPT-2 (small) has 12 such outputs, which are concatenated to form a single vector of the original size (768 numbers).

', + '

Each head multiplies its attention scores with the Value embeddings to produce a head-specific attention output—a refined representation of each token after considering context.

GPT-2 (small) concatenates the 12 head outputs, then applies the attention block\'s learned output projection matrix (c_proj) before sending the result to the next layer. This projection is part of the computation even though it is not drawn as a separate matrix here.

', on: function () { this.timeoutId = setTimeout( () => {