Fix scaling when embeddings are tied (#591)

2024-03-18 21:41:07 +01:00 · 2024-03-18 21:41:07 +01:00 · e05e502c34
commit e05e502c34
parent e4b19bb9e1
1 changed files with 1 additions and 1 deletions
--- a/t5/t5.py
+++ b/t5/t5.py
@ -315,9 +315,9 @@ class T5(nn.Module):
            inputs, memory=memory, mask=mask, memory_mask=None, cache=cache
        )
        if not self.tie_word_embeddings:
-            y *= self.model_dim**-0.5
            y = self.lm_head(y)
        else:
+            y *= self.model_dim**-0.5
            y = y @ self.wte.weight.T
        return y, cache