diff options
| -rw-r--r-- | gpt2/config.go | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/gpt2/config.go b/gpt2/config.go index 9f1a65f..7a5f470 100644 --- a/gpt2/config.go +++ b/gpt2/config.go @@ -5,6 +5,7 @@ type Config struct { NumLayers int NumHeads int HeadDim int + HiddenDim int NumPositions int } @@ -20,6 +21,7 @@ func ConfigDefault() Config { NumLayers: 12, NumHeads: 12, HeadDim: 64, + HiddenDim: 768, NumPositions: 1024, } } @@ -30,6 +32,7 @@ func ConfigLarge() Config { NumLayers: 36, NumHeads: 20, HeadDim: 64, + HiddenDim: 1280, NumPositions: 1024, } } |