diff --git a/combo/default_model.py b/combo/default_model.py
index d074e6298ed856c748e63ed43d24ec4267a553e7..f02786b8d3c2f1160f36c59f7df7c765170ad158 100644
--- a/combo/default_model.py
+++ b/combo/default_model.py
@@ -19,7 +19,7 @@ from combo.modules.morpho import MorphologicalFeatures
 from combo.modules.parser import DependencyRelationModel, HeadPredictionModel
 from combo.modules.text_field_embedders import BasicTextFieldEmbedder
 from combo.modules.token_embedders import CharacterBasedWordEmbedder, TransformersWordEmbedder
-from combo.nn.activations import ReLUActivation, TanhActivation, LinearActivation
+from combo.nn.activations import ReLUActivation, TanhActivation, LinearActivation, GELUActivation
 from combo.modules import FeedForwardPredictor
 from combo.nn.base import Linear
 from combo.nn.regularizers.regularizers import L2Regularizer
@@ -128,10 +128,10 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary) -> C
         ),
         lemmatizer=LemmatizerModel(
             vocabulary=vocabulary,
-            activations=[ReLUActivation(), ReLUActivation(), ReLUActivation(), LinearActivation()],
+            activations=[GELUActivation(), GELUActivation(), GELUActivation(), LinearActivation()],
             char_vocab_namespace="token_characters",
             dilation=[1, 2, 4, 1],
-            embedding_dim=256,
+            embedding_dim=300,
             filters=[256, 256, 256],
             input_projection_layer=Linear(
                 activation=TanhActivation(),
@@ -183,7 +183,7 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary) -> C
                 "char": CharacterBasedWordEmbedder(
                     vocabulary=vocabulary,
                     dilated_cnn_encoder=DilatedCnnEncoder(
-                        activations=[ReLUActivation(), ReLUActivation(), LinearActivation()],
+                        activations=[GELUActivation(), GELUActivation(), LinearActivation()],
                         dilation=[1, 2, 4],
                         filters=[512, 256, 64],
                         input_dim=64,
diff --git a/combo/nn/activations.py b/combo/nn/activations.py
index f4e5d43e2a4d1ca70078d5f2bcea9f2a758ba8b7..60f64aa7304f7c701c120e1f9dcea3c52ee20cae 100644
--- a/combo/nn/activations.py
+++ b/combo/nn/activations.py
@@ -39,7 +39,7 @@ class ReLUActivation(Activation):
 
 
 @Registry.register('gelu')
-class ReLUActivation(Activation):
+class GELUActivation(Activation):
     def __init__(self):
         super().__init__()
         self.__torch_activation = torch.nn.GELU()