diff --git a/combo/config.template.json b/combo/config.template.json
index 93332c54d3cbc0c0cba97084e7853556258e1495..c6bf59762fe616134895398a2b6ef8f1c7424a78 100644
--- a/combo/config.template.json
+++ b/combo/config.template.json
@@ -10,7 +10,7 @@
             "parameters": {
               "activation": { "type": "tanh", "parameters": {} },
               "dropout_rate": 0.25,
-              "in_features": 1024,
+              "in_features": 164,
               "out_features": 128
             }
           },
@@ -22,7 +22,7 @@
                 "type": "linear_layer",
                 "parameters": {
                   "activation": { "type": "tanh", "parameters": {} },
-                  "in_features": 1024,
+                  "in_features": 164,
                   "out_features": 512
                 }
               },
@@ -30,7 +30,7 @@
                 "type": "linear_layer",
                 "parameters": {
                   "activation": { "type": "tanh", "parameters": {} },
-                  "in_features": 1024,
+                  "in_features": 164,
                   "out_features": 512
                 }
               }
@@ -41,7 +41,7 @@
             "parameters": {
               "activation": { "type": "tanh", "parameters": {} },
               "dropout_rate": 0.25,
-              "in_features": 1024,
+              "in_features": 164,
               "out_features": 128
             }
           },
@@ -66,7 +66,7 @@
             "parameters": {
               "activation": { "type": "tanh", "parameters": {} },
               "dropout_rate": 0.25,
-              "in_features": 1024,
+              "in_features": 164,
               "out_features": 32
             }
           },
@@ -94,7 +94,7 @@
           ],
           "dropout": [0.25, 0.0],
           "hidden_dims": [128],
-          "input_dim": 1024,
+          "input_dim": 164,
           "num_layers": 2,
           "vocab_namespace": "feats_labels"
         }
@@ -123,19 +123,17 @@
         }
       },
       "seq_encoder": {
-        "type": "combo_encoder",
+        "type": "combo_transformer_encoder",
         "parameters": {
           "layer_dropout_probability": 0.33,
-          "stacked_bilstm": {
-            "type": "combo_stacked_bilstm",
-            "parameters": {
-              "hidden_size": 512,
-              "input_size": 164,
-              "layer_dropout_probability": 0.33,
-              "num_layers": 2,
-              "recurrent_dropout_probability": 0.33
-            }
-          }
+          "input_dim": 164,
+          "num_layers": 2,
+          "feedforward_hidden_dim": 2048,
+          "num_attention_heads": 4,
+          "positional_encoding": null,
+          "positional_embedding_size": 512,
+          "dropout_prob": 0.1,
+          "activation": "relu"
         }
       },
       "text_field_embedder": {
@@ -175,7 +173,7 @@
         "type": "feedforward_predictor_from_vocab",
         "parameters": {
           "vocab_namespace": "upostag_labels",
-          "input_dim": 1024,
+          "input_dim": 164,
           "num_layers": 2,
           "hidden_dims": [64],
           "activations": [
@@ -189,7 +187,7 @@
         "type": "feedforward_predictor_from_vocab",
         "parameters": {
           "vocab_namespace": "xpostag_labels",
-          "input_dim": 1024,
+          "input_dim": 164,
           "num_layers": 2,
           "hidden_dims": [64],
           "activations": [
diff --git a/combo/default_model.py b/combo/default_model.py
index 61be958707c01d1e79fe39c77b2bbc64af99310f..47d2b14ff935c50cd6a519b95a6b6df5013e99a0 100644
--- a/combo/default_model.py
+++ b/combo/default_model.py
@@ -131,26 +131,26 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary, use_
                 dependency_projection_layer=Linear(
                     activation=TanhActivation(),
                     dropout_rate=0.25,
-                    in_features=1024,
+                    in_features=164,
                     out_features=128
                 ),
                 head_predictor=HeadPredictionModel(
                     cycle_loss_n=0,
                     dependency_projection_layer=Linear(
                         activation=TanhActivation(),
-                        in_features=1024,
+                        in_features=164,
                         out_features=512
                     ),
                     head_projection_layer=Linear(
                         activation=TanhActivation(),
-                        in_features=1024,
+                        in_features=164,
                         out_features=512
                     )
                 ),
                 head_projection_layer=Linear(
                     activation=TanhActivation(),
                     dropout_rate=0.25,
-                    in_features=1024,
+                    in_features=164,
                     out_features=128
                 ),
                 vocab_namespace="deprel_labels"
@@ -160,12 +160,12 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary, use_
                 activations=[GELUActivation(), GELUActivation(), GELUActivation(), LinearActivation()],
                 char_vocab_namespace="token_characters",
                 dilation=[1, 2, 4, 1],
-                embedding_dim=300,
+                embedding_dim=256,
                 filters=[256, 256, 256],
                 input_projection_layer=Linear(
                     activation=TanhActivation(),
                     dropout_rate=0.25,
-                    in_features=1024,
+                    in_features=164,
                     out_features=32
                 ),
                 kernel_size=[3, 3, 3, 1],
@@ -187,7 +187,7 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary, use_
                 activations=[TanhActivation(), LinearActivation()],
                 dropout=[0.25, 0.],
                 hidden_dims=[128],
-                input_dim=1024,
+                input_dim=164,
                 num_layers=2,
                 vocab_namespace="feats_labels"
             ),
@@ -231,7 +231,7 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary, use_
                 activations=[TanhActivation(), LinearActivation()],
                 dropout=[0.25, 0.],
                 hidden_dims=[64],
-                input_dim=1024,
+                input_dim=164,
                 num_layers=2,
                 vocab_namespace="upostag_labels"
             ),
@@ -240,7 +240,7 @@ def default_model(pretrained_transformer_name: str, vocabulary: Vocabulary, use_
                 activations=[TanhActivation(), LinearActivation()],
                 dropout=[0.25, 0.],
                 hidden_dims=[64],
-                input_dim=1024,
+                input_dim=164,
                 num_layers=2,
                 vocab_namespace="xpostag_labels"
             )