Enable weighted average of LM embeddings.

combo/config.graph.template.jsonnet

@@ -49,7 +49,7 @@ local lemma_char_dim = 64;
 # Character embedding dim, int
 local char_dim = 64;
 # Word embedding projection dim, int
-local projected_embedding_dim = 100;
+local projected_embedding_dim = 768;
 # Loss weights, dict[str, int]
 local loss_weights = {
     xpostag: 0.05,
@@ -202,6 +202,7 @@ assert pretrained_tokens == null || pretrained_transformer_name == null: "Can't
                 },
                 token: if use_transformer then {
                     type: "transformers_word_embeddings",
+                    last_layer_only: false,
                     model_name: pretrained_transformer_name,
                     projection_dim: projected_embedding_dim,
                     tokenizer_kwargs: if std.startsWith(pretrained_transformer_name, "allegro/herbert")

combo/models/embeddings.py

@@ -111,10 +111,12 @@ class TransformersWordEmbedder(token_embedders.PretrainedTransformerMismatchedEm
                  projection_activation: Optional[allen_nn.Activation] = lambda x: x,
                  projection_dropout_rate: Optional[float] = 0.0,
                  freeze_transformer: bool = True,
+                 last_layer_only: bool = True,
                  tokenizer_kwargs: Optional[Dict[str, Any]] = None,
                  transformer_kwargs: Optional[Dict[str, Any]] = None):
         super().__init__(model_name,
                          train_parameters=not freeze_transformer,
+                         last_layer_only=last_layer_only,
                          tokenizer_kwargs=tokenizer_kwargs,
                          transformer_kwargs=transformer_kwargs)
         if projection_dim: