Skip to content
Snippets Groups Projects
Select Git revision
  • deanonimzer
  • master default protected
  • v2 protected
  • v1 protected
  • develop protected
5 results
Blame History Permalink
wiktor_ner.py 3.36 KiB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113 






"""Module for parsing WiktorNER files."""

import json
from typing import List, Tuple

from src.annotations import Annotation, MorphosyntacticAnnotation, NerAnnotation
from src.input_parsers.interface import InputParser


class WiktorNERInputParser(InputParser):
    """Parser for WiktorNER files.

    Example WiktorNER file:
    {
        "filename": "greeting-5b1401",
        "text": "Hello Tom!",
        "tokens": [
            {
                "index": 1,
                "position": [0,5],
                "orth": "Hello",
                "lex": [
                    {
                        "lemma": "hello",
                        "mstag": "interj"
                    }
                ]
            },
            {
                "index": 2,
                "position": [6,9],
                "orth": "Tom",
                "lex": [
                    {
                        "lemma": "Tom",
                        "mstag": "noun"
                    }
                ]
            },
            {
                "index": 3,
                "position": [9,10],
                "orth": "!",
                "lex": [
                    {
                        "lemma": "!",
                        "mstag": "interp"
                    }
                ]
            }
        ],
        "entities": [
            {
                "text": "Tom",
                "type": "nam_prs_human",
                "tokens": [2],
                "positions": [6,9]
            }
        ]
    }
    """

    def parse(self, content: str) -> Tuple[str, List[Tuple[int, int, Annotation]]]:
        """Parse wiktorner file into text and annotations.

        Annotations are returned as a dictionary with channel name as a key and list of
        tuples.

        Args:
            co z  (str): Path to file containing CCL.

        Returns:
            Tuple[str, List[Tuple[int, int, Annotation]]]: Text and annotations.

        """
        content_parsed = json.loads(content)

        if "text" in content_parsed:
            text = content_parsed["text"]
        else:
            text = ""

        annotations = []

        # Morphosyntactic annotations
        if "tokens" in content_parsed:
            for token in content_parsed["tokens"]:
                if "position" in token:
                    token_start, token_end = token["position"]
                    if "lexemes" in token:
                        for lexeme in token["lexemes"]:
                            if "disamb" in lexeme and lexeme["disamb"] is True:
                                if "mstag" in lexeme:
                                    annotations.append(
                                        (
                                            token_start,
                                            token_end,
                                            MorphosyntacticAnnotation(lexeme["mstag"]),
                                        )
                                    )

        # NER annotations
        if "entities" in content_parsed:
            for entity in content_parsed["entities"]:
                if "positions" in entity:
                    entity_start, entity_end = entity["positions"]
                    if "type" in entity:
                        annotations.append(
                            (entity_start, entity_end, NerAnnotation(entity["type"]))
                        )

        return text, annotations