Select Git revision
conlleval.py
Michał Marcińczuk authored
conlleval.py 7.37 KiB
"""
Source: https://github.com/sighsmile/conlleval
This script applies to IOB2 or IOBES tagging scheme.
If you are using a different scheme, please convert to IOB2 or IOBES.
IOB2:
- B = begin,
- I = inside but not the first,
- O = outside
e.g.
John lives in New York City .
B-PER O O B-LOC I-LOC I-LOC O
IOBES:
- B = begin,
- E = end,
- S = singleton,
- I = inside but not the first or the last,
- O = outside
e.g.
John lives in New York City .
S-PER O O B-LOC I-LOC E-LOC O
prefix: IOBES
chunk_type: PER, LOC, etc.
"""
from __future__ import division, print_function, unicode_literals
import sys
from collections import defaultdict
def split_tag(chunk_tag):
"""
split chunk tag into IOBES prefix and chunk_type
e.g.
B-PER -> (B, PER)
O -> (O, None)
"""
if chunk_tag == 'O':
return ('O', None)
return chunk_tag.split('-', maxsplit=1)
def is_chunk_end(prev_tag, tag):
"""
check if the previous chunk ended between the previous and current word
e.g.
(B-PER, I-PER) -> False
(B-LOC, O) -> True
Note: in case of contradicting tags, e.g. (B-PER, I-LOC)
this is considered as (B-PER, B-LOC)
"""
prefix1, chunk_type1 = split_tag(prev_tag)
prefix2, chunk_type2 = split_tag(tag)
if prefix1 == 'O':
return False
if prefix2 == 'O':
return prefix1 != 'O'
if chunk_type1 != chunk_type2:
return True
return prefix2 in ['B', 'S'] or prefix1 in ['E', 'S']
def is_chunk_start(prev_tag, tag):
"""
check if a new chunk started between the previous and current word
"""
prefix1, chunk_type1 = split_tag(prev_tag)
prefix2, chunk_type2 = split_tag(tag)
if prefix2 == 'O':
return False
if prefix1 == 'O':
return prefix2 != 'O'
if chunk_type1 != chunk_type2:
return True
return prefix2 in ['B', 'S'] or prefix1 in ['E', 'S']
def calc_metrics(tp, p, t, percent=True):
"""
compute overall precision, recall and FB1 (default values are 0.0)
if percent is True, return 100 * original decimal value
"""
precision = tp / p if p else 0
recall = tp / t if t else 0
fb1 = 2 * precision * recall / (precision + recall) if precision + recall else 0
if percent:
return 100 * precision, 100 * recall, 100 * fb1
else:
return precision, recall, fb1
def count_chunks(true_seqs, pred_seqs):
"""
true_seqs: a list of true tags
pred_seqs: a list of predicted tags
return:
correct_chunks: a dict (counter),
key = chunk types,
value = number of correctly identified chunks per type
true_chunks: a dict, number of true chunks per type
pred_chunks: a dict, number of identified chunks per type
correct_counts, true_counts, pred_counts: similar to above, but for tags
"""
correct_chunks = defaultdict(int)
true_chunks = defaultdict(int)
pred_chunks = defaultdict(int)
correct_counts = defaultdict(int)
true_counts = defaultdict(int)
pred_counts = defaultdict(int)
prev_true_tag, prev_pred_tag = 'O', 'O'
correct_chunk = None
for true_tag, pred_tag in zip(true_seqs, pred_seqs):
if true_tag == pred_tag:
correct_counts[true_tag] += 1
true_counts[true_tag] += 1
pred_counts[pred_tag] += 1
_, true_type = split_tag(true_tag)
_, pred_type = split_tag(pred_tag)
if correct_chunk is not None:
true_end = is_chunk_end(prev_true_tag, true_tag)
pred_end = is_chunk_end(prev_pred_tag, pred_tag)
if pred_end and true_end:
correct_chunks[correct_chunk] += 1
correct_chunk = None
elif pred_end != true_end or true_type != pred_type:
correct_chunk = None
true_start = is_chunk_start(prev_true_tag, true_tag)
pred_start = is_chunk_start(prev_pred_tag, pred_tag)
if true_start and pred_start and true_type == pred_type:
correct_chunk = true_type
if true_start:
true_chunks[true_type] += 1
if pred_start:
pred_chunks[pred_type] += 1
prev_true_tag, prev_pred_tag = true_tag, pred_tag
if correct_chunk is not None:
correct_chunks[correct_chunk] += 1
return (correct_chunks, true_chunks, pred_chunks,
correct_counts, true_counts, pred_counts)
def get_result(correct_chunks, true_chunks, pred_chunks,
correct_counts, true_counts, pred_counts, verbose=True):
"""
if verbose, print overall performance, as well as preformance per chunk type;
otherwise, simply return overall prec, rec, f1 scores
"""
# sum counts
sum_correct_chunks = sum(correct_chunks.values())
sum_true_chunks = sum(true_chunks.values())
sum_pred_chunks = sum(pred_chunks.values())
sum_correct_counts = sum(correct_counts.values())
sum_true_counts = sum(true_counts.values())
nonO_correct_counts = sum(v for k, v in correct_counts.items() if k != 'O')
nonO_true_counts = sum(v for k, v in true_counts.items() if k != 'O')
chunk_types = sorted(list(set(list(true_chunks) + list(pred_chunks))))
# compute overall precision, recall and FB1 (default values are 0.0)
prec, rec, f1 = calc_metrics(sum_correct_chunks, sum_pred_chunks, sum_true_chunks)
res = (prec, rec, f1)
if not verbose:
return res
# print overall performance, and performance per chunk type
print("processed %i tokens with %i phrases; " % (sum_true_counts, sum_true_chunks), end='')
print("found: %i phrases; correct: %i.\n" % (sum_pred_chunks, sum_correct_chunks), end='')
print("accuracy: %6.2f%%; (non-O)" % (100*nonO_correct_counts/nonO_true_counts))
print("accuracy: %6.2f%%; " % (100*sum_correct_counts/sum_true_counts), end='')
print("precision: %6.2f%%; recall: %6.2f%%; FB1: %6.2f" % (prec, rec, f1))
# for each chunk type, compute precision, recall and FB1 (default values are 0.0)
for t in chunk_types:
prec, rec, f1 = calc_metrics(correct_chunks[t], pred_chunks[t], true_chunks[t])
print("%17s: " %t , end='')
print("precision: %6.2f%%; recall: %6.2f%%; FB1: %6.2f" %
(prec, rec, f1), end='')
print(" %d" % pred_chunks[t])
return res
# you can generate LaTeX output for tables like in
# http://cnts.uia.ac.be/conll2003/ner/example.tex
# but I'm not implementing this
def evaluate(true_seqs, pred_seqs, verbose=True):
(correct_chunks, true_chunks, pred_chunks,
correct_counts, true_counts, pred_counts) = count_chunks(true_seqs, pred_seqs)
result = get_result(correct_chunks, true_chunks, pred_chunks,
correct_counts, true_counts, pred_counts, verbose=verbose)
return result
def evaluate_conll_file(fileIterator):
true_seqs, pred_seqs = [], []
for line in fileIterator:
cols = line.strip().split()
# each non-empty line must contain >= 3 columns
if not cols:
true_seqs.append('O')
pred_seqs.append('O')
elif len(cols) < 3:
raise IOError("conlleval: too few columns in line %s\n" % line)
else:
# extract tags from last 2 columns
true_seqs.append(cols[-2])
pred_seqs.append(cols[-1])
return evaluate(true_seqs, pred_seqs)
if __name__ == '__main__':
"""
usage: conlleval < file
"""
evaluate_conll_file(sys.stdin)