|
|
|
|
|
from collections import Counter |
|
|
|
|
|
|
|
|
def gt_label2entity(gt_infos): |
|
|
"""Get all entities from ground truth infos. |
|
|
Args: |
|
|
gt_infos (list[dict]): Ground-truth information contains text and |
|
|
label. |
|
|
Returns: |
|
|
gt_entities (list[list]): Original labeled entities in groundtruth. |
|
|
[[category,start_position,end_position]] |
|
|
""" |
|
|
gt_entities = [] |
|
|
for gt_info in gt_infos: |
|
|
line_entities = [] |
|
|
label = gt_info['label'] |
|
|
for key, value in label.items(): |
|
|
for _, places in value.items(): |
|
|
for place in places: |
|
|
line_entities.append([key, place[0], place[1]]) |
|
|
gt_entities.append(line_entities) |
|
|
return gt_entities |
|
|
|
|
|
|
|
|
def _compute_f1(origin, found, right): |
|
|
"""Calculate recall, precision, f1-score. |
|
|
|
|
|
Args: |
|
|
origin (int): Original entities in groundtruth. |
|
|
found (int): Predicted entities from model. |
|
|
right (int): Predicted entities that |
|
|
can match to the original annotation. |
|
|
Returns: |
|
|
recall (float): Metric of recall. |
|
|
precision (float): Metric of precision. |
|
|
f1 (float): Metric of f1-score. |
|
|
""" |
|
|
recall = 0 if origin == 0 else (right / origin) |
|
|
precision = 0 if found == 0 else (right / found) |
|
|
f1 = 0. if recall + precision == 0 else (2 * precision * recall) / ( |
|
|
precision + recall) |
|
|
return recall, precision, f1 |
|
|
|
|
|
|
|
|
def compute_f1_all(pred_entities, gt_entities): |
|
|
"""Calculate precision, recall and F1-score for all categories. |
|
|
|
|
|
Args: |
|
|
pred_entities: The predicted entities from model. |
|
|
gt_entities: The entities of ground truth file. |
|
|
Returns: |
|
|
class_info (dict): precision,recall, f1-score in total |
|
|
and each categories. |
|
|
""" |
|
|
origins = [] |
|
|
founds = [] |
|
|
rights = [] |
|
|
for i, _ in enumerate(pred_entities): |
|
|
origins.extend(gt_entities[i]) |
|
|
founds.extend(pred_entities[i]) |
|
|
rights.extend([ |
|
|
pre_entity for pre_entity in pred_entities[i] |
|
|
if pre_entity in gt_entities[i] |
|
|
]) |
|
|
|
|
|
class_info = {} |
|
|
origin_counter = Counter([x[0] for x in origins]) |
|
|
found_counter = Counter([x[0] for x in founds]) |
|
|
right_counter = Counter([x[0] for x in rights]) |
|
|
for type_, count in origin_counter.items(): |
|
|
origin = count |
|
|
found = found_counter.get(type_, 0) |
|
|
right = right_counter.get(type_, 0) |
|
|
recall, precision, f1 = _compute_f1(origin, found, right) |
|
|
class_info[type_] = { |
|
|
'precision': precision, |
|
|
'recall': recall, |
|
|
'f1-score': f1 |
|
|
} |
|
|
origin = len(origins) |
|
|
found = len(founds) |
|
|
right = len(rights) |
|
|
recall, precision, f1 = _compute_f1(origin, found, right) |
|
|
class_info['all'] = { |
|
|
'precision': precision, |
|
|
'recall': recall, |
|
|
'f1-score': f1 |
|
|
} |
|
|
return class_info |
|
|
|
|
|
|
|
|
def eval_ner_f1(results, gt_infos): |
|
|
"""Evaluate for ner task. |
|
|
|
|
|
Args: |
|
|
results (list): Predict results of entities. |
|
|
gt_infos (list[dict]): Ground-truth information which contains |
|
|
text and label. |
|
|
Returns: |
|
|
class_info (dict): precision,recall, f1-score of total |
|
|
and each catogory. |
|
|
""" |
|
|
assert len(results) == len(gt_infos) |
|
|
gt_entities = gt_label2entity(gt_infos) |
|
|
pred_entities = [] |
|
|
for i, gt_info in enumerate(gt_infos): |
|
|
line_entities = [] |
|
|
for result in results[i]: |
|
|
line_entities.append(result) |
|
|
pred_entities.append(line_entities) |
|
|
assert len(pred_entities) == len(gt_entities) |
|
|
class_info = compute_f1_all(pred_entities, gt_entities) |
|
|
|
|
|
return class_info |
|
|
|
