Skip to content

Evaluation

AlignmentEvaluator()

Class that provides evaluation metrics for alignment.

Source code in src/deeponto/align/evaluation.py
23
24
def __init__(self):
    pass

precision(prediction_mappings, reference_mappings) staticmethod

The percentage of correct predictions.

P=|MpredMref||Mpred|
Source code in src/deeponto/align/evaluation.py
26
27
28
29
30
31
32
33
34
@staticmethod
def precision(prediction_mappings: List[EntityMapping], reference_mappings: List[ReferenceMapping]) -> float:
    r"""The percentage of correct predictions.

    $$P = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{pred}|}$$
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    return len(set(preds).intersection(set(refs))) / len(set(preds))

recall(prediction_mappings, reference_mappings) staticmethod

The percentage of correct retrievals.

R=|MpredMref||Mref|
Source code in src/deeponto/align/evaluation.py
36
37
38
39
40
41
42
43
44
@staticmethod
def recall(prediction_mappings: List[EntityMapping], reference_mappings: List[ReferenceMapping]) -> float:
    r"""The percentage of correct retrievals.

    $$R = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{ref}|}$$
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    return len(set(preds).intersection(set(refs))) / len(set(refs))

f1(prediction_mappings, reference_mappings, null_reference_mappings=[]) staticmethod

Compute the F1 score given the prediction and reference mappings.

F1=2PRP+R

null_reference_mappings is an additional set whose elements should be ignored in the calculation, i.e., neither positive nor negative. Specifically, both Mpred and Mref will substract Mnull from them.

Source code in src/deeponto/align/evaluation.py
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
@staticmethod
def f1(
    prediction_mappings: List[EntityMapping],
    reference_mappings: List[ReferenceMapping],
    null_reference_mappings: List[ReferenceMapping] = [],
):
    r"""Compute the F1 score given the prediction and reference mappings.

    $$F_1 = \frac{2 P R}{P + R}$$

    `null_reference_mappings` is an additional set whose elements
    should be **ignored** in the calculation, i.e., **neither positive nor negative**.
    Specifically, both $\mathcal{M}_{pred}$ and $\mathcal{M}_{ref}$ will **substract**
    $\mathcal{M}_{null}$ from them.
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    null_refs = [n.to_tuple() for n in null_reference_mappings]
    # elements in the {null_set} are removed from both {pred} and {ref} (ignored)
    if null_refs:
        preds = set(preds) - set(null_refs)
        refs = set(refs) - set(null_refs)
    P = len(set(preds).intersection(set(refs))) / len(set(preds))
    R = len(set(preds).intersection(set(refs))) / len(set(refs))
    F1 = 2 * P * R / (P + R)

    return {"P": round(P, 3), "R": round(R, 3), "F1": round(F1, 3)}

hits_at_K(reference_and_candidates, K) staticmethod

Compute Hits@K for a list of (reference_mapping, candidate_mappings) pair.

It is computed as the number of a reference_mapping existed in the first K ranked candidate_mappings, divided by the total number of input pairs.

Hits@K=iNIrankik/N
Source code in src/deeponto/align/evaluation.py
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def hits_at_K(reference_and_candidates: List[Tuple[ReferenceMapping, List[EntityMapping]]], K: int):
    r"""Compute $Hits@K$ for a list of `(reference_mapping, candidate_mappings)` pair.

    It is computed as the number of a `reference_mapping` existed in the first $K$ ranked `candidate_mappings`,
    divided by the total number of input pairs.

    $$Hits@K = \sum_i^N \mathbb{I}_{rank_i \leq k} / N$$
    """
    n_hits = 0
    for pred, cands in reference_and_candidates:
        ordered_candidates = [c.to_tuple() for c in EntityMapping.sort_entity_mappings_by_score(cands, k=K)]
        if pred.to_tuple() in ordered_candidates:
            n_hits += 1
    return n_hits / len(reference_and_candidates)

mean_reciprocal_rank(reference_and_candidates) staticmethod

Compute MRR for a list of (reference_mapping, candidate_mappings) pair.

MRR=iNranki1/N
Source code in src/deeponto/align/evaluation.py
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
@staticmethod
def mean_reciprocal_rank(reference_and_candidates: List[Tuple[ReferenceMapping, List[EntityMapping]]]):
    r"""Compute $MRR$ for a list of `(reference_mapping, candidate_mappings)` pair.

    $$MRR = \sum_i^N rank_i^{-1} / N$$
    """
    sum_inverted_ranks = 0
    for pred, cands in reference_and_candidates:
        ordered_candidates = [c.to_tuple() for c in EntityMapping.sort_entity_mappings_by_score(cands)]
        if pred.to_tuple() in ordered_candidates:
            rank = ordered_candidates.index(pred.to_tuple()) + 1
        else:
            rank = math.inf
        sum_inverted_ranks += 1 / rank
    return sum_inverted_ranks / len(reference_and_candidates)

Last update: February 1, 2023
Created: January 22, 2023
GitHub: @Lawhy   Personal Page: yuanhe.wiki