Skip to content

Evaluation

AlignmentEvaluator()

Class that provides evaluation metrics for alignment.

Source code in src/deeponto/align/evaluation.py 
23
24
def __init__(self):
    pass

precision(prediction_mappings, reference_mappings) staticmethod

The percentage of correct predictions.

\[P = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{pred}|}\]
Source code in src/deeponto/align/evaluation.py 
26
27
28
29
30
31
32
33
34
@staticmethod
def precision(prediction_mappings: List[EntityMapping], reference_mappings: List[ReferenceMapping]) -> float:
    r"""The percentage of correct predictions.

    $$P = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{pred}|}$$
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    return len(set(preds).intersection(set(refs))) / len(set(preds))

recall(prediction_mappings, reference_mappings) staticmethod

The percentage of correct retrievals.

\[R = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{ref}|}\]
Source code in src/deeponto/align/evaluation.py 
36
37
38
39
40
41
42
43
44
@staticmethod
def recall(prediction_mappings: List[EntityMapping], reference_mappings: List[ReferenceMapping]) -> float:
    r"""The percentage of correct retrievals.

    $$R = \frac{|\mathcal{M}_{pred} \cap \mathcal{M}_{ref}|}{|\mathcal{M}_{ref}|}$$
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    return len(set(preds).intersection(set(refs))) / len(set(refs))

f1(prediction_mappings, reference_mappings, null_reference_mappings=[]) staticmethod

Compute the F1 score given the prediction and reference mappings.

\[F_1 = \frac{2 P R}{P + R}\]

null_reference_mappings is an additional set whose elements should be ignored in the calculation, i.e., neither positive nor negative. Specifically, both \(\mathcal{M}_{pred}\) and \(\mathcal{M}_{ref}\) will substract \(\mathcal{M}_{null}\) from them.

Source code in src/deeponto/align/evaluation.py 
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
@staticmethod
def f1(
    prediction_mappings: List[EntityMapping],
    reference_mappings: List[ReferenceMapping],
    null_reference_mappings: List[ReferenceMapping] = [],
):
    r"""Compute the F1 score given the prediction and reference mappings.

    $$F_1 = \frac{2 P R}{P + R}$$

    `null_reference_mappings` is an additional set whose elements
    should be **ignored** in the calculation, i.e., **neither positive nor negative**.
    Specifically, both $\mathcal{M}_{pred}$ and $\mathcal{M}_{ref}$ will **substract**
    $\mathcal{M}_{null}$ from them.
    """
    preds = [p.to_tuple() for p in prediction_mappings]
    refs = [r.to_tuple() for r in reference_mappings]
    null_refs = [n.to_tuple() for n in null_reference_mappings]
    # elements in the {null_set} are removed from both {pred} and {ref} (ignored)
    if null_refs:
        preds = set(preds) - set(null_refs)
        refs = set(refs) - set(null_refs)
    P = len(set(preds).intersection(set(refs))) / len(set(preds))
    R = len(set(preds).intersection(set(refs))) / len(set(refs))
    F1 = 2 * P * R / (P + R)

    return {"P": round(P, 3), "R": round(R, 3), "F1": round(F1, 3)}

hits_at_K(reference_and_candidates, K) staticmethod

Compute \(Hits@K\) for a list of (reference_mapping, candidate_mappings) pair.

It is computed as the number of a reference_mapping existed in the first \(K\) ranked candidate_mappings, divided by the total number of input pairs.

\[Hits@K = \sum_i^N \mathbb{I}_{rank_i \leq k} / N\]
Source code in src/deeponto/align/evaluation.py 
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
@staticmethod
def hits_at_K(reference_and_candidates: List[Tuple[ReferenceMapping, List[EntityMapping]]], K: int):
    r"""Compute $Hits@K$ for a list of `(reference_mapping, candidate_mappings)` pair.

    It is computed as the number of a `reference_mapping` existed in the first $K$ ranked `candidate_mappings`,
    divided by the total number of input pairs.

    $$Hits@K = \sum_i^N \mathbb{I}_{rank_i \leq k} / N$$
    """
    n_hits = 0
    for pred, cands in reference_and_candidates:
        ordered_candidates = [c.to_tuple() for c in EntityMapping.sort_entity_mappings_by_score(cands, k=K)]
        if pred.to_tuple() in ordered_candidates:
            n_hits += 1
    return n_hits / len(reference_and_candidates)

mean_reciprocal_rank(reference_and_candidates) staticmethod

Compute \(MRR\) for a list of (reference_mapping, candidate_mappings) pair.

\[MRR = \sum_i^N rank_i^{-1} / N\]
Source code in src/deeponto/align/evaluation.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
@staticmethod
def mean_reciprocal_rank(reference_and_candidates: List[Tuple[ReferenceMapping, List[EntityMapping]]]):
    r"""Compute $MRR$ for a list of `(reference_mapping, candidate_mappings)` pair.

    $$MRR = \sum_i^N rank_i^{-1} / N$$
    """
    sum_inverted_ranks = 0
    for pred, cands in reference_and_candidates:
        ordered_candidates = [c.to_tuple() for c in EntityMapping.sort_entity_mappings_by_score(cands)]
        if pred.to_tuple() in ordered_candidates:
            rank = ordered_candidates.index(pred.to_tuple()) + 1
        else:
            rank = math.inf
        sum_inverted_ranks += 1 / rank
    return sum_inverted_ranks / len(reference_and_candidates)
Last update: February 1, 2023
Created: January 22, 2023
GitHub: @Lawhy   Personal Page: yuanhe.wiki