fmeval.transforms.semantic_robustness_metrics

  1import evaluate as hf_evaluate
  2from typing import List, Dict, Any, Tuple
  3
  4import numpy as np
  5
  6from fmeval.util import require
  7from fmeval.transforms.common import Mean
  8from fmeval.transforms.transform import Transform
  9from fmeval.transforms.util import validate_call
 10
 11
 12class BertScoreDissimilarity(Transform):
 13    """This Transform augments its input record with the BERTScore Dissimilarity metric.
 14
 15    BERTScore Dissimilarity is simply 1 - BERTScore
 16    (https://huggingface.co/spaces/evaluate-metric/bertscore).
 17    This Transform uses the mean of a list of BERTScore values as the BERTScore
 18    in the formula above.
 19    """
 20
 21    def __init__(self, bert_score_keys: List[str], output_key: str):
 22        """BertScoreDissimilarity initializer.
 23
 24        :param bert_score_keys: The keys corresponding to the BERTScore values.
 25        :param output_key: The key corresponding to the output of this transform.
 26        """
 27        super().__init__(bert_score_keys, output_key)
 28        self.register_input_output_keys(bert_score_keys, [output_key])
 29        self.bert_score_keys = bert_score_keys
 30        self.output_key = output_key
 31
 32    @validate_call
 33    def __call__(self, record: Dict[str, Any]) -> Dict[str, Any]:
 34        """Augment the input record with the computed BERTScore Dissimilarity metric.
 35
 36        :param record: The input record.
 37        :returns: The input record with the BERTScore Dissimilarity metric added in.
 38        """
 39        add_mean_bert_score = Mean(
 40            self.bert_score_keys,
 41            self.output_key,
 42        )
 43        record = add_mean_bert_score(record)
 44        # Override the intermediate value corresponding to self.output_key
 45        # (i.e. the mean bert score) to 1 - mean.
 46        record[self.output_key] = 1 - record[self.output_key]
 47        return record
 48
 49
 50class WER(Transform):
 51    """This Transform computes the Word Error Rate metric and augments its input record with the computed value.
 52
 53    Word Error Rate measures syntactic differences, that is, changes in the words, whereas BERTScore Dissimilarity
 54    measures semantic differences. Semantic differences account for cases when the precise words in the output
 55    change but the meaning is the same. For example, consider the outputs "it is pouring down today" vs.
 56    "it is very rainy today".
 57    """
 58
 59    def __init__(self, prediction_keys: List[str], reference_keys: List[str], output_key: str):
 60        """WER initializer.
 61
 62        Note that the order of elements in `prediction_keys` and `reference_keys` matters;
 63        the kth element of `prediction_keys` should correspond to the kth element of
 64        `reference_keys`.
 65
 66        :param prediction_keys: The record keys corresponding to model predictions.
 67        :param reference_keys: The record keys corresponding ot reference, aka target, values.
 68        :param output_key: The output key to assign the computed WER value.
 69        """
 70        require(
 71            len(prediction_keys) == len(reference_keys),
 72            "prediction_keys and reference_keys should have the same number of elements. "
 73            f"prediction_keys has {len(prediction_keys)} elements while reference_keys has "
 74            f"{len(reference_keys)} elements.",
 75        )
 76        super().__init__(prediction_keys, reference_keys, output_key)
 77        self.register_input_output_keys(prediction_keys + reference_keys, [output_key], allow_duplicates=True)
 78        self.prediction_keys = prediction_keys
 79        self.reference_keys = reference_keys
 80        self.output_key = output_key
 81        self.wer = hf_evaluate.load("wer")
 82
 83    @validate_call
 84    def __call__(self, record: Dict[str, Any]) -> Dict[str, Any]:
 85        """Augment the input record with the computed WER metric.
 86
 87        :param record: The input record.
 88        :returns: The input record with the WER metric added in.
 89        """
 90        wer_metric = self.wer.compute(
 91            predictions=[record[prediction_key] for prediction_key in self.prediction_keys],
 92            references=[record[reference_key] for reference_key in self.reference_keys],
 93        )
 94        record[self.output_key] = wer_metric
 95        return record
 96
 97
 98class MeanDeltaScores(Transform):
 99    """This transform augments an input record with mean delta scores.
100
101    Given
102        1) An "original score", which is a score that was computed using
103            an "original" i.e. unperturbed input
104        2) A series of "perturbed scores", which are scores computed using
105            perturbations of the original input
106    the delta score for a particular perturbed score is computed using the
107    formula: abs(original_score - perturbed_score), and the mean delta score
108    is simply the arithmetic mean of all delta scores for the series of
109    perturbed scores.
110    """
111
112    def __init__(self, key_mapping: Dict[str, Tuple[List[str], str]]):
113        """MeanDeltaScores initializer.
114
115        :param key_mapping: Maps an original score key to a tuple of the form
116            (perturbed_score_keys, output_key). output_key will be used
117            as the output key corresponding to the mean delta score computed
118            using the original score and perturbed scores.
119        """
120        super().__init__(key_mapping)
121        original_score_keys = list(key_mapping.keys())
122        perturbed_score_keys = [key for tup in key_mapping.values() for key in tup[0]]
123        self.register_input_output_keys(
124            input_keys=original_score_keys + perturbed_score_keys,
125            output_keys=[tup[1] for tup in key_mapping.values()],
126        )
127        self.key_mapping = key_mapping
128
129    @validate_call
130    def __call__(self, record: Dict[str, Any]) -> Dict[str, Any]:
131        """Augment the input record with the computed mean delta scores.
132
133        :param record: The input record.
134        :returns: The input record with the mean delta scores added in.
135        """
136        for original_score_key, tup in self.key_mapping.items():
137            perturbed_score_keys, output_key = tup
138            record[output_key] = np.mean(
139                [
140                    abs(record[original_score_key] - record[perturbed_score_key])
141                    for perturbed_score_key in perturbed_score_keys
142                ]
143            )
144        return record