fmeval.eval_algo_mapping

 1from typing import Dict, Type
 2
 3from fmeval.eval_algorithms import EvalAlgorithm
 4from fmeval.eval_algorithms.classification_accuracy_semantic_robustness import (
 5    ClassificationAccuracySemanticRobustness,
 6)
 7from fmeval.eval_algorithms.eval_algorithm import EvalAlgorithmInterface
 8from fmeval.eval_algorithms.factual_knowledge import FactualKnowledge
 9from fmeval.eval_algorithms.general_semantic_robustness import GeneralSemanticRobustness
10from fmeval.eval_algorithms.prompt_stereotyping import PromptStereotyping
11from fmeval.eval_algorithms.qa_accuracy import QAAccuracy
12from fmeval.eval_algorithms.qa_accuracy_semantic_robustness import QAAccuracySemanticRobustness
13from fmeval.eval_algorithms.qa_toxicity import QAToxicity
14from fmeval.eval_algorithms.summarization_accuracy import SummarizationAccuracy
15from fmeval.eval_algorithms.classification_accuracy import ClassificationAccuracy
16from fmeval.eval_algorithms.summarization_accuracy_semantic_robustness import (
17    SummarizationAccuracySemanticRobustness,
18)
19from fmeval.eval_algorithms.summarization_toxicity import SummarizationToxicity
20from fmeval.eval_algorithms.toxicity import Toxicity
21
22EVAL_ALGORITHMS: Dict[str, Type["EvalAlgorithmInterface"]] = {
23    EvalAlgorithm.CLASSIFICATION_ACCURACY.value: ClassificationAccuracy,
24    EvalAlgorithm.CLASSIFICATION_ACCURACY_SEMANTIC_ROBUSTNESS.value: ClassificationAccuracySemanticRobustness,
25    EvalAlgorithm.FACTUAL_KNOWLEDGE.value: FactualKnowledge,
26    EvalAlgorithm.GENERAL_SEMANTIC_ROBUSTNESS.value: GeneralSemanticRobustness,
27    EvalAlgorithm.PROMPT_STEREOTYPING.value: PromptStereotyping,
28    EvalAlgorithm.QA_ACCURACY.value: QAAccuracy,
29    EvalAlgorithm.QA_ACCURACY_SEMANTIC_ROBUSTNESS.value: QAAccuracySemanticRobustness,
30    EvalAlgorithm.QA_TOXICITY.value: QAToxicity,
31    EvalAlgorithm.SUMMARIZATION_ACCURACY.value: SummarizationAccuracy,
32    EvalAlgorithm.SUMMARIZATION_ACCURACY_SEMANTIC_ROBUSTNESS.value: SummarizationAccuracySemanticRobustness,
33    EvalAlgorithm.SUMMARIZATION_TOXICITY.value: SummarizationToxicity,
34    EvalAlgorithm.TOXICITY.value: Toxicity,
35}