fmeval.eval 

 1import inspect
 2import json
 3from typing import Dict, Optional, Union
 4
 5from fmeval.eval_algo_mapping import EVAL_ALGORITHMS
 6from fmeval.eval_algorithms.eval_algorithm import EvalAlgorithmInterface, EvalAlgorithmConfig
 7from fmeval.exceptions import EvalAlgorithmClientError
 8
 9
10def get_eval_algorithm(
11    eval_name: str, eval_algorithm_config: Optional[Union[Dict, EvalAlgorithmConfig]] = None
12) -> EvalAlgorithmInterface:
13    """
14    Get eval algorithm class with name
15
16    :param eval_name: eval algorithm name
17    :return: eval algorithm class
18    """
19    if eval_name in EVAL_ALGORITHMS:
20        if isinstance(eval_algorithm_config, EvalAlgorithmConfig):
21            eval_algorithm_config = json.loads(json.dumps(eval_algorithm_config, default=vars))
22        try:
23            config_parameters = inspect.signature(EVAL_ALGORITHMS[eval_name]).parameters.get("eval_algorithm_config")
24            return (
25                EVAL_ALGORITHMS[eval_name](config_parameters.annotation(**eval_algorithm_config))
26                if eval_algorithm_config and config_parameters
27                else EVAL_ALGORITHMS[eval_name]()
28            )
29        except TypeError as e:
30            raise EvalAlgorithmClientError(
31                f"Unable to create algorithm for eval_name {eval_name} with config {eval_algorithm_config}: Error {e}"
32            )
33    else:
34        raise EvalAlgorithmClientError(f"Unknown eval algorithm {eval_name}")
def get_eval_algorithm( eval_name: str, eval_algorithm_config: Union[Dict, fmeval.eval_algorithms.eval_algorithm.EvalAlgorithmConfig, NoneType] = None) -> fmeval.eval_algorithms.eval_algorithm.EvalAlgorithmInterface: 
11def get_eval_algorithm(
12    eval_name: str, eval_algorithm_config: Optional[Union[Dict, EvalAlgorithmConfig]] = None
13) -> EvalAlgorithmInterface:
14    """
15    Get eval algorithm class with name
16
17    :param eval_name: eval algorithm name
18    :return: eval algorithm class
19    """
20    if eval_name in EVAL_ALGORITHMS:
21        if isinstance(eval_algorithm_config, EvalAlgorithmConfig):
22            eval_algorithm_config = json.loads(json.dumps(eval_algorithm_config, default=vars))
23        try:
24            config_parameters = inspect.signature(EVAL_ALGORITHMS[eval_name]).parameters.get("eval_algorithm_config")
25            return (
26                EVAL_ALGORITHMS[eval_name](config_parameters.annotation(**eval_algorithm_config))
27                if eval_algorithm_config and config_parameters
28                else EVAL_ALGORITHMS[eval_name]()
29            )
30        except TypeError as e:
31            raise EvalAlgorithmClientError(
32                f"Unable to create algorithm for eval_name {eval_name} with config {eval_algorithm_config}: Error {e}"
33            )
34    else:
35        raise EvalAlgorithmClientError(f"Unknown eval algorithm {eval_name}")

Get eval algorithm class with name

Parameters
  • eval_name: eval algorithm name
Returns

eval algorithm class