fmeval.model_runners.model_runner

 1from abc import ABC, abstractmethod
 2from typing import Optional, Tuple, List, Union
 3
 4from fmeval.constants import MIME_TYPE_JSON
 5from fmeval.model_runners.composers import create_content_composer
 6from fmeval.model_runners.extractors import create_extractor
 7
 8
 9class ModelRunner(ABC):
10    """
11    This class is responsible for running the model and extracting the model output.
12
13    It handles everything related to the model, including: model deployment, payload construction for invocations,
14    and making sense of the model output.
15    """
16
17    def __init__(
18        self,
19        content_template: Optional[str] = None,
20        output: Optional[str] = None,
21        log_probability: Optional[str] = None,
22        embedding: Optional[str] = None,
23        content_type: str = MIME_TYPE_JSON,
24        accept_type: str = MIME_TYPE_JSON,
25        **kwargs
26    ):
27        """
28        :param content_template: String template to compose the model input from the prompt
29        :param output: JMESPath expression of output in the model output
30        :param log_probability: JMESPath expression of log probability in the model output
31        :param embedding: JMESPath expression of embedding in the model output
32        :param content_type: The content type of the request sent to the model for inference
33        :param accept_type: The accept type of the request sent to the model for inference
34        """
35        self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs)
36        self._extractor = create_extractor(
37            model_accept_type=accept_type,
38            output_location=output,
39            log_probability_location=log_probability,
40            embedding_location=embedding,
41            **kwargs,
42        )
43
44    @abstractmethod
45    def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]:
46        """
47        Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model
48        expects, and extracting the output and log probability from the model response. The response of the ModelRunner
49        will be a tuple of (output, log_probability) or embedding
50
51        :param prompt: the prompt
52        :return: the tuple containing model output string and the log probability, or embedding
53        """
class ModelRunner(abc.ABC):
10class ModelRunner(ABC):
11    """
12    This class is responsible for running the model and extracting the model output.
13
14    It handles everything related to the model, including: model deployment, payload construction for invocations,
15    and making sense of the model output.
16    """
17
18    def __init__(
19        self,
20        content_template: Optional[str] = None,
21        output: Optional[str] = None,
22        log_probability: Optional[str] = None,
23        embedding: Optional[str] = None,
24        content_type: str = MIME_TYPE_JSON,
25        accept_type: str = MIME_TYPE_JSON,
26        **kwargs
27    ):
28        """
29        :param content_template: String template to compose the model input from the prompt
30        :param output: JMESPath expression of output in the model output
31        :param log_probability: JMESPath expression of log probability in the model output
32        :param embedding: JMESPath expression of embedding in the model output
33        :param content_type: The content type of the request sent to the model for inference
34        :param accept_type: The accept type of the request sent to the model for inference
35        """
36        self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs)
37        self._extractor = create_extractor(
38            model_accept_type=accept_type,
39            output_location=output,
40            log_probability_location=log_probability,
41            embedding_location=embedding,
42            **kwargs,
43        )
44
45    @abstractmethod
46    def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]:
47        """
48        Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model
49        expects, and extracting the output and log probability from the model response. The response of the ModelRunner
50        will be a tuple of (output, log_probability) or embedding
51
52        :param prompt: the prompt
53        :return: the tuple containing model output string and the log probability, or embedding
54        """

This class is responsible for running the model and extracting the model output.

It handles everything related to the model, including: model deployment, payload construction for invocations, and making sense of the model output.

ModelRunner( content_template: Optional[str] = None, output: Optional[str] = None, log_probability: Optional[str] = None, embedding: Optional[str] = None, content_type: str = 'application/json', accept_type: str = 'application/json', **kwargs)
18    def __init__(
19        self,
20        content_template: Optional[str] = None,
21        output: Optional[str] = None,
22        log_probability: Optional[str] = None,
23        embedding: Optional[str] = None,
24        content_type: str = MIME_TYPE_JSON,
25        accept_type: str = MIME_TYPE_JSON,
26        **kwargs
27    ):
28        """
29        :param content_template: String template to compose the model input from the prompt
30        :param output: JMESPath expression of output in the model output
31        :param log_probability: JMESPath expression of log probability in the model output
32        :param embedding: JMESPath expression of embedding in the model output
33        :param content_type: The content type of the request sent to the model for inference
34        :param accept_type: The accept type of the request sent to the model for inference
35        """
36        self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs)
37        self._extractor = create_extractor(
38            model_accept_type=accept_type,
39            output_location=output,
40            log_probability_location=log_probability,
41            embedding_location=embedding,
42            **kwargs,
43        )
Parameters
  • content_template: String template to compose the model input from the prompt
  • output: JMESPath expression of output in the model output
  • log_probability: JMESPath expression of log probability in the model output
  • embedding: JMESPath expression of embedding in the model output
  • content_type: The content type of the request sent to the model for inference
  • accept_type: The accept type of the request sent to the model for inference
@abstractmethod
def predict( self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]:
45    @abstractmethod
46    def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]:
47        """
48        Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model
49        expects, and extracting the output and log probability from the model response. The response of the ModelRunner
50        will be a tuple of (output, log_probability) or embedding
51
52        :param prompt: the prompt
53        :return: the tuple containing model output string and the log probability, or embedding
54        """

Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model expects, and extracting the output and log probability from the model response. The response of the ModelRunner will be a tuple of (output, log_probability) or embedding

Parameters
  • prompt: the prompt
Returns

the tuple containing model output string and the log probability, or embedding