fmeval.model_runners.model_runner
1from abc import ABC, abstractmethod 2from typing import Optional, Tuple, List, Union 3 4from fmeval.constants import MIME_TYPE_JSON 5from fmeval.model_runners.composers import create_content_composer 6from fmeval.model_runners.extractors import create_extractor 7 8 9class ModelRunner(ABC): 10 """ 11 This class is responsible for running the model and extracting the model output. 12 13 It handles everything related to the model, including: model deployment, payload construction for invocations, 14 and making sense of the model output. 15 """ 16 17 def __init__( 18 self, 19 content_template: Optional[str] = None, 20 output: Optional[str] = None, 21 log_probability: Optional[str] = None, 22 embedding: Optional[str] = None, 23 content_type: str = MIME_TYPE_JSON, 24 accept_type: str = MIME_TYPE_JSON, 25 **kwargs 26 ): 27 """ 28 :param content_template: String template to compose the model input from the prompt 29 :param output: JMESPath expression of output in the model output 30 :param log_probability: JMESPath expression of log probability in the model output 31 :param embedding: JMESPath expression of embedding in the model output 32 :param content_type: The content type of the request sent to the model for inference 33 :param accept_type: The accept type of the request sent to the model for inference 34 """ 35 self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs) 36 self._extractor = create_extractor( 37 model_accept_type=accept_type, 38 output_location=output, 39 log_probability_location=log_probability, 40 embedding_location=embedding, 41 **kwargs, 42 ) 43 44 @abstractmethod 45 def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]: 46 """ 47 Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model 48 expects, and extracting the output and log probability from the model response. The response of the ModelRunner 49 will be a tuple of (output, log_probability) or embedding 50 51 :param prompt: the prompt 52 :return: the tuple containing model output string and the log probability, or embedding 53 """
class
ModelRunner(abc.ABC):
10class ModelRunner(ABC): 11 """ 12 This class is responsible for running the model and extracting the model output. 13 14 It handles everything related to the model, including: model deployment, payload construction for invocations, 15 and making sense of the model output. 16 """ 17 18 def __init__( 19 self, 20 content_template: Optional[str] = None, 21 output: Optional[str] = None, 22 log_probability: Optional[str] = None, 23 embedding: Optional[str] = None, 24 content_type: str = MIME_TYPE_JSON, 25 accept_type: str = MIME_TYPE_JSON, 26 **kwargs 27 ): 28 """ 29 :param content_template: String template to compose the model input from the prompt 30 :param output: JMESPath expression of output in the model output 31 :param log_probability: JMESPath expression of log probability in the model output 32 :param embedding: JMESPath expression of embedding in the model output 33 :param content_type: The content type of the request sent to the model for inference 34 :param accept_type: The accept type of the request sent to the model for inference 35 """ 36 self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs) 37 self._extractor = create_extractor( 38 model_accept_type=accept_type, 39 output_location=output, 40 log_probability_location=log_probability, 41 embedding_location=embedding, 42 **kwargs, 43 ) 44 45 @abstractmethod 46 def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]: 47 """ 48 Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model 49 expects, and extracting the output and log probability from the model response. The response of the ModelRunner 50 will be a tuple of (output, log_probability) or embedding 51 52 :param prompt: the prompt 53 :return: the tuple containing model output string and the log probability, or embedding 54 """
This class is responsible for running the model and extracting the model output.
It handles everything related to the model, including: model deployment, payload construction for invocations, and making sense of the model output.
ModelRunner( content_template: Optional[str] = None, output: Optional[str] = None, log_probability: Optional[str] = None, embedding: Optional[str] = None, content_type: str = 'application/json', accept_type: str = 'application/json', **kwargs)
18 def __init__( 19 self, 20 content_template: Optional[str] = None, 21 output: Optional[str] = None, 22 log_probability: Optional[str] = None, 23 embedding: Optional[str] = None, 24 content_type: str = MIME_TYPE_JSON, 25 accept_type: str = MIME_TYPE_JSON, 26 **kwargs 27 ): 28 """ 29 :param content_template: String template to compose the model input from the prompt 30 :param output: JMESPath expression of output in the model output 31 :param log_probability: JMESPath expression of log probability in the model output 32 :param embedding: JMESPath expression of embedding in the model output 33 :param content_type: The content type of the request sent to the model for inference 34 :param accept_type: The accept type of the request sent to the model for inference 35 """ 36 self._composer = create_content_composer(content_type=content_type, template=content_template, **kwargs) 37 self._extractor = create_extractor( 38 model_accept_type=accept_type, 39 output_location=output, 40 log_probability_location=log_probability, 41 embedding_location=embedding, 42 **kwargs, 43 )
Parameters
- content_template: String template to compose the model input from the prompt
- output: JMESPath expression of output in the model output
- log_probability: JMESPath expression of log probability in the model output
- embedding: JMESPath expression of embedding in the model output
- content_type: The content type of the request sent to the model for inference
- accept_type: The accept type of the request sent to the model for inference
@abstractmethod
def
predict( self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]:
45 @abstractmethod 46 def predict(self, prompt: str) -> Union[Tuple[Optional[str], Optional[float]], List[float]]: 47 """ 48 Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model 49 expects, and extracting the output and log probability from the model response. The response of the ModelRunner 50 will be a tuple of (output, log_probability) or embedding 51 52 :param prompt: the prompt 53 :return: the tuple containing model output string and the log probability, or embedding 54 """
Runs the model on the given prompt. This includes updating the prompt to fit the request format that the model expects, and extracting the output and log probability from the model response. The response of the ModelRunner will be a tuple of (output, log_probability) or embedding
Parameters
- prompt: the prompt
Returns
the tuple containing model output string and the log probability, or embedding