fmeval.data_loaders.data_config

 1from typing import Optional
 2from dataclasses import dataclass
 3from fmeval.util import require
 4from fmeval.constants import SUPPORTED_MIME_TYPES
 5
 6
 7@dataclass
 8class DataConfig:
 9    """
10    Configures the information required by data-loading components.
11
12    Note that the term "location" used below refers to a string
13    that can be used to locate the data that comprises a single
14    column in the to-be-produced Ray Dataset. As an example,
15    when the dataset MIME type is JSON or JSON Lines, the "location"
16    is a JMESPath query.
17
18    **Note**:
19        Parsing logic used by data loaders make the assumption that
20        attributes in this class with the suffix "_location" correspond
21        to a "location" (defined above). When adding new attributes to this class,
22        if an attribute corresponds to a location, the attribute name must end
23        with "_location"
24
25    :param dataset_name: the dataset name
26    :param dataset_uri: either a local path or s3 URI representing where the dataset is stored
27    :param dataset_mime_type: the MIME type of the dataset file
28    :param model_input_location: the location  for model inputs
29    :param model_output_location: the location for model outputs
30    :param target_output_location: the location for target outputs
31    :param category_location: the location for categories
32    :param sent_more_input_location: the location for the "sent more"
33            inputs (used by the Prompt Stereotyping evaluation algorithm)
34    :param sent_less_input_location: the location for the "sent less"
35            inputs (used by the Prompt Stereotyping evaluation algorithm)
36    :param sent_more_log_prob_location: the location for the "sent more"
37            input log probability (used by the Prompt Stereotyping evaluation algorithm)
38    :param sent_less_log_prob_location: the location for the "sent less"
39            input log probability (used by the Prompt Stereotyping evaluation algorithm).
40    :param context_location: the location of the context for RAG evaluations.
41    """
42
43    dataset_name: str
44    dataset_uri: str
45    dataset_mime_type: str
46    model_input_location: Optional[str] = None
47    model_output_location: Optional[str] = None
48    target_output_location: Optional[str] = None
49    category_location: Optional[str] = None
50    sent_more_input_location: Optional[str] = None
51    sent_less_input_location: Optional[str] = None
52    sent_more_log_prob_location: Optional[str] = None
53    sent_less_log_prob_location: Optional[str] = None
54    context_location: Optional[str] = None
55
56    def __post_init__(self):
57        require(
58            self.dataset_mime_type in SUPPORTED_MIME_TYPES,
59            f"Unsupported MIME type: {self.dataset_mime_type}. "
60            f"The following mime types are supported: {SUPPORTED_MIME_TYPES}.",
61        )