Skip to content

Schema

AggregationFunction

Bases: str, Enum

Enum of supported aggregation functions

Source code in src/autolabel/schema.py 
291
292
293
294
295
class AggregationFunction(str, Enum):
    """Enum of supported aggregation functions"""

    MAX = "max"
    MEAN = "mean"

ConfidenceCacheEntry

Bases: BaseModel

Source code in src/autolabel/schema.py 
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
class ConfidenceCacheEntry(BaseModel):
    prompt: Optional[str] = ""
    raw_response: Optional[str] = ""
    logprobs: Optional[list] = None
    score_type: Optional[str] = "logprob_average"
    creation_time_ms: Optional[int] = -1
    ttl_ms: Optional[int] = -1

    class Config:
        orm_mode = True

    def get_id(self) -> str:
        """
        Generates a unique ID for the given confidence cache configuration
        """
        return calculate_md5([self.prompt, self.raw_response, self.score_type])

    def get_serialized_output(self) -> str:
        """
        Returns the serialized cache entry output
        """
        return json.dumps(self.logprobs)

    def deserialize_output(self, output: str) -> Dict[str, float]:
        """
        Deserializes the cache entry output
        """
        return json.loads(output)

deserialize_output(output)

Deserializes the cache entry output

Source code in src/autolabel/schema.py 
268
269
270
271
272
def deserialize_output(self, output: str) -> Dict[str, float]:
    """
    Deserializes the cache entry output
    """
    return json.loads(output)

get_id()

Generates a unique ID for the given confidence cache configuration

Source code in src/autolabel/schema.py 
256
257
258
259
260
def get_id(self) -> str:
    """
    Generates a unique ID for the given confidence cache configuration
    """
    return calculate_md5([self.prompt, self.raw_response, self.score_type])

get_serialized_output()

Returns the serialized cache entry output

Source code in src/autolabel/schema.py 
262
263
264
265
266
def get_serialized_output(self) -> str:
    """
    Returns the serialized cache entry output
    """
    return json.dumps(self.logprobs)

Dataset

Bases: BaseModel

Contains Dataset parameters, including input file path, indexes for state management (e.g. job batching and retries), and a unique ID

Source code in src/autolabel/schema.py 
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
class Dataset(BaseModel):
    """Contains Dataset parameters, including input file path, indexes for state management (e.g. job batching and retries), and a unique ID"""

    id: str
    input_file: str
    start_index: int
    end_index: int

    class Config:
        orm_mode = True

    @classmethod
    def create_id(
        self,
        dataset: Union[str, pd.DataFrame],
        config: AutolabelConfig,
        start_index: int,
        max_items: int,
    ) -> str:
        """
        Generates a unique ID for the given Dataset configuration
        Args:
            dataset: either 1) input file name or 2) pandas Dataframe
            config:  AutolabelConfig object containing project settings
            start_index: index to begin labeling job at (used for job batching, retries, state management)
            max_items: number of data points to label, beginning at start_index

        Returns:
            filehash: a unique ID generated from an MD5 hash of the functions parameters
        """
        if isinstance(dataset, str):
            filehash = calculate_md5(
                [open(dataset, "rb"), config._dataset_config, start_index, max_items]
            )
        else:
            filehash = calculate_md5(
                [dataset.to_csv(), config._dataset_config, start_index, max_items]
            )
        return filehash

create_id(dataset, config, start_index, max_items) classmethod

Generates a unique ID for the given Dataset configuration Args: dataset: either 1) input file name or 2) pandas Dataframe config: AutolabelConfig object containing project settings start_index: index to begin labeling job at (used for job batching, retries, state management) max_items: number of data points to label, beginning at start_index

Returns:

Name Type Description
filehash str

a unique ID generated from an MD5 hash of the functions parameters
Source code in src/autolabel/schema.py 
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
@classmethod
def create_id(
    self,
    dataset: Union[str, pd.DataFrame],
    config: AutolabelConfig,
    start_index: int,
    max_items: int,
) -> str:
    """
    Generates a unique ID for the given Dataset configuration
    Args:
        dataset: either 1) input file name or 2) pandas Dataframe
        config:  AutolabelConfig object containing project settings
        start_index: index to begin labeling job at (used for job batching, retries, state management)
        max_items: number of data points to label, beginning at start_index

    Returns:
        filehash: a unique ID generated from an MD5 hash of the functions parameters
    """
    if isinstance(dataset, str):
        filehash = calculate_md5(
            [open(dataset, "rb"), config._dataset_config, start_index, max_items]
        )
    else:
        filehash = calculate_md5(
            [dataset.to_csv(), config._dataset_config, start_index, max_items]
        )
    return filehash

ErrorType

Bases: str, Enum

Enum of supported error types

Source code in src/autolabel/schema.py 
 94
 95
 96
 97
 98
 99
100
class ErrorType(str, Enum):
    """Enum of supported error types"""

    LLM_PROVIDER_ERROR = "llm_provider_error"
    PARSING_ERROR = "parsing_error"
    OUTPUT_GUIDELINES_NOT_FOLLOWED_ERROR = "output_guidelines_not_followed_error"
    EMPTY_RESPONSE_ERROR = "empty_response_error"

FewShotAlgorithm

Bases: str, Enum

Enum of supported algorithms for choosing which examples to provide the LLM in its instruction prompt

Source code in src/autolabel/schema.py 
39
40
41
42
43
44
45
46
class FewShotAlgorithm(str, Enum):
    """Enum of supported algorithms for choosing which examples to provide the LLM in its instruction prompt"""

    FIXED = "fixed"
    SEMANTIC_SIMILARITY = "semantic_similarity"
    MAX_MARGINAL_RELEVANCE = "max_marginal_relevance"
    LABEL_DIVERSITY_RANDOM = "label_diversity_random"
    LABEL_DIVERSITY_SIMILARITY = "label_diversity_similarity"

GenerationCacheEntry

Bases: BaseModel

Source code in src/autolabel/schema.py 
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
class GenerationCacheEntry(BaseModel):
    model_name: str
    prompt: str
    model_params: str
    generations: Optional[List[Union[Generation, ChatGeneration]]] = None
    creation_time_ms: Optional[int] = -1
    ttl_ms: Optional[int] = -1

    class Config:
        orm_mode = True

    def get_id(self) -> str:
        """
        Generates a unique ID for the given generation cache configuration
        """
        return calculate_md5([self.model_name, self.model_params, self.prompt])

    def get_serialized_output(self) -> str:
        """
        Returns the serialized cache entry output
        """
        return json.dumps([gen.dict() for gen in self.generations])

    def deserialize_output(
        self, output: str
    ) -> List[Union[Generation, ChatGeneration]]:
        """
        Deserializes the cache entry output
        """
        generations = [
            Generation(**gen) if gen["type"] == "Generation" else ChatGeneration(**gen)
            for gen in json.loads(output)
        ]
        return generations

deserialize_output(output)

Deserializes the cache entry output

Source code in src/autolabel/schema.py 
232
233
234
235
236
237
238
239
240
241
242
def deserialize_output(
    self, output: str
) -> List[Union[Generation, ChatGeneration]]:
    """
    Deserializes the cache entry output
    """
    generations = [
        Generation(**gen) if gen["type"] == "Generation" else ChatGeneration(**gen)
        for gen in json.loads(output)
    ]
    return generations

get_id()

Generates a unique ID for the given generation cache configuration

Source code in src/autolabel/schema.py 
220
221
222
223
224
def get_id(self) -> str:
    """
    Generates a unique ID for the given generation cache configuration
    """
    return calculate_md5([self.model_name, self.model_params, self.prompt])

get_serialized_output()

Returns the serialized cache entry output

Source code in src/autolabel/schema.py 
226
227
228
229
230
def get_serialized_output(self) -> str:
    """
    Returns the serialized cache entry output
    """
    return json.dumps([gen.dict() for gen in self.generations])

LLMAnnotation

Bases: BaseModel

Contains label information of a given data point, including the generated label, the prompt given to the LLM, and the LLMs response. Optionally includes a confidence_score if supported by the model

Source code in src/autolabel/schema.py 
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
class LLMAnnotation(BaseModel):
    """Contains label information of a given data point, including the generated label, the prompt given to the LLM, and the LLMs response. Optionally includes a confidence_score if supported by the model"""

    successfully_labeled: bool
    label: Any
    curr_sample: Optional[bytes] = ""
    confidence_score: Optional[float] = None
    generation_info: Optional[Dict[str, Any]] = None
    raw_response: Optional[str] = ""
    explanation: Optional[str] = ""
    prompt: Optional[str] = ""
    confidence_prompt: Optional[str] = ""
    input_tokens: Optional[int] = None
    output_tokens: Optional[int] = None
    cost: Optional[float] = None
    latency: Optional[float] = None
    error: Optional[LabelingError] = None

LabelingError

Bases: BaseModel

Contains information about an error that occurred during the labeling process

Source code in src/autolabel/schema.py 
103
104
105
106
107
class LabelingError(BaseModel):
    """Contains information about an error that occurred during the labeling process"""

    error_type: ErrorType
    error_message: str

MetricResult

Bases: BaseModel

Contains performance metrics gathered from autolabeler runs

Source code in src/autolabel/schema.py 
86
87
88
89
90
91
class MetricResult(BaseModel):
    """Contains performance metrics gathered from autolabeler runs"""

    name: str
    value: Any
    show_running: Optional[bool] = True

MetricType

Bases: str, Enum

Enum of supported performance metrics. Some metrics are always available (task agnostic), while others are only supported by certain types of tasks

Source code in src/autolabel/schema.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
class MetricType(str, Enum):
    """Enum of supported performance metrics. Some metrics are always available (task agnostic), while others are only supported by certain types of tasks"""

    # Task agnostic
    SUPPORT = "support"
    COMPLETION_RATE = "completion_rate"
    # Classification metrics
    ACCURACY = "accuracy"
    CONFUSION_MATRIX = "confusion_matrix"
    LABEL_DISTRIBUTION = "label_distribution"
    F1 = "f1"
    F1_MICRO = "f1_micro"
    F1_MACRO = "f1_macro"
    F1_WEIGHTED = "f1_weighted"
    TEXT_PARTIAL_MATCH = "text_partial_match"
    # Token Classification metrics
    F1_EXACT = "f1_exact"
    F1_STRICT = "f1_strict"
    F1_PARTIAL = "f1_partial"
    F1_ENT_TYPE = "f1_ent_type"
    # Confidence metrics
    AUROC = "auroc"
    THRESHOLD = "threshold"

    # Aggregate Metrics
    CLASSIFICATION_REPORT = "classification_report"

ModelProvider

Bases: str, Enum

Enum containing all LLM providers currently supported by autolabeler

Source code in src/autolabel/schema.py 
14
15
16
17
18
19
20
21
22
23
24
25
class ModelProvider(str, Enum):
    """Enum containing all LLM providers currently supported by autolabeler"""

    OPENAI = "openai"
    OPENAI_VISION = "openai_vision"
    ANTHROPIC = "anthropic"
    HUGGINGFACE_PIPELINE = "huggingface_pipeline"
    HUGGINGFACE_PIPELINE_VISION = "huggingface_pipeline_vision"
    REFUEL = "refuel"
    GOOGLE = "google"
    COHERE = "cohere"
    CUSTOM = "custom"

RefuelLLMResult

Bases: BaseModel

List of generated outputs. This is a List[List[]] because each input could have multiple candidate generations.

Source code in src/autolabel/schema.py 
275
276
277
278
279
280
281
282
283
284
285
286
287
288
class RefuelLLMResult(BaseModel):
    """List of generated outputs. This is a List[List[]] because
    each input could have multiple candidate generations."""

    generations: List[List[Union[Generation, ChatGeneration]]]

    """Errors encountered while running the labeling job"""
    errors: List[Optional[LabelingError]]

    """Costs incurred during the labeling job"""
    costs: Optional[List[float]] = []

    """Latencies incurred during the labeling job"""
    latencies: Optional[List[float]] = []

costs: Optional[List[float]] = [] class-attribute instance-attribute

Latencies incurred during the labeling job

errors: List[Optional[LabelingError]] instance-attribute

Costs incurred during the labeling job

generations: List[List[Union[Generation, ChatGeneration]]] instance-attribute

Errors encountered while running the labeling job

TaskType

Bases: str, Enum

Enum containing all the types of tasks that autolabeler currently supports

Source code in src/autolabel/schema.py 
28
29
30
31
32
33
34
35
36
class TaskType(str, Enum):
    """Enum containing all the types of tasks that autolabeler currently supports"""

    CLASSIFICATION = "classification"
    NAMED_ENTITY_RECOGNITION = "named_entity_recognition"
    QUESTION_ANSWERING = "question_answering"
    ENTITY_MATCHING = "entity_matching"
    MULTILABEL_CLASSIFICATION = "multilabel_classification"
    ATTRIBUTE_EXTRACTION = "attribute_extraction"