Skip to content

Flan-T5 Models for Text Generation Tasks

This task pool provides a set of text generation tasks from the GLUE benchmark for the Flan-T5 model. Each task is associated with a dataset. We report the exact match accuracy metric for CoLA, MNLI, MRPC, QNLI, QQP, RTE, and SST2, and spearman's rho for STSB.

References

flan_t5_glue_text_generation

FlanT5GLUETextGenerationTaskPool

Bases: LightningFabricMixin, TaskPool

A task pool for FlanT5 GLUE text generation tasks. This class manages the tasks and provides methods for loading and evaluating tasks.

Source code in fusion_bench/compat/taskpool/flan_t5_glue_text_generation.py 
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
class FlanT5GLUETextGenerationTaskPool(LightningFabricMixin, TaskPool):
    """
    A task pool for FlanT5 GLUE text generation tasks.
    This class manages the tasks and provides methods for loading and evaluating tasks.
    """

    _tokenizer_instance = None

    @property
    def tokenizer(self):
        """
        Returns the tokenizer. If it's not already initialized, it initializes it using the config's tokenizer.
        """
        if self._tokenizer_instance is None:
            self._tokenizer_instance = AutoTokenizer.from_pretrained(
                self.config.tokenizer
            )
        return self._tokenizer_instance

    def load_task(self, task_name_or_config: str | DictConfig):
        """
        Loads a task given a task name or config. If the task name is in `CLASSIFICATION_TASKS`, it creates a `FlanT5GLUETextGenerationClassificationTask`.
        If the task name is in `REGRESSION_TASKS`, it creates a `FlanT5GLUETextGenerationRegressionTask`. Otherwise, it raises a `ValueError`.
        """
        if isinstance(task_name_or_config, str):
            task_config = self.get_task_config(task_name_or_config)
        else:
            task_config = task_name_or_config

        if task_config.name in CLASSIFICATION_TASKS:
            task = FlanT5GLUETextGenerationClassificationTask(task_config)
            task._taskpool = self
            return task
        elif task_config.name in REGRESSION_TASKS:
            task = FlanT5GLUETextGenerationRegressionTask(task_config)
            task._taskpool = self
            return task
        else:
            raise ValueError(f"Unknown task {task_config.name}")

    def evaluate(self, model: T5ForConditionalGeneration, name: str = None):
        """
        Evaluate the model on the FlanT5 GLUE text generation tasks.

        Args:
            model (T5ForConditionalGeneration): The model to evaluate.
            name (str, optional): The name of the model. Defaults to None. This is used to identify the model in the report.

        Returns:
            dict: A dictionary containing the evaluation results for each task.
        """
        if not isinstance(model, T5ForConditionalGeneration):
            log.warning(
                f"Model is not an instance of T5ForConditionalGeneration, but {type(model)}"
            )
        report = {}
        training_params, all_params = count_parameters(model)
        report["model_info"] = {
            "trainable_params": training_params,
            "all_params": all_params,
            "trainable_percentage": training_params / all_params,
        }
        if name is not None:
            report["model_info"]["name"] = name
        model = self.fabric.setup(model)
        report.update(super().evaluate(model))
        log.info(f"evaluation report: {report}")
        return report
tokenizer property

Returns the tokenizer. If it's not already initialized, it initializes it using the config's tokenizer.

evaluate(model, name=None)

Evaluate the model on the FlanT5 GLUE text generation tasks.

Parameters:

  • model (T5ForConditionalGeneration) –

    The model to evaluate.

  • name (str, default: None ) –

    The name of the model. Defaults to None. This is used to identify the model in the report.

Returns:

  • dict –

    A dictionary containing the evaluation results for each task.

Source code in fusion_bench/compat/taskpool/flan_t5_glue_text_generation.py 
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
def evaluate(self, model: T5ForConditionalGeneration, name: str = None):
    """
    Evaluate the model on the FlanT5 GLUE text generation tasks.

    Args:
        model (T5ForConditionalGeneration): The model to evaluate.
        name (str, optional): The name of the model. Defaults to None. This is used to identify the model in the report.

    Returns:
        dict: A dictionary containing the evaluation results for each task.
    """
    if not isinstance(model, T5ForConditionalGeneration):
        log.warning(
            f"Model is not an instance of T5ForConditionalGeneration, but {type(model)}"
        )
    report = {}
    training_params, all_params = count_parameters(model)
    report["model_info"] = {
        "trainable_params": training_params,
        "all_params": all_params,
        "trainable_percentage": training_params / all_params,
    }
    if name is not None:
        report["model_info"]["name"] = name
    model = self.fabric.setup(model)
    report.update(super().evaluate(model))
    log.info(f"evaluation report: {report}")
    return report
load_task(task_name_or_config)

Loads a task given a task name or config. If the task name is in CLASSIFICATION_TASKS, it creates a FlanT5GLUETextGenerationClassificationTask. If the task name is in REGRESSION_TASKS, it creates a FlanT5GLUETextGenerationRegressionTask. Otherwise, it raises a ValueError.

Source code in fusion_bench/compat/taskpool/flan_t5_glue_text_generation.py 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
def load_task(self, task_name_or_config: str | DictConfig):
    """
    Loads a task given a task name or config. If the task name is in `CLASSIFICATION_TASKS`, it creates a `FlanT5GLUETextGenerationClassificationTask`.
    If the task name is in `REGRESSION_TASKS`, it creates a `FlanT5GLUETextGenerationRegressionTask`. Otherwise, it raises a `ValueError`.
    """
    if isinstance(task_name_or_config, str):
        task_config = self.get_task_config(task_name_or_config)
    else:
        task_config = task_name_or_config

    if task_config.name in CLASSIFICATION_TASKS:
        task = FlanT5GLUETextGenerationClassificationTask(task_config)
        task._taskpool = self
        return task
    elif task_config.name in REGRESSION_TASKS:
        task = FlanT5GLUETextGenerationRegressionTask(task_config)
        task._taskpool = self
        return task
    else:
        raise ValueError(f"Unknown task {task_config.name}")