Merge remote-tracking branch 'origin' into kylesayrs/consolidate-saving

Author: kylesayrs

src/llmcompressor/args/dataset_arguments.py

@@ -70,7 +70,7 @@ class CustomDatasetArguments(DVCDatasetArguments):
 class DatasetArguments(CustomDatasetArguments):
     """
     Arguments pertaining to what data we are going to input our model for
-    calibration, training or eval
+    calibration, training
 
     Using `HfArgumentParser` we can turn this class into argparse
     arguments to be able to specify them on the command line
@@ -150,13 +150,6 @@ class DatasetArguments(CustomDatasetArguments):
             "of training examples to this value if set."
         },
     )
-    max_eval_samples: Optional[int] = field(
-        default=None,
-        metadata={
-            "help": "For debugging purposes or quicker training, truncate the number "
-            "of evaluation examples to this value if set."
-        },
-    )
     max_predict_samples: Optional[int] = field(
         default=None,
         metadata={

src/llmcompressor/transformers/finetune/__init__.py

@@ -2,4 +2,4 @@
 
 from .data import TextGenerationDataset
 from .session_mixin import SessionManagerMixIn
-from .text_generation import apply, compress, eval, oneshot, train
+from .text_generation import apply, compress, oneshot, train

src/llmcompressor/transformers/finetune/data/data_helpers.py

@@ -97,17 +97,15 @@ def get_raw_dataset(
 def make_dataset_splits(
     tokenized_datasets: Dict[str, Any],
     do_train: bool = False,
-    do_eval: bool = False,
     do_predict: bool = False,
     do_oneshot: bool = False,
 ) -> Dict[str, Dataset]:
     """
     Restructures the datasets dictionary based on what tasks will be run
-    (train, eval, predict)
+    (train, predict)
 
     :param tokenized_datasets: dictionary of processed datasets
     :param do_train: Whether to store the train dataset
-    :param do_eval: Whether to store the validation dataset
     :param do_predict: Whether to store the test dataset
     :param do_oneshot: Whether to store the calibration dataset
     :return: Datasets to be used by the requested tasks
@@ -119,16 +117,12 @@ def make_dataset_splits(
         if isinstance(tokenized_datasets, Dataset):
             tokenized_datasets = {"train": tokenized_datasets}
 
-    train_split = eval_split = predict_split = calib_split = None
+    train_split = predict_split = calib_split = None
 
     if do_train:
         if "train" not in tokenized_datasets:
             raise ValueError("--do_train requires a train dataset")
         train_split = tokenized_datasets["train"]
-    if do_eval:
-        if "validation" not in tokenized_datasets:
-            raise ValueError("--do_eval requires a validation dataset")
-        eval_split = tokenized_datasets["validation"]
     if do_predict:
         if "test" not in tokenized_datasets:
             raise ValueError("--do_predict requires a test dataset")
@@ -142,7 +136,6 @@ def make_dataset_splits(
 
     split_datasets = {
         "train": train_split,
-        "validation": eval_split,
         "test": predict_split,
         "calibration": calib_split,
     }
@@ -222,7 +215,7 @@ def transform_dataset_keys(data_files: Dict[str, Any]):
     Transform dict keys to `train`, `val` or `test` for the given input dict
     if matches exist with the existing keys. Note that there can only be one
     matching file name.
-    Ex. Folder(train_eval.json)          -> Folder(train.json)
+    Ex. Folder(train_foo.json)           -> Folder(train.json)
         Folder(train1.json, train2.json) -> Same
 
     :param data_files: The dict where keys will be transformed

src/llmcompressor/transformers/finetune/runner.py

@@ -31,14 +31,14 @@
 
 class StageRunner:
     """
-    Launcher class for train, eval and one_shot flows. Manages data splits for each
+    Launcher class for train, and one_shot flows. Manages data splits for each
     flow and configurations. In the future this class will also handle alternating
     between the different flows
 
     LifeCycle
         - populate_datasets()
         - set_trainer()
-        - train() / evaluate() / predict()
+        - train() / predict()
 
     :param model_args: Arguments pertaining to model/config/processor
     :param data_args: Arguments pertaining to what data to use for different flows
@@ -121,7 +121,6 @@ def _get_split_name(inp_str):
         self.datasets = make_dataset_splits(
             tokenized_datasets,
             do_train=self._training_args.do_train,
-            do_eval=self._training_args.do_eval,
             do_predict=self._training_args.do_predict,
             do_oneshot=self._training_args.do_oneshot,
         )
@@ -156,17 +155,6 @@ def train(self, checkpoint: str, stage: Optional[str] = None):
         # this includes saving the state, optimizer and scheduler
         self.trainer.save_model(output_dir=self._output_dir)
 
-    def evaluate(self):
-        """
-        Run trainer's evaluation loop on eval_dataset, logging the desired metrics
-        """
-        logger.info("*** Evaluate ***")
-        metrics = self.trainer.evaluate(self.get_dataset_split("validation"))
-
-        metrics["eval_samples"] = len(self.get_dataset_split("validation"))
-        self.trainer.log_metrics("eval", metrics)
-        self.trainer.save_metrics("eval", metrics)
-
     def predict(self):
         """
         Run trainer's prediction loop on predict_dataset, logging the desired metrics

src/llmcompressor/transformers/finetune/session_mixin.py

@@ -42,7 +42,6 @@
 TRAINER_STATE_NAME = "trainer_state.json"
 METADATA_ARGS = [
     "per_device_train_batch_size",
-    "per_device_eval_batch_size",
     "max_seq_length",
     "save_safetensors",
     "fp16",
@@ -406,22 +405,6 @@ def train(self, *args, stage: Optional[str] = None, **kwargs):
 
         return output
 
-    def evaluate(self, *args, **kwargs):
-        """
-        Run a sparsification evaluation cycle.
-        Runs initialize_structure for the sparse session before calling
-        super().evaluate() and finalization of the session after.
-        :param args: positional args to pass to super().evaluate()
-        :param kwargs: keyword args to pass to super().evaluate()
-        :return: the output from super.evaluate()
-        """
-        self.initialize_structure()
-
-        output = super().evaluate(*args, **kwargs)
-        self.finalize_session()
-
-        return output
-
     def predict(self, *args, **kwargs):
         """
         Run a sparsification prediction cycle.

src/llmcompressor/transformers/finetune/text_generation.py

@@ -98,7 +98,7 @@ def oneshot(**kwargs) -> None:
 
 def apply(**kwargs):
     """
-    CLI entrypoint for any of training, eval, predict or oneshot
+    CLI entrypoint for any of training, predict or oneshot
     """
     report_to = kwargs.get("report_to", None)
     model_args, data_args, recipe_args, training_args = parse_args(**kwargs)
@@ -323,12 +323,12 @@ def main(
         - Trainer()
             - SessionMixIn()
             - HFTransformersTrainer()
-        - StageRunner.train() and/or evaluate() and/or predict() and/or oneshot()
+        - StageRunner.train() and/or predict() and/or oneshot()
 
     :param model_args: Arguments pertaining to which model/config/tokenizer we are
     going to fine-tune from
     :param data_args: Arguments pertaining to what data we are going to input our model
-    for training and eval
+    for training
     :param training_args: Arguments pertaining to training loop configuration
     """
 
@@ -358,7 +358,7 @@ def main(
         f"distributed training: {bool(training_args.local_rank != -1)}, "
         f"16-bits training: {training_args.fp16}"
     )
-    logger.info(f"Training/evaluation parameters {training_args}")
+    logger.info(f"Training parameters {training_args}")
 
     # Detecting last checkpoint.
     last_checkpoint = None
@@ -397,7 +397,6 @@ def main(
     add_labels = training_args.do_train or training_args.run_stages
     stage_runner.populate_datasets(processor=processor, add_labels=add_labels)
     train_dataset = stage_runner.get_dataset_split("train")
-    eval_dataset = stage_runner.get_dataset_split("validation")
     calib_dataset = stage_runner.get_dataset_split("calibration")
 
     trainer = Trainer(
@@ -409,7 +408,6 @@ def main(
         model_args=model_args,
         data_args=data_args,
         train_dataset=train_dataset or calib_dataset,
-        eval_dataset=eval_dataset,
         processing_class=processor,
         data_collator=data_args.data_collator,
     )
@@ -443,10 +441,6 @@ def main(
             checkpoint = last_checkpoint
         stage_runner.train(checkpoint)
 
-    # Evaluation
-    if training_args.do_eval:
-        stage_runner.evaluate()
-
     # Prediction
     if training_args.do_predict:
         stage_runner.predict()

tests/llmcompressor/transformers/finetune/data/test_dataset_helpers.py

@@ -15,18 +15,12 @@ def test_combined_datasets():
     raw_wikitext2 = get_raw_dataset(data_args)
     datasets = {"all": raw_wikitext2}
 
-    split_datasets = make_dataset_splits(
-        datasets, do_train=True, do_eval=True, do_predict=True
-    )
+    split_datasets = make_dataset_splits(datasets, do_train=True, do_predict=True)
     assert split_datasets.get("train") is not None
-    assert split_datasets.get("validation") is not None
     assert split_datasets.get("test") is not None
 
-    split_datasets = make_dataset_splits(
-        datasets, do_train=True, do_eval=False, do_predict=True
-    )
+    split_datasets = make_dataset_splits(datasets, do_train=True, do_predict=True)
     assert split_datasets.get("train") is not None
-    assert split_datasets.get("validation") is None
     assert split_datasets.get("test") is not None
 
 
@@ -41,15 +35,10 @@ def test_separate_datasets():
         raw_wikitext2 = get_raw_dataset(data_args, split=split_str)
         datasets[split_name] = raw_wikitext2
 
-    split_datasets = make_dataset_splits(
-        datasets, do_train=True, do_eval=True, do_predict=False
-    )
+    split_datasets = make_dataset_splits(datasets, do_train=True, do_predict=False)
     assert split_datasets.get("train") is not None
-    assert split_datasets.get("validation") is not None
     assert split_datasets.get("test") is None
 
     with pytest.raises(ValueError):
         # fails due to no test split specified
-        split_datasets = make_dataset_splits(
-            datasets, do_train=True, do_eval=True, do_predict=True
-        )
+        split_datasets = make_dataset_splits(datasets, do_train=True, do_predict=True)

tests/llmcompressor/transformers/finetune/test_session_mixin.py

@@ -49,13 +49,11 @@ def mixin_trainer():
     model = AutoModelForCausalLM.from_pretrained(model_state_path)
     recipe = "tests/llmcompressor/transformers/finetune/test_quantization.yaml"
     train_dataset = "open-platypus"
-    eval_dataset = "open-platypus"
 
     return MixInTest(
         model=model,
         recipe=recipe,
         train_dataset=train_dataset,
-        eval_dataset=eval_dataset,
     )