Relaxing requirements for trl (#1342)

Author: hjh0119

requirements/framework.txt

@@ -18,4 +18,4 @@ tensorboard
 tqdm
 transformers>=4.33,<4.43
 transformers_stream_generator
-trl>=0.9.6
+trl>=0.9.4

swift/llm/rlhf.py

@@ -20,6 +20,13 @@
 
 
 def llm_rlhf(args: RLHFArguments) -> Dict[str, Any]:
+    if args.rlhf_type == 'simpo':
+        import trl
+        from packaging import version
+        assert version.parse(trl.__version__) <= version.parse('0.9.4'), \
+            'Please ensure to update `trl` to the latest version using the following command:' \
+            'pip install trl==0.9.6 --index-url https://pypi.org/simple.'
+
     logger.info(f'args: {args}')
     seed_everything(args.seed)
     training_args = args.training_args

swift/llm/utils/argument.py

@@ -1495,6 +1495,11 @@ def check_loss_type(self):
             'cpo': ['sigmoid', 'hinge', 'ipo', 'simpo'],
             'kto': ['kto', 'bco']
         }
+        if self.loss_type == 'kto_pair':
+            import trl
+            from packaging import version
+            if version.parse(trl.__version__) <= version.parse('0.9.4'):
+                return
         if self.rlhf_type in supported_loss_types:
             assert self.loss_type in supported_loss_types.get(self.rlhf_type), \
                 f"algo {self.rlhf_type} doesn't support loss type {self.loss_type}"