🚀 Supporting deepspeed>=0.16.4's rename (#2963)

* Added else clause to avoid NameError on optimizer_offload

* Accounted for deepspeed's renaming in 0.16.4

* Switched to packaging.version.parse over the (broken) tuple split

* Moved from NotImplementedError to RuntimeError in else clause

Author: jamesbraza

trl/models/utils.py

@@ -19,6 +19,7 @@
 from typing import TYPE_CHECKING, Literal, Optional, Union
 
 from accelerate.utils import is_deepspeed_available
+from packaging import version
 from transformers import PreTrainedModel, PreTrainedTokenizer
 
 from .modeling_value_head import AutoModelForCausalLMWithValueHead, AutoModelForSeq2SeqLMWithValueHead
@@ -141,6 +142,8 @@ def remove_hooks(model: "DeepSpeedEngine") -> None:
         optimizer_offload = model.optimizer.parameter_offload
     elif model.optimizer is not None:
         optimizer_offload = model.optimizer
+    else:
+        raise RuntimeError("The model optimizer is None, which is not yet supported.")
 
     for param in iter_params(optimizer_offload.module, recurse=True):
         param.ds_active_sub_modules.clear()
@@ -170,7 +173,13 @@ def add_hooks(model: "DeepSpeedEngine") -> None:
         optimizer_offload = model.optimizer.parameter_offload
     elif model.optimizer is not None:
         optimizer_offload = model.optimizer
-    optimizer_offload._register_hooks_recursively(optimizer_offload.module)
+    else:
+        raise RuntimeError("The model optimizer is None, which is not yet supported.")
+    if version.parse(deepspeed.__version__) >= version.parse("0.16.4"):
+        # Account for renaming in https://github.com/deepspeedai/DeepSpeed/pull/6847
+        optimizer_offload._register_deepspeed_module(optimizer_offload.module)
+    else:
+        optimizer_offload._register_hooks_recursively(optimizer_offload.module)
 
 
 @contextmanager