fix(transformers): update layer normalization patching in apply_liger_kernel_to_glm4v function

Author: vvvdwbvvv

src/liger_kernel/transformers/monkey_patch.py

@@ -1910,14 +1910,14 @@ def apply_liger_kernel_to_glm4v(
         if vision_model is not None:
             for vision_block in vision_model.blocks:
                 if rms_norm:
-                    _patch_layer_norm_module(vision_block.norm1)
-                    _patch_layer_norm_module(vision_block.norm2)
+                    _patch_rms_norm_module(vision_block.norm1)
+                    _patch_rms_norm_module(vision_block.norm2)
                 if swiglu:
                     _patch_swiglu_module(vision_block.mlp, LigerSwiGLUMLP)
 
         if text_model is not None:
             if rms_norm:
-                _patch_layer_norm_module(text_model.norm)
+                _patch_rms_norm_module(text_model.norm)
             for decoder_layer in text_model.layers:
                 if swiglu:
                     _patch_swiglu_module(decoder_layer.mlp, LigerPhi3SwiGLUMLP)
@@ -1935,7 +1935,7 @@ def apply_liger_kernel_to_glm4v(
     "gemma3_text": apply_liger_kernel_to_gemma3_text,
     "gemma3": apply_liger_kernel_to_gemma3,
     "glm4": apply_liger_kernel_to_glm4,
-    "glm4.1v": apply_liger_kernel_to_glm4v,
+    "glm4v": apply_liger_kernel_to_glm4v,
     "llama": apply_liger_kernel_to_llama,
     "llama4_text": apply_liger_kernel_to_llama4,
     "llama4": apply_liger_kernel_to_llama4,

test/transformers/test_monkey_patch.py

@@ -1655,60 +1655,6 @@ def test_apply_liger_kernel_to_instance_for_glm4():
         except Exception as e:
             pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
 
-
-@pytest.mark.skipif(not is_glm4v_available(), reason="glm4v module not available")
-def test_apply_liger_kernel_to_instance_for_glm4v():
-    # Ensure any monkey patching is cleaned up for subsequent tests
-    with patch("transformers.models.glm4v.modeling_glm4v"):
-        from liger_kernel.transformers.model.glm4v import lce_forward as glm4v_lce_forward
-
-        # Instantiate a dummy model
-        config = transformers.models.glm4v.configuration_glm4v.Glm4vConfig(
-            torch_dtype=torch.bfloat16,
-            text_config={
-                "num_hidden_layers": 2,
-                "rms_norm_eps": 1e-5,
-                "hidden_size": 32,
-                "intermediate_size": 64,
-                "hidden_act": "silu",
-            },
-            vision_config={
-                "num_hidden_layers": 2,
-                "layer_norm_eps": 1e-5,
-                "hidden_size": 48,
-                "intermediate_size": 64,
-            },
-        )
-        dummy_model_instance = AutoModelForCausalLM.from_config(config)
-
-        # Check that model instance variables are not yet patched with Liger modules
-        assert inspect.getsource(dummy_model_instance.forward) != inspect.getsource(glm4v_lce_forward)
-        assert inspect.getsource(dummy_model_instance.model.norm.forward) != inspect.getsource(LigerRMSNorm.forward)
-        for layer in dummy_model_instance.model.layers:
-            assert inspect.getsource(layer.mlp.forward) != inspect.getsource(LigerPhi3SwiGLUMLP.forward)
-            assert inspect.getsource(layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_attention_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_self_attn_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_mlp_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
-
-        # Test applying kernels to the model instance
-        _apply_liger_kernel_to_instance(model=dummy_model_instance)
-
-        # Check that the model's instance variables were correctly patched with Liger modules
-        assert inspect.getsource(dummy_model_instance.forward) == inspect.getsource(glm4v_lce_forward)
-        assert inspect.getsource(dummy_model_instance.model.norm.forward) == inspect.getsource(LigerRMSNorm.forward)
-        for layer in dummy_model_instance.model.layers:
-            assert inspect.getsource(layer.mlp.forward) == inspect.getsource(LigerPhi3SwiGLUMLP.forward)
-            assert inspect.getsource(layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_attention_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_self_attn_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
-            assert inspect.getsource(layer.post_mlp_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)
-        try:
-            print(dummy_model_instance)
-        except Exception as e:
-            pytest.fail(f"An exception occured in extra_expr: {type(e).__name__} - {e}")
-
-
 @pytest.mark.skipif(not is_glm4v_available(), reason="glm4v module not available")
 def test_apply_liger_kernel_to_instance_for_glm4v():
     # Ensure any monkey patching is cleaned up for subsequent tests
@@ -1729,7 +1675,7 @@ def test_apply_liger_kernel_to_instance_for_glm4v():
             },
             vision_config={
                 "num_hidden_layers": 2,
-                "layer_norm_eps": 1e-5,
+                "rms_norm_eps": 1e-5,
                 "hidden_size": 48,
                 "intermediate_size": 64,
             },
@@ -1739,9 +1685,7 @@ def test_apply_liger_kernel_to_instance_for_glm4v():
 
         # Check that model instance variables are not yet patched with Liger modules
         assert inspect.getsource(dummy_model_instance.forward) != inspect.getsource(glm4v_lce_forward)
-        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) != inspect.getsource(
-            LigerRMSNorm.forward
-        )
+        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) != inspect.getsource(LigerRMSNorm.forward)
         for layer in dummy_model_instance.language_model.layers:
             assert inspect.getsource(layer.mlp.forward) != inspect.getsource(LigerPhi3SwiGLUMLP.forward)
             assert inspect.getsource(layer.input_layernorm.forward) != inspect.getsource(LigerRMSNorm.forward)
@@ -1757,10 +1701,8 @@ def test_apply_liger_kernel_to_instance_for_glm4v():
         _apply_liger_kernel_to_instance(model=dummy_model_instance)
 
         # Check that the model's instance variables were correctly patched with Liger modules
-        assert inspect.getsource(dummy_model_instance.forward) != inspect.getsource(glm4v_lce_forward)
-        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) != inspect.getsource(
-            LigerRMSNorm.forward
-        )
+        assert inspect.getsource(dummy_model_instance.forward) == inspect.getsource(glm4v_lce_forward)
+        assert inspect.getsource(dummy_model_instance.language_model.norm.forward) == inspect.getsource(LigerRMSNorm.forward)
         for layer in dummy_model_instance.language_model.layers:
             assert inspect.getsource(layer.mlp.forward) == inspect.getsource(LigerPhi3SwiGLUMLP.forward)
             assert inspect.getsource(layer.input_layernorm.forward) == inspect.getsource(LigerRMSNorm.forward)