4.53.0 Transformers Pin (#421)

* higher transfomers pin

* unit tests fix

Author: farah-shadid

requirements.txt

@@ -1,6 +1,6 @@
 spacy>=3.5.0,<4.1.0
 numpy>=1.15.0
-transformers[sentencepiece]>=3.4.0,<4.42.0
+transformers[sentencepiece]>=3.4.0,<4.53.3
 torch>=1.8.0
 srsly>=2.4.0,<3.0.0
 dataclasses>=0.6,<1.0; python_version < "3.7"

setup.cfg

@@ -33,7 +33,7 @@ install_requires =
     spacy>=3.5.0,<4.1.0
     numpy>=1.15.0; python_version < "3.9"
     numpy>=1.19.0; python_version >= "3.9"
-    transformers>=3.4.0,<4.50.0
+    transformers>=3.4.0,< 4.53.3
     torch>=1.8.0
     srsly>=2.4.0,<3.0.0
     dataclasses>=0.6,<1.0; python_version < "3.7"

spacy_transformers/tests/regression/test_spacy_issue6401.py

@@ -66,4 +66,8 @@ def test_transformer_pipeline_textcat():
         nlp2 = spacy.load(file_path)
         doc2 = nlp2("We're interested at underwater basket weaving.")
         cats2 = doc2.cats
-        assert cats1 == cats2
+        # Compare floating point values with tolerance for small numerical differences
+        assert len(cats1) == len(cats2)
+        for key in cats1:
+            assert key in cats2
+            assert abs(cats1[key] - cats2[key]) < 1e-5, f"Values for '{key}' differ: {cats1[key]} vs {cats2[key]}"

spacy_transformers/tests/test_serialize.py

@@ -141,6 +141,8 @@ def test_transformer_pipeline_todisk_settings():
     # clobbered, so do both for this test
     trf.model.tokenizer.model_max_length = 499
     # add attentions on-the-fly
+    # Need to set attn_implementation to "eager" to support output_attentions=True
+    trf.model.transformer.config._attn_implementation = "eager"
     trf.model.transformer.config.output_attentions = True
     assert nlp("test")._.trf_data.model_output.attentions is not None
     with make_tempdir() as d:

spacy_transformers/tests/util.py

@@ -18,7 +18,14 @@ def _assert_equal_tensors(tensors1, tensors2):
     for i in range(len(tensors1)):
         t1 = ops.asarray(tensors1[i])
         t2 = ops.asarray(tensors2[i])
-        assert_array_equal(ops.to_numpy(t1), ops.to_numpy(t2))
+        # Use assert_allclose for floating point comparison with tolerance
+        # to handle tiny numerical differences between runs
+        numpy.testing.assert_allclose(
+            ops.to_numpy(t1), 
+            ops.to_numpy(t2), 
+            rtol=1e-4, 
+            atol=1e-5
+        )
 
 
 class DummyTokenizer: