Add tests to remind us of removing deprecated compatibility code

Signed-off-by: Max de Bayser <[email protected]>

Author: maxdebayser

.github/workflows/test.yml

@@ -50,6 +50,9 @@ jobs:
           - name: "worker and utils"
             markers: "not e2e"
             flags: "--timeout=300"
+          - name: "compatibility"
+            markers: "compat"
+            flags: "--timeout=300"
 
     name: "${{ matrix.test_suite.name }} (${{ matrix.vllm_version.name }})"
 
@@ -90,6 +93,7 @@ jobs:
         if: (steps.changed-src-files.outputs.any_changed == 'true' && matrix.vllm_version.repo)
         run: |
           uv add ${{ matrix.vllm_version.repo }}
+          echo "TEST_VLLM_VERSION=main" >> $GITHUB_ENV
 
       - name: "Install vLLM with Spyre plugin"
         if: steps.changed-src-files.outputs.any_changed == 'true'

tests/utils/upstream_compatibility.py

@@ -0,0 +1,21 @@
+import pytest
+pytestmark = pytest.mark.compat
+import os
+
+VLLM_VERSION = os.getenv("TEST_VLLM_VERSION", "default")
+
+def test_vllm_bert_support(monkeypatch):
+    '''
+    Test if the vllm version under test already has Bert support for V1
+    '''
+
+    from vllm.model_executor.models.interfaces import SupportsV0Only
+    from vllm.model_executor.models.bert import BertEmbeddingModel
+
+    bert_supports_v0_only = issubclass(BertEmbeddingModel, MamSupportsV0Onlymal)
+
+    if VLLM_VERSION == "main":
+        assert not bert_supports_v0_only
+    else:
+        assert bert_supports_v0_only, ("The currently supported vLLM version already"
+            "supports Bert in V1. Remove the compatibility workarounds.")