Upgrade pydantic to v2 (PR1) (#1191)

Author: papa99do

.github/workflows/run_required_checks.yml

@@ -24,10 +24,17 @@ jobs:
       - name: Check for Non-Markdown Changes
         id: check_changes
         run: |
-          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -vE '\.md$'; then
-            echo "non_documentation_changes_found=true" >> $GITHUB_OUTPUT
+          BEFORE="${{ github.event.before }}"
+          AFTER="${{ github.sha }}"
+          
+          if [ -z "$BEFORE" ] || \
+             git diff --name-only "$BEFORE" "$AFTER" | grep -qvE '\.md$'
+          then
+            # either BEFORE was empty, or we found at least one non‑.md change
+            echo "non_documentation_changes_found=true" >> "$GITHUB_OUTPUT"
           else
-            echo "non_documentation_changes_found=false" >> $GITHUB_OUTPUT
+            # BEFORE was set and all changed files end in .md
+            echo "non_documentation_changes_found=false" >> "$GITHUB_OUTPUT"
           fi
 
   Run-integration-tests:

requirements.txt

@@ -1,5 +1,15 @@
 # Marqo's requirements files are stored in marqo-base repo: https://github.com/marqo-ai/marqo-base/tree/main/requirements
+# Every time we upgrade dependencies, we need to build a new base image. This process often requires upgrading Vespa
+# as well (since Vespa regularly removes old version from their dns repo). In the long run, we'll merge marqo-base repo
+# back into marqo repo, and improve the image building process. In the short term, we will add new platform-agnostic
+# dependencies (or upgrades of these dependencies) in this file. Please not if the dependency is platform-relevant,
+# meaning you need to use one version for arm64 and another for amd64, you still need to follow the process in
+# marqo-base repo and build a new base image.
 
 msgpack==1.1.0
 msgpack_numpy==0.4.8
-python-json-logger==3.3.0
+python-json-logger==3.3.0
+pydantic==2.11.1
+fastapi==0.115.12
+fastapi-utils==0.8.0
+uvicorn==0.34.0

src/marqo/api/models/add_docs_objects.py

@@ -1,8 +1,8 @@
 from typing import List, Dict
 from typing import Optional, Any, Sequence
 
-from pydantic import BaseModel, root_validator
-from pydantic import Field
+from pydantic.v1 import BaseModel, root_validator
+from pydantic.v1 import Field
 
 from marqo.tensor_search.enums import EnvVars
 from marqo.tensor_search.models.private_models import ModelAuth
@@ -25,7 +25,7 @@ class Config:
     mappings: Optional[dict] = None
     documents: Sequence[Dict[str, Any]]
     imageDownloadThreadCount: int = Field(default_factory=lambda: read_env_vars_and_defaults_ints(EnvVars.MARQO_IMAGE_DOWNLOAD_THREAD_COUNT_PER_REQUEST))
-    mediaDownloadThreadCount: Optional[int]
+    mediaDownloadThreadCount: Optional[int] = None
     textChunkPrefix: Optional[str] = None
 
     @root_validator

src/marqo/api/models/embed_request.py

@@ -5,8 +5,7 @@
 """
 from typing import Union, List, Dict, Optional
 
-import pydantic
-from pydantic import Field, root_validator
+from pydantic.v1 import Field, root_validator, validator
 
 from marqo.base_model import MarqoBaseModel
 from marqo.core.embed.embed import EmbedContentType
@@ -21,7 +20,7 @@ class EmbedRequest(MarqoBaseModel):
     modelAuth: Optional[ModelAuth] = None
     content_type: Optional[EmbedContentType] = Field(default=EmbedContentType.Query, alias="contentType")
 
-    @pydantic.validator('content')
+    @validator('content')
     def validate_content(cls, value):
         # Iterate through content list items
         if (isinstance(value, list) or isinstance(value, dict)) and len(value) == 0:

src/marqo/api/models/get_batch_documents_request.py

@@ -1,4 +1,4 @@
-from pydantic import Field, conlist
+from pydantic.v1 import Field, conlist
 
 from marqo.base_model import MarqoBaseModel
 

src/marqo/api/models/recommend_query.py

@@ -2,7 +2,7 @@
 
 from marqo.core.models.interpolation_method import InterpolationMethod
 from marqo.tensor_search.models.api_models import BaseMarqoModel
-from pydantic import root_validator
+from pydantic.v1 import root_validator
 from marqo.tensor_search.models.score_modifiers_object import ScoreModifierLists
 
 

src/marqo/api/models/update_documents.py

@@ -1,6 +1,6 @@
 from typing import Dict, Any, List
 
-from pydantic import validator
+from pydantic.v1 import validator
 
 from marqo.base_model import ImmutableStrictBaseModel
 from marqo.api.exceptions import BadRequestError

src/marqo/base_model.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel
+from pydantic.v1 import BaseModel
 
 
 class MarqoBaseModel(BaseModel):

src/marqo/core/embed/embed.py

@@ -2,27 +2,24 @@
 from timeit import default_timer as timer
 from typing import List, Optional, Union, Dict
 
-import pydantic
-
-import marqo.api.exceptions as api_exceptions
-import marqo.s2_inference.errors as s2_inference_errors
 from marqo import exceptions as base_exceptions
 from marqo.core.index_management.index_management import IndexManagement
 from marqo.core.inference.api import Inference
-from marqo.tensor_search import utils
+from marqo.logging import get_logger
 from marqo.tensor_search.models.api_models import BulkSearchQueryEntity
 from marqo.tensor_search.models.private_models import ModelAuth
 from marqo.tensor_search.models.search import Qidx
 from marqo.tensor_search.telemetry import RequestMetricsStore
-from marqo.logging import get_logger
 from marqo.vespa.vespa_client import VespaClient
 
 logger = get_logger(__name__)
 
+
 class EmbedContentType(str, Enum):
     Query = "query"
     Document = "document"
 
+
 class Embed:
     def __init__(self, vespa_client: VespaClient, index_management: IndexManagement, inference: Inference):
         self.vespa_client = vespa_client

src/marqo/core/inference/api/inference.py

@@ -1,9 +1,8 @@
 from abc import ABC, abstractmethod
 from typing import Optional, Dict, Any, List, Tuple, Union
 
-import pydantic
 from numpy import ndarray
-from pydantic import StrictStr, root_validator
+from pydantic.v1 import StrictStr, root_validator, Field
 
 from marqo.base_model import ImmutableBaseModel
 from marqo.core.inference.api import Modality, PreprocessingConfigType
@@ -12,21 +11,21 @@
 
 
 class ModelConfig(ImmutableBaseModel):
-    model_name: StrictStr = pydantic.Field(alias='modelName')
-    model_properties: Optional[Dict[str, Any]] = pydantic.Field(default=None, alias='modelProperties')
-    model_auth: Optional[ModelAuth] = pydantic.Field(default=None, alias='modelAuth')
-    normalize_embeddings: bool = pydantic.Field(default=True, alias='normalizeEmbeddings')
+    model_name: StrictStr = Field(alias='modelName')
+    model_properties: Optional[Dict[str, Any]] = Field(default=None, alias='modelProperties')
+    model_auth: Optional[ModelAuth] = Field(default=None, alias='modelAuth')
+    normalize_embeddings: bool = Field(default=True, alias='normalizeEmbeddings')
 
 
 class InferenceRequest(ImmutableBaseModel):
     modality: Modality
-    contents: List[str] = pydantic.Field(min_items=1)
-    device: Optional[str] = pydantic.Field(default=None)
-    model_config: ModelConfig = pydantic.Field(alias='modelConfig')
-    preprocessing_config: PreprocessingConfigType = pydantic.Field(alias='preprocessingConfig')
-    use_inference_cache: bool = pydantic.Field(default=False, alias='useInferenceCache')
+    contents: List[str] = Field(min_items=1)
+    device: Optional[str] = Field(default=None)
+    model_config: ModelConfig = Field(alias='modelConfig')
+    preprocessing_config: PreprocessingConfigType = Field(alias='preprocessingConfig')
+    use_inference_cache: bool = Field(default=False, alias='useInferenceCache')
     # whether we should return error for individual content, when set to false, any error should fail the whole batch
-    return_individual_error: bool = pydantic.Field(default=True, alias='returnIndividualError')
+    return_individual_error: bool = Field(default=True, alias='returnIndividualError')
 
     @root_validator(pre=False)
     def check_preprocessing_config_matches_modality(cls, values):
@@ -47,8 +46,8 @@ class InferenceErrorModel(ImmutableBaseModel):
     """
     A model class to store error information for each individual content
     """
-    status_code: int = pydantic.Field(default=400)
-    error_code: str = pydantic.Field(default='inference_error')
+    status_code: int = Field(default=400)
+    error_code: str = Field(default='inference_error')
     error_message: str