Add buffer to image, audio, video and document blocks

CHANGELOG.md

@@ -5,52 +5,67 @@
 ## [2025-10-26]
 
 ### llama-index-core [0.14.6]
+
 - Add allow_parallel_tool_calls for non-streaming ([#20117](https://github.com/run-llama/llama_index/pull/20117))
 - Fix invalid use of field-specific metadata ([#20122](https://github.com/run-llama/llama_index/pull/20122))
 - update doc for SemanticSplitterNodeParser ([#20125](https://github.com/run-llama/llama_index/pull/20125))
 - fix rare cases when sentence splits are larger than chunk size ([#20147](https://github.com/run-llama/llama_index/pull/20147))
 
 ### llama-index-embeddings-bedrock [0.7.0]
+
 - Fix BedrockEmbedding to support Cohere v4 response format ([#20094](https://github.com/run-llama/llama_index/pull/20094))
 
 ### llama-index-embeddings-isaacus [0.1.0]
+
 - feat: Isaacus embeddings integration ([#20124](https://github.com/run-llama/llama_index/pull/20124))
 
 ### llama-index-embeddings-oci-genai [0.4.2]
+
 - Update OCI GenAI cohere models ([#20146](https://github.com/run-llama/llama_index/pull/20146))
 
 ### llama-index-llms-anthropic [0.9.7]
+
 - Fix double token stream in anthropic llm ([#20108](https://github.com/run-llama/llama_index/pull/20108))
 - Ensure anthropic content delta only has user facing response ([#20113](https://github.com/run-llama/llama_index/pull/20113))
 
 ### llama-index-llms-baseten [0.1.7]
+
 - add GLM ([#20121](https://github.com/run-llama/llama_index/pull/20121))
 
 ### llama-index-llms-helicone [0.1.0]
+
 - integrate helicone to llama-index ([#20131](https://github.com/run-llama/llama_index/pull/20131))
 
 ### llama-index-llms-oci-genai [0.6.4]
+
 - Update OCI GenAI cohere models ([#20146](https://github.com/run-llama/llama_index/pull/20146))
 
 ### llama-index-llms-openai [0.6.5]
+
 - chore: openai vbump ([#20095](https://github.com/run-llama/llama_index/pull/20095))
 
 ### llama-index-readers-imdb-review [0.4.2]
+
 - chore: Update selenium dependency in imdb-review reader ([#20105](https://github.com/run-llama/llama_index/pull/20105))
 
 ### llama-index-retrievers-bedrock [0.5.0]
+
 - feat(bedrock): add async support for AmazonKnowledgeBasesRetriever ([#20114](https://github.com/run-llama/llama_index/pull/20114))
 
 ### llama-index-retrievers-superlinked [0.1.3]
+
 - Update README.md ([#19829](https://github.com/run-llama/llama_index/pull/19829))
 
 ### llama-index-storage-kvstore-postgres [0.4.2]
+
 - fix: Replace raw SQL string interpolation with proper SQLAlchemy parameterized APIs in PostgresKVStore ([#20104](https://github.com/run-llama/llama_index/pull/20104))
 
 ### llama-index-tools-mcp [0.4.3]
+
 - Fix BasicMCPClient resource signatures ([#20118](https://github.com/run-llama/llama_index/pull/20118))
 
 ### llama-index-vector-stores-postgres [0.7.1]
+
 - Add GIN index support for text array metadata in PostgreSQL vector store ([#20130](https://github.com/run-llama/llama_index/pull/20130))
 
 ## [2025-10-15]

llama-index-core/llama_index/core/base/llms/types.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
 import base64
-import filetype
 from binascii import Error as BinasciiError
 from enum import Enum
-from io import BytesIO
+from io import IOBase
 from pathlib import Path
 from typing import (
     Annotated,
@@ -61,12 +60,14 @@ class ImageBlock(BaseModel):
     """A representation of image data to directly pass to/from the LLM."""
 
     block_type: Literal["image"] = "image"
-    image: bytes | None = None
+    image: bytes | IOBase | None = None
     path: FilePath | None = None
     url: AnyUrl | str | None = None
     image_mimetype: str | None = None
     detail: str | None = None
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @field_validator("url", mode="after")
     @classmethod
     def urlstr_to_anyurl(cls, url: str | AnyUrl | None) -> AnyUrl | None:
@@ -78,6 +79,16 @@ def urlstr_to_anyurl(cls, url: str | AnyUrl | None) -> AnyUrl | None:
 
         return AnyUrl(url=url)
 
+    @field_serializer("image")
+    def serialize_image(self, image: bytes | IOBase | None) -> bytes | None:
+        """Serialize the image field."""
+        if isinstance(image, bytes):
+            return image
+        if isinstance(image, IOBase):
+            image.seek(0)
+            return image.read()
+        return None
+
     @model_validator(mode="after")
     def image_to_base64(self) -> Self:
         """
@@ -87,7 +98,7 @@ def image_to_base64(self) -> Self:
         we try to guess it using the filetype library. To avoid resource-intense
         operations, we won't load the path or the URL to guess the mimetype.
         """
-        if not self.image:
+        if not self.image or not isinstance(self.image, bytes):
             if not self.image_mimetype:
                 path = self.path or self.url
                 if path:
@@ -115,19 +126,23 @@ def _guess_mimetype(self, img_data: bytes) -> None:
             guess = filetype.guess(img_data)
             self.image_mimetype = guess.mime if guess else None
 
-    def resolve_image(self, as_base64: bool = False) -> BytesIO:
+    def resolve_image(self, as_base64: bool = False) -> IOBase:
         """
         Resolve an image such that PIL can read it.
 
         Args:
             as_base64 (bool): whether the resolved image should be returned as base64-encoded bytes
 
         """
-        data_buffer = resolve_binary(
-            raw_bytes=self.image,
-            path=self.path,
-            url=str(self.url) if self.url else None,
-            as_base64=as_base64,
+        data_buffer = (
+            self.image
+            if isinstance(self.image, IOBase)
+            else resolve_binary(
+                raw_bytes=self.image,
+                path=self.path,
+                url=str(self.url) if self.url else None,
+                as_base64=as_base64,
+            )
         )
 
         # Check size by seeking to end and getting position
@@ -144,11 +159,13 @@ class AudioBlock(BaseModel):
     """A representation of audio data to directly pass to/from the LLM."""
 
     block_type: Literal["audio"] = "audio"
-    audio: bytes | None = None
+    audio: bytes | IOBase | None = None
     path: FilePath | None = None
     url: AnyUrl | str | None = None
     format: str | None = None
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @field_validator("url", mode="after")
     @classmethod
     def urlstr_to_anyurl(cls, url: str | AnyUrl) -> AnyUrl:
@@ -157,6 +174,16 @@ def urlstr_to_anyurl(cls, url: str | AnyUrl) -> AnyUrl:
             return url
         return AnyUrl(url=url)
 
+    @field_serializer("audio")
+    def serialize_audio(self, audio: bytes | IOBase | None) -> bytes | None:
+        """Serialize the audio field."""
+        if isinstance(audio, bytes):
+            return audio
+        if isinstance(audio, IOBase):
+            audio.seek(0)
+            return audio.read()
+        return None
+
     @model_validator(mode="after")
     def audio_to_base64(self) -> Self:
         """
@@ -166,7 +193,7 @@ def audio_to_base64(self) -> Self:
         we try to guess it using the filetype library. To avoid resource-intense
         operations, we won't load the path or the URL to guess the mimetype.
         """
-        if not self.audio:
+        if not self.audio or not isinstance(self.audio, bytes):
             return self
 
         try:
@@ -186,41 +213,47 @@ def _guess_format(self, audio_data: bytes) -> None:
             guess = filetype.guess(audio_data)
             self.format = guess.extension if guess else None
 
-    def resolve_audio(self, as_base64: bool = False) -> BytesIO:
+    def resolve_audio(self, as_base64: bool = False) -> IOBase:
         """
         Resolve an audio such that PIL can read it.
 
         Args:
             as_base64 (bool): whether the resolved audio should be returned as base64-encoded bytes
 
         """
-        data_buffer = resolve_binary(
-            raw_bytes=self.audio,
-            path=self.path,
-            url=str(self.url) if self.url else None,
-            as_base64=as_base64,
+        data_buffer = (
+            self.audio
+            if isinstance(self.audio, IOBase)
+            else resolve_binary(
+                raw_bytes=self.audio,
+                path=self.path,
+                url=str(self.url) if self.url else None,
+                as_base64=as_base64,
+            )
         )
         # Check size by seeking to end and getting position
         data_buffer.seek(0, 2)  # Seek to end
         size = data_buffer.tell()
         data_buffer.seek(0)  # Reset to beginning
 
         if size == 0:
-            raise ValueError("resolve_image returned zero bytes")
+            raise ValueError("resolve_audio returned zero bytes")
         return data_buffer
 
 
 class VideoBlock(BaseModel):
     """A representation of video data to directly pass to/from the LLM."""
 
     block_type: Literal["video"] = "video"
-    video: bytes | None = None
+    video: bytes | IOBase | None = None
     path: FilePath | None = None
     url: AnyUrl | str | None = None
     video_mimetype: str | None = None
     detail: str | None = None
     fps: int | None = None
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @field_validator("url", mode="after")
     @classmethod
     def urlstr_to_anyurl(cls, url: str | AnyUrl | None) -> AnyUrl | None:
@@ -231,14 +264,24 @@ def urlstr_to_anyurl(cls, url: str | AnyUrl | None) -> AnyUrl | None:
             return None
         return AnyUrl(url=url)
 
+    @field_serializer("video")
+    def serialize_video(self, video: bytes | IOBase | None) -> bytes | None:
+        """Serialize the video field."""
+        if isinstance(video, bytes):
+            return video
+        if isinstance(video, IOBase):
+            video.seek(0)
+            return video.read()
+        return None
+
     @model_validator(mode="after")
     def video_to_base64(self) -> "VideoBlock":
         """
         Store the video as base64 and guess the mimetype when possible.
 
         If video data is passed but no mimetype is provided, try to infer it.
         """
-        if not self.video:
+        if not self.video or not isinstance(self.video, bytes):
             if not self.video_mimetype:
                 path = self.path or self.url
                 if path:
@@ -263,19 +306,23 @@ def _guess_mimetype(self, vid_data: bytes) -> None:
             if guess and guess.mime.startswith("video/"):
                 self.video_mimetype = guess.mime
 
-    def resolve_video(self, as_base64: bool = False) -> BytesIO:
+    def resolve_video(self, as_base64: bool = False) -> IOBase:
         """
-        Resolve a video file to a BytesIO buffer.
+        Resolve a video file to a IOBase buffer.
 
         Args:
             as_base64 (bool): whether to return the video as base64-encoded bytes
 
         """
-        data_buffer = resolve_binary(
-            raw_bytes=self.video,
-            path=self.path,
-            url=str(self.url) if self.url else None,
-            as_base64=as_base64,
+        data_buffer = (
+            self.video
+            if isinstance(self.video, IOBase)
+            else resolve_binary(
+                raw_bytes=self.video,
+                path=self.path,
+                url=str(self.url) if self.url else None,
+                as_base64=as_base64,
+            )
         )
 
         # Check size by seeking to end and getting position
@@ -292,21 +339,23 @@ class DocumentBlock(BaseModel):
     """A representation of a document to directly pass to the LLM."""
 
     block_type: Literal["document"] = "document"
-    data: Optional[bytes] = None
+    data: bytes | IOBase | None = None
     path: Optional[Union[FilePath | str]] = None
     url: Optional[str] = None
     title: Optional[str] = None
     document_mimetype: Optional[str] = None
 
+    model_config = ConfigDict(arbitrary_types_allowed=True)
+
     @model_validator(mode="after")
     def document_validation(self) -> Self:
         self.document_mimetype = self.document_mimetype or self._guess_mimetype()
 
         if not self.title:
             self.title = "input_document"
 
-        # skip data validation if it's not provided
-        if not self.data:
+        # skip data validation if no byte is provided
+        if not self.data or not isinstance(self.data, bytes):
             return self
 
         try:
@@ -316,35 +365,49 @@ def document_validation(self) -> Self:
 
         return self
 
-    def resolve_document(self) -> BytesIO:
+    @field_serializer("data")
+    def serialize_data(self, data: bytes | IOBase | None) -> bytes | None:
+        """Serialize the data field."""
+        if isinstance(data, bytes):
+            return data
+        if isinstance(data, IOBase):
+            data.seek(0)
+            return data.read()
+        return None
+
+    def resolve_document(self) -> IOBase:
         """
         Resolve a document such that it is represented by a BufferIO object.
         """
-        data_buffer = resolve_binary(
-            raw_bytes=self.data,
-            path=self.path,
-            url=str(self.url) if self.url else None,
-            as_base64=False,
+        data_buffer = (
+            self.data
+            if isinstance(self.data, IOBase)
+            else resolve_binary(
+                raw_bytes=self.data,
+                path=self.path,
+                url=str(self.url) if self.url else None,
+                as_base64=False,
+            )
         )
         # Check size by seeking to end and getting position
         data_buffer.seek(0, 2)  # Seek to end
         size = data_buffer.tell()
         data_buffer.seek(0)  # Reset to beginning
 
         if size == 0:
-            raise ValueError("resolve_image returned zero bytes")
+            raise ValueError("resolve_document returned zero bytes")
         return data_buffer
 
-    def _get_b64_string(self, data_buffer: BytesIO) -> str:
+    def _get_b64_string(self, data_buffer: IOBase) -> str:
         """
-        Get base64-encoded string from a BytesIO buffer.
+        Get base64-encoded string from a IOBase buffer.
         """
         data = data_buffer.read()
         return base64.b64encode(data).decode("utf-8")
 
-    def _get_b64_bytes(self, data_buffer: BytesIO) -> bytes:
+    def _get_b64_bytes(self, data_buffer: IOBase) -> bytes:
         """
-        Get base64-encoded bytes from a BytesIO buffer.
+        Get base64-encoded bytes from a IOBase buffer.
         """
         data = data_buffer.read()
         return base64.b64encode(data)