Add exclude parameter to S3 pull functionality

src/art/local/backend.py

@@ -48,6 +48,7 @@
 from art.utils.s3 import (
     pull_model_from_s3,
     push_model_to_s3,
+    ExcludableOption,
 )
 
 
@@ -443,6 +444,7 @@ async def _experimental_pull_from_s3(
         prefix: str | None = None,
         verbose: bool = False,
         delete: bool = False,
+        exclude: list[ExcludableOption] | None = None,
     ) -> None:
         """Download the model directory from S3 into local Backend storage. Right now this can be used to pull trajectory logs for processing or model checkpoints.
         Args:
@@ -452,7 +454,9 @@ async def _experimental_pull_from_s3(
             prefix: The prefix to pull from S3. If None, the model name will be used.
             verbose: Whether to print verbose output.
             delete: Whether to delete the local model directory.
+            exclude: List of directories to exclude from sync. Valid options: "checkpoints", "logs", "trajectories".
         """
+
         await pull_model_from_s3(
             model_name=model.name,
             project=model.project,
@@ -462,6 +466,7 @@ async def _experimental_pull_from_s3(
             verbose=verbose,
             delete=delete,
             art_path=self._path,
+            exclude=exclude,
         )
 
     async def _experimental_push_to_s3(

src/art/utils/benchmarking/load_trajectories.py

@@ -238,6 +238,7 @@ async def pull_model_trajectories(model: ArtModel) -> None:
             model,
             s3_bucket=bucket,
             verbose=True,
+            exclude=["checkpoints", "logs"],
         )
 
         print("Finished pulling trajectories.", flush=True)

src/art/utils/s3.py

@@ -4,7 +4,7 @@
 import asyncio
 from asyncio.subprocess import DEVNULL
 import tempfile
-from typing import Optional, Sequence
+from typing import Optional, Sequence, Literal
 import zipfile
 
 from art.errors import ForbiddenBucketCreationError
@@ -17,6 +17,8 @@
 
 __all__: Sequence[str] = ("s3_sync",)
 
+ExcludableOption = Literal["checkpoints", "logs", "trajectories"]
+
 
 class S3SyncError(RuntimeError):
     """Raised when the underlying *aws s3 sync* command exits with a non‑zero status."""
@@ -67,6 +69,7 @@ async def s3_sync(
     profile: Optional[str] = None,
     verbose: bool = False,
     delete: bool = False,
+    exclude: list[ExcludableOption] | None = None,
 ) -> None:
     """Synchronise *source* and *destination* using the AWS CLI.
 
@@ -82,6 +85,7 @@ async def s3_sync(
         profile: Optional AWS profile name to pass to the CLI.
         verbose: When *True*, the output of the AWS CLI is streamed to the
             calling process; otherwise it is suppressed.
+        exclude: List of directories to exclude from sync.
 
     Raises:
         S3SyncError: If the *aws s3 sync* command exits with a non‑zero status.
@@ -100,6 +104,12 @@ async def s3_sync(
 
     if delete:
         cmd.append("--delete")
+
+    # Add exclude patterns for each excluded directory
+    if exclude:
+        for excluded_dir in exclude:
+            cmd.extend(["--exclude", f"{excluded_dir}/*"])
+
     cmd += [source, destination]
 
     # Suppress output unless verbose mode is requested.
@@ -156,6 +166,7 @@ async def pull_model_from_s3(
     verbose: bool = False,
     delete: bool = False,
     art_path: str | None = None,
+    exclude: list[ExcludableOption] | None = None,
 ) -> str:
     """Pull a model from S3 to the local directory.
 
@@ -169,6 +180,7 @@ async def pull_model_from_s3(
             calling process; otherwise it is suppressed.
         delete: When *True*, delete the local model directory if it exists.
         art_path: The path to the ART directory.
+        exclude: List of directories to exclude from sync.
 
     Returns:
         The local directory path.
@@ -196,7 +208,7 @@ async def pull_model_from_s3(
         prefix=prefix,
     )
     await ensure_bucket_exists(s3_bucket)
-    await s3_sync(s3_path, local_dir, verbose=verbose, delete=delete)
+    await s3_sync(s3_path, local_dir, verbose=verbose, delete=delete, exclude=exclude)
 
     # After pulling, migrate to new structure if needed
     if step is not None: