Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 14 additions & 1 deletion verl/utils/fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ def copy_to_local(
"""Copy files/directories from HDFS to local cache with validation.

Args:
src (str): Source path - HDFS path (hdfs://...) or local filesystem path
src (str): Source path - HDFS path (hdfs://...), local filesystem path, or Hugging Face model ID
cache_dir (str, optional): Local directory for cached files. Uses system tempdir if None
filelock (str): Base name for file lock. Defaults to ".file.lock"
verbose (bool): Enable copy operation logging. Defaults to False
Expand All @@ -210,6 +210,19 @@ def copy_to_local(
"""
# Save to a local path for persistence.
local_path = copy_local_path_from_hdfs(src, cache_dir, filelock, verbose, always_recopy)

if use_shm and isinstance(local_path, str) and not os.path.exists(local_path):
try:
from huggingface_hub import snapshot_download

resolved = snapshot_download(local_path)
if isinstance(resolved, str) and os.path.exists(resolved):
local_path = resolved
except ImportError:
pass
except Exception as e:
print(f"WARNING: Failed to download model from Hugging Face: {e}")

# Load into shm to improve efficiency.
if use_shm:
return copy_to_shm(local_path)
Expand Down
Loading