adding fine tune example with s3 as the dataset store (kubeflow/trainer#2006)

* s3 as dataset source code review changes

Signed-off-by: deepanker13 <[email protected]>

* fixing python black test

Signed-off-by: deepanker13 <[email protected]>

* removing conflicts in example file

Signed-off-by: deepanker13 <[email protected]>

* retriggering CI

Signed-off-by: deepanker13 <[email protected]>

* removing dummy keys

Signed-off-by: deepanker13 <[email protected]>

* code review change for adding s3 keys block

Signed-off-by: deepanker13 <[email protected]>

---------

Signed-off-by: deepanker13 <[email protected]>

Author: deepanker13

python/kubeflow/storage_initializer/s3.py

@@ -42,18 +42,31 @@ def download_dataset(self):
         import boto3
 
         # Create an S3 client for Nutanix Object Store/S3
-        s3_client = boto3.client(
-            "s3",
+        s3_client = boto3.Session(
             aws_access_key_id=self.config.access_key,
             aws_secret_access_key=self.config.secret_key,
-            endpoint_url=self.config.endpoint_url,
             region_name=self.config.region_name,
         )
+        s3_resource = s3_client.resource("s3", endpoint_url=self.config.endpoint_url)
+        # Get the bucket object
+        bucket = s3_resource.Bucket(self.config.bucket_name)
 
-        # Download the file
-        s3_client.download_file(
-            self.config.bucket_name,
-            self.config.file_key,
-            os.path.join(VOLUME_PATH_DATASET, self.config.file_key),
-        )
-        print(f"File downloaded to: {VOLUME_PATH_DATASET}")
+        # Filter objects with the specified prefix
+        objects = bucket.objects.filter(Prefix=self.config.file_key)
+        # Iterate over filtered objects
+        for obj in objects:
+            # Extract the object key (filename)
+            obj_key = obj.key
+            path_components = obj_key.split(os.path.sep)
+            path_excluded_first_last_parts = os.path.sep.join(path_components[1:-1])
+
+            # Create directories if they don't exist
+            os.makedirs(
+                os.path.join(VOLUME_PATH_DATASET, path_excluded_first_last_parts),
+                exist_ok=True,
+            )
+
+            # Download the file
+            file_path = os.path.sep.join(path_components[1:])
+            bucket.download_file(obj_key, os.path.join(VOLUME_PATH_DATASET, file_path))
+        print(f"Files downloaded")