feat: Add max_file_size to Snowflake config (feast-dev#5377)

Add max_file_size to Snowflake config

Signed-off-by: hkuepers <[email protected]>
Co-authored-by: hkuepers <[email protected]>
Signed-off-by: Jacob Weinhold <[email protected]>

Author: 2 people

sdk/python/feast/infra/offline_stores/snowflake.py

@@ -130,6 +130,9 @@ class SnowflakeOfflineStoreConfig(FeastConfigBaseModel):
 
     convert_timestamp_columns: Optional[bool] = None
     """ Convert timestamp columns on export to a Parquet-supported format """
+
+    max_file_size: Optional[int] = None
+    """ Upper size limit (in bytes) of each file that is offloaded. Default: 16777216"""
     model_config = ConfigDict(populate_by_name=True)
 
 
@@ -616,6 +619,9 @@ def to_remote_storage(self) -> List[str]:
               DETAILED_OUTPUT = TRUE
               HEADER = TRUE
         """
+        if (max_file_size := self.config.offline_store.max_file_size) is not None:
+            query += f"\nMAX_FILE_SIZE = {max_file_size}"
+
         cursor = execute_snowflake_statement(self.snowflake_conn, query)
         # s3gov schema is used by Snowflake in AWS govcloud regions
         # remove gov portion from schema and pass it to online store upload