Skip to content

Commit 1752a4f

Browse files
astronautasntkathole
authored andcommitted
add pull_all_from_table_or_query for clickhouse, to align with new materialization logic (calling it)
Signed-off-by: lukas.valatka <[email protected]>
1 parent 366e5a8 commit 1752a4f

File tree

2 files changed

+37
-1
lines changed

2 files changed

+37
-1
lines changed

sdk/python/feast/infra/offline_stores/contrib/clickhouse_offline_store/clickhouse.py

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,6 +191,43 @@ def pull_latest_from_table_or_query(
191191
on_demand_feature_views=None,
192192
)
193193

194+
@staticmethod
195+
def pull_all_from_table_or_query(
196+
config: RepoConfig,
197+
data_source: DataSource,
198+
join_key_columns: List[str],
199+
feature_name_columns: List[str],
200+
timestamp_field: str,
201+
created_timestamp_column: Optional[str] = None,
202+
start_date: Optional[datetime] = None,
203+
end_date: Optional[datetime] = None,
204+
) -> RetrievalJob:
205+
assert isinstance(config.offline_store, ClickhouseOfflineStoreConfig)
206+
assert isinstance(data_source, ClickhouseSource)
207+
208+
from_expression = data_source.get_table_query_string()
209+
210+
timestamp_fields = [timestamp_field]
211+
212+
if created_timestamp_column:
213+
timestamp_fields.append(created_timestamp_column)
214+
215+
field_string = ", ".join(
216+
join_key_columns + feature_name_columns + timestamp_fields
217+
)
218+
219+
query = f"""
220+
SELECT {field_string}
221+
FROM {from_expression}
222+
WHERE {timestamp_field} BETWEEN parseDateTimeBestEffort('{start_date}') AND parseDateTimeBestEffort('{end_date}')
223+
"""
224+
225+
return ClickhouseRetrievalJob(
226+
query=query,
227+
config=config,
228+
full_feature_names=False,
229+
)
230+
194231

195232
class ClickhouseRetrievalJob(PostgreSQLRetrievalJob):
196233
def _to_df_internal(self, timeout: Optional[int] = None) -> pd.DataFrame:

sdk/python/tests/integration/registration/test_universal_types.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,7 +343,6 @@ def offline_types_test_fixtures(request, environment):
343343
if (
344344
environment.data_source_creator.__class__.__name__
345345
== "ClickhouseDataSourceCreator"
346-
and config.feature_dtype in {"float", "datetime", "bool"}
347346
and config.feature_is_list
348347
and not config.has_empty_list
349348
):

0 commit comments

Comments
 (0)