Skip to content

Commit c8d5326

Browse files
committed
[BUG] Apply create_partitions to historical validate (#303)
* Apply create_partitions to historical validate. * Remove comments and adjusts.
1 parent 569e4fb commit c8d5326

File tree

2 files changed

+4
-3
lines changed

2 files changed

+4
-3
lines changed

butterfree/load/writers/historical_feature_store_writer.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,6 @@ def validate(
225225
Raises:
226226
AssertionError: if count of written data doesn't match count in current
227227
feature set dataframe.
228-
229228
"""
230229
table_name = (
231230
f"{feature_set.name}"
@@ -240,7 +239,9 @@ def validate(
240239
written_count = (
241240
spark_client.read(
242241
self.db_config.format_,
243-
path=self.db_config.get_path_with_partitions(table_name, dataframe),
242+
path=self.db_config.get_path_with_partitions(
243+
table_name, self._create_partitions(dataframe)
244+
),
244245
).count()
245246
if self.interval_mode and not self.debug_mode
246247
else spark_client.read_table(table_name).count()

setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
from setuptools import find_packages, setup
22

33
__package_name__ = "butterfree"
4-
__version__ = "1.2.0.dev3"
4+
__version__ = "1.2.0.dev4"
55
__repository_url__ = "https://github.com/quintoandar/butterfree"
66

77
with open("requirements.txt") as f:

0 commit comments

Comments
 (0)