fix(mlop-2726): removing the last unecessary validation (#413)

Author: albjoaov

butterfree/validations/basic_validaton.py

@@ -26,7 +26,6 @@ def check(self) -> None:
             ValueError: if any of the verifications fail
 
         """
-        self.validate_df_is_spark_df()
         self.validate_column_ts()
         self.validate_df_is_empty()
 
@@ -53,17 +52,3 @@ def validate_df_is_empty(self) -> None:
             raise ValueError("DataFrame can't be None.")
         if (not self.dataframe.isStreaming) and self.dataframe.rdd.isEmpty():
             raise ValueError("DataFrame can't be empty.")
-
-    def validate_df_is_spark_df(self) -> None:
-        """Check type of dataframe object.
-
-        Raises:
-            ValueError: if dataframe is not instance of pyspark.sql.DataFrame.
-
-        """
-        if not self.dataframe:
-            raise ValueError("DataFrame can't be None.")
-        if not isinstance(self.dataframe, DataFrame):
-            raise ValueError(
-                "dataframe needs to be a instance of pyspark.sql.DataFrame"
-            )

tests/unit/butterfree/load/test_sink.py

@@ -90,28 +90,6 @@ def test_flush(self, feature_set_dataframe, mocker):
         for w in writer:
             w.write.assert_called_once()
 
-    def test_flush_with_invalid_df(self, not_feature_set_dataframe, mocker):
-        # given
-        spark_client = SparkClient()
-        writer = [
-            HistoricalFeatureStoreWriter(),
-            OnlineFeatureStoreWriter(),
-        ]
-        feature_set = mocker.stub("feature_set")
-        feature_set.entity = "house"
-        feature_set.name = "test"
-
-        # when
-        sink = Sink(writers=writer)
-
-        # then
-        with pytest.raises(ValueError):
-            sink.flush(
-                dataframe=not_feature_set_dataframe,
-                feature_set=feature_set,
-                spark_client=spark_client,
-            )
-
     def test_flush_with_writers_list_empty(self):
         # given
         writer = []

tests/unit/butterfree/validations/test_basic_validation.py

@@ -15,10 +15,3 @@ def test_validate_empty(self, feature_set_empty):
 
         with pytest.raises(ValueError):
             check.validate_df_is_empty()
-
-    def test_validate_not_spark_df(self):
-        df_writer = "not a spark df writer"
-        check = BasicValidation(df_writer)
-
-        with pytest.raises(ValueError):
-            check.validate_df_is_spark_df()