FIX-#4023: Fall back to pandas in case of MultiIndex columns (#5149)

Signed-off-by: Andrey Pavlenko <[email protected]>

Author: AndreyPavlenko

modin/experimental/core/execution/native/implementations/hdk_on_native/dataframe/dataframe.py

@@ -2385,14 +2385,22 @@ def from_pandas(cls, df):
         """
         new_index = df.index
         new_columns = df.columns
+
+        if isinstance(new_columns, MultiIndex):
+            # MultiIndex columns are not supported by the HDK backend.
+            # We just print this warning here and fall back to pandas.
+            index_cols = None
+            ErrorMessage.single_warning(
+                "MultiIndex columns are not currently supported by the HDK backend."
+            )
         # If there is non-trivial index, we put it into columns.
         # If the index is trivial, but there are no columns, we put
         # it into columns either because, otherwise, we don't know
         # the number of rows and, thus, unable to restore the index.
         # That's what we usually have for arrow tables and execution
         # result. Unnamed index is renamed to __index__. Also all
         # columns get 'F_' prefix to handle names unsupported in HDK.
-        if len(new_index) == 0 or (
+        elif len(new_index) == 0 or (
             len(new_columns) != 0 and cls._is_trivial_index(new_index)
         ):
             index_cols = None

modin/experimental/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py

@@ -163,6 +163,8 @@ def _get_unsupported_cols(cls, obj):
 
             if obj.empty:
                 unsupported_cols = []
+            elif isinstance(obj.columns, pandas.MultiIndex):
+                unsupported_cols = [str(c) for c in obj.columns]
             else:
                 cols = [name for name, col in obj.dtypes.items() if col == "object"]
                 type_samples = obj.iloc[0][cols]

modin/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py

@@ -447,10 +447,7 @@ def applier(lib):
         eval_general(pd, pandas, applier)
 
     @pytest.mark.parametrize("is_multiindex", [True, False])
-    @pytest.mark.parametrize(
-        "column_names", [None, ["level1", None], ["level1", "level2"]]
-    )
-    def test_reset_index_multicolumns(self, is_multiindex, column_names):
+    def test_reset_index_multicolumns(self, is_multiindex):
         index = (
             pandas.MultiIndex.from_tuples(
                 [(i, j, k) for i in range(2) for j in range(3) for k in range(4)],
@@ -459,9 +456,6 @@ def test_reset_index_multicolumns(self, is_multiindex, column_names):
             if is_multiindex
             else pandas.Index(np.arange(1, len(self.data["a"]) + 1), name="index")
         )
-        columns = pandas.MultiIndex.from_tuples(
-            [("a", "b"), ("b", "c")], names=column_names
-        )
         data = np.array(list(self.data.values())).T
 
         def applier(df, **kwargs):
@@ -471,7 +465,7 @@ def applier(df, **kwargs):
         run_and_compare(
             fn=applier,
             data=data,
-            constructor_kwargs={"index": index, "columns": columns},
+            constructor_kwargs={"index": index},
         )
 
     def test_set_index_name(self):
@@ -498,6 +492,27 @@ def test_set_index_names(self):
 
         df_equals(pandas_df, modin_df)
 
+    def test_rename(self):
+        index = pandas.MultiIndex.from_tuples(
+            [("foo1", "bar1"), ("foo2", "bar2")], names=["foo", "bar"]
+        )
+        columns = pandas.MultiIndex.from_tuples(
+            [("fizz1", "buzz1"), ("fizz2", "buzz2")], names=["fizz", "buzz"]
+        )
+
+        def rename(df, **kwargs):
+            return df.rename(
+                index={"foo1": "foo3", "bar2": "bar3"},
+                columns={"fizz1": "fizz3", "buzz2": "buzz3"},
+            )
+
+        run_and_compare(
+            fn=rename,
+            data=[(0, 0), (1, 1)],
+            constructor_kwargs={"index": index, "columns": columns},
+            force_lazy=False,
+        )
+
 
 class TestFillna:
     data = {"a": [1, 1, None], "b": [None, None, 2], "c": [3, None, None]}