SDK: change list apis to return objects as default (#1630)

* SDK: change list apis to return objects as default

- change list_trials, list_experiments to return list of objects as
 a default
- also, give 'in_short' parameter for who wants only name and status
 as before

* [enh]: change return type from List[dict] to List[V1beta1Experiment]

* [enh]: deserialize dict to katib's custom class

* [docs]: refactor KatibClient docs

* change deserialize method location to utils

* remove useless import

* Add objects necessary to deserilization in swagger

* use fakeresponse rather than duplicating codes

* Update sdk/python/v1beta1/kubeflow/katib/utils/utils.py

Co-authored-by: Andrey Velichkevich <[email protected]>

* Update sdk/python/v1beta1/kubeflow/katib/utils/utils.py

Co-authored-by: Andrey Velichkevich <[email protected]>

Co-authored-by: Andrey Velichkevich <[email protected]>

Author: anencore94

hack/gen-python-sdk/post_gen.py

@@ -50,6 +50,8 @@ def _rewrite_helper(input_file, output_file, rewrite_rules):
         lines.append("from kubernetes.client import V1ListMeta\n")
         lines.append("from kubernetes.client import V1Container\n")
         lines.append("from kubernetes.client import V1HTTPGetAction\n")
+        lines.append("from kubernetes.client import V1ManagedFieldsEntry\n")
+        lines.append("from kubernetes.client import V1OwnerReference\n")
 
     with open(output_file, 'w') as f:
         f.writelines(lines)

hack/gen-python-sdk/swagger_config.json

@@ -6,7 +6,9 @@
     "V1Container": "from kubernetes.client import V1Container",
     "V1ListMeta": "from kubernetes.client import V1ListMeta",
     "V1ObjectMeta": "from kubernetes.client import V1ObjectMeta",
-    "V1HTTPGetAction": "from kubernetes.client import V1HTTPGetAction"
+    "V1HTTPGetAction": "from kubernetes.client import V1HTTPGetAction",
+    "V1ManagedFieldsEntry": "from kubernetes.client import V1ManagedFieldsEntry",
+    "V1OwnerReference": "from kubernetes.client import V1OwnerReference"
   },
   "typeMappings": {
     "V1Time": "datetime",

sdk/python/v1beta1/docs/KatibClient.md

@@ -116,7 +116,7 @@ dict
 List all Katib Experiments.
 If the namespace is `None`, it takes "default" namespace.
 
-Return list of Experiment names with the statuses.
+Return list of Experiment objects.
 
 ### Parameters
 
@@ -126,7 +126,7 @@ Return list of Experiment names with the statuses.
 
 ### Return type
 
-list[dict]
+list[V1beta1Experiment]
 
 ## get_experiment_status
 
@@ -175,7 +175,7 @@ bool
 List all Experiment's Trials.
 If the namespace is `None`, it takes "default" namespace.
 
-Return list of Trial names with the statuses.
+Return list of Trial objects
 
 ### Parameters
 
@@ -186,7 +186,7 @@ Return list of Trial names with the statuses.
 
 ### Return type
 
-list[dict]
+list[V1beta1Trial]
 
 ## get_success_trial_details
 

sdk/python/v1beta1/kubeflow/katib/api/katib_client.py

@@ -14,10 +14,12 @@
 
 import multiprocessing
 
-from kubernetes import client, config
-
+from kubeflow.katib import V1beta1Experiment
+from kubeflow.katib import V1beta1Trial
+from kubeflow.katib.api_client import ApiClient
 from kubeflow.katib.constants import constants
 from kubeflow.katib.utils import utils
+from kubernetes import client, config
 
 
 class KatibClient(object):
@@ -45,6 +47,7 @@ def __init__(self, config_file=None, context=None,
             self.in_cluster = True
 
         self.api_instance = client.CustomObjectsApi()
+        self.api_client = ApiClient()
 
     def _is_ipython(self):
         """Returns whether we are running in notebook."""
@@ -251,8 +254,8 @@ def list_experiments(self, namespace=None):
         :param namespace: Experiments namespace.
         If the namespace is None, it takes "default" namespace.
 
-        :return: List of Experiment names with the statuses.
-        :rtype: list[dict]
+        :return: List of Experiment objects.
+        :rtype: list[V1beta1Experiment]
         """
         if namespace is None:
             namespace = utils.get_default_target_namespace()
@@ -267,13 +270,11 @@ def list_experiments(self, namespace=None):
         katibexp = None
         try:
             katibexp = thread.get(constants.APISERVER_TIMEOUT)
-            result = []
-            for i in katibexp.get("items"):
-                output = {}
-                output["name"] = i.get("metadata", {}).get("name")
-                output["status"] = i.get("status", {}).get(
-                    "conditions", [])[-1].get("type")
-                result.append(output)
+            result = [
+                self.api_client.deserialize(utils.FakeResponse(item), V1beta1Experiment)
+                for item in katibexp.get("items")
+            ]
+
         except multiprocessing.TimeoutError:
             raise RuntimeError("Timeout trying to get katib experiment.")
         except client.rest.ApiException as e:
@@ -282,8 +283,8 @@ def list_experiments(self, namespace=None):
           %s\n" % e)
         except Exception as e:
             raise RuntimeError(
-                "There was a problem to get experiments in namespace {1}. Exception: \
-          {2} ".format(namespace, e))
+                "There was a problem to get experiments in namespace {0}. Exception: \
+          {1} ".format(namespace, e))
         return result
 
     def get_experiment_status(self, name, namespace=None):
@@ -324,8 +325,8 @@ def list_trials(self, name=None, namespace=None):
         :param namespace: Experiments namespace.
         If the namespace is None, it takes "default" namespace.
 
-        :return: List of Trial names with the statuses.
-        :rtype: list[dict]
+        :return: List of Trial objects
+        :rtype: list[V1beta1Trial]
         """
         if namespace is None:
             namespace = utils.get_default_target_namespace()
@@ -340,14 +341,10 @@ def list_trials(self, name=None, namespace=None):
         katibtrial = None
         try:
             katibtrial = thread.get(constants.APISERVER_TIMEOUT)
-            result = []
-            for i in katibtrial.get("items"):
-                output = {}
-                if i.get("metadata", {}).get("ownerReferences")[0].get("name") == name:
-                    output["name"] = i.get("metadata", {}).get("name")
-                    output["status"] = i.get("status", {}).get(
-                        "conditions", [])[-1].get("type")
-                    result.append(output)
+            result = [
+                self.api_client.deserialize(utils.FakeResponse(item), V1beta1Trial)
+                for item in katibtrial.get("items")
+            ]
         except multiprocessing.TimeoutError:
             raise RuntimeError("Timeout trying to getkatib experiment.")
         except client.rest.ApiException as e:

sdk/python/v1beta1/kubeflow/katib/models/__init__.py

@@ -61,3 +61,5 @@
 from kubernetes.client import V1ListMeta
 from kubernetes.client import V1Container
 from kubernetes.client import V1HTTPGetAction
+from kubernetes.client import V1ManagedFieldsEntry
+from kubernetes.client import V1OwnerReference

sdk/python/v1beta1/kubeflow/katib/utils/utils.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 
 
@@ -34,3 +35,11 @@ def set_katib_namespace(katib):
     katib_namespace = katib.metadata.namespace
     namespace = katib_namespace or get_default_target_namespace()
     return namespace
+
+class FakeResponse:
+    """Fake object of RESTResponse to deserialize
+    Ref) https://github.com/kubeflow/katib/pull/1630#discussion_r697877815
+    Ref) https://github.com/kubernetes-client/python/issues/977#issuecomment-592030030
+    """
+    def __init__(self, obj):
+        self.data = json.dumps(obj)