fixed ckan_agent result processing

ckanext/chat/bot/agent.py

@@ -297,6 +297,7 @@ class CKANResult(BaseModel):
     "  - Present updates and changes, requesting user confirmation before proceeding, when running actions that chnage the data.\n"
     "  - Request confirmation if SSL verification is disabled (`ssl_verify=False` for downloads).\n"
     "Guidelines:\n"
+    "- if `ckan_run` fails adopt your call by the suggestions made in the response, add default parameters as necessarry.\n"
     "- CKAN entities are organized like following: Datasets or Packages contain Resources that can be Files or Links, Every Dataset lives in exactly one Organisation, but can be associated with multiple Groups."
     " Views are attached to Resources and render them dependent on the necessaties of the resource format and user needs.\n"
     "- use 'get_ckan_actions' to find a dict with keys of action names and values the functions signature.\n"
@@ -357,12 +358,6 @@ class CKANResult(BaseModel):
     # model_settings=OpenAIModelSettings(openai_reasoning_effort= "low")
 )
 
-
-class SliceModel(BaseModel):
-    start: int
-    end: int
-
-
 doc_agent = Agent(
     model=model,
     deps_type=TextResource,
@@ -510,8 +505,9 @@ def run_action(ctx: RunContext[Deps], action_name: str, parameters: Dict) -> Any
         response = toolkit.get_action(action_name)(context, parameters)
     except Exception as e:
         return {"error": str(e)}
-    clean_response = unpack_lazy_json(response)
-    clean_response = process_entity(clean_response)
+    unpacked_response = unpack_lazy_json(response)
+    log.debug(type(unpacked_response))
+    clean_response = process_entity(unpacked_response)
     log.debug(clean_response)
     #log.debug("{} -> {}".format(len(str(response)), len(str(clean_response))))
     return clean_response
@@ -824,7 +820,7 @@ async def literature_analyse(doc: TextResource, question: str, ssl_verify=True)
                 deps=doc,
                 usage_limits=UsageLimits(request_limit=50),
             ),
-            timeout=30
+            timeout=60
         )
     except asyncio.TimeoutError:
         msg="Timeout on literature_analyse attempt, retrying..."

ckanext/chat/bot/utils.py

@@ -243,20 +243,24 @@ def unpack_lazy_json(obj):
     return obj
 
 
-def process_entity(data: Any, depth: int = 0, max_depth: int = 2) -> Any:
+def process_entity(data: Any, depth: int = 0, max_depth: int = 4) -> Any:
+    log.debug(f"{type(data)},{depth},{max_depth}")
     if depth > max_depth:
         log.warning("Max recursion depth reached")
-        return data
+        #data=truncate_by_depth(data,max_depth)
+        return None
 
     if isinstance(data, dict):
         data = unpack_lazy_json(data)
+        #log.debug(data.keys())
         if "resources" in data:
             try:
+                #log.debug("Dataset")
                 dataset_dict = DynamicDataset(**data).model_dump(
                     exclude_unset=True, exclude_defaults=False, exclude_none=True
                 )
                 dataset_dict = {k: v for k, v in dataset_dict.items() if bool(v)}
-                return process_entity(dataset_dict, depth + 1, max_depth)
+                return truncate_by_depth(dataset_dict,max_depth-depth)
             except ValidationError as validation_error:
                 log.warning(
                     f"Validation error converting to DynamicDataset: {validation_error.json()}"
@@ -265,24 +269,26 @@ def process_entity(data: Any, depth: int = 0, max_depth: int = 2) -> Any:
                 log.warning(f"Conversion to DynamicDataset failed: {ex}")
         elif "package_id" in data or "url" in data:
             try:
+                #log.debug("Resource")
                 resource_dict = DynamicResource(**data).model_dump(
                     exclude_unset=True, exclude_defaults=False, exclude_none=True
                 )
                 resource_dict = {k: v for k, v in resource_dict.items() if bool(v)}
-                return process_entity(resource_dict, depth + 1, max_depth)
+                return  truncate_by_depth(resource_dict,max_depth-depth)
             except ValidationError as validation_error:
                 log.warning(
                     f"Validation error converting to DynamicResource: {validation_error.json()}"
                 )
             except Exception as ex:
                 log.warning(f"Conversion to DynamicResource failed: {ex}")
-
-        new_dict = {}
-        for key, value in data.items():
-            processed_value = process_entity(value, depth + 1, max_depth)
-            if processed_value not in ([], {}, "", None):
-                new_dict[key] = processed_value
-        return new_dict
+        else:
+            #log.debug("Dictionary")
+            new_dict = {}
+            for key, value in data.items():
+                processed_value = process_entity(value, depth + 1, max_depth)
+                if processed_value not in ([], {}, "", None):
+                    new_dict[key] = processed_value
+            return new_dict
 
     elif isinstance(data, list):
         new_list = []

ckanext/chat/plugin.py

@@ -34,6 +34,7 @@ def declare_config_options(self, declaration: Declaration, key: Key):
         declaration.declare(group.api_token, "your-api-token")
         declaration.declare(group.embedding_model, "text-embedding-3-small")
         declaration.declare(group.embedding_api, "")
+        declaration.declare(group.milvus_url, "")
         declaration.declare(group.collection_name, "")
 
     # IBlueprint