Merge pull request #104 from dlt-hub/feat/better-detection-fallbacks

set paginator to auto if not detected, do not set json_path if not detected

Author: sh-rp

dlt_init_openapi/parser/endpoints.py

@@ -97,9 +97,18 @@ def render_pagination_args(self) -> Optional[Dict[str, Union[str, int]]]:
             return None
         return self.detected_pagination.paginator_config if self.detected_pagination else None
 
+    @property
+    def render_auto_paginator(self) -> bool:
+        """if we could not figure out the paginator, set it to auto"""
+        # we do not set auto if there is a global paginator, this is a bug in the underlying layer, we have to fix this
+        # once rest_api is fixed
+        if self.detected_global_pagination:
+            return False
+        return not (self.detected_pagination)
+
     @property
     def data_json_path(self) -> str:
-        return self.payload.json_path if self.payload else "$"
+        return self.payload.json_path if self.payload else None
 
     @property
     def transformer(self) -> Optional[TransformerSetting]:

dlt_init_openapi/renderer/default/templates/source.py.j2

@@ -51,7 +51,9 @@ def {{ source_name }}(
             {% endif %}
             {% endif %}
             "endpoint": {
+                {% if endpoint.data_json_path %}
                 "data_selector": "{{ endpoint.data_json_path }}",
+                {% endif %}
                 "path": "{{endpoint.path }}",
                 {% if endpoint.transformer or endpoint.unresolvable_path_param_names or endpoint.unresolvable_query_param_names %}
                 "params": {
@@ -84,6 +86,9 @@ def {{ source_name }}(
                     {% endfor %}
                 },
                 {% endif %}
+                {% if endpoint.render_auto_paginator %}
+                "paginator": "auto",
+                {% endif %}
             }
         },
         {% endfor %}

tests/integration/basics/test_transformer.py

@@ -17,7 +17,11 @@ def test_simple_transformer(resources: Dict[str, Any]) -> None:
         "table_name": "collection",
         "primary_key": "id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/collection/"},
+        "endpoint": {
+            "data_selector": "$",
+            "path": "/collection/",
+            "paginator": "auto",
+        },
     }
     assert resources["single_collection"] == {
         "name": "single_collection",
@@ -26,6 +30,7 @@ def test_simple_transformer(resources: Dict[str, Any]) -> None:
         "write_disposition": "merge",
         "endpoint": {
             "data_selector": "$",
+            "paginator": "auto",
             "path": "/collection/{collection_id}",
             "params": {"collection_id": {"type": "resolve", "resource": "collections", "field": "id"}},
         },
@@ -47,7 +52,7 @@ def test_simple_transformer_with_deselected_parent() -> None:
         "primary_key": "id",
         "write_disposition": "merge",
         "selected": False,
-        "endpoint": {"data_selector": "$", "path": "/collection/"},
+        "endpoint": {"data_selector": "$", "path": "/collection/", "paginator": "auto"},
     }
     assert resources["single_collection"] == {
         "name": "single_collection",
@@ -58,6 +63,7 @@ def test_simple_transformer_with_deselected_parent() -> None:
             "data_selector": "$",
             "path": "/collection/{collection_id}",
             "params": {"collection_id": {"type": "resolve", "resource": "collections", "field": "id"}},
+            "paginator": "auto",
         },
     }
 
@@ -68,7 +74,7 @@ def test_match_by_path_var_only(resources: Dict[str, Any]) -> None:
         "table_name": "user",
         "primary_key": "user_id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/users/"},
+        "endpoint": {"data_selector": "$", "path": "/users/", "paginator": "auto"},
     }
     assert resources["single_user"] == {
         "name": "single_user",
@@ -77,6 +83,7 @@ def test_match_by_path_var_only(resources: Dict[str, Any]) -> None:
         "write_disposition": "merge",
         "endpoint": {
             "data_selector": "$",
+            "paginator": "auto",
             "path": "/users/{user_id}",
             "params": {"user_id": {"type": "resolve", "resource": "users", "field": "user_id"}},
         },
@@ -89,16 +96,23 @@ def test_match_singularized_path(resources: Dict[str, Any]) -> None:
         "table_name": "invoice",
         "primary_key": "invoice_id",
         "write_disposition": "merge",
-        "endpoint": {"data_selector": "$", "path": "/invoices/"},
+        "endpoint": {"data_selector": "$", "path": "/invoices/", "paginator": "auto"},
     }
     assert resources["single_invoice"] == {
         "name": "single_invoice",
         "table_name": "invoice",
         "primary_key": "invoice_id",
         "write_disposition": "merge",
         "endpoint": {
+            "paginator": "auto",
             "data_selector": "$",
             "path": "/invoice/{invoice_id}",
-            "params": {"invoice_id": {"type": "resolve", "resource": "invoices", "field": "invoice_id"}},
+            "params": {
+                "invoice_id": {
+                    "type": "resolve",
+                    "resource": "invoices",
+                    "field": "invoice_id",
+                }
+            },
         },
     }

tests/integration/extracted/test_dotastats_cases.py

@@ -21,6 +21,5 @@ def test_simple_dotastats_load() -> None:
         "table_name": "team",
         "endpoint": {
             "path": "/teams",
-            "data_selector": "$",
         },
     }

tests/integration/extracted/test_pokemon_cases.py

@@ -11,7 +11,7 @@ def test_simple_poke_load() -> None:
     assert source["resources"][0] == {
         "name": "pokemon",
         "table_name": "pokemon",
-        "endpoint": {"path": "/api/v2/pokemon/", "data_selector": "$"},
+        "endpoint": {"path": "/api/v2/pokemon/", "paginator": "auto"},
     }
 
 
@@ -72,6 +72,7 @@ def test_simple_child_table_poke_load() -> None:
         "primary_key": "id",
         "write_disposition": "merge",
         "endpoint": {
+            # "paginator": "auto",
             "path": "/api/v2/pokemon/{name}/",
             "data_selector": "$",
             "params": {

tests/integration/extracted/test_quotesapi_cases.py

@@ -23,7 +23,6 @@ def test_simple_quotesapi_load() -> None:
         "name": "list",
         "table_name": "list",
         "endpoint": {
-            "data_selector": "$",
             "path": "/quote/list",
         },
     }