Add Geolocation and User Agent type params to universal scraper, remove parse parameter for universal scraper, update tests

Author: oxy-rostyslav

CHANGELOG.md

@@ -1,11 +1,12 @@
 # Changelog
 
-## [0.2.0] - 2025-05-12
+## [0.2.0] - 2025-05-13
 
 ### Added
 
 - Changelog
 - E2E tests
+- Geolocation and User Agent type parameters to universal scraper
 
 ### Changed
 
@@ -16,3 +17,4 @@
 ### Removed
 
 - WebUnblocker tool
+- Parse parameter for universal scraper

src/oxylabs_mcp/server.py

@@ -16,8 +16,9 @@
 async def universal_scraper(
     ctx: Context,  # type: ignore[type-arg]
     url: url_params.URL_PARAM,
-    parse: url_params.PARSE_PARAM = False,  # noqa: FBT002
     render: url_params.RENDER_PARAM = "",
+    user_agent_type: url_params.USER_AGENT_TYPE_PARAM = "",
+    geo_location: url_params.GEO_LOCATION_PARAM = "",
     output_format: url_params.OUTPUT_FORMAT_PARAM = "",
 ) -> str:
     """Get a content of any webpage.
@@ -28,16 +29,19 @@ async def universal_scraper(
     try:
         async with oxylabs_client(ctx) as client:
             payload: dict[str, Any] = {"url": url}
-            if parse:
-                payload["parse"] = parse
+
             if render:
                 payload["render"] = render
+            if user_agent_type:
+                payload["user_agent_type"] = user_agent_type
+            if geo_location:
+                payload["geo_location"] = geo_location
 
             response = await client.post(settings.OXYLABS_SCRAPER_URL, json=payload)
 
             response.raise_for_status()
 
-            return get_content(response, parse=parse, output_format=output_format)
+            return get_content(response, output_format=output_format)
     except MCPServerError as e:
         return e.stringify()
 

src/oxylabs_mcp/utils.py

@@ -208,18 +208,18 @@ def extract_links_with_text(html: str, base_url: str | None = None) -> list[str]
     return links
 
 
-def get_content(response: Response, *, parse: bool, output_format: str) -> str:
+def get_content(response: Response, *, output_format: str, parse: bool = False) -> str:
     """Extract content from response and convert to a proper format."""
     content = response.json()["results"][0]["content"]
     if parse and isinstance(content, dict):
         return json.dumps(content)
     if output_format == "html":
         return str(content)
-    if output_format == "md":
-        striped_html = strip_html(str(content))
-        return markdownify(striped_html)  # type: ignore[no-any-return]
     if output_format == "links":
         links = extract_links_with_text(str(content))
         return "\n".join(links)
+    if output_format in ("md", ""):
+        striped_html = strip_html(str(content))
+        return markdownify(striped_html)  # type: ignore[no-any-return]
 
     return str(content)

tests/e2e/test_llm_agent.py

@@ -86,6 +86,15 @@ async def oxylabs_mcp_server():
             },
             "iPhone 16",
         ),
+        (
+            "Search for iPhone 16 in google with browser rendering",
+            "google_search_scraper",
+            {
+                "query": "iPhone 16",
+                "render": "html",
+            },
+            "iPhone 16",
+        ),
         (
             "Search for iPhone 16 in google with user agent type mobile",
             "google_search_scraper",

tests/integration/params.py

@@ -8,21 +8,28 @@
     {"query": "Generic query"},
     does_not_raise(),
     {"results": [{"content": "Mocked content"}]},
-    "Mocked content",
+    "\n\nMocked content\n\n",
     id="query-only-args",
 )
 PARSE_ENABLED = pytest.param(
     {"query": "Generic query", "parse": True},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="parse-enabled-args",
 )
-RENDER_HTML = pytest.param(
+RENDER_HTML_WITH_QUERY = pytest.param(
     {"query": "Generic query", "render": "html"},
     does_not_raise(),
     {"results": [{"content": "Mocked content"}]},
-    "Mocked content",
+    "\n\nMocked content\n\n",
+    id="render-enabled-args",
+)
+RENDER_INVALID_WITH_QUERY = pytest.param(
+    {"query": "Generic query", "render": "png"},
+    pytest.raises(ToolError),
+    {},
+    None,
     id="render-enabled-args",
 )
 OUTPUT_FORMATS = [
@@ -54,12 +61,33 @@
         id="html-output-format-args",
     ),
 ]
-USER_AGENTS = [
+USER_AGENTS_WITH_QUERY = [
     pytest.param(
-        {"query": "Generic query", "user_agent_type": "mobile"},
+        {"query": "Generic query", "user_agent_type": uat},
         does_not_raise(),
         {"results": [{"content": "Mocked content"}]},
-        "Mocked content",
+        "\n\nMocked content\n\n",
+        id=f"{uat}-user-agent-specified-args",
+    )
+    for uat in [
+        "desktop",
+        "desktop_chrome",
+        "desktop_firefox",
+        "desktop_safari",
+        "desktop_edge",
+        "desktop_opera",
+        "mobile",
+        "mobile_ios",
+        "mobile_android",
+        "tablet",
+    ]
+]
+USER_AGENTS_WITH_URL = [
+    pytest.param(
+        {"url": "https://example.com", "user_agent_type": uat},
+        does_not_raise(),
+        {"results": [{"content": "Mocked content"}]},
+        "\n\nMocked content\n\n",
         id=f"{uat}-user-agent-specified-args",
     )
     for uat in [
@@ -85,91 +113,126 @@
 START_PAGE_SPECIFIED = pytest.param(
     {"query": "Generic query", "start_page": 2},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="start-page-specified-args",
 )
 START_PAGE_INVALID = pytest.param(
     {"query": "Generic query", "start_page": -1},
     pytest.raises(ToolError),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="start-page-invalid-args",
 )
 PAGES_SPECIFIED = pytest.param(
     {"query": "Generic query", "pages": 20},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="pages-specified-args",
 )
 PAGES_INVALID = pytest.param(
     {"query": "Generic query", "pages": -10},
     pytest.raises(ToolError),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="pages-invalid-args",
 )
 LIMIT_SPECIFIED = pytest.param(
     {"query": "Generic query", "limit": 100},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="limit-specified-args",
 )
 LIMIT_INVALID = pytest.param(
     {"query": "Generic query", "limit": 0},
     pytest.raises(ToolError),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="limit-invalid-args",
 )
 DOMAIN_SPECIFIED = pytest.param(
     {"query": "Generic query", "domain": "io"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="domain-specified-args",
 )
-GEO_LOCATION_SPECIFIED = pytest.param(
+GEO_LOCATION_SPECIFIED_WITH_QUERY = pytest.param(
     {"query": "Generic query", "geo_location": "Miami, Florida"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="geo-location-specified-args",
 )
+GEO_LOCATION_SPECIFIED_WITH_URL = pytest.param(
+    {"url": "https://example.com", "geo_location": "Miami, Florida"},
+    does_not_raise(),
+    {"results": [{"content": "Mocked content"}]},
+    "\n\nMocked content\n\n",
+    id="geo-location-specified-args",
+)
 LOCALE_SPECIFIED = pytest.param(
     {"query": "Generic query", "locale": "ja_JP"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="locale-specified-args",
 )
 CATEGORY_SPECIFIED = pytest.param(
     {"query": "Man's T-shirt", "category_id": "QE21R9AV"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="category-id-specified-args",
 )
 MERCHANT_ID_SPECIFIED = pytest.param(
     {"query": "Man's T-shirt", "merchant_id": "QE21R9AV"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="merchant-id-specified-args",
 )
 CURRENCY_SPECIFIED = pytest.param(
     {"query": "Man's T-shirt", "currency": "USD"},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="currency-specified-args",
 )
 AUTOSELECT_VARIANT_ENABLED = pytest.param(
     {"query": "B0BVF87BST", "autoselect_variant": True},
     does_not_raise(),
-    {"results": [{"content": '{"data": "value"}'}]},
+    {"results": [{"content": {"data": "value"}}]},
     '{"data": "value"}',
     id="autoselect-variant-enabled-args",
 )
+URL_ONLY = pytest.param(
+    {"url": "https://example.com"},
+    does_not_raise(),
+    {"results": [{"content": "Mocked content"}]},
+    "\n\nMocked content\n\n",
+    id="url-only-args",
+)
+NO_URL = pytest.param(
+    {},
+    pytest.raises(ToolError),
+    {"results": [{"content": "Mocked content"}]},
+    "\n\nMocked content\n\n",
+    id="no-url-args",
+)
+RENDER_HTML_WITH_URL = pytest.param(
+    {"url": "https://example.com", "render": "html"},
+    does_not_raise(),
+    {"results": [{"content": "Mocked content"}]},
+    "\n\nMocked content\n\n",
+    id="render-enabled-args",
+)
+RENDER_INVALID_WITH_URL = pytest.param(
+    {"url": "https://example.com", "render": "png"},
+    pytest.raises(ToolError),
+    {},
+    None,
+    id="render-enabled-args",
+)