Merge pull request #10 from apilytics/request-response-sizes

Send request and response body size information with metrics

Author: ruohola

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Send request and response body size information with metrics.
+
+### Changed
+
+- Change `status_code` into an optional parameter in `ApilyticsSender.set_response_info`.
+
 ## [1.1.0] - 2022-01-16
 
 ### Added

README.md

@@ -37,7 +37,8 @@ import os
 APILYTICS_API_KEY = os.getenv("APILYTICS_API_KEY")
 
 MIDDLEWARE = [
-    "apilytics.django.ApilyticsMiddleware",
+    "apilytics.django.ApilyticsMiddleware",  # Ideally the first middleware in the list.
+    # ...
 ]
 ```
 
@@ -53,6 +54,7 @@ from fastapi import FastAPI
 
 app = FastAPI()
 
+# Ideally the first middleware you add.
 app.add_middleware(ApilyticsMiddleware, api_key=os.getenv("APILYTICS_API_KEY"))
 ```
 
@@ -78,9 +80,13 @@ def my_apilytics_middleware(request, get_response):
         path=request.path,
         query=request.query_string,
         method=request.method,
+        request_size=len(request.body),
     ) as sender:
         response = get_response(request)
-        sender.set_response_info(status_code=response.status_code)
+        sender.set_response_info(
+            status_code=response.status_code,
+            response_size=len(response.body),
+        )
     return response
 ```
 

apilytics/core.py

@@ -29,9 +29,13 @@ class ApilyticsSender:
                 path=request.path,
                 query=request.query_string,
                 method=request.method,
+                request_size=len(request.body),
             ) as sender:
                 response = get_response(request)
-                sender.set_response_info(status_code=response.status_code)
+                sender.set_response_info(
+                    status_code=response.status_code,
+                    response_size=len(response.body),
+                )
     """
 
     _executor: ClassVar[concurrent.futures.Executor]
@@ -47,6 +51,7 @@ def __init__(
         path: str,
         method: str,
         query: Optional[str] = None,
+        request_size: Optional[int] = None,
         apilytics_integration: Optional[str] = None,
         integrated_library: Optional[str] = None,
     ) -> None:
@@ -59,6 +64,7 @@ def __init__(
             method: Method of the user's HTTP request, e.g. "GET".
             query: Optional query string of the user's HTTP request e.g. "key=val&other=123".
                 An empty string and None are treated equally. Can have an optional "?" at the start.
+            request_size: Size of the user's HTTP request's body in bytes.
             apilytics_integration: Name of the Apilytics integration that's calling this,
                 e.g. "apilytics-python-django". No need to pass this when calling from user code.
             integrated_library: Name and version of the integration that this is used in,
@@ -68,6 +74,9 @@ def __init__(
         self._path = path
         self._method = method
         self._query = query
+        self._request_size = request_size
+
+        self._response_size: Optional[int] = None
         self._status_code: Optional[int] = None
 
         self._apilytics_version = self._apilytics_version_template.format(
@@ -97,16 +106,21 @@ def __exit__(
             )
         self._executor.submit(self._send_metrics)
 
-    def set_response_info(self, *, status_code: int) -> None:
+    def set_response_info(
+        self, *, status_code: Optional[int] = None, response_size: Optional[int] = None
+    ) -> None:
         """
         Update the context manager with info from the HTTP response object.
 
         Should be called before the context manager's block ends.
 
         Args:
-            status_code: Status code for the HTTP response.
+            status_code: Status code for the HTTP response. Can be omitted (or None)
+                if the middleware could not get the status code.
+            response_size: Size of the body of the sent HTTP response in bytes.
         """
         self._status_code = status_code
+        self._response_size = response_size
 
     def _send_metrics(self) -> None:
         request = urllib.request.Request(
@@ -120,10 +134,24 @@ def _send_metrics(self) -> None:
         )
         data = {
             "path": self._path,
-            **({"query": self._query} if self._query else {}),
             "method": self._method,
-            "statusCode": self._status_code,
             "timeMillis": (self._end_time_ns - self._start_time_ns) // 1_000_000,
+            **({"query": self._query} if self._query else {}),  # Don't send empty str.
+            **(
+                {"statusCode": self._status_code}
+                if self._status_code is not None
+                else {}
+            ),
+            **(
+                {"requestSize": self._request_size}
+                if self._request_size is not None
+                else {}
+            ),
+            **(
+                {"responseSize": self._response_size}
+                if self._response_size is not None
+                else {}
+            ),
         }
         try:
             urllib.request.urlopen(url=request, data=json.dumps(data).encode())

apilytics/django.py

@@ -14,6 +14,8 @@ class ApilyticsMiddleware:
     """
     Django middleware that sends API analytics data to Apilytics (https://apilytics.io).
 
+    This should ideally be the first middleware in your ``settings.MIDDLEWARE`` list.
+
     Requires your Apilytics origin's API key to be set as ``APILYTICS_API_KEY`` in
     ``settings.py`` for this to do anything. The variable can be unset (or ``None``)
     e.g. in a test environment where data should not be sent.
@@ -25,6 +27,7 @@ class ApilyticsMiddleware:
 
             MIDDLEWARE = [
                 "apilytics.django.ApilyticsMiddleware",
+                # ...
             ]
     """
 
@@ -44,9 +47,13 @@ def __call__(self, request: django.http.HttpRequest) -> django.http.HttpResponse
             path=request.path,
             query=request.META.get("QUERY_STRING"),
             method=request.method or "",  # Typed as Optional, should never be None.
+            request_size=int(request.META.get("CONTENT_LENGTH", 0)),
             apilytics_integration="apilytics-python-django",
             integrated_library=f"django/{django.__version__}",
         ) as sender:
             response = self.get_response(request)
-            sender.set_response_info(status_code=response.status_code)
+            sender.set_response_info(
+                status_code=response.status_code,
+                response_size=len(getattr(response, "content", ())),
+            )
         return response

apilytics/fastapi.py

@@ -12,6 +12,8 @@ class ApilyticsMiddleware(starlette.middleware.base.BaseHTTPMiddleware):
     """
     FastAPI middleware that sends API analytics data to Apilytics (https://apilytics.io).
 
+    This should ideally be the first middleware you add to your app.
+
     Examples:
         main.py::
 
@@ -53,9 +55,13 @@ async def dispatch(
             path=request.url.path,
             query=request.url.query,
             method=request.method,
+            request_size=len(await request.body()),
             apilytics_integration="apilytics-python-fastapi",
             integrated_library=f"fastapi/{fastapi.__version__}",
         ) as sender:
             response = await call_next(request)
-            sender.set_response_info(status_code=response.status_code)
+            sender.set_response_info(
+                status_code=response.status_code,
+                response_size=int(response.headers.get("content-length", 0)),
+            )
         return response

tests/django/app/urls.py

@@ -4,5 +4,7 @@
 
 urlpatterns = [
     django.urls.re_path(r"^error/?$", tests.django.app.views.error_view),
+    django.urls.re_path(r"^empty/?$", tests.django.app.views.no_body_view),
+    django.urls.re_path(r"^streaming/?$", tests.django.app.views.streaming_view),
     django.urls.re_path(r"^.*$", tests.django.app.views.ok_view),
 ]

tests/django/app/views.py

@@ -3,11 +3,23 @@
 import django.http
 
 
-def ok_view(request: django.http.HttpRequest) -> django.http.HttpResponse:
-    if request.method == "POST":
-        return django.http.HttpResponse(status=201)
+def error_view(request: django.http.HttpRequest) -> NoReturn:
+    raise RuntimeError
+
+
+def no_body_view(request: django.http.HttpRequest) -> django.http.HttpResponse:
     return django.http.HttpResponse(status=200)
 
 
-def error_view(request: django.http.HttpRequest) -> NoReturn:
-    raise RuntimeError
+def streaming_view(
+    request: django.http.HttpRequest,
+) -> django.http.StreamingHttpResponse:
+    return django.http.StreamingHttpResponse(
+        status=200, streaming_content=(b"first", b"second")
+    )
+
+
+def ok_view(request: django.http.HttpRequest) -> django.http.HttpResponse:
+    if request.method == "POST":
+        return django.http.HttpResponse(status=201, content=b"created")
+    return django.http.HttpResponse(status=200, content=b"ok")

tests/django/test_django.py

@@ -36,10 +36,19 @@ def test_middleware_should_call_apilytics_api(
     }
 
     data = tests.conftest.decode_request_data(call_kwargs["data"])
-    assert data.keys() == {"path", "method", "statusCode", "timeMillis"}
+    assert data.keys() == {
+        "path",
+        "method",
+        "statusCode",
+        "requestSize",
+        "responseSize",
+        "timeMillis",
+    }
     assert data["path"] == "/"
     assert data["method"] == "GET"
     assert data["statusCode"] == 200
+    assert data["requestSize"] == 0
+    assert data["responseSize"] > 0
     assert isinstance(data["timeMillis"], int)
 
 
@@ -57,6 +66,64 @@ def test_middleware_should_send_query_params(
     assert data["path"] == "/dummy/123/path/"
     assert data["query"] == "param=foo&param2=bar"
     assert data["statusCode"] == 201
+    assert data["requestSize"] == 20  # Empty form data POST adds a 20 boundary string.
+    assert data["responseSize"] > 0
+    assert isinstance(data["timeMillis"], int)
+
+
+def test_middleware_should_send_zero_request_and_response_sizes(
+    mocked_urlopen: unittest.mock.MagicMock,
+) -> None:
+    client.handler.load_middleware()
+    response = client.post("/empty?some=query", content_type="application/json")
+    assert response.status_code == 200
+
+    assert mocked_urlopen.call_count == 1
+    __, call_kwargs = mocked_urlopen.call_args
+    data = tests.conftest.decode_request_data(call_kwargs["data"])
+    assert data["requestSize"] == 2  # Django makes it `b"{}"` for empty JSON POSTs.
+    assert data["responseSize"] == 0
+
+
+def test_middleware_should_send_non_zero_request_and_response_sizes(
+    mocked_urlopen: unittest.mock.MagicMock,
+) -> None:
+    client.handler.load_middleware()
+    response = client.post(
+        "/dummy?some=query", data={"hello": "world"}, content_type="application/json"
+    )
+    assert response.status_code == 201
+
+    assert mocked_urlopen.call_count == 1
+    __, call_kwargs = mocked_urlopen.call_args
+    data = tests.conftest.decode_request_data(call_kwargs["data"])
+    assert data["requestSize"] == 18
+    assert data["responseSize"] == 7  # `len(b"created")`
+
+
+def test_middleware_should_work_with_streaming_response(
+    mocked_urlopen: unittest.mock.MagicMock,
+) -> None:
+    client.handler.load_middleware()
+    response = client.get("/streaming")
+    assert response.status_code == 200
+
+    assert mocked_urlopen.call_count == 1
+    __, call_kwargs = mocked_urlopen.call_args
+    data = tests.conftest.decode_request_data(call_kwargs["data"])
+    assert data.keys() == {
+        "path",
+        "method",
+        "statusCode",
+        "requestSize",
+        "responseSize",
+        "timeMillis",
+    }
+    assert data["path"] == "/streaming"
+    assert data["method"] == "GET"
+    assert data["statusCode"] == 200
+    assert data["requestSize"] == 0
+    assert data["responseSize"] == 0  # Can't get body size from a streaming response.
     assert isinstance(data["timeMillis"], int)
 
 
@@ -84,7 +151,17 @@ def test_middleware_should_send_data_even_on_errors(
 
     __, call_kwargs = mocked_urlopen.call_args
     data = tests.conftest.decode_request_data(call_kwargs["data"])
+    assert data.keys() == {
+        "method",
+        "path",
+        "timeMillis",
+        "statusCode",
+        "requestSize",
+        "responseSize",
+    }
     assert data["method"] == "GET"
     assert data["path"] == "/error"
     assert data["statusCode"] == 500
+    assert data["requestSize"] == 0
+    assert data["responseSize"] > 0
     assert isinstance(data["timeMillis"], int)

tests/fastapi/app.py

@@ -2,6 +2,7 @@
 from typing import NoReturn
 
 import fastapi
+import starlette.responses
 
 import apilytics.fastapi
 
@@ -15,8 +16,24 @@ async def error_route(request: fastapi.Request) -> NoReturn:
     raise RuntimeError
 
 
+@app.api_route("/empty", methods=["POST"])
+async def no_body_route(request: fastapi.Request) -> fastapi.Response:
+    return fastapi.Response(status_code=fastapi.status.HTTP_200_OK)
+
+
+@app.api_route("/streaming", methods=["GET"])
+async def streaming_route(
+    request: fastapi.Request,
+) -> starlette.responses.StreamingResponse:
+    return starlette.responses.StreamingResponse(
+        status_code=fastapi.status.HTTP_200_OK, content=(b"first", b"second")
+    )
+
+
 @app.api_route("/{path:path}", methods=["GET", "POST"])
 async def ok_route(request: fastapi.Request) -> fastapi.Response:
     if request.method == "POST":
-        return fastapi.Response(status_code=fastapi.status.HTTP_201_CREATED)
-    return fastapi.Response(status_code=fastapi.status.HTTP_200_OK)
+        return fastapi.Response(
+            status_code=fastapi.status.HTTP_201_CREATED, content=b"created"
+        )
+    return fastapi.Response(status_code=fastapi.status.HTTP_200_OK, content=b"ok")