update to connexion 3.x (#3426)

* Replace uwsgi with uvicorn

* Update pinned dependencies

* admin: port connexion App creation to connexion 3

- use SwaggerUIOptions
- remove BalrogRequestBodyValidator, let connexion do the validation

* public: update app instanciation for connexion 3

* web: work around TypeError from connexion's ServerErrorMiddleware

See spec-first/connexion#2059

* tests: remove check for error text in testDeleteRequiredSignoff

connexion now rejects those requests due to extra params, so the error is
different.

* tests: remove 404 test for paths that now work

* test: don't send control characters

the httpx client rejects those urls

* tests: port to new APIs

The test client for the connexion app is a starlette TestClient instead of a flask client, with a different API for responses:
- `text` attribute instead of `get_data()` method or `data` attribute
- `json()` method instead of `get_json()` method or `json` attribute
- no `mimetype` attribute, so we look up the `content-type` header instead
and requests:
- `params` argument replaces `query_string`

* tests: propagate exceptions through connexion middlewares

* admin: move CORS handling to starlette CORSMiddleware

CORS checks need to happen before routing, otherwise preflight requests
get a 405.

* admin: port statsd timers to a middleware

* public: port statsd request handler to a middleware

Author: jcristau

Dockerfile

@@ -6,11 +6,10 @@ ENV LC_ALL C.UTF-8
 
 LABEL maintainer="[email protected]"
 
-# uwsgi needs libpcre3 for routing support to be enabled.
 # default-libmysqlclient-dev is required to use SQLAlchemy with MySQL, which we do in production.
 # xz-utils is needed to compress production database dumps
 RUN apt-get -q update \
-    && apt-get -q --yes install libpcre3-dev default-libmysqlclient-dev mariadb-client xz-utils pkg-config \
+    && apt-get -q --yes install default-libmysqlclient-dev mariadb-client xz-utils pkg-config \
     && apt-get clean
 
 WORKDIR /app
@@ -29,7 +28,7 @@ RUN apt-get install -q --yes gcc && \
 # Copying Balrog to /app instead of installing it means that production can run
 # it, and we can bind mount to override it for local development.
 COPY src/ /app/src/
-COPY uwsgi/ /app/uwsgi/
+COPY uvicorn/ /app/uvicorn/
 COPY scripts/manage-db.py scripts/run-batch-deletes.sh scripts/run.sh scripts/reset-stage-db.sh scripts/get-prod-db-dump.py /app/scripts/
 COPY MANIFEST.in pyproject.toml setup.py version.json version.txt /app/
 

Dockerfile.test

@@ -6,14 +6,13 @@ ENV LC_ALL C.UTF-8
 LABEL maintainer="[email protected]"
 
 # netcat is needed for health checks
-# Some versions of the python:3.8 Docker image remove libpcre3, which uwsgi needs for routing support to be enabled.
 # default-libmysqlclient-dev is required to use SQLAlchemy with MySQL
 # mariadb-client is needed to import sample data
 # curl is needed to pull sample data
 # gcc is needed to compile some python packages
 # xz-utils is needed to unpack sampled ata
 RUN apt-get -q update \
-    && apt-get -q --yes install g++ netcat-traditional libpcre3 libpcre3-dev default-libmysqlclient-dev mariadb-client curl gcc xz-utils pkg-config \
+    && apt-get -q --yes install g++ netcat-traditional default-libmysqlclient-dev mariadb-client curl gcc xz-utils pkg-config \
     && apt-get clean
 
 WORKDIR /app
@@ -26,7 +25,7 @@ RUN pip install --no-deps -r requirements/local.txt
 COPY src/ /app/src/
 COPY tests/ /app/tests/
 COPY scripts/ /app/scripts/
-COPY uwsgi/ /app/uwsgi/
+COPY uvicorn/ /app/uvicorn/
 COPY MANIFEST.in setup.py pyproject.toml tox.ini version.json version.txt /app/
 
 RUN pip install -e .

MANIFEST.in

@@ -6,7 +6,7 @@ include version.txt
 
 recursive-include src *
 recursive-include requirements *.txt
-recursive-include uwsgi *
+recursive-include uvicorn *
 
 exclude .dirschema.yml
 exclude .editorconfig

requirements/base.in

@@ -1,7 +1,7 @@
 aiohttp
 arrow
 auth0-python>=4
-connexion<3
+connexion[flask,swagger-ui,uvicorn]>3
 deepmerge
 ecdsa
 flask
@@ -10,6 +10,7 @@ gcloud-aio-storage
 google-api-core
 google-auth
 google-cloud-storage
+gunicorn
 jsonschema>=4.5.0
 mozilla-version
 mysqlclient
@@ -25,4 +26,3 @@ sqlalchemy-migrate
 sqlalchemy<2
 statsd
 swagger-spec-validator
-uwsgi

requirements/base.txt

@@ -1,7 +1,7 @@
 aiohttp
 arrow
 auth0-python
-connexion<3  # Just to be consistent with base.in
+connexion[flask,swagger-ui,uvicorn]>3
 deepmerge
 flask
 flask_wtf

requirements/docs.in

@@ -1,9 +1,9 @@
 #!/bin/bash
 
 if [ $1 == "public" ]; then
-   exec uwsgi --ini /app/uwsgi/public.ini --python-autoreload 1
+    exec gunicorn -k uvicorn.workers.UvicornWorker -b "${HOST}:${PORT}" --chdir /app/uvicorn --reload public:connexion_app
 elif [ $1 == "admin" ]; then
-   exec uwsgi --ini /app/uwsgi/admin.ini --python-autoreload 1
+    exec gunicorn -k uvicorn.workers.UvicornWorker -b "${HOST}:${PORT}" --chdir /app/uvicorn --reload admin:connexion_app
 elif [ $1 == "create-db" ]; then
     if [ -z "${DBURI}" ]; then
         echo "\${DBURI} must be set!"

requirements/docs.txt

@@ -3,18 +3,22 @@
 from os import path
 
 import connexion
-from flask import g, request
+from connexion.middleware import MiddlewarePosition
+from connexion.options import SwaggerUIOptions
+from flask import request
 from sentry_sdk import capture_exception
 from specsynthase.specbuilder import SpecBuilder
+from starlette.middleware.cors import CORSMiddleware
 from statsd.defaults.env import statsd
 
 import auslib
+import auslib.web.admin.views.validators  # noqa
 from auslib.db import ChangeScheduledError, OutdatedDataError, UpdateMergeError
 from auslib.dockerflow import create_dockerflow_endpoints
 from auslib.errors import BlobValidationError, PermissionDeniedError, ReadOnlyError, SignoffRequiredError
 from auslib.util.auth import AuthError, verified_userinfo
 from auslib.web.admin.views.problem import problem
-from auslib.web.admin.views.validators import BalrogRequestBodyValidator
+from auslib.web.common import middlewares
 
 log = logging.getLogger(__name__)
 
@@ -29,50 +33,54 @@
     .add_spec(path.join(web_dir, "common/swagger/responses.yml"))
 )
 
-validator_map = {"body": BalrogRequestBodyValidator}
-
-
-def should_time_request():
-    # don't time OPTIONS requests
-    if request.method == "OPTIONS":
-        return False
-    # don't time requests that don't match a valid route
-    if request.url_rule is None:
-        return False
-    # don't time dockerflow endpoints
-    if request.path.startswith("/__"):
-        return False
-
-    return True
-
-
-def create_app():
-    connexion_app = connexion.App(__name__, debug=False, options={"swagger_ui": False})
-    connexion_app.add_api(spec, validator_map=validator_map, strict_validation=True)
+swagger_ui_options = SwaggerUIOptions(swagger_ui=False)
+
+
+class StatsdMiddleware:
+    def __init__(self, app):
+        self.app = app
+
+    def metric_name(self, scope):
+        if scope["method"] == "OPTIONS":
+            return
+        op = scope.get("extensions", {}).get("connexion_routing", {}).get("operation_id")
+        if op is None:
+            return
+        # do some massaging to get the metric name right
+        # * remove various module prefixes
+        # * add a common prefix to ensure that we can mark these metrics as gauges for
+        #   statsd
+        metric = op.replace(".", "_").removeprefix("auslib_web_admin_views_").removeprefix("auslib_web_admin_").removeprefix("auslib_web_common_")
+        return f"endpoint_{metric}"
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] != "http":
+            await self.app(scope, receive, send)
+            return
+
+        metric = self.metric_name(scope)
+        if not metric:
+            await self.app(scope, receive, send)
+            return
+
+        timer = statsd.timer(metric)
+        timer.start()
+        try:
+            await self.app(scope, receive, send)
+        finally:
+            timer.stop()
+
+
+def create_app(allow_origins=None):
+    connexion_app = connexion.App(__name__, swagger_ui_options=swagger_ui_options, middlewares=middlewares[:])
+    connexion_app.app.debug = False
+    connexion_app.add_api(spec, strict_validation=True)
     connexion_app.add_api(path.join(current_dir, "swagger", "api_v2.yml"), base_path="/v2", strict_validation=True, validate_responses=True)
+    connexion_app.add_middleware(StatsdMiddleware, MiddlewarePosition.BEFORE_VALIDATION)
     flask_app = connexion_app.app
 
     create_dockerflow_endpoints(flask_app)
 
-    @flask_app.before_request
-    def setup_timer():
-        g.request_timer = None
-        if should_time_request():
-            # do some massaging to get the metric name right
-            # * get rid of the `/v2` prefix on v2 endpoints added by `base_path` further up
-            # * remove various module prefixes
-            # * add a common prefix to ensure that we can mark these metrics as gauges for
-            #   statsd
-            metric = (
-                request.url_rule.endpoint.removeprefix("/v2.")
-                .removeprefix("auslib_web_admin_views_")
-                .removeprefix("auslib_web_admin_")
-                .removeprefix("auslib_web_common_")
-            )
-            metric = f"endpoint_{metric}"
-            g.request_timer = statsd.timer(metric)
-            g.request_timer.start()
-
     @flask_app.before_request
     def setup_request():
         if request.full_path.startswith("/v2"):
@@ -182,12 +190,6 @@ def add_security_headers(response):
         response.headers["X-Frame-Options"] = "DENY"
         response.headers["X-Content-Type-Options"] = "nosniff"
         response.headers["Strict-Transport-Security"] = flask_app.config.get("STRICT_TRANSPORT_SECURITY", "max-age=31536000;")
-        response.headers["Access-Control-Allow-Headers"] = "Authorization, Content-Type"
-        response.headers["Access-Control-Allow-Methods"] = "OPTIONS, GET, POST, PUT, DELETE"
-        if "*" in flask_app.config["CORS_ORIGINS"]:
-            response.headers["Access-Control-Allow-Origin"] = "*"
-        elif "Origin" in request.headers and request.headers["Origin"] in flask_app.config["CORS_ORIGINS"]:
-            response.headers["Access-Control-Allow-Origin"] = request.headers["Origin"]
         if re.match("^/ui/", request.path):
             # This enables swagger-ui to dynamically fetch and
             # load the swagger specification JSON file containing API definition and examples.
@@ -196,14 +198,13 @@ def add_security_headers(response):
             response.headers["Content-Security-Policy"] = flask_app.config.get("CONTENT_SECURITY_POLICY", "default-src 'none'; frame-ancestors 'none'")
         return response
 
-    # this is specifically set-up last before after_request handlers are called
-    # in reverse order of registering, and we want this one to be called first
-    # to avoid it being skipped if another one raises an exception
-    @flask_app.after_request
-    def send_stats(response):
-        if hasattr(g, "request_timer") and g.request_timer:
-            g.request_timer.stop()
-
-        return response
+    if allow_origins:
+        connexion_app.add_middleware(
+            CORSMiddleware,
+            MiddlewarePosition.BEFORE_ROUTING,
+            allow_origins=allow_origins,
+            allow_headers=["Authorization", "Content-Type"],
+            allow_methods=["OPTIONS", "GET", "POST", "PUT", "DELETE"],
+        )
 
     return connexion_app

scripts/run.sh

@@ -1,43 +1,7 @@
-import logging
-from typing import AnyStr, Union
-
-import jsonschema
-from connexion.decorators.validation import RequestBodyValidator
-from connexion.exceptions import BadRequestProblem
-from connexion.lifecycle import ConnexionResponse
-from connexion.utils import is_null
-
 # To enable shared jsonschema validators
 import auslib.util.jsonschema_validators  # noqa
 from auslib.util.timestamp import getMillisecondTimestamp
 
-logger = logging.getLogger(__name__)
-
-
-class BalrogRequestBodyValidator(RequestBodyValidator):
-    def validate_schema(self, data: dict, url: AnyStr) -> Union[ConnexionResponse, None]:
-        """This function is largely based on https://github.com/zalando/connexion/blob/master/connexion/decorators/validation.py
-        and should largely be kept in line with it."""
-        if self.is_null_value_valid and is_null(data):
-            return None
-        try:
-            self.validator.validate(data)
-        except jsonschema.ValidationError as exception:
-            # Add field name to the error response
-            exception_field = ""
-            for i in exception.path:
-                exception_field = i + ": "
-            if exception.__cause__ is not None:
-                exception_message = exception.__cause__.message + " " + exception_field + exception.message
-            else:
-                exception_message = exception_field + exception.message
-            # Some exceptions could contain unicode characters - if we don't replace them
-            # we could end up with a UnicodeEncodeError.
-            logger.error("{url} validation error: {error}".format(url=url, error=exception_message.encode("utf-8", "replace")))
-            raise BadRequestProblem(detail=exception_message)
-
-        return None
-
 
 def is_when_present_and_in_past_validator(what):
     """Validates if scheduled_change_time value i.e. 'when' field value is present in