Add mTLS support to generator (#359)

arithmetic1728 · web-flow · commit da0510a03387 · 2020-04-08T14:54:23.000-07:00
Add preliminary support for mTLS to the generated surface.
mTLS provides mutual authentication between a client and a service using certificates.
Unless an alternative or custom endpoint is provided, the client surface assumes that the mTLS endpoint for a service is the same as the non-mtls variant with the 'mtls' moniker prepended to the domain.

Client certificates can be passed explicitly or yielded via a callback.
diff --git a/packages/gapic-generator/.circleci/config.yml b/packages/gapic-generator/.circleci/config.yml
@@ -253,7 +253,7 @@ jobs:
             ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
       - run:
           name: Run showcase tests.
-          command: nox -s showcase_alternative_templates          
+          command: nox -s showcase_alternative_templates
   showcase-unit-3.6:
     docker:
       - image: python:3.6-slim
@@ -263,7 +263,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
@@ -287,7 +287,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
@@ -311,7 +311,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
@@ -335,7 +335,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
@@ -359,7 +359,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
@@ -383,7 +383,7 @@ jobs:
           name: Install system dependencies.
           command: |
             apt-get update
-            apt-get install -y curl pandoc unzip
+            apt-get install -y curl pandoc unzip git
       - run:
           name: Install protoc 3.7.1.
           command: |
diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/client.py.j2
@@ -2,7 +2,8 @@
 
 {% block content %}
 from collections import OrderedDict
-from typing import Dict, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union
+import re
+from typing import Callable, Dict, {% if service.any_server_streaming %}Iterable, {% endif %}{% if service.any_client_streaming %}Iterator, {% endif %}Sequence, Tuple, Type, Union
 import pkg_resources
 
 import google.api_core.client_options as ClientOptions # type: ignore
@@ -57,7 +58,40 @@ class {{ service.client_name }}Meta(type):
 class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
     """{{ service.meta.doc|rst(width=72, indent=4) }}"""
 
-    DEFAULT_OPTIONS = ClientOptions.ClientOptions({% if service.host %}api_endpoint='{{ service.host }}'{% endif %})
+    @staticmethod
+    def _get_default_mtls_endpoint(api_endpoint):
+        """Convert api endpoint to mTLS endpoint.
+        Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to
+        "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively.
+        Args:
+            api_endpoint (Optional[str]): the api endpoint to convert.
+        Returns:
+            str: converted mTLS api endpoint.
+        """
+        if not api_endpoint:
+            return api_endpoint
+
+        mtls_endpoint_re = re.compile(
+            r"(?P<name>[^.]+)(?P<mtls>\.mtls)?(?P<sandbox>\.sandbox)?(?P<googledomain>\.googleapis\.com)?"
+        )
+
+        m = mtls_endpoint_re.match(api_endpoint)
+        name, mtls, sandbox, googledomain = m.groups()
+        if mtls or not googledomain:
+            return api_endpoint
+
+        if sandbox:
+            return api_endpoint.replace(
+                "sandbox.googleapis.com", "mtls.sandbox.googleapis.com"
+            )
+
+        return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com")
+
+    DEFAULT_ENDPOINT = {% if service.host %}'{{ service.host }}'{% else %}None{% endif %}
+    DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__(  # type: ignore
+        DEFAULT_ENDPOINT
+    )
+    DEFAULT_OPTIONS = ClientOptions.ClientOptions(api_endpoint=DEFAULT_ENDPOINT)
 
     @classmethod
     def from_service_account_file(cls, filename: str, *args, **kwargs):
@@ -106,23 +140,56 @@ class {{ service.client_name }}(metaclass={{ service.client_name }}Meta):
                 transport to use. If set to None, a transport is chosen
                 automatically.
             client_options (ClientOptions): Custom options for the client.
+                (1) The ``api_endpoint`` property can be used to override the
+                default endpoint provided by the client.
+                (2) If ``transport`` argument is None, ``client_options`` can be
+                used to create a mutual TLS transport. If ``api_endpoint`` is
+                provided and different from the default endpoint, or the
+                ``client_cert_source`` property is provided,  mutual TLS
+                transport will be created if client SSL credentials are found.
+                Client SSL credentials are obtained from ``client_cert_source``
+                or application default SSL credentials.
+
+        Raises:
+            google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport
+                creation failed for any reason.
         """
         if isinstance(client_options, dict):
             client_options = ClientOptions.from_dict(client_options)
 
+        # Set default api endpoint if not set.
+        if client_options.api_endpoint is None:
+            client_options.api_endpoint = self.DEFAULT_ENDPOINT
+
         # Save or instantiate the transport.
         # Ordinarily, we provide the transport, but allowing a custom transport
         # instance provides an extensibility point for unusual situations.
         if isinstance(transport, {{ service.name }}Transport):
+            # transport is a {{ service.name }}Transport instance.
             if credentials:
                 raise ValueError('When providing a transport instance, '
                                  'provide its credentials directly.')
             self._transport = transport
-        else:
+        elif transport is not None or (
+            client_options.api_endpoint == self.DEFAULT_ENDPOINT
+            and client_options.client_cert_source is None
+        ):
+            # Don't trigger mTLS.
             Transport = type(self).get_transport_class(transport)
             self._transport = Transport(
+                credentials=credentials, host=client_options.api_endpoint
+            )
+        else:
+            # Trigger mTLS. If the user overrides endpoint, use it as the mTLS
+            # endpoint, otherwise use the default mTLS endpoint.
+            option_endpoint = client_options.api_endpoint
+            api_mtls_endpoint = self.DEFAULT_MTLS_ENDPOINT if option_endpoint == self.DEFAULT_ENDPOINT else option_endpoint
+
+            self._transport = {{ service.name }}GrpcTransport(
                 credentials=credentials,
-                host=client_options.api_endpoint{% if service.host %} or '{{ service.host }}'{% endif %},
+                host=client_options.api_endpoint,
+                api_mtls_endpoint=api_mtls_endpoint,
+                client_cert_source=client_options.client_cert_source,
             )
 
     {% for method in service.methods.values() -%}
diff --git a/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2 b/packages/gapic-generator/gapic/templates/%namespace/%name_%version/%sub/services/%service/transports/grpc.py.j2
@@ -1,13 +1,15 @@
 {% extends '_base.py.j2' %}
 
 {% block content %}
-from typing import Callable, Dict
+from typing import Callable, Dict, Tuple
 
 from google.api_core import grpc_helpers   # type: ignore
 {%- if service.has_lro %}
 from google.api_core import operations_v1  # type: ignore
 {%- endif %}
 from google.auth import credentials        # type: ignore
+from google.auth.transport.grpc import SslCredentials  # type: ignore
+
 
 import grpc  # type: ignore
 
@@ -35,7 +37,9 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
     def __init__(self, *,
             host: str{% if service.host %} = '{{ service.host }}'{% endif %},
             credentials: credentials.Credentials = None,
-            channel: grpc.Channel = None) -> None:
+            channel: grpc.Channel = None,
+            api_mtls_endpoint: str = None,
+            client_cert_source: Callable[[], Tuple[bytes, bytes]] = None) -> None:
         """Instantiate the transport.
 
         Args:
@@ -49,19 +53,51 @@ class {{ service.name }}GrpcTransport({{ service.name }}Transport):
                 This argument is ignored if ``channel`` is provided.
             channel (Optional[grpc.Channel]): A ``Channel`` instance through
                 which to make calls.
+            api_mtls_endpoint (Optional[str]): The mutual TLS endpoint. If
+                provided, it overrides the ``host`` argument and tries to create
+                a mutual TLS channel with client SSL credentials from
+                ``client_cert_source`` or applicatin default SSL credentials.
+            client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): A
+                callback to provide client SSL certificate bytes and private key
+                bytes, both in PEM format. It is ignored if ``api_mtls_endpoint``
+                is None.
+
+        Raises:
+          google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport
+              creation failed for any reason.
         """
-        # Sanity check: Ensure that channel and credentials are not both
-        # provided.
         if channel:
+            # Sanity check: Ensure that channel and credentials are not both
+            # provided.
             credentials = False
 
+            # If a channel was explicitly provided, set it.
+            self._grpc_channel = channel
+        elif api_mtls_endpoint:
+            host = api_mtls_endpoint if ":" in api_mtls_endpoint else api_mtls_endpoint + ":443"
+
+            # Create SSL credentials with client_cert_source or application
+            # default SSL credentials.
+            if client_cert_source:
+                cert, key = client_cert_source()
+                ssl_credentials = grpc.ssl_channel_credentials(
+                    certificate_chain=cert, private_key=key
+                )
+            else:
+                ssl_credentials = SslCredentials().ssl_credentials
+
+            # create a new channel. The provided one is ignored.
+            self._grpc_channel = grpc_helpers.create_channel(
+                host,
+                credentials=credentials,
+                ssl_credentials=ssl_credentials,
+                scopes=self.AUTH_SCOPES,
+            )
+
         # Run the base constructor.
         super().__init__(host=host, credentials=credentials)
         self._stubs = {}  # type: Dict[str, Callable]
 
-        # If a channel was explicitly provided, set it.
-        if channel:
-            self._grpc_channel = channel
 
     @classmethod
     def create_channel(cls,
diff --git a/packages/gapic-generator/gapic/templates/setup.py.j2 b/packages/gapic-generator/gapic/templates/setup.py.j2
@@ -16,6 +16,7 @@ setuptools.setup(
     platforms='Posix; MacOS X; Windows',
     include_package_data=True,
     install_requires=(
+        'google-auth >= 1.13.1',
         'google-api-core >= 1.8.0, < 2.0.0dev',
         'googleapis-common-protos >= 1.5.8',
         'grpcio >= 1.10.0',
diff --git a/packages/gapic-generator/gapic/templates/tests/unit/%name_%version/%sub/test_%service.py.j2 b/packages/gapic-generator/gapic/templates/tests/unit/%name_%version/%sub/test_%service.py.j2
diff --git a/packages/gapic-generator/noxfile.py b/packages/gapic-generator/noxfile.py
