Add Async Background Thread transport

Refactors handlers into separate package

Adds background threaded transport

Adds fix to Batch commit to properly set log
name

Author: Bill Prin

docs/index.rst

@@ -107,6 +107,9 @@
   logging-metric
   logging-sink
   logging-handlers
+  logging-transports-sync
+  logging-transports-thread
+  logging-transports-base
 
 .. toctree::
   :maxdepth: 0

docs/logging-handlers.rst

@@ -1,6 +1,6 @@
 Python Logging Module Handler
 ==============================
 
-.. automodule:: gcloud.logging.handlers
+.. automodule:: gcloud.logging.handlers.handlers
   :members:
   :show-inheritance:

docs/logging-transports-base.rst

@@ -0,0 +1,6 @@
+Python Logging Handler Sync Transport
+======================================
+
+.. automodule:: gcloud.logging.handlers.transports.base
+  :members:
+  :show-inheritance:

docs/logging-transports-sync.rst

@@ -0,0 +1,6 @@
+Python Logging Handler Sync Transport
+======================================
+
+.. automodule:: gcloud.logging.handlers.transports.sync
+  :members:
+  :show-inheritance:

docs/logging-transports-thread.rst

@@ -0,0 +1,7 @@
+Python Logging Handler Threaded Transport
+=========================================
+
+
+.. automodule:: gcloud.logging.handlers.transports.background_thread
+  :members:
+  :show-inheritance:

docs/logging-usage.rst

@@ -402,12 +402,21 @@ Logging client.
     >>> cloud_logger = logging.getLogger('cloudLogger')
     >>> cloud_logger.setLevel(logging.INFO) # defaults to WARN
     >>> cloud_logger.addHandler(handler)
-    >>> cloud_logger.error('bad news') # API call
+    >>> cloud_logger.error('bad news')
 
 .. note::
 
-    This handler currently only supports a synchronous API call, which means each logging statement
-    that uses this handler will require an API call.
+    This handler by default uses an asynchronous transport that sends log entries on a background
+     thread. However, the API call will still be made in the same process. For other transport
+     options, see the transports section.
+
+All logs will go to a single custom log, which defaults to "python". The name of the Python
+logger will be included in the structured log entry under the "python_logger" field. You can
+change it by providing a name to the handler:
+
+.. doctest::
+
+    >>> handler = CloudLoggingHandler(client, name="mycustomlog")
 
 It is also possible to attach the handler to the root Python logger, so that for example a plain
 `logging.warn` call would be sent to Cloud Logging, as well as any other loggers created. However,
@@ -424,4 +433,24 @@ this automatically:
     >>> handler = CloudLoggingHandler(client)
     >>> logging.getLogger().setLevel(logging.INFO) # defaults to WARN
     >>> setup_logging(handler)
-    >>> logging.error('bad news') # API call
+    >>> logging.error('bad news')
+
+You can also exclude certain loggers:
+
+.. doctest::
+
+   >>> setup_logging(handler, excluded_loggers=('werkzeug',)))
+
+
+
+Python logging handler transports
+==================================
+
+The Python logging handler can use different transports. The default is
+:class:`gcloud.logging.handlers.BackgroundThreadTransport`.
+
+ 1. :class:`gcloud.logging.handlers.BackgroundThreadTransport` this is the default. It writes
+ entries on a background :class:`python.threading.Thread`.
+
+ 1. :class:`gcloud.logging.handlers.SyncTransport` this handler does a direct API call on each
+ logging statement to write the entry.

gcloud/logging/handlers/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Python :mod:`logging` handlers for Google Cloud Logging."""
+
+from gcloud.logging.handlers.handlers import CloudLoggingHandler
+from gcloud.logging.handlers.handlers import setup_logging

gcloud/logging/handlers.py gcloud/logging/handlers/handlers.pygcloud/logging/handlers.py renamed to gcloud/logging/handlers/handlers.py

@@ -16,18 +16,22 @@
 
 import logging
 
+from gcloud.logging.handlers.transports import BackgroundThreadTransport
+
+
 EXCLUDE_LOGGER_DEFAULTS = (
     'gcloud',
-    'oauth2client.client'
+    'oauth2client'
 )
 
+DEFAULT_LOGGER_NAME = 'python'
+
 
-class CloudLoggingHandler(logging.StreamHandler, object):
-    """Python standard logging handler to log messages to the Google Cloud
-    Logging API.
+class CloudLoggingHandler(logging.StreamHandler):
+    """Python standard ``logging`` handler.
 
-    This handler can be used to route Python standard logging messages to
-    Google Cloud logging.
+    This handler can be used to route Python standard logging messages
+    directly to the Google Cloud Logging API.
 
     Note that this handler currently only supports a synchronous API call,
     which means each logging statement that uses this handler will require
@@ -37,6 +41,18 @@ class CloudLoggingHandler(logging.StreamHandler, object):
     :param client: the authenticated gcloud logging client for this handler
                    to use
 
+    :type name: str
+    :param name: the name of the custom log in Stackdriver Logging. Defaults
+                 to 'python'. The name of the Python logger will be represented
+                 in the ``python_logger`` field.
+
+    :type transport: type
+    :param transport: Class for creating new transport objects. It should
+                      extend from the base :class:`.Transport` type and
+                      implement :meth`.Transport.send`. Defaults to
+                      :class:`.BackgroundThreadTransport`. The other
+                      option is :class:`.SyncTransport`.
+
     Example:
 
     .. doctest::
@@ -51,30 +67,37 @@ class CloudLoggingHandler(logging.StreamHandler, object):
         cloud_logger.setLevel(logging.INFO)
         cloud_logger.addHandler(handler)
 
-        cloud.logger.error("bad news") # API call
+        cloud.logger.error('bad news')  # API call
 
     """
 
-    def __init__(self, client):
+    def __init__(self, client,
+                 name=DEFAULT_LOGGER_NAME,
+                 transport=BackgroundThreadTransport):
         super(CloudLoggingHandler, self).__init__()
+        self.name = name
         self.client = client
+        self.transport = transport(client, name)
 
     def emit(self, record):
-        """
-        Overrides the default emit behavior of StreamHandler.
+        """Actually log the specified logging record.
+
+        Overrides the default emit behavior of ``StreamHandler``.
 
         See: https://docs.python.org/2/library/logging.html#handler-objects
+
+        :type record: :class:`logging.LogRecord`
+        :param record: The record to be logged.
         """
         message = super(CloudLoggingHandler, self).format(record)
-        logger = self.client.logger(record.name)
-        logger.log_struct({"message": message},
-                          severity=record.levelname)
+        self.transport.send(record, message)
 
 
 def setup_logging(handler, excluded_loggers=EXCLUDE_LOGGER_DEFAULTS):
-    """Helper function to attach the CloudLoggingAPI handler to the Python
-    root logger, while excluding loggers this library itself uses to avoid
-    infinite recursion
+    """Attach the ``CloudLogging`` handler to the Python root logger
+
+    Excludes loggers that this library itself uses to avoid
+    infinite recursion.
 
     :type handler: :class:`logging.handler`
     :param handler: the handler to attach to the global handler
@@ -90,14 +113,14 @@ def setup_logging(handler, excluded_loggers=EXCLUDE_LOGGER_DEFAULTS):
 
         import logging
         import gcloud.logging
-        from gcloud.logging.handlers import CloudLoggingAPIHandler
+        from gcloud.logging.handlers import CloudLoggingHandler
 
         client = gcloud.logging.Client()
         handler = CloudLoggingHandler(client)
-        setup_logging(handler)
+        gcloud.logging.setup_logging(handler)
         logging.getLogger().setLevel(logging.DEBUG)
 
-        logging.error("bad news") # API call
+        logging.error('bad news')  # API call
 
     """
     all_excluded_loggers = set(excluded_loggers + EXCLUDE_LOGGER_DEFAULTS)

gcloud/logging/test_handlers.py gcloud/logging/handlers/test_handlers.pygcloud/logging/test_handlers.py renamed to gcloud/logging/handlers/test_handlers.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 # Copyright 2016 Google Inc. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,32 +21,32 @@ class TestCloudLoggingHandler(unittest.TestCase):
     PROJECT = 'PROJECT'
 
     def _getTargetClass(self):
-        from gcloud.logging.handlers import CloudLoggingHandler
+        from gcloud.logging.handlers.handlers import CloudLoggingHandler
         return CloudLoggingHandler
 
     def _makeOne(self, *args, **kw):
         return self._getTargetClass()(*args, **kw)
 
     def test_ctor(self):
         client = _Client(self.PROJECT)
-        handler = self._makeOne(client)
+        handler = self._makeOne(client, transport=_Transport)
         self.assertEqual(handler.client, client)
 
     def test_emit(self):
         client = _Client(self.PROJECT)
-        handler = self._makeOne(client)
+        handler = self._makeOne(client, transport=_Transport)
         LOGNAME = 'loggername'
         MESSAGE = 'hello world'
         record = _Record(LOGNAME, logging.INFO, MESSAGE)
         handler.emit(record)
-        self.assertEqual(client.logger(LOGNAME).log_struct_called_with,
-                         ({'message': MESSAGE}, logging.INFO))
+
+        self.assertEqual(handler.transport.send_called_with, (record, MESSAGE))
 
 
 class TestSetupLogging(unittest.TestCase):
 
     def _callFUT(self, handler, excludes=None):
-        from gcloud.logging.handlers import setup_logging
+        from gcloud.logging.handlers.handlers import setup_logging
         if excludes:
             return setup_logging(handler, excluded_loggers=excludes)
         else:
@@ -94,20 +93,10 @@ def release(self):
         pass  # pragma: NO COVER
 
 
-class _Logger(object):
-
-    def log_struct(self, message, severity=None):
-        self.log_struct_called_with = (message, severity)
-
-
 class _Client(object):
 
     def __init__(self, project):
         self.project = project
-        self.logger_ = _Logger()
-
-    def logger(self, _):  # pylint: disable=unused-argument
-        return self.logger_
 
 
 class _Record(object):
@@ -122,3 +111,12 @@ def __init__(self, name, level, message):
 
     def getMessage(self):
         return self.message
+
+
+class _Transport(object):
+
+    def __init__(self, client, name):
+        pass
+
+    def send(self, record, message):
+        self.send_called_with = (record, message)

gcloud/logging/handlers/transports/__init__.py

@@ -0,0 +1,26 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Transport classes for Python logging integration.
+
+Currently two options are provided, a synchronous transport that makes
+an API call for each log statement, and an asynchronous handler that
+sends the API using a :class:`~gcloud.logging.logger.Batch` object in
+the background.
+"""
+
+from gcloud.logging.handlers.transports.base import Transport
+from gcloud.logging.handlers.transports.sync import SyncTransport
+from gcloud.logging.handlers.transports.background_thread import (
+    BackgroundThreadTransport)