Implementing storage regression tests to match gcloud-node.

Note meant to be merged. Some missing features were
uncovered or hard to use APIs and I want to discuss here
and then map out the plan (i.e. how to break this up).

Author: dhermes

CONTRIBUTING.rst

@@ -160,7 +160,10 @@ Running Regression Tests
   so you'll need to provide some environment variables to facilitate
   authentication to your project:
 
+  - ``GCLOUD_TESTS_PROJECT_ID``: Developers Console project ID (e.g.
+    bamboo-shift-455).
   - ``GCLOUD_TESTS_DATASET_ID``: The name of the dataset your tests connect to.
+    This is typically the same as ``GCLOUD_TESTS_PROJECT_ID``.
   - ``GCLOUD_TESTS_CLIENT_EMAIL``: The email for the service account you're
     authenticating with
   - ``GCLOUD_TESTS_KEY_FILE``: The path to an encrypted key file.

gcloud/storage/bucket.py

@@ -252,7 +252,8 @@ def upload_file(self, filename, key=None):
         if key is None:
             key = os.path.basename(filename)
         key = self.new_key(key)
-        return key.set_contents_from_filename(filename)
+        key.set_contents_from_filename(filename)
+        return key
 
     def upload_file_object(self, file_obj, key=None):
         """Shortcut method to upload a file object into this bucket.

gcloud/storage/iterator.py

@@ -40,11 +40,12 @@ class Iterator(object):
     :type path: string
     :param path: The path to query for the list of items.
     """
-    def __init__(self, connection, path):
+    def __init__(self, connection, path, extra_params=None):
         self.connection = connection
         self.path = path
         self.page_number = 0
         self.next_page_token = None
+        self.extra_params = extra_params
 
     def __iter__(self):
         """Iterate through the list of items."""
@@ -71,8 +72,13 @@ def get_query_params(self):
         :rtype: dict or None
         :returns: A dictionary of query parameters or None if there are none.
         """
+        result = None
         if self.next_page_token:
-            return {'pageToken': self.next_page_token}
+            result = {'pageToken': self.next_page_token}
+        if self.extra_params is not None:
+            result = result or {}
+            result.update(self.extra_params)
+        return result
 
     def get_next_page_response(self):
         """Requests the next page from the path provided.

gcloud/storage/key.py

@@ -442,10 +442,11 @@ class KeyIterator(Iterator):
     :type bucket: :class:`gcloud.storage.bucket.Bucket`
     :param bucket: The bucket from which to list keys.
     """
-    def __init__(self, bucket):
+    def __init__(self, bucket, extra_params=None):
         self.bucket = bucket
         super(KeyIterator, self).__init__(
-            connection=bucket.connection, path=bucket.path + '/o')
+            connection=bucket.connection, path=bucket.path + '/o',
+            extra_params=extra_params)
 
     def get_items_from_response(self, response):
         """Factory method, yields :class:`.storage.key.Key` items from response.

gcloud/storage/test_iterator.py

@@ -75,6 +75,13 @@ def test_get_query_params_w_token(self):
         self.assertEqual(iterator.get_query_params(),
                          {'pageToken': TOKEN})
 
+    def test_get_query_params_extra_params(self):
+        connection = _Connection()
+        PATH = '/foo'
+        extra_params = {'key': 'val'}
+        iterator = self._makeOne(connection, PATH, extra_params=extra_params)
+        self.assertEqual(iterator.get_query_params(), extra_params)
+
     def test_get_next_page_response_new_no_token_in_response(self):
         PATH = '/foo'
         TOKEN = 'token'

regression/data/CloudPlatform_128px_Retina.png

@@ -1,3 +1,4 @@
-export GCLOUD_TESTS_DATASET_ID="my-dataset"
+export GCLOUD_TESTS_PROJECT_ID="my-project"
+export GCLOUD_TESTS_DATASET_ID=${GCLOUD_TESTS_PROJECT_ID}
 export GCLOUD_TESTS_CLIENT_EMAIL="[email protected]"
 export GCLOUD_TESTS_KEY_FILE="path.key"

regression/data/five-mb-file.zip

@@ -2,37 +2,58 @@
 import sys
 
 from gcloud import datastore
+from gcloud import storage
 
 
 # Defaults from shell environ. May be None.
+PROJECT_ID = os.getenv('GCLOUD_TESTS_PROJECT_ID')
 DATASET_ID = os.getenv('GCLOUD_TESTS_DATASET_ID')
 CLIENT_EMAIL = os.getenv('GCLOUD_TESTS_CLIENT_EMAIL')
 KEY_FILENAME = os.getenv('GCLOUD_TESTS_KEY_FILE')
-DATASETS = {}
+CACHED_RETURN_VALS = {}
 
 ENVIRON_ERROR_MSG = """\
 To run the regression tests, you need to set some environment variables.
 Please check the Contributing guide for instructions.
 """
 
 
-def get_environ():
-    if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
-        print >> sys.stderr, ENVIRON_ERROR_MSG
-        sys.exit(1)
+def get_environ(require_datastore=False, require_storage=False):
+    if require_datastore:
+        if DATASET_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
+            print >> sys.stderr, ENVIRON_ERROR_MSG
+            sys.exit(1)
+
+    if require_storage:
+        if PROJECT_ID is None or CLIENT_EMAIL is None or KEY_FILENAME is None:
+            print >> sys.stderr, ENVIRON_ERROR_MSG
+            sys.exit(1)
 
     return {
+        'project_id': PROJECT_ID,
         'dataset_id': DATASET_ID,
         'client_email': CLIENT_EMAIL,
         'key_filename': KEY_FILENAME,
     }
 
 
 def get_dataset():
-    environ = get_environ()
+    environ = get_environ(require_datastore=True)
     get_dataset_args = (environ['dataset_id'], environ['client_email'],
                         environ['key_filename'])
-    if get_dataset_args not in DATASETS:
+    key = ('get_dataset', get_dataset_args)
+    if key not in CACHED_RETURN_VALS:
+        # Cache return value for the environment.
+        CACHED_RETURN_VALS[key] = datastore.get_dataset(*get_dataset_args)
+    return CACHED_RETURN_VALS[key]
+
+
+def get_storage_connection():
+    environ = get_environ(require_storage=True)
+    get_connection_args = (environ['project_id'], environ['client_email'],
+                           environ['key_filename'])
+    key = ('get_storage_connection', get_connection_args)
+    if key not in CACHED_RETURN_VALS:
         # Cache return value for the environment.
-        DATASETS[get_dataset_args] = datastore.get_dataset(*get_dataset_args)
-    return DATASETS[get_dataset_args]
+        CACHED_RETURN_VALS[key] = storage.get_connection(*get_connection_args)
+    return CACHED_RETURN_VALS[key]

regression/local_test_setup.sample

@@ -11,7 +11,7 @@ def get_parser():
     parser = argparse.ArgumentParser(
         description='GCloud test runner against actual project.')
     parser.add_argument('--package', dest='package',
-                        choices=('datastore',),
+                        choices=('datastore', 'storage'),
                         default='datastore', help='Package to be tested.')
     return parser
 
@@ -27,7 +27,10 @@ def main():
     parser = get_parser()
     args = parser.parse_args()
     # Make sure environ is set before running test.
-    regression_utils.get_environ()
+    if args.package == 'datastore':
+        regression_utils.get_environ(require_datastore=True)
+    elif args.package == 'storage':
+        regression_utils.get_environ(require_storage=True)
     test_result = run_module_tests(args.package)
     if not test_result.wasSuccessful():
         sys.exit(1)