Increase maximum file upload to 100MB, use cloudstorage python library

app/handlers/cloud_storage.py

 # Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
 """Utility functions for dealing with Google Cloud Storage."""
 
 from cStringIO import StringIO
 from base64 import b64encode
 from urlparse import urlparse, parse_qs
 from xml.etree import ElementTree
 import cherrypy
 import handlers
 import json
 import routes
 import urllib
 import time
 
 from google.appengine.api import app_identity
 from google.appengine.api import files
 from google.appengine.api import namespace_manager
 from google.appengine.api import urlfetch
 
 from models.private_key import PrivateKey
 
+import cloudstorage
+
 # The Google Cloud Storage bucket for this app
 _BUCKET = "pub.dartlang.org"
 
 # From https://code.google.com/apis/console
 _ACCESS_KEY = "818368855108@developer.gserviceaccount.com"
 
 # From https://developers.google.com/storage/docs/authentication
 _FULL_CONTROL_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control"
 
 # The maximum size (in bytes) of a chunk that can be read from cloud storage at
@@ skipping 171 matching lines @@
     raise handlers.http_error(500, "Cloud storage %s error: %s\n%s" % (
         response.status_code,
         xml.find('Code').text,
         xml.find('Message').text
     ))
 
 def delete_object(obj):
     """Deletes an object from cloud storage."""
     files.delete(_appengine_object_path(obj))
 
 def open(obj):

[Bob Nystrom, 2015/03/23 19:41:13]

Might be good to leave a comment that this is deprecated, or just remove it
outright if it's not being used anymore.

[kustermann, 2015/03/24 18:28:20]

Done.

It's still used -- I've added a deprecation comment.

     """Opens an object in cloud storage."""
     return files.open(_appengine_object_path(obj), 'r')
 
+def open_with_gcs(obj):
+    """Opens an object in cloud storage with the GCS library."""
+    return cloudstorage.open(_gcs_appengine_object_path(obj), 'r')
+
 def read(obj):
     """Consumes and returns all data in an object in cloud storage.
 
     The data is returned as a StringIO instance, so it may be used in place of a
     file.
 
     This can be necessary since many operations on the file object itself
     require a network round trip."""
 
-    with open(obj) as f:
+    with open_with_gcs(obj) as f:
         io = StringIO()
         data = f.read(_CHUNK_SIZE)
         while data:
             io.write(data)
             data = f.read(_CHUNK_SIZE)
         io.seek(0)
         return io
 
 def object_url(obj):
     """Returns the URL for an object in cloud storage."""
     if handlers.is_production():
         return 'https://storage.googleapis.com/' + _object_path(obj)
     else:
         return '/gs_/' + urllib.quote(obj)
 
 def _object_path(obj):
     """Returns the path for an object in cloud storage."""
     ns = namespace_manager.get_namespace()
     if ns == "": return _BUCKET + '/' + obj
     return _BUCKET + '/ns/' + ns + '/' + obj
 
 def _appengine_object_path(obj):
     """Returns the path for an object for use with the AppEngine APIs."""
     return '/gs/' + _object_path(obj)
 
+def _gcs_appengine_object_path(obj):
+    """Returns the path for an object for use with the GCS APIs."""
+    return '/' + _object_path(obj)
+
 def _iso8601(secs):
     """Returns the ISO8601 representation of the given time.
 
     The time should be in seconds past the epoch."""
     return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(secs))