OLD | NEW |
1 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | 1 # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
2 # for details. All rights reserved. Use of this source code is governed by a | 2 # for details. All rights reserved. Use of this source code is governed by a |
3 # BSD-style license that can be found in the LICENSE file. | 3 # BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 """Utility functions for dealing with Google Cloud Storage.""" | 5 """Utility functions for dealing with Google Cloud Storage.""" |
6 | 6 |
7 from cStringIO import StringIO | 7 from cStringIO import StringIO |
8 from base64 import b64encode | 8 from base64 import b64encode |
9 from urlparse import urlparse, parse_qs | 9 from urlparse import urlparse, parse_qs |
10 from xml.etree import ElementTree | 10 from xml.etree import ElementTree |
11 import cherrypy | 11 import cherrypy |
12 import handlers | 12 import handlers |
13 import json | 13 import json |
14 import routes | 14 import routes |
15 import urllib | 15 import urllib |
16 import time | 16 import time |
17 | 17 |
18 from google.appengine.api import app_identity | 18 from google.appengine.api import app_identity |
19 from google.appengine.api import files | 19 from google.appengine.api import files |
20 from google.appengine.api import namespace_manager | 20 from google.appengine.api import namespace_manager |
21 from google.appengine.api import urlfetch | 21 from google.appengine.api import urlfetch |
22 | 22 |
23 from models.private_key import PrivateKey | 23 from models.private_key import PrivateKey |
24 | 24 |
| 25 import cloudstorage |
| 26 |
25 # The Google Cloud Storage bucket for this app | 27 # The Google Cloud Storage bucket for this app |
26 _BUCKET = "pub.dartlang.org" | 28 _BUCKET = "pub.dartlang.org" |
27 | 29 |
28 # From https://code.google.com/apis/console | 30 # From https://code.google.com/apis/console |
29 _ACCESS_KEY = "818368855108@developer.gserviceaccount.com" | 31 _ACCESS_KEY = "818368855108@developer.gserviceaccount.com" |
30 | 32 |
31 # From https://developers.google.com/storage/docs/authentication | 33 # From https://developers.google.com/storage/docs/authentication |
32 _FULL_CONTROL_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control" | 34 _FULL_CONTROL_SCOPE = "https://www.googleapis.com/auth/devstorage.full_control" |
33 | 35 |
34 # The maximum size (in bytes) of a chunk that can be read from cloud storage at | 36 # The maximum size (in bytes) of a chunk that can be read from cloud storage at |
(...skipping 172 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
207 response.status_code, | 209 response.status_code, |
208 xml.find('Code').text, | 210 xml.find('Code').text, |
209 xml.find('Message').text | 211 xml.find('Message').text |
210 )) | 212 )) |
211 | 213 |
212 def delete_object(obj): | 214 def delete_object(obj): |
213 """Deletes an object from cloud storage.""" | 215 """Deletes an object from cloud storage.""" |
214 files.delete(_appengine_object_path(obj)) | 216 files.delete(_appengine_object_path(obj)) |
215 | 217 |
216 def open(obj): | 218 def open(obj): |
217 """Opens an object in cloud storage.""" | 219 """Opens an object in cloud storage. |
| 220 |
| 221 DEPRECATED: For larger files, please consider using `open_with_gcs()`. |
| 222 """ |
218 return files.open(_appengine_object_path(obj), 'r') | 223 return files.open(_appengine_object_path(obj), 'r') |
219 | 224 |
| 225 def open_with_gcs(obj): |
| 226 """Opens an object in cloud storage with the GCS library.""" |
| 227 return cloudstorage.open(_gcs_appengine_object_path(obj), 'r') |
| 228 |
220 def read(obj): | 229 def read(obj): |
221 """Consumes and returns all data in an object in cloud storage. | 230 """Consumes and returns all data in an object in cloud storage. |
222 | 231 |
223 The data is returned as a StringIO instance, so it may be used in place of a | 232 The data is returned as a StringIO instance, so it may be used in place of a |
224 file. | 233 file. |
225 | 234 |
226 This can be necessary since many operations on the file object itself | 235 This can be necessary since many operations on the file object itself |
227 require a network round trip.""" | 236 require a network round trip.""" |
228 | 237 |
229 with open(obj) as f: | 238 with open_with_gcs(obj) as f: |
230 io = StringIO() | 239 io = StringIO() |
231 data = f.read(_CHUNK_SIZE) | 240 data = f.read(_CHUNK_SIZE) |
232 while data: | 241 while data: |
233 io.write(data) | 242 io.write(data) |
234 data = f.read(_CHUNK_SIZE) | 243 data = f.read(_CHUNK_SIZE) |
235 io.seek(0) | 244 io.seek(0) |
236 return io | 245 return io |
237 | 246 |
238 def object_url(obj): | 247 def object_url(obj): |
239 """Returns the URL for an object in cloud storage.""" | 248 """Returns the URL for an object in cloud storage.""" |
240 if handlers.is_production(): | 249 if handlers.is_production(): |
241 return 'https://storage.googleapis.com/' + _object_path(obj) | 250 return 'https://storage.googleapis.com/' + _object_path(obj) |
242 else: | 251 else: |
243 return '/gs_/' + urllib.quote(obj) | 252 return '/gs_/' + urllib.quote(obj) |
244 | 253 |
245 def _object_path(obj): | 254 def _object_path(obj): |
246 """Returns the path for an object in cloud storage.""" | 255 """Returns the path for an object in cloud storage.""" |
247 ns = namespace_manager.get_namespace() | 256 ns = namespace_manager.get_namespace() |
248 if ns == "": return _BUCKET + '/' + obj | 257 if ns == "": return _BUCKET + '/' + obj |
249 return _BUCKET + '/ns/' + ns + '/' + obj | 258 return _BUCKET + '/ns/' + ns + '/' + obj |
250 | 259 |
251 def _appengine_object_path(obj): | 260 def _appengine_object_path(obj): |
252 """Returns the path for an object for use with the AppEngine APIs.""" | 261 """Returns the path for an object for use with the AppEngine APIs.""" |
253 return '/gs/' + _object_path(obj) | 262 return '/gs/' + _object_path(obj) |
254 | 263 |
| 264 def _gcs_appengine_object_path(obj): |
| 265 """Returns the path for an object for use with the GCS APIs.""" |
| 266 return '/' + _object_path(obj) |
| 267 |
255 def _iso8601(secs): | 268 def _iso8601(secs): |
256 """Returns the ISO8601 representation of the given time. | 269 """Returns the ISO8601 representation of the given time. |
257 | 270 |
258 The time should be in seconds past the epoch.""" | 271 The time should be in seconds past the epoch.""" |
259 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(secs)) | 272 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(secs)) |
OLD | NEW |