| OLD | NEW |
|---|
| (Empty) |
| 1 # Copyright 2010 Google Inc. | |
| 2 # | |
| 3 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 4 # copy of this software and associated documentation files (the | |
| 5 # "Software"), to deal in the Software without restriction, including | |
| 6 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 7 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 8 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 9 # lowing conditions: | |
| 10 # | |
| 11 # The above copyright notice and this permission notice shall be included | |
| 12 # in all copies or substantial portions of the Software. | |
| 13 # | |
| 14 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 15 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 16 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 17 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 18 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 19 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 20 # IN THE SOFTWARE. | |
| 21 | |
| 22 """Implementation of wildcarding over StorageUris. | |
| 23 | |
| 24 StorageUri is an abstraction that Google introduced in the boto library, | |
| 25 for representing storage provider-independent bucket and object names with | |
| 26 a shorthand URI-like syntax (see boto/boto/storage_uri.py) The current | |
| 27 class provides wildcarding support for StorageUri objects (including both | |
| 28 bucket and file system objects), allowing one to express collections of | |
| 29 objects with syntax like the following: | |
| 30 gs://mybucket/images/*.png | |
| 31 file:///tmp/???abc??? | |
| 32 | |
| 33 We provide wildcarding support as part of gsutil rather than as part | |
| 34 of boto because wildcarding is really part of shell command-like | |
| 35 functionality. | |
| 36 | |
| 37 A comment about wildcard semantics: In a hierarchical file system it's common | |
| 38 to distinguish recursive from single path component wildcards (e.g., using | |
| 39 '**' for the former and '*' for the latter). For example, | |
| 40 /opt/eclipse/*/*.html | |
| 41 would enumerate HTML files one directory down from /opt/eclipse, while | |
| 42 /opt/eclipse/**/*.html | |
| 43 would enumerate HTML files in all subdirectories of /opt/eclipse. We provide | |
| 44 '**' wildcarding support for file system directories but '*' and '**' behave | |
| 45 the same for bucket URIs because the bucket namespace is flat (i.e., | |
| 46 there's no meaningful distinction between '*' and '**' for buckets). | |
| 47 Thus, for example, if you were to upload data using the following command: | |
| 48 % gsutil cp -r /opt/eclipse gs://bucket/eclipse | |
| 49 it would create a set of objects mirroring the filename hierarchy, and | |
| 50 the following two commands would yield identical results: | |
| 51 % gsutil ls gs://bucket/eclipse/*/*.html | |
| 52 % gsutil ls gs://bucket/eclipse/**/*.html | |
| 53 | |
| 54 Note also that if you use file system wildcards it's likely your shell | |
| 55 interprets the wildcarding before passing the command to gsutil. For example: | |
| 56 % gsutil cp /opt/eclipse/*/*.html gs://bucket/eclipse | |
| 57 would likely be expanded by the shell into the following before running gsutil: | |
| 58 % gsutil cp /opt/eclipse/RUNNING.html gs://bucket/eclipse | |
| 59 | |
| 60 Note also that some shells (e.g., bash) don't support '**' wildcarding. If | |
| 61 you want to use '**' wildcarding with such a shell you can single quote | |
| 62 each wildcarded string, so it gets passed uninterpreted by the shell to | |
| 63 gsutil (at which point gsutil will perform the wildcarding expansion): | |
| 64 % gsutil cp '/opt/eclipse/**/*.html' gs://bucket/eclipse | |
| 65 """ | |
| 66 | |
| 67 import fnmatch | |
| 68 import glob | |
| 69 import os | |
| 70 import re | |
| 71 import time | |
| 72 import urllib | |
| 73 import boto | |
| 74 from boto.storage_uri import BucketStorageUri | |
| 75 | |
| 76 WILDCARD_REGEX = re.compile('[*?\[\]]') | |
| 77 WILDCARD_OBJECT_ITERATOR = 'wildcard_object_iterator' | |
| 78 WILDCARD_BUCKET_ITERATOR = 'wildcard_bucket_iterator' | |
| 79 | |
| 80 | |
| 81 # Enum class for specifying what to return from each iteration. | |
| 82 class ResultType(object): | |
| 83 KEYS = 'KEYS' | |
| 84 URIS = 'URIS' | |
| 85 | |
| 86 | |
| 87 class WildcardIterator(object): | |
| 88 """Base class for wildcarding over StorageUris. | |
| 89 | |
| 90 This class implements support for iterating over StorageUris that | |
| 91 contain wildcards, such as 'gs://bucket/abc*' and 'file://directory/abc*'. | |
| 92 | |
| 93 The base class is abstract; you should instantiate using the | |
| 94 wildcard_iterator() static factory method, which chooses the right | |
| 95 implementation depending on the StorageUri. | |
| 96 """ | |
| 97 | |
| 98 def __repr__(self): | |
| 99 """Returns string representation of WildcardIterator.""" | |
| 100 return 'WildcardIterator(%s, %s)' % (self.wildcard_uri, self.result_type) | |
| 101 | |
| 102 | |
| 103 class CloudWildcardIterator(WildcardIterator): | |
| 104 """WildcardIterator subclass for buckets and objects. | |
| 105 | |
| 106 Iterates over Keys or URIs matching the StorageUri wildcard. It's more | |
| 107 efficient to use this method to iterate keys if you want to get metadata | |
| 108 that's available in the Bucket (for example to get the name and size of | |
| 109 each object), because that information is available in the bucket GET | |
| 110 results. If you were to iterate over URIs for such cases and then get | |
| 111 the name and size info from each resulting StorageUri, it would cause | |
| 112 an additional object GET request for each of the result URIs. | |
| 113 """ | |
| 114 | |
| 115 def __init__(self, wildcard_uri, proj_id_handler, result_type, | |
| 116 bucket_storage_uri_class=BucketStorageUri, | |
| 117 headers=None, debug=0): | |
| 118 """Instantiate an iterator over keys matching given wildcard URI. | |
| 119 | |
| 120 Args: | |
| 121 wildcard_uri: StorageUri that contains the wildcard to iterate. | |
| 122 proj_id_handler: ProjectIdHandler to use for current command. | |
| 123 result_type: ResultType object specifying what to iterate. | |
| 124 bucket_storage_uri_class: BucketStorageUri interface. | |
| 125 Settable for testing/mocking. | |
| 126 headers: dictionary containing optional HTTP headers to pass to boto. | |
| 127 debug: debug level to pass in to boto connection (range 0..3). | |
| 128 | |
| 129 Raises: | |
| 130 WildcardException: for invalid result_type. | |
| 131 """ | |
| 132 self.wildcard_uri = wildcard_uri | |
| 133 self.result_type = result_type | |
| 134 if result_type != ResultType.KEYS and result_type != ResultType.URIS: | |
| 135 raise WildcardException('Invalid ResultType (%s)' % result_type) | |
| 136 # Make a copy of the headers so any updates we make during wildcard | |
| 137 # expansion aren't left in the input params (specifically, so we don't | |
| 138 # include the x-goog-project-id header needed by a subset of cases, in | |
| 139 # the data returned to caller, which could then be used in other cases | |
| 140 # where that header must not be passed). | |
| 141 self.headers = headers.copy() | |
| 142 self.proj_id_handler = proj_id_handler | |
| 143 self.debug = debug | |
| 144 self.bucket_storage_uri_class = bucket_storage_uri_class | |
| 145 | |
| 146 def __NeededResultType(self, obj, uri, headers): | |
| 147 """Helper function to generate needed ResultType, per constructor param. | |
| 148 | |
| 149 Args: | |
| 150 obj: Key form of object to return, or None if not available. | |
| 151 uri: StorageUri form of object to return. | |
| 152 headers: dictionary containing optional HTTP headers to pass to boto. | |
| 153 | |
| 154 Returns: | |
| 155 StorageUri or subclass of boto.s3.key.Key, depending on constructor param. | |
| 156 | |
| 157 Raises: | |
| 158 WildcardException: for bucket-only uri with ResultType.KEYS. | |
| 159 """ | |
| 160 if self.result_type == ResultType.URIS: | |
| 161 return uri | |
| 162 # Else ResultType.KEYS. | |
| 163 if not obj: | |
| 164 if not uri.object_name: | |
| 165 raise WildcardException('Bucket-only URI (%s) with ResultType.KEYS ' | |
| 166 'iteration request' % uri) | |
| 167 # This case happens when we do gsutil ls -l on a object name-ful | |
| 168 # StorageUri with no object-name wildcard. Since the ListCommand | |
| 169 # implementation only reads bucket info we need to read the object | |
| 170 # for this case. | |
| 171 obj = uri.get_key(validate=False, headers=headers) | |
| 172 # When we retrieve the object this way its last_modified timestamp | |
| 173 # is formatted in RFC 1123 format, which is different from when we | |
| 174 # retrieve from the bucket listing (which uses ISO 8601 format), so | |
| 175 # convert so we consistently return ISO 8601 format. | |
| 176 tuple_time = (time.strptime(obj.last_modified, '%a, %d %b %Y %H:%M:%S %Z')
) | |
| 177 obj.last_modified = time.strftime('%Y-%m-%dT%H:%M:%S', tuple_time) | |
| 178 return obj | |
| 179 | |
| 180 def __iter__(self): | |
| 181 """Python iterator that gets called when iterating over cloud wildcard. | |
| 182 | |
| 183 Yields: | |
| 184 StorageUri or Key, per constructor param. | |
| 185 | |
| 186 Raises: | |
| 187 WildcardException: If there were no matches for the given wildcard. | |
| 188 """ | |
| 189 some_matched = False | |
| 190 # First handle bucket wildcarding, if any. | |
| 191 if ContainsWildcard(self.wildcard_uri.bucket_name): | |
| 192 regex = fnmatch.translate(self.wildcard_uri.bucket_name) | |
| 193 bucket_uris = [] | |
| 194 prog = re.compile(regex) | |
| 195 self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_BUCKET_ITERATOR, | |
| 196 self.wildcard_uri, | |
| 197 self.headers) | |
| 198 for b in self.wildcard_uri.get_all_buckets(headers=self.headers): | |
| 199 if prog.match(b.name): | |
| 200 # Use str(b.name) because get_all_buckets() returns Unicode | |
| 201 # string, which when used to construct x-goog-copy-src metadata | |
| 202 # requests for object-to-object copies, causes pathname '/' chars | |
| 203 # to be entity-encoded (bucket%2Fdir instead of bucket/dir), | |
| 204 # which causes the request to fail. | |
| 205 uri_str = '%s://%s' % (self.wildcard_uri.scheme, | |
| 206 urllib.quote_plus(str(b.name))) | |
| 207 bucket_uris.append( | |
| 208 boto.storage_uri( | |
| 209 uri_str, debug=self.debug, | |
| 210 bucket_storage_uri_class=self.bucket_storage_uri_class)) | |
| 211 else: | |
| 212 bucket_uris = [self.wildcard_uri.clone_replace_name('')] | |
| 213 | |
| 214 # Now iterate over bucket(s), and handle object wildcarding, if any. | |
| 215 self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_OBJECT_ITERATOR, | |
| 216 self.wildcard_uri, | |
| 217 self.headers) | |
| 218 for bucket_uri in bucket_uris: | |
| 219 if not self.wildcard_uri.object_name: | |
| 220 # Bucket-only URI. | |
| 221 some_matched = True | |
| 222 yield self.__NeededResultType(None, bucket_uri, self.headers) | |
| 223 else: | |
| 224 # URI contains an object name. If there's no wildcard just yield | |
| 225 # the needed URI. | |
| 226 if not ContainsWildcard(self.wildcard_uri.object_name): | |
| 227 some_matched = True | |
| 228 uri_to_yield = bucket_uri.clone_replace_name( | |
| 229 self.wildcard_uri.object_name) | |
| 230 yield self.__NeededResultType(None, uri_to_yield, self.headers) | |
| 231 else: | |
| 232 # Add the input URI's object name part to the bucket we're | |
| 233 # currently listing. For example if the request was to iterate | |
| 234 # gs://*/*.txt, bucket_uris will contain a list of all the user's | |
| 235 # buckets, and for each we'll add *.txt to the end so we iterate | |
| 236 # the matching files from each bucket in turn. | |
| 237 uri_to_list = bucket_uri.clone_replace_name( | |
| 238 self.wildcard_uri.object_name) | |
| 239 # URI contains an object wildcard. | |
| 240 for obj in self.__ListObjsInBucket(uri_to_list): | |
| 241 regex = fnmatch.translate(self.wildcard_uri.object_name) | |
| 242 prog = re.compile(regex) | |
| 243 if prog.match(obj.name): | |
| 244 some_matched = True | |
| 245 expanded_uri = uri_to_list.clone_replace_name(obj.name) | |
| 246 yield self.__NeededResultType(obj, expanded_uri, self.headers) | |
| 247 | |
| 248 if not some_matched: | |
| 249 raise WildcardException('No matches for "%s"' % self.wildcard_uri) | |
| 250 | |
| 251 def __ListObjsInBucket(self, uri): | |
| 252 """Helper function to get a list of objects in a bucket. | |
| 253 | |
| 254 This function does not provide the complete wildcard match; instead | |
| 255 it uses the server request prefix (if applicable) to reduce server | |
| 256 and network load and returns the underlying boto bucket iterator, | |
| 257 against which remaining wildcard filtering must be applied by the | |
| 258 caller. For example, for StorageUri('gs://bucket/abc*xyz') this | |
| 259 method returns the iterator from doing a prefix='abc' bucket GET | |
| 260 request; and subsequently a regex needs to be applied to subset the | |
| 261 'abc'-prefix matches down to the subset matching 'abc*xyz'. | |
| 262 | |
| 263 Args: | |
| 264 uri: StorageUri to list. | |
| 265 | |
| 266 Returns: | |
| 267 An instance of a boto.s3.BucketListResultSet that handles paging, etc. | |
| 268 """ | |
| 269 | |
| 270 # Generate a request prefix if the object name part of the | |
| 271 # wildcard starts with a non-regex string (e.g., that's true for | |
| 272 # 'gs://bucket/abc*xyz'). | |
| 273 match = WILDCARD_REGEX.search(uri.object_name) | |
| 274 if match and match.start() > 0: | |
| 275 # Glob occurs at beginning of object name, so construct a prefix | |
| 276 # string to send to server. | |
| 277 prefix = uri.object_name[:match.start()] | |
| 278 else: | |
| 279 prefix = None | |
| 280 return uri.get_bucket(validate=False, headers=self.headers).list( | |
| 281 prefix=prefix, headers=self.headers) | |
| 282 | |
| 283 | |
| 284 class FileWildcardIterator(WildcardIterator): | |
| 285 """WildcardIterator subclass for files and directories. | |
| 286 | |
| 287 If you use recursive wildcards ('**') only a single such wildcard is | |
| 288 supported. For example you could use the wildcard '**/*.txt' to list all .txt | |
| 289 files in any subdirectory of the current directory, but you couldn't use a | |
| 290 wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt | |
| 291 files in any subdirectory named 'abc'). | |
| 292 """ | |
| 293 | |
| 294 def __init__(self, wildcard_uri, result_type, headers=None, debug=0): | |
| 295 """Instantiate an iterator over keys matching given wildcard URI. | |
| 296 | |
| 297 Args: | |
| 298 wildcard_uri: StorageUri that contains the wildcard to iterate. | |
| 299 result_type: ResultType object specifying what to iterate. | |
| 300 headers: dictionary containing optional HTTP headers to pass to boto. | |
| 301 debug: debug level to pass in to boto connection (range 0..3). | |
| 302 | |
| 303 Raises: | |
| 304 WildcardException: for invalid result_type. | |
| 305 """ | |
| 306 self.wildcard_uri = wildcard_uri | |
| 307 self.result_type = result_type | |
| 308 if result_type != ResultType.KEYS and result_type != ResultType.URIS: | |
| 309 raise WildcardException('Invalid ResultType (%s)' % result_type) | |
| 310 self.headers = headers | |
| 311 self.debug = debug | |
| 312 | |
| 313 def __iter__(self): | |
| 314 wildcard = self.wildcard_uri.object_name | |
| 315 match = re.search('\*\*', wildcard) | |
| 316 if match: | |
| 317 # Recursive wildcarding request ('.../**/...'). | |
| 318 # Example input: wildcard = '/tmp/tmp2pQJAX/**/*' | |
| 319 base_dir = wildcard[:match.start()-1] | |
| 320 remaining_wildcard = wildcard[match.start()+2:] | |
| 321 # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and | |
| 322 # remaining_wildcard = '/*' | |
| 323 if remaining_wildcard.startswith('*'): | |
| 324 raise WildcardException('Invalid wildcard with more than 2 consecutive ' | |
| 325 '*s (%s)' % wildcard) | |
| 326 # If there was no remaining wildcard past the recursive wildcard, | |
| 327 # treat it as if it were a '*'. For example, file://tmp/** is equivalent | |
| 328 # to file://tmp/**/* | |
| 329 if not remaining_wildcard: | |
| 330 remaining_wildcard = '*' | |
| 331 # Skip slash(es). | |
| 332 remaining_wildcard = remaining_wildcard.lstrip('/') | |
| 333 filepaths = [] | |
| 334 for dirpath, unused_dirnames, filenames in os.walk(base_dir): | |
| 335 filepaths.extend( | |
| 336 os.path.join(dirpath, f) for f in fnmatch.filter(filenames, | |
| 337 remaining_wildcard) | |
| 338 ) | |
| 339 else: | |
| 340 # Not a recursive wildcarding request. | |
| 341 filepaths = glob.glob(wildcard) | |
| 342 for filepath in filepaths: | |
| 343 expanded_uri = self.wildcard_uri.clone_replace_name(filepath) | |
| 344 yield expanded_uri | |
| 345 | |
| 346 | |
| 347 class WildcardException(StandardError): | |
| 348 """Exception thrown for invalid wildcard URIs.""" | |
| 349 | |
| 350 def __init__(self, reason): | |
| 351 StandardError.__init__(self) | |
| 352 self.reason = reason | |
| 353 | |
| 354 def __repr__(self): | |
| 355 return 'WildcardException: %s' % self.reason | |
| 356 | |
| 357 def __str__(self): | |
| 358 return 'WildcardException: %s' % self.reason | |
| 359 | |
| 360 | |
| 361 def wildcard_iterator(uri_or_str, proj_id_handler, | |
| 362 result_type=ResultType.URIS, | |
| 363 bucket_storage_uri_class=BucketStorageUri, | |
| 364 headers=None, debug=0): | |
| 365 """Instantiate a WildCardIterator for the given StorageUri. | |
| 366 | |
| 367 Args: | |
| 368 uri_or_str: StorageUri or URI string naming wildcard objects to iterate. | |
| 369 proj_id_handler: ProjectIdHandler to use for current command. | |
| 370 result_type: ResultType object specifying what to iterate. | |
| 371 bucket_storage_uri_class: BucketStorageUri interface. | |
| 372 Settable for testing/mocking. | |
| 373 headers: dictionary containing optional HTTP headers to pass to boto. | |
| 374 debug: debug level to pass in to boto connection (range 0..3). | |
| 375 | |
| 376 Returns: | |
| 377 A WildcardIterator that handles the requested iteration. | |
| 378 | |
| 379 Raises: | |
| 380 WildcardException: if invalid result_type. | |
| 381 """ | |
| 382 | |
| 383 if isinstance(uri_or_str, basestring): | |
| 384 # Disable enforce_bucket_naming, to allow bucket names containing | |
| 385 # wildcard chars. | |
| 386 uri = boto.storage_uri( | |
| 387 uri_or_str, debug=debug, validate=False, | |
| 388 bucket_storage_uri_class=bucket_storage_uri_class) | |
| 389 else: | |
| 390 uri = uri_or_str | |
| 391 | |
| 392 if uri.is_cloud_uri(): | |
| 393 return CloudWildcardIterator(uri, proj_id_handler, result_type, | |
| 394 bucket_storage_uri_class, headers, debug) | |
| 395 elif uri.is_file_uri(): | |
| 396 return FileWildcardIterator(uri, result_type, headers=headers, debug=debug) | |
| 397 else: | |
| 398 raise WildcardException('Unexpected type of StorageUri (%s)' % uri) | |
| 399 | |
| 400 | |
| 401 def ContainsWildcard(uri_str): | |
| 402 """Checks whether given URI contains a wildcard. | |
| 403 | |
| 404 Args: | |
| 405 uri_str: string to check. | |
| 406 | |
| 407 Returns: | |
| 408 True or False. | |
| 409 """ | |
| 410 | |
| 411 return WILDCARD_REGEX.search(uri_str) is not None | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10199002:
Upgrade gsutil to 3.4 (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart