| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # | |
| 3 # Copyright 2010 Google Inc. | |
| 4 # | |
| 5 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 6 # copy of this software and associated documentation files (the | |
| 7 # "Software"), to deal in the Software without restriction, including | |
| 8 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 9 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 10 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 11 # lowing conditions: | |
| 12 # | |
| 13 # The above copyright notice and this permission notice shall be included | |
| 14 # in all copies or substantial portions of the Software. | |
| 15 # | |
| 16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 22 # IN THE SOFTWARE. | |
| 23 | |
| 24 """Unit tests for gslib wildcard_iterator""" | |
| 25 | |
| 26 import os | |
| 27 import shutil | |
| 28 import sys | |
| 29 import tempfile | |
| 30 import time | |
| 31 import unittest | |
| 32 | |
| 33 # Put local libs at front of path so tests will run latest lib code rather | |
| 34 # than whatever code is found on user's PYTHONPATH. | |
| 35 sys.path.insert(0, '.') | |
| 36 sys.path.insert(0, 'boto') | |
| 37 from boto import InvalidUriError | |
| 38 from gslib import test_util | |
| 39 from gslib import wildcard_iterator | |
| 40 from wildcard_iterator import ResultType | |
| 41 | |
| 42 | |
| 43 class CloudWildcardIteratorTests(unittest.TestCase): | |
| 44 """CloudWildcardIterator test suite""" | |
| 45 | |
| 46 def GetSuiteDescription(self): | |
| 47 return 'CloudWildcardIterator test suite' | |
| 48 | |
| 49 @classmethod | |
| 50 def SetUpClass(cls): | |
| 51 """Creates 2 mock buckets, each containing 3 objects""" | |
| 52 | |
| 53 cls.base_uri_str = 'gs://gslib_test_%d' % int(time.time()) | |
| 54 cls.test_bucket0_uri, cls.test_bucket0_obj_uri_strs = ( | |
| 55 cls.__SetUpOneMockBucket(0) | |
| 56 ) | |
| 57 cls.test_bucket1_uri, cls.test_bucket1_obj_uri_strs = ( | |
| 58 cls.__SetUpOneMockBucket(1) | |
| 59 ) | |
| 60 cls.created_test_data = True | |
| 61 | |
| 62 @classmethod | |
| 63 def __SetUpOneMockBucket(cls, bucket_num): | |
| 64 """Creates a mock bucket containing 3 objects. | |
| 65 | |
| 66 Args: | |
| 67 bucket_num: number for building bucket name. | |
| 68 | |
| 69 Returns: | |
| 70 tuple: (bucket name, set of object URI strings) | |
| 71 """ | |
| 72 | |
| 73 bucket_uri = test_util.test_storage_uri( | |
| 74 '%s_%s' % (cls.base_uri_str, bucket_num)) | |
| 75 bucket_uri.create_bucket() | |
| 76 obj_uri_strs = set() | |
| 77 for obj_name in ['abcd', 'abdd', 'ade$']: | |
| 78 obj_uri = test_util.test_storage_uri('%s%s' % (bucket_uri, obj_name)) | |
| 79 key = obj_uri.new_key() | |
| 80 key.set_contents_from_string('') | |
| 81 obj_uri_strs.add(str(obj_uri)) | |
| 82 return (bucket_uri, obj_uri_strs) | |
| 83 | |
| 84 @classmethod | |
| 85 def TearDownClass(cls): | |
| 86 """Cleans up bucket and objects created by SetUpClass""" | |
| 87 | |
| 88 if hasattr(cls, 'created_test_data'): | |
| 89 for test_obj_uri_str in cls.test_bucket0_obj_uri_strs: | |
| 90 test_util.test_storage_uri(test_obj_uri_str).delete_key() | |
| 91 for test_obj_uri_str in cls.test_bucket1_obj_uri_strs: | |
| 92 test_util.test_storage_uri(test_obj_uri_str).delete_key() | |
| 93 cls.test_bucket0_uri.delete_bucket() | |
| 94 cls.test_bucket1_uri.delete_bucket() | |
| 95 | |
| 96 def TestNoOpObjectIterator(self): | |
| 97 """Tests that bucket-only URI iterates just that one URI""" | |
| 98 | |
| 99 results = list(test_util.test_wildcard_iterator(self.test_bucket0_uri, | |
| 100 ResultType.URIS)) | |
| 101 self.assertEqual(1, len(results)) | |
| 102 self.assertEqual(str(self.test_bucket0_uri), str(results[0])) | |
| 103 | |
| 104 def TestMatchingAllObjects(self): | |
| 105 """Tests matching all objects, based on wildcard""" | |
| 106 | |
| 107 actual_obj_uri_strs = set( | |
| 108 str(u) for u in test_util.test_wildcard_iterator( | |
| 109 self.test_bucket0_uri.clone_replace_name('*'), ResultType.URIS)) | |
| 110 self.assertEqual(self.test_bucket0_obj_uri_strs, actual_obj_uri_strs) | |
| 111 | |
| 112 def TestMatchingObjectSubset(self): | |
| 113 """Tests matching a subset of objects, based on wildcard""" | |
| 114 | |
| 115 exp_obj_uri_strs = set( | |
| 116 [str(self.test_bucket0_uri.clone_replace_name('abcd')), | |
| 117 str(self.test_bucket0_uri.clone_replace_name('abdd'))]) | |
| 118 actual_obj_uri_strs = set( | |
| 119 str(u) for u in test_util.test_wildcard_iterator( | |
| 120 self.test_bucket0_uri.clone_replace_name('ab??'), ResultType.URIS)) | |
| 121 self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) | |
| 122 | |
| 123 def TestMatchingNonWildcardedUri(self): | |
| 124 """Tests matching a single named object""" | |
| 125 | |
| 126 exp_obj_uri_strs = set([str(self.test_bucket0_uri.clone_replace_name('abcd') | |
| 127 )]) | |
| 128 actual_obj_uri_strs = set( | |
| 129 str(u) for u in test_util.test_wildcard_iterator( | |
| 130 self.test_bucket0_uri.clone_replace_name('abcd'), ResultType.URIS)) | |
| 131 self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) | |
| 132 | |
| 133 def TestWildcardedObjectUriWithVsWithoutPrefix(self): | |
| 134 """Tests that wildcarding w/ and w/o server prefix get same result""" | |
| 135 | |
| 136 with_prefix_uri_strs = set( | |
| 137 str(u) for u in test_util.test_wildcard_iterator( | |
| 138 self.test_bucket0_uri.clone_replace_name('abcd'), ResultType.URIS)) | |
| 139 # By including a wildcard at the start of the string no prefix can be | |
| 140 # used in server request. | |
| 141 no_prefix_uri_strs = set( | |
| 142 str(u) for u in test_util.test_wildcard_iterator( | |
| 143 self.test_bucket0_uri.clone_replace_name('?bcd'), ResultType.URIS)) | |
| 144 self.assertEqual(with_prefix_uri_strs, no_prefix_uri_strs) | |
| 145 | |
| 146 def TestNoMatchingWildcardedObjectUri(self): | |
| 147 """Tests that we raise an exception for non-matching wildcarded URI""" | |
| 148 | |
| 149 try: | |
| 150 for unused_ in test_util.test_wildcard_iterator( | |
| 151 self.test_bucket0_uri.clone_replace_name('*x0'), ResultType.URIS): | |
| 152 self.fail('Expected WildcardException not raised.') | |
| 153 except wildcard_iterator.WildcardException, e: | |
| 154 # Expected behavior. | |
| 155 self.assertTrue(str(e).find('No matches') != -1) | |
| 156 | |
| 157 def TestWildcardedInvalidObjectUri(self): | |
| 158 """Tests that we raise an exception for wildcarded invalid URI""" | |
| 159 | |
| 160 try: | |
| 161 for unused_ in test_util.test_wildcard_iterator( | |
| 162 'badscheme://asdf', ResultType.URIS): | |
| 163 self.assertFalse('Expected InvalidUriError not raised.') | |
| 164 except InvalidUriError, e: | |
| 165 # Expected behavior. | |
| 166 self.assertTrue(e.message.find('Unrecognized scheme') != -1) | |
| 167 | |
| 168 def TestWildcardedInvalidResultType(self): | |
| 169 """Tests that we raise an exception for wildcard with invalid ResultType""" | |
| 170 | |
| 171 try: | |
| 172 test_util.test_wildcard_iterator('gs://asdf/*', 'invalid') | |
| 173 self.fail('Expected WildcardException not raised.') | |
| 174 except wildcard_iterator.WildcardException, e: | |
| 175 # Expected behavior. | |
| 176 self.assertTrue(str(e).find('Invalid ResultType') != -1) | |
| 177 | |
| 178 def TestSingleMatchWildcardedBucketUri(self): | |
| 179 """Tests matching a single bucket based on a wildcarded bucket URI""" | |
| 180 | |
| 181 exp_obj_uri_strs = set(['%s_1/' % self.base_uri_str]) | |
| 182 actual_obj_uri_strs = set( | |
| 183 str(u) for u in test_util.test_wildcard_iterator('%s*1' % | |
| 184 self.base_uri_str, | |
| 185 ResultType.URIS)) | |
| 186 self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) | |
| 187 | |
| 188 def TestMultiMatchWildcardedBucketUri(self): | |
| 189 """Tests matching a multiple buckets based on a wildcarded bucket URI""" | |
| 190 | |
| 191 exp_obj_uri_strs = set(['%s_%s/' % | |
| 192 (self.base_uri_str, i) for i in range(2)]) | |
| 193 actual_obj_uri_strs = set( | |
| 194 str(u) for u in test_util.test_wildcard_iterator('%s*' % | |
| 195 self.base_uri_str, | |
| 196 ResultType.URIS)) | |
| 197 self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) | |
| 198 | |
| 199 def TestMultiLevelWildcardUri(self): | |
| 200 """Tests matching with both bucket and object wildcards""" | |
| 201 | |
| 202 exp_obj_uri_strs = set([str(self.test_bucket0_uri.clone_replace_name('abcd' | |
| 203 ))]) | |
| 204 actual_obj_uri_strs = set( | |
| 205 str(u) for u in test_util.test_wildcard_iterator('%s_0*/abc*' % | |
| 206 self.base_uri_str, | |
| 207 ResultType.URIS)) | |
| 208 self.assertEqual(exp_obj_uri_strs, actual_obj_uri_strs) | |
| 209 | |
| 210 def TestBucketOnlyWildcardWithResultTypeKeys(self): | |
| 211 """Tests that bucket-only wildcard with ResultType.KEYS raises exception""" | |
| 212 | |
| 213 try: | |
| 214 for unused_ in test_util.test_wildcard_iterator( | |
| 215 '%s*1' % self.base_uri_str, ResultType.KEYS): | |
| 216 self.fail('Expected WildcardException not raised.') | |
| 217 except wildcard_iterator.WildcardException, e: | |
| 218 # Expected behavior. | |
| 219 self.assertTrue(str(e).find('with ResultType.KEYS iteration') != -1) | |
| 220 | |
| 221 | |
| 222 class FileIteratorTests(unittest.TestCase): | |
| 223 """FileWildcardIterator test suite""" | |
| 224 | |
| 225 def GetSuiteDescription(self): | |
| 226 return 'FileWildcardIterator test suite' | |
| 227 | |
| 228 @classmethod | |
| 229 def SetUpClass(cls): | |
| 230 """Creates a test dir containing 3 files and one nested subdirectory + file. | |
| 231 """ | |
| 232 | |
| 233 # Create the test directories. | |
| 234 cls.test_dir = tempfile.mkdtemp() | |
| 235 nested_subdir = '%s%sdir1%sdir2' % (cls.test_dir, os.sep, os.sep) | |
| 236 os.makedirs(nested_subdir) | |
| 237 | |
| 238 # Create the test files. | |
| 239 immed_child_filenames = ['abcd', 'abdd', 'ade$', 'dir1'] | |
| 240 immed_child_filepaths = ['%s%s%s' % (cls.test_dir, os.sep, f) | |
| 241 for f in immed_child_filenames] | |
| 242 filenames = ['abcd', 'abdd', 'ade$', 'dir1%sdir2%szzz' % (os.sep, os.sep)] | |
| 243 filepaths = ['%s%s%s' % (cls.test_dir, os.sep, f) for f in filenames] | |
| 244 for filepath in filepaths: | |
| 245 open(filepath, 'w') | |
| 246 | |
| 247 # Set up global test variables. | |
| 248 cls.immed_child_uri_strs = set( | |
| 249 os.path.join('file://%s' % f) for f in immed_child_filepaths | |
| 250 ) | |
| 251 | |
| 252 cls.all_file_uri_strs = set( | |
| 253 [('file://%s' % o) for o in filepaths] | |
| 254 ) | |
| 255 | |
| 256 cls.all_uri_strs = set( | |
| 257 ['file://%s' % nested_subdir] | |
| 258 ).union(cls.all_file_uri_strs) | |
| 259 | |
| 260 @classmethod | |
| 261 def TearDownClass(cls): | |
| 262 """Cleans up test dir and file created by SetUpClass""" | |
| 263 | |
| 264 if hasattr(cls, 'test_dir'): | |
| 265 shutil.rmtree(cls.test_dir) | |
| 266 | |
| 267 def TestNoOpDirectoryIterator(self): | |
| 268 """Tests that directory-only URI iterates just that one URI""" | |
| 269 | |
| 270 results = list(test_util.test_wildcard_iterator('file:///tmp/', | |
| 271 ResultType.URIS)) | |
| 272 self.assertEqual(1, len(results)) | |
| 273 self.assertEqual('file:///tmp/', str(results[0])) | |
| 274 | |
| 275 def TestMatchingAllFiles(self): | |
| 276 """Tests matching all files, based on wildcard""" | |
| 277 | |
| 278 uri = test_util.test_storage_uri('file://%s/*' % self.test_dir) | |
| 279 actual_uri_strs = set(str(u) for u in | |
| 280 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 281 ) | |
| 282 self.assertEqual(self.immed_child_uri_strs, actual_uri_strs) | |
| 283 | |
| 284 def TestMatchingFileSubset(self): | |
| 285 """Tests matching a subset of files, based on wildcard""" | |
| 286 | |
| 287 exp_uri_strs = set( | |
| 288 ['file://%s/abcd' % self.test_dir, 'file://%s/abdd' % self.test_dir] | |
| 289 ) | |
| 290 uri = test_util.test_storage_uri('file://%s/ab??' % self.test_dir) | |
| 291 actual_uri_strs = set(str(u) for u in | |
| 292 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 293 ) | |
| 294 self.assertEqual(exp_uri_strs, actual_uri_strs) | |
| 295 | |
| 296 def TestMatchingNonWildcardedUri(self): | |
| 297 """Tests matching a single named file""" | |
| 298 | |
| 299 exp_uri_strs = set(['file://%s/abcd' % self.test_dir]) | |
| 300 uri = test_util.test_storage_uri('file://%s/abcd' % self.test_dir) | |
| 301 actual_uri_strs = set(str(u) for u in | |
| 302 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 303 ) | |
| 304 self.assertEqual(exp_uri_strs, actual_uri_strs) | |
| 305 | |
| 306 def TestMatchingFilesIgnoringOtherRegexChars(self): | |
| 307 """Tests ignoring non-wildcard regex chars (e.g., ^ and $)""" | |
| 308 | |
| 309 exp_uri_strs = set(['file://%s/ade$' % self.test_dir]) | |
| 310 uri = test_util.test_storage_uri('file://%s/ad*$' % self.test_dir) | |
| 311 actual_uri_strs = set(str(u) for u in | |
| 312 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 313 ) | |
| 314 self.assertEqual(exp_uri_strs, actual_uri_strs) | |
| 315 | |
| 316 def TestRecursiveDirectoryOnlyWildcarding(self): | |
| 317 """Tests recusive expansion of directory-only '**' wildcard""" | |
| 318 | |
| 319 uri = test_util.test_storage_uri('file://%s/**' % self.test_dir) | |
| 320 actual_uri_strs = set(str(u) for u in | |
| 321 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 322 ) | |
| 323 self.assertEqual(self.all_file_uri_strs, actual_uri_strs) | |
| 324 | |
| 325 def TestRecursiveDirectoryPlusFileWildcarding(self): | |
| 326 """Tests recusive expansion of '**' directory plus '*' wildcard""" | |
| 327 | |
| 328 uri = test_util.test_storage_uri('file://%s/**/*' % self.test_dir) | |
| 329 actual_uri_strs = set(str(u) for u in | |
| 330 test_util.test_wildcard_iterator(uri, ResultType.KEYS) | |
| 331 ) | |
| 332 self.assertEqual(self.all_file_uri_strs, actual_uri_strs) | |
| 333 | |
| 334 def TestInvalidRecursiveDirectoryWildcard(self): | |
| 335 """Tests that wildcard containing '***' raises exception""" | |
| 336 | |
| 337 try: | |
| 338 uri = test_util.test_storage_uri('file://%s/***/abcd' % self.test_dir) | |
| 339 for unused_ in test_util.test_wildcard_iterator(uri, ResultType.KEYS): | |
| 340 self.fail('Expected WildcardException not raised.') | |
| 341 except wildcard_iterator.WildcardException, e: | |
| 342 # Expected behavior. | |
| 343 self.assertTrue(str(e).find('more than 2 consecutive') != -1) | |
| 344 | |
| 345 def TestMissingDir(self): | |
| 346 """Tests that wildcard raises exception when directory doesn't exist""" | |
| 347 | |
| 348 try: | |
| 349 for unused_ in test_util.test_wildcard_iterator('file://no_such_dir/*', | |
| 350 ResultType.KEYS): | |
| 351 self.fail('Expected WildcardException not raised.') | |
| 352 except wildcard_iterator.WildcardException, e: | |
| 353 # Expected behavior. | |
| 354 self.assertTrue(str(e).find('No matches') != -1) | |
| 355 | |
| 356 def TestExistingDirNoFileMatch(self): | |
| 357 """Tests that wildcard raises exception when there's no match""" | |
| 358 | |
| 359 try: | |
| 360 uri = test_util.test_storage_uri( | |
| 361 'file://%s/non_existent*' % self.test_dir) | |
| 362 for unused_ in test_util.test_wildcard_iterator(uri, ResultType.KEYS): | |
| 363 self.fail('Expected WildcardException not raised.') | |
| 364 except wildcard_iterator.WildcardException, e: | |
| 365 # Expected behavior. | |
| 366 self.assertTrue(str(e).find('No matches') != -1) | |
| 367 | |
| 368 | |
| 369 if __name__ == '__main__': | |
| 370 if sys.version_info[:3] < (2, 5, 1): | |
| 371 sys.exit('These tests must be run on at least Python 2.5.1\n') | |
| 372 test_loader = unittest.TestLoader() | |
| 373 test_loader.testMethodPrefix = 'Test' | |
| 374 for suite in (test_loader.loadTestsFromTestCase(CloudWildcardIteratorTests), | |
| 375 test_loader.loadTestsFromTestCase(FileIteratorTests)): | |
| 376 # Seems like there should be a cleaner way to find the test_class. | |
| 377 test_class = suite.__getattribute__('_tests')[0] | |
| 378 # We call SetUpClass() and TearDownClass() ourselves because we | |
| 379 # don't assume the user has Python 2.7 (which supports classmethods | |
| 380 # that do it, with camelCase versions of these names). | |
| 381 try: | |
| 382 print 'Setting up %s...' % test_class.GetSuiteDescription() | |
| 383 test_class.SetUpClass() | |
| 384 print 'Running %s...' % test_class.GetSuiteDescription() | |
| 385 unittest.TextTestRunner(verbosity=2).run(suite) | |
| 386 finally: | |
| 387 print 'Cleaning up after %s...' % test_class.GetSuiteDescription() | |
| 388 test_class.TearDownClass() | |
| 389 print '' | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10199002:
Upgrade gsutil to 3.4 (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart