Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(134)

Side by Side Diff: third_party/gsutil/20110627/gslib/test_command.py

Issue 10199002: Upgrade gsutil to 3.4 (Closed) Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart
Patch Set: Addressed comments Created 8 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 #
3 # Copyright 2010 Google Inc.
4 #
5 # Permission is hereby granted, free of charge, to any person obtaining a
6 # copy of this software and associated documentation files (the
7 # "Software"), to deal in the Software without restriction, including
8 # without limitation the rights to use, copy, modify, merge, publish, dis-
9 # tribute, sublicense, and/or sell copies of the Software, and to permit
10 # persons to whom the Software is furnished to do so, subject to the fol-
11 # lowing conditions:
12 #
13 # The above copyright notice and this permission notice shall be included
14 # in all copies or substantial portions of the Software.
15 #
16 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
18 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
19 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 # IN THE SOFTWARE.
23
24 """Unit tests for gsutil command methods"""
25
26 import os
27 import shutil
28 import sys
29 import tempfile
30 import time
31 import unittest
32
33 # Put local libs at front of path so tests will run latest lib code rather
34 # than whatever code is found on user's PYTHONPATH.
35 sys.path.insert(0, '.')
36 sys.path.insert(0, 'boto')
37 import boto
38 from tests.s3 import mock_storage_service
39 from gslib import test_util
40 from gslib.command import Command
41 from gslib.exception import CommandException
42 import wildcard_iterator
43
44 command_inst = Command(
45 '.', '.', '', '',
46 bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri)
47
48 # Constant option for specifying a recursive copy.
49 RECURSIVE = [('-r', '')]
50
51
52 class GsutilCpTests(unittest.TestCase):
53 """gsutil command method test suite"""
54
55 def GetSuiteDescription(self):
56 return 'gsutil command method test suite'
57
58 @classmethod
59 def tearDown(cls):
60 """Deletes any objects or files created by last test run"""
61 try:
62 for key_uri in test_util.test_wildcard_iterator('%s*' %
63 cls.dst_bucket_uri):
64 key_uri.delete_key()
65 # For some reason trying to catch except
66 # wildcard_iterator.WildcardException doesn't work here.
67 except Exception:
68 # Ignore cleanup failures.
69 pass
70 # Recursively delete dst dir and then re-create it, so in effect we
71 # remove all dirs and files under that directory.
72 shutil.rmtree(cls.dst_dir_root)
73 os.mkdir(cls.dst_dir_root)
74
75 @classmethod
76 def SrcFile(cls, fname):
77 """Returns path for given test src file"""
78 for path in cls.all_src_file_paths:
79 if path.find(fname) != -1:
80 return path
81 raise Exception('SrcFile(%s): no match' % fname)
82
83 @classmethod
84 def CreateEmptyObject(cls, uri):
85 """Creates an empty object for the given StorageUri"""
86 key = uri.new_key()
87 key.set_contents_from_string('')
88
89 @classmethod
90 def SetUpClass(cls):
91 """Initializes test suite.
92
93 Creates a source bucket containing 3 objects;
94 a source directory containing a subdirectory and file;
95 and a destination bucket and directory.
96 """
97 cls.uri_base_str = 'gs://gsutil_test_%s' % int(time.time())
98 # Use a designated tmpdir prefix to make it easy to find the end of
99 # the tmp path.
100 cls.tmpdir_prefix = 'tmp_gstest'
101
102 # Create the test buckets.
103 cls.src_bucket_uri = test_util.test_storage_uri('%s_src' % cls.uri_base_str)
104 cls.dst_bucket_uri = test_util.test_storage_uri('%s_dst' % cls.uri_base_str)
105 cls.src_bucket_uri.create_bucket()
106 cls.dst_bucket_uri.create_bucket()
107
108 # Create the test objects in src bucket.
109 cls.all_src_obj_uris = []
110 for i in range(3):
111 obj_uri = test_util.test_storage_uri('%sobj%d' % (cls.src_bucket_uri, i))
112 cls.CreateEmptyObject(obj_uri)
113 cls.all_src_obj_uris.append(obj_uri)
114
115 # Create the test directories.
116 cls.src_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
117 os.sep)
118 nested_subdir = '%sdir0%sdir1' % (cls.src_dir_root, os.sep)
119 os.makedirs(nested_subdir)
120 cls.dst_dir_root = '%s%s' % (tempfile.mkdtemp(prefix=cls.tmpdir_prefix),
121 os.sep)
122
123 # Create the test files in src directory.
124 cls.all_src_file_paths = []
125 cls.nested_child_file_paths = ['f0', 'f1', 'f2.txt', 'dir0/dir1/nested']
126 cls.non_nested_file_names = ['f0', 'f1', 'f2.txt']
127 file_names = ['f0', 'f1', 'f2.txt', 'dir0%sdir1%snested' % (os.sep, os.sep)]
128 file_paths = ['%s%s' % (cls.src_dir_root, f) for f in file_names]
129 for file_path in file_paths:
130 open(file_path, 'w')
131 cls.all_src_file_paths.append(file_path)
132
133 cls.created_test_data = True
134
135 @classmethod
136 def TearDownClass(cls):
137 """Cleans up buckets and directories created by SetUpClass"""
138
139 if not hasattr(cls, 'created_test_data'):
140 return
141 # Call cls.tearDown() in case the tests got interrupted, to ensure
142 # dst objects and files get deleted.
143 cls.tearDown()
144 # Now delete src objects and files, and all buckets and dirs.
145 try:
146 for key_uri in test_util.test_wildcard_iterator('%s*' %
147 cls.src_bucket_uri):
148 key_uri.delete_key()
149 except wildcard_iterator.WildcardException:
150 # Ignore cleanup failures.
151 pass
152 try:
153 for key_uri in test_util.test_wildcard_iterator('%s**' %
154 cls.src_dir_root):
155 key_uri.delete_key()
156 except wildcard_iterator.WildcardException:
157 # Ignore cleanup failures.
158 pass
159 cls.src_bucket_uri.delete_bucket()
160 cls.dst_bucket_uri.delete_bucket()
161 shutil.rmtree(cls.src_dir_root)
162 shutil.rmtree(cls.dst_dir_root)
163
164 def TestCopyingTopLevelFileToBucket(self):
165 """Tests copying one top-level file to a bucket"""
166 src_file = self.SrcFile('f0')
167 command_inst.CopyObjsCommand([src_file, self.dst_bucket_uri.uri],
168 headers={})
169 actual = list(test_util.test_wildcard_iterator('%s*' %
170 self.dst_bucket_uri.uri))
171 self.assertEqual(1, len(actual))
172 self.assertEqual('f0', actual[0].object_name)
173
174 def TestCopyingNestedFileToBucket(self):
175 """Tests copying one nested file to a bucket"""
176 src_file = self.SrcFile('nested')
177 command_inst.CopyObjsCommand([src_file, self.dst_bucket_uri.uri],
178 headers={})
179 actual = list(test_util.test_wildcard_iterator('%s*' %
180 self.dst_bucket_uri.uri))
181 self.assertEqual(1, len(actual))
182 # File should be final comp ('nested'), w/o nested path (dir0/...).
183 self.assertEqual('nested', actual[0].object_name)
184
185 def TestCopyingDirToBucket(self):
186 """Tests copying top-level directory to a bucket"""
187 command_inst.CopyObjsCommand([self.src_dir_root, self.dst_bucket_uri.uri],
188 RECURSIVE, headers={})
189 actual = set(str(u) for u in
190 test_util.test_wildcard_iterator('%s*' %
191 self.dst_bucket_uri.uri))
192 expected = set()
193 for file_path in self.all_src_file_paths:
194 start_tmp_pos = file_path.find(self.tmpdir_prefix)
195 file_path_sans_top_tmp_dir = file_path[start_tmp_pos:]
196 expected.add('%s%s' % (self.dst_bucket_uri.uri,
197 file_path_sans_top_tmp_dir))
198 self.assertEqual(expected, actual)
199
200 def TestCopyingDirContainingOneFileToBucket(self):
201 """Tests copying a directory containing 1 file to a bucket
202
203 We test this case to ensure that correct bucket handling isn't dependent
204 on the copy being treated as a multi-source copy.
205 """
206 command_inst.CopyObjsCommand(['%sdir0%sdir1' % (self.src_dir_root, os.sep),
207 self.dst_bucket_uri.uri],
208 RECURSIVE, headers={})
209 actual = list((str(u) for u in
210 test_util.test_wildcard_iterator('%s*' %
211 self.dst_bucket_uri.uri)))
212 self.assertEqual(1, len(actual))
213 self.assertEqual('%sdir1%snested' % (self.dst_bucket_uri.uri, os.sep),
214 actual[0])
215
216 def TestCopyingBucketToDir(self):
217 """Tests copying from a bucket to a directory"""
218 command_inst.CopyObjsCommand([self.src_bucket_uri.uri,
219 self.dst_dir_root],
220 RECURSIVE, headers={})
221 actual = set(
222 str(u) for u in test_util.test_wildcard_iterator('%s**' %
223 self.dst_dir_root))
224 expected = set()
225 for uri in self.all_src_obj_uris:
226 expected.add('file://%s%s/%s' % (self.dst_dir_root, uri.bucket_name,
227 uri.object_name))
228 self.assertEqual(expected, actual)
229
230 def TestCopyingBucketToBucket(self):
231 """Tests copying from a bucket-only URI to a bucket"""
232 command_inst.CopyObjsCommand([self.src_bucket_uri.uri,
233 self.dst_bucket_uri.uri],
234 RECURSIVE, headers={})
235 actual = set(str(u) for u in
236 test_util.test_wildcard_iterator('%s*' %
237 self.dst_bucket_uri.uri))
238 expected = set()
239 for uri in self.all_src_obj_uris:
240 expected.add('%s%s' % (self.dst_bucket_uri.uri, uri.object_name))
241 self.assertEqual(expected, actual)
242
243 def TestCopyingDirToDir(self):
244 """Tests copying from a directory to a directory"""
245 command_inst.CopyObjsCommand([self.src_dir_root, self.dst_dir_root],
246 RECURSIVE, headers={})
247 actual = set(
248 str(u) for u in test_util.test_wildcard_iterator('%s**' %
249 self.dst_dir_root))
250 expected = set()
251 for file_path in self.all_src_file_paths:
252 start_tmp_pos = file_path.find(self.tmpdir_prefix)
253 file_path_sans_top_tmp_dir = file_path[start_tmp_pos:]
254 expected.add('file://%s%s' % (self.dst_dir_root,
255 file_path_sans_top_tmp_dir))
256 self.assertEqual(expected, actual)
257
258 def TestCopyingFilesAndDirNonRecursive(self):
259 """Tests copying containing files and a directory without -r"""
260 command_inst.CopyObjsCommand(['%s*' % self.src_dir_root, self.dst_dir_root],
261 headers={})
262 actual = set(
263 str(u) for u in test_util.test_wildcard_iterator('%s**' %
264 self.dst_dir_root))
265 expected = set(['file://%s%s' % (self.dst_dir_root, f)
266 for f in self.non_nested_file_names])
267 self.assertEqual(expected, actual)
268
269 def TestCopyingFileToDir(self):
270 """Tests copying one file to a directory"""
271 src_file = self.SrcFile('nested')
272 command_inst.CopyObjsCommand([src_file, self.dst_dir_root], headers={})
273 actual = list(test_util.test_wildcard_iterator('%s*' % self.dst_dir_root))
274 self.assertEqual(1, len(actual))
275 self.assertEqual('file://%s%s' % (self.dst_dir_root, 'nested'),
276 actual[0].uri)
277
278 def TestCopyingCompressedFileToBucket(self):
279 """Tests copying one file with compression to a bucket"""
280 src_file = self.SrcFile('f2.txt')
281 command_inst.CopyObjsCommand([src_file, self.dst_bucket_uri.uri],
282 sub_opts=[('-z', 'txt')], headers={})
283 actual = list(
284 str(u) for u in test_util.test_wildcard_iterator(
285 '%s*' % self.dst_bucket_uri.uri))
286 self.assertEqual(1, len(actual))
287 expected_dst_uri = self.dst_bucket_uri.clone_replace_name('f2.txt')
288 self.assertEqual(expected_dst_uri.uri, actual[0])
289 dst_key = expected_dst_uri.get_key()
290 dst_key.open_read()
291 self.assertEqual('gzip', dst_key.content_encoding)
292
293 def TestCopyingObjectToObject(self):
294 """Tests copying an object to an object"""
295 command_inst.CopyObjsCommand(['%sobj1' % self.src_bucket_uri.uri,
296 self.dst_bucket_uri.uri], headers={})
297 actual = list(test_util.test_wildcard_iterator('%s*' %
298 self.dst_bucket_uri.uri))
299 self.assertEqual(1, len(actual))
300 self.assertEqual('obj1', actual[0].object_name)
301
302 def TestCopyingObjsAndFilesToDir(self):
303 """Tests copying objects and files to a directory"""
304 command_inst.CopyObjsCommand(['%s*' % self.src_bucket_uri.uri,
305 '%s*' % self.src_dir_root,
306 self.dst_dir_root],
307 RECURSIVE, headers={})
308 actual = set(str(u) for u in test_util.test_wildcard_iterator(
309 '%s**' % self.dst_dir_root))
310 expected = set()
311 for uri in self.all_src_obj_uris:
312 expected.add('file://%s%s' % (self.dst_dir_root, uri.object_name))
313 for file_path in self.nested_child_file_paths:
314 expected.add('file://%s%s' % (self.dst_dir_root, file_path))
315 self.assertEqual(expected, actual)
316
317 def TestCopyingObjsAndFilesToBucket(self):
318 """Tests copying objects and files to a bucket"""
319 command_inst.CopyObjsCommand(['%s*' % self.src_bucket_uri.uri,
320 '%s*' % self.src_dir_root,
321 self.dst_bucket_uri.uri],
322 RECURSIVE, headers={})
323 actual = set(str(u) for u in
324 test_util.test_wildcard_iterator(
325 '%s*' % self.dst_bucket_uri.uri))
326 expected = set()
327 for uri in self.all_src_obj_uris:
328 expected.add('%s%s' % (self.dst_bucket_uri.uri, uri.object_name))
329 for file_path in self.nested_child_file_paths:
330 expected.add('%s%s' % (self.dst_bucket_uri.uri, file_path))
331 self.assertEqual(expected, actual)
332
333 def TestAttemptDirCopyWithoutRecursion(self):
334 """Tests copying a directory without -r"""
335 try:
336 command_inst.CopyObjsCommand([self.src_dir_root, self.dst_dir_root],
337 headers={})
338 self.fail('Did not get expected CommandException')
339 except CommandException, e:
340 self.assertNotEqual(e.reason.find('Nothing to copy'), -1)
341
342 def TestAttemptCopyingProviderOnlySrc(self):
343 """Attempts to copy a src specified as a provider-only URI"""
344 try:
345 command_inst.CopyObjsCommand(['gs://', self.src_bucket_uri.uri],
346 headers={})
347 self.fail('Did not get expected CommandException')
348 except CommandException, e:
349 self.assertNotEqual(e.reason.find('Provider-only'), -1)
350
351 def TestAttemptCopyingOverlappingSrcDst(self):
352 """Attempts to an object atop itself"""
353 obj_uri = test_util.test_storage_uri('%sobj' % self.dst_bucket_uri)
354 self.CreateEmptyObject(obj_uri)
355 try:
356 command_inst.CopyObjsCommand(['%s*' % self.dst_bucket_uri.uri,
357 self.dst_bucket_uri.uri], headers={})
358 self.fail('Did not get expected CommandException')
359 except CommandException, e:
360 self.assertNotEqual(e.reason.find('are the same object - abort'), -1)
361
362 def TestAttemptCopyingToMultiMatchWildcard(self):
363 """Attempts to copy where dst wildcard matches >1 obj"""
364 try:
365 command_inst.CopyObjsCommand(['%sobj0' % self.src_bucket_uri.uri,
366 '%s*' % self.src_bucket_uri.uri],
367 headers={})
368 self.fail('Did not get expected CommandException')
369 except CommandException, e:
370 self.assertNotEqual(e.reason.find('matches more than 1'), -1)
371
372 def TestAttemptCopyingMultiObjsToFile(self):
373 """Attempts to copy multiple objects to a file"""
374 # Use src_dir_root so we can point to an existing file for this test.
375 try:
376 command_inst.CopyObjsCommand(['%s*' % self.src_bucket_uri.uri,
377 '%sf0' % self.src_dir_root],
378 RECURSIVE, headers={})
379 self.fail('Did not get expected CommandException')
380 except CommandException, e:
381 self.assertNotEqual(e.reason.find('must name a bucket or '), -1)
382
383 def TestAttemptCopyingWithFileDirConflict(self):
384 """Attempts to copy objects that cause a file/directory conflict"""
385 # Create objects with name conflicts (a/b and a). Use 'dst' bucket because
386 # it gets cleared after each test.
387 self.CreateEmptyObject(test_util.test_storage_uri(
388 '%sa/b' % self.dst_bucket_uri))
389 self.CreateEmptyObject(test_util.test_storage_uri(
390 '%sa' % self.dst_bucket_uri))
391 try:
392 command_inst.CopyObjsCommand([self.dst_bucket_uri.uri, self.dst_dir_root],
393 RECURSIVE, headers={})
394 self.fail('Did not get expected CommandException')
395 except CommandException, e:
396 self.assertNotEqual(e.reason.find(
397 'exists where a directory needs to be created'), -1)
398
399 def TestAttemptCopyingWithDirFileConflict(self):
400 """Attempts to copy objects that cause a directory/file conflict"""
401 # Create subdir in dest dir.
402 os.mkdir('%ssubdir' % self.dst_dir_root)
403 # Create an object that conflicts with this dest subdir. Use 'dst' bucket
404 # because it gets cleared after each test.
405 self.CreateEmptyObject(test_util.test_storage_uri(
406 '%ssubdir' % self.dst_bucket_uri))
407 try:
408 command_inst.CopyObjsCommand([self.dst_bucket_uri.uri, self.dst_dir_root],
409 RECURSIVE, headers={})
410 self.fail('Did not get expected CommandException')
411 except CommandException, e:
412 self.assertNotEqual(e.reason.find(
413 'where the file needs to be created'), -1)
414
415 def TestWildcardMoveWithinBucket(self):
416 """Attempts to move using src wildcard that overlaps dest object.
417
418 We want to ensure that this doesn't stomp the result data. See the
419 comment starting with "Expand wildcards before" in MoveObjsCommand
420 for details.
421 """
422 # Create a single object; use 'dst' bucket because it gets cleared after
423 # each test.
424 self.CreateEmptyObject(
425 test_util.test_storage_uri('%sold' % self.dst_bucket_uri))
426 command_inst.MoveObjsCommand(['%s*' % self.dst_bucket_uri.uri,
427 '%snew' % self.dst_bucket_uri.uri],
428 headers={})
429 actual = list(
430 test_util.test_wildcard_iterator('%s*' % self.dst_bucket_uri.uri))
431 self.assertEqual(1, len(actual))
432 self.assertEqual('new', actual[0].object_name)
433
434 # The remaining tests are pretty minimal - most just ensure a
435 # basic use of each command runs, without checking more detailed
436 # conditions/expectations.
437
438 def TestCatCommmandRuns(self):
439 """Test that the cat command basically runs"""
440 command_inst.CatCommand(['%sobj1' % self.src_bucket_uri.uri], headers={})
441
442 def TestGetAclCommmandRuns(self):
443 """Test that the GetAcl command basically runs"""
444 command_inst.GetAclCommand([self.src_bucket_uri.uri], headers={})
445
446 def TestListCommandRuns(self):
447 """Test that the ListCommand basically runs"""
448 command_inst.ListCommand([self.src_bucket_uri.uri], headers={})
449
450 def TestMakeBucketsCommand(self):
451 """Test MakeBucketsCommand on existing bucket"""
452 try:
453 command_inst.MakeBucketsCommand([self.dst_bucket_uri.uri], headers={})
454 self.fail('Did not get expected StorageCreateError')
455 except boto.exception.StorageCreateError, e:
456 self.assertEqual(e.status, 409)
457
458 def TestRemoveBucketsCommand(self):
459 """Test RemoveBucketsCommand on non-existent bucket"""
460 try:
461 command_inst.RemoveBucketsCommand(
462 ['gs://non_existent_%s' % self.dst_bucket_uri.bucket_name],
463 headers={})
464 self.fail('Did not get expected StorageResponseError')
465 except boto.exception.StorageResponseError, e:
466 self.assertEqual(e.status, 404)
467
468 def TestRemoveObjsCommand(self):
469 """Test RemoveObjsCommand on non-existent object"""
470 try:
471 command_inst.RemoveObjsCommand(['%snon_existent' %
472 self.dst_bucket_uri.uri],
473 headers={})
474 self.fail('Did not get expected WildcardException')
475 # For some reason if we catch this as "WildcardException" it doesn't
476 # work right. Python bug?
477 except Exception, e:
478 self.assertNotEqual(e.reason.find('Not Found'), -1)
479
480 def TestSetAclCommmandRuns(self):
481 """Test that the SetAcl command basically runs"""
482 command_inst.SetAclCommand(['private', self.src_bucket_uri.uri], headers={})
483
484 def TestVerCommmandRuns(self):
485 """Test that the Ver command basically runs"""
486 command_inst.VerCommand([])
487
488 if __name__ == '__main__':
489 if sys.version_info[:3] < (2, 5, 1):
490 sys.exit('These tests must be run on at least Python 2.5.1\n')
491 test_loader = unittest.TestLoader()
492 test_loader.testMethodPrefix = 'Test'
493 suite = test_loader.loadTestsFromTestCase(GsutilCpTests)
494 # Seems like there should be a cleaner way to find the test_class.
495 test_class = suite.__getattribute__('_tests')[0]
496 # We call SetUpClass() and TearDownClass() ourselves because we
497 # don't assume the user has Python 2.7 (which supports classmethods
498 # that do it, with camelCase versions of these names).
499 try:
500 print 'Setting up %s...' % test_class.GetSuiteDescription()
501 test_class.SetUpClass()
502 print 'Running %s...' % test_class.GetSuiteDescription()
503 unittest.TextTestRunner(verbosity=2).run(suite)
504 finally:
505 print 'Cleaning up after %s...' % test_class.GetSuiteDescription()
506 test_class.TearDownClass()
507 print ''
OLDNEW
« no previous file with comments | « third_party/gsutil/20110627/gslib/project_id.py ('k') | third_party/gsutil/20110627/gslib/test_util.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698