| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ | |
| 3 # | |
| 4 # Permission is hereby granted, free of charge, to any person obtaining a | |
| 5 # copy of this software and associated documentation files (the | |
| 6 # "Software"), to deal in the Software without restriction, including | |
| 7 # without limitation the rights to use, copy, modify, merge, publish, dis- | |
| 8 # tribute, sublicense, and/or sell copies of the Software, and to permit | |
| 9 # persons to whom the Software is furnished to do so, subject to the fol- | |
| 10 # lowing conditions: | |
| 11 # | |
| 12 # The above copyright notice and this permission notice shall be included | |
| 13 # in all copies or substantial portions of the Software. | |
| 14 # | |
| 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS | |
| 16 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- | |
| 17 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT | |
| 18 # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, | |
| 19 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | |
| 20 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS | |
| 21 # IN THE SOFTWARE. | |
| 22 # | |
| 23 import getopt, sys, os | |
| 24 import boto | |
| 25 from boto.exception import S3ResponseError | |
| 26 | |
| 27 usage_string = """ | |
| 28 SYNOPSIS | |
| 29 s3put [-a/--access_key <access_key>] [-s/--secret_key <secret_key>] | |
| 30 -b/--bucket <bucket_name> [-c/--callback <num_cb>] | |
| 31 [-d/--debug <debug_level>] [-i/--ignore <ignore_dirs>] | |
| 32 [-n/--no_op] [-p/--prefix <prefix>] [-q/--quiet] | |
| 33 [-g/--grant grant] [-w/--no_overwrite] path | |
| 34 | |
| 35 Where | |
| 36 access_key - Your AWS Access Key ID. If not supplied, boto will | |
| 37 use the value of the environment variable | |
| 38 AWS_ACCESS_KEY_ID | |
| 39 secret_key - Your AWS Secret Access Key. If not supplied, boto | |
| 40 will use the value of the environment variable | |
| 41 AWS_SECRET_ACCESS_KEY | |
| 42 bucket_name - The name of the S3 bucket the file(s) should be | |
| 43 copied to. | |
| 44 path - A path to a directory or file that represents the items | |
| 45 to be uploaded. If the path points to an individual file, | |
| 46 that file will be uploaded to the specified bucket. If the | |
| 47 path points to a directory, s3_it will recursively traverse | |
| 48 the directory and upload all files to the specified bucket. | |
| 49 debug_level - 0 means no debug output (default), 1 means normal | |
| 50 debug output from boto, and 2 means boto debug output | |
| 51 plus request/response output from httplib | |
| 52 ignore_dirs - a comma-separated list of directory names that will | |
| 53 be ignored and not uploaded to S3. | |
| 54 num_cb - The number of progress callbacks to display. The default | |
| 55 is zero which means no callbacks. If you supplied a value | |
| 56 of "-c 10" for example, the progress callback would be | |
| 57 called 10 times for each file transferred. | |
| 58 prefix - A file path prefix that will be stripped from the full | |
| 59 path of the file when determining the key name in S3. | |
| 60 For example, if the full path of a file is: | |
| 61 /home/foo/bar/fie.baz | |
| 62 and the prefix is specified as "-p /home/foo/" the | |
| 63 resulting key name in S3 will be: | |
| 64 /bar/fie.baz | |
| 65 The prefix must end in a trailing separator and if it | |
| 66 does not then one will be added. | |
| 67 grant - A canned ACL policy that will be granted on each file | |
| 68 transferred to S3. The value of provided must be one | |
| 69 of the "canned" ACL policies supported by S3: | |
| 70 private|public-read|public-read-write|authenticated-read | |
| 71 no_overwrite - No files will be overwritten on S3, if the file/key | |
| 72 exists on s3 it will be kept. This is useful for | |
| 73 resuming interrupted transfers. Note this is not a | |
| 74 sync, even if the file has been updated locally if | |
| 75 the key exists on s3 the file on s3 will not be | |
| 76 updated. | |
| 77 | |
| 78 If the -n option is provided, no files will be transferred to S3 but | |
| 79 informational messages will be printed about what would happen. | |
| 80 """ | |
| 81 def usage(): | |
| 82 print usage_string | |
| 83 sys.exit() | |
| 84 | |
| 85 def submit_cb(bytes_so_far, total_bytes): | |
| 86 print '%d bytes transferred / %d bytes total' % (bytes_so_far, total_bytes) | |
| 87 | |
| 88 def get_key_name(fullpath, prefix): | |
| 89 key_name = fullpath[len(prefix):] | |
| 90 l = key_name.split(os.sep) | |
| 91 return '/'.join(l) | |
| 92 | |
| 93 def main(): | |
| 94 try: | |
| 95 opts, args = getopt.getopt(sys.argv[1:], 'a:b:c::d:g:hi:np:qs:vw', | |
| 96 ['access_key', 'bucket', 'callback', 'debug',
'help', 'grant', | |
| 97 'ignore', 'no_op', 'prefix', 'quiet', 'secre
t_key', 'no_overwrite']) | |
| 98 except: | |
| 99 usage() | |
| 100 ignore_dirs = [] | |
| 101 aws_access_key_id = None | |
| 102 aws_secret_access_key = None | |
| 103 bucket_name = '' | |
| 104 total = 0 | |
| 105 debug = 0 | |
| 106 cb = None | |
| 107 num_cb = 0 | |
| 108 quiet = False | |
| 109 no_op = False | |
| 110 prefix = '/' | |
| 111 grant = None | |
| 112 no_overwrite = False | |
| 113 for o, a in opts: | |
| 114 if o in ('-h', '--help'): | |
| 115 usage() | |
| 116 sys.exit() | |
| 117 if o in ('-a', '--access_key'): | |
| 118 aws_access_key_id = a | |
| 119 if o in ('-b', '--bucket'): | |
| 120 bucket_name = a | |
| 121 if o in ('-c', '--callback'): | |
| 122 num_cb = int(a) | |
| 123 cb = submit_cb | |
| 124 if o in ('-d', '--debug'): | |
| 125 debug = int(a) | |
| 126 if o in ('-g', '--grant'): | |
| 127 grant = a | |
| 128 if o in ('-i', '--ignore'): | |
| 129 ignore_dirs = a.split(',') | |
| 130 if o in ('-n', '--no_op'): | |
| 131 no_op = True | |
| 132 if o in ('w', '--no_overwrite'): | |
| 133 no_overwrite = True | |
| 134 if o in ('-p', '--prefix'): | |
| 135 prefix = a | |
| 136 if prefix[-1] != os.sep: | |
| 137 prefix = prefix + os.sep | |
| 138 if o in ('-q', '--quiet'): | |
| 139 quiet = True | |
| 140 if o in ('-s', '--secret_key'): | |
| 141 aws_secret_access_key = a | |
| 142 if len(args) != 1: | |
| 143 print usage() | |
| 144 path = os.path.expanduser(args[0]) | |
| 145 path = os.path.expandvars(path) | |
| 146 path = os.path.abspath(path) | |
| 147 if bucket_name: | |
| 148 c = boto.connect_s3(aws_access_key_id=aws_access_key_id, | |
| 149 aws_secret_access_key=aws_secret_access_key) | |
| 150 c.debug = debug | |
| 151 b = c.get_bucket(bucket_name) | |
| 152 if os.path.isdir(path): | |
| 153 if no_overwrite: | |
| 154 if not quiet: | |
| 155 print 'Getting list of existing keys to check against' | |
| 156 keys = [] | |
| 157 for key in b.list(): | |
| 158 keys.append(key.name) | |
| 159 for root, dirs, files in os.walk(path): | |
| 160 for ignore in ignore_dirs: | |
| 161 if ignore in dirs: | |
| 162 dirs.remove(ignore) | |
| 163 for file in files: | |
| 164 fullpath = os.path.join(root, file) | |
| 165 key_name = get_key_name(fullpath, prefix) | |
| 166 copy_file = True | |
| 167 if no_overwrite: | |
| 168 if key_name in keys: | |
| 169 copy_file = False | |
| 170 if not quiet: | |
| 171 print 'Skipping %s as it exists in s3' % file | |
| 172 if copy_file: | |
| 173 if not quiet: | |
| 174 print 'Copying %s to %s/%s' % (file, bucket_name, ke
y_name) | |
| 175 if not no_op: | |
| 176 k = b.new_key(key_name) | |
| 177 k.set_contents_from_filename(fullpath, cb=cb, | |
| 178 num_cb=num_cb, polic
y=grant) | |
| 179 total += 1 | |
| 180 elif os.path.isfile(path): | |
| 181 key_name = os.path.split(path)[1] | |
| 182 copy_file = True | |
| 183 if no_overwrite: | |
| 184 if b.get_key(key_name): | |
| 185 copy_file = False | |
| 186 if not quiet: | |
| 187 print 'Skipping %s as it exists in s3' % path | |
| 188 if copy_file: | |
| 189 k = b.new_key(key_name) | |
| 190 k.set_contents_from_filename(path, cb=cb, num_cb=num_cb, policy=
grant) | |
| 191 else: | |
| 192 print usage() | |
| 193 | |
| 194 if __name__ == "__main__": | |
| 195 main() | |
| 196 | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10199002:
Upgrade gsutil to 3.4 (Closed)
Base URL: https://dart.googlecode.com/svn/branches/bleeding_edge/dart