OLD | NEW |
---|---|
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
3 # Use of this source code is governed by a BSD-style license that can be | 3 # Use of this source code is governed by a BSD-style license that can be |
4 # found in the LICENSE file. | 4 # found in the LICENSE file. |
5 | 5 |
6 """Does one of the following depending on the --mode argument: | 6 """Does one of the following depending on the --mode argument: |
7 check verify all the inputs exist, touches the file specified with | 7 check Verifies all the inputs exist, touches the file specified with |
8 --result and exits. | 8 --result and exits. |
9 hashtable puts a manifest file and hard links each of the inputs into the | 9 hashtable Puts a manifest file and hard links each of the inputs into the |
10 output directory. | 10 output directory. |
11 remap stores all the inputs files in a directory without running the | 11 remap Stores all the inputs files in a directory without running the |
12 executable. | 12 executable. |
13 run recreates a tree with all the inputs files and run the executable | 13 run Recreates a tree with all the inputs files and run the executable |
14 in it. | 14 in it. |
15 | 15 |
16 See more information at | 16 See more information at |
17 http://dev.chromium.org/developers/testing/isolated-testing | 17 http://dev.chromium.org/developers/testing/isolated-testing |
18 """ | 18 """ |
19 | 19 |
20 import hashlib | |
21 import json | 20 import json |
22 import logging | 21 import logging |
23 import optparse | 22 import optparse |
24 import os | 23 import os |
25 import re | 24 import re |
26 import shutil | |
27 import subprocess | 25 import subprocess |
28 import sys | 26 import sys |
29 import tempfile | 27 import tempfile |
30 import time | |
31 | 28 |
32 import tree_creator | 29 import tree_creator |
33 | 30 |
34 # Needs to be coherent with the file's docstring above. | 31 # Needs to be coherent with the file's docstring above. |
35 VALID_MODES = ('check', 'hashtable', 'remap', 'run') | 32 VALID_MODES = ('check', 'hashtable', 'remap', 'run') |
36 | 33 |
37 | 34 |
38 def touch(filename): | |
39 """Implements the equivalent of the 'touch' command.""" | |
40 if not os.path.exists(filename): | |
41 open(filename, 'a').close() | |
42 os.utime(filename, None) | |
43 | |
44 | |
45 def rmtree(root): | |
46 """Wrapper around shutil.rmtree() to retry automatically on Windows.""" | |
47 if sys.platform == 'win32': | |
48 for i in range(3): | |
49 try: | |
50 shutil.rmtree(root) | |
51 break | |
52 except WindowsError: # pylint: disable=E0602 | |
53 delay = (i+1)*2 | |
54 print >> sys.stderr, ( | |
55 'The test has subprocess outliving it. Sleep %d seconds.' % delay) | |
56 time.sleep(delay) | |
57 else: | |
58 shutil.rmtree(root) | |
59 | |
60 | |
61 def relpath(path, root): | 35 def relpath(path, root): |
62 """os.path.relpath() that keeps trailing slash.""" | 36 """os.path.relpath() that keeps trailing slash.""" |
63 out = os.path.relpath(path, root) | 37 out = os.path.relpath(path, root) |
64 if path.endswith('/'): | 38 if path.endswith('/'): |
65 out += '/' | 39 out += '/' |
66 return out | 40 return out |
67 | 41 |
68 | 42 |
69 def separate_inputs_command(args, root): | 43 def separate_inputs_command(args, root, files): |
70 """Strips off the command line from the inputs. | 44 """Strips off the command line from the inputs. |
71 | 45 |
72 gyp provides input paths relative to cwd. Convert them to be relative to root. | 46 gyp provides input paths relative to cwd. Convert them to be relative to root. |
47 OptionParser kindly strips off '--' from sys.argv if it's provided and that's | |
48 the first non-arg value. Manually look up if it was present in sys.argv. | |
73 """ | 49 """ |
74 cmd = [] | 50 cmd = [] |
75 if '--' in args: | 51 if '--' in args: |
csharp
2012/03/08 21:53:39
Will optparse always be messing with us? Will this
M-A Ruel
2012/03/08 21:56:41
Yes, it depends if the first non-argument is '--'
| |
76 i = args.index('--') | 52 i = args.index('--') |
77 cmd = args[i+1:] | 53 cmd = args[i+1:] |
78 args = args[:i] | 54 args = args[:i] |
55 elif '--' in sys.argv: | |
56 # optparse is messing with us. Fix it manually. | |
57 cmd = args | |
58 args = [] | |
59 if files: | |
60 args = [ | |
61 i.decode('utf-8') for i in open(files, 'rb').read().splitlines() if i | |
62 ] + args | |
79 cwd = os.getcwd() | 63 cwd = os.getcwd() |
80 return [relpath(os.path.join(cwd, arg), root) for arg in args], cmd | 64 return [relpath(os.path.join(cwd, arg), root) for arg in args], cmd |
81 | 65 |
82 | 66 |
83 def isolate(outdir, resultfile, indir, infiles, mode, read_only, cmd): | 67 def isolate(outdir, resultfile, indir, infiles, mode, read_only, cmd): |
84 """Main function to isolate a target with its dependencies. | 68 """Main function to isolate a target with its dependencies. |
85 | 69 |
86 It's behavior depends on |mode|. | 70 It's behavior depends on |mode|. |
87 """ | 71 """ |
88 if mode == 'run': | 72 mode_fn = getattr(sys.modules[__name__], 'MODE' + mode) |
89 return run(outdir, resultfile, indir, infiles, read_only, cmd) | 73 assert mode_fn |
90 | 74 |
91 if mode == 'hashtable': | 75 infiles = tree_creator.expand_directories( |
92 return hashtable(outdir, resultfile, indir, infiles) | 76 indir, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x)) |
93 | 77 |
94 assert mode in ('check', 'remap'), mode | 78 if not cmd: |
95 if mode == 'remap': | 79 cmd = [infiles[0]] |
96 if not outdir: | 80 if cmd[0].endswith('.py'): |
97 outdir = tempfile.mkdtemp(prefix='isolate') | 81 cmd.insert(0, sys.executable) |
98 tree_creator.recreate_tree( | |
99 outdir, indir, infiles, tree_creator.HARDLINK) | |
100 if read_only: | |
101 tree_creator.make_writable(outdir, True) | |
102 | 82 |
103 if resultfile: | 83 # Only hashtable mode really needs the sha-1. |
104 # Signal the build tool that the test succeeded. | 84 dictfiles = tree_creator.process_inputs( |
85 indir, infiles, mode == 'hashtable', read_only) | |
86 | |
87 if not outdir: | |
88 outdir = os.path.dirname(resultfile) | |
89 result = mode_fn(outdir, indir, dictfiles, read_only, cmd) | |
90 | |
91 if result == 0: | |
92 # Saves the resulting file. | |
93 out = { | |
94 'command': cmd, | |
95 'files': dictfiles, | |
96 } | |
105 with open(resultfile, 'wb') as f: | 97 with open(resultfile, 'wb') as f: |
106 for infile in infiles: | 98 json.dump(out, f) |
107 f.write(infile.encode('utf-8')) | 99 return result |
108 f.write('\n') | |
109 | 100 |
110 | 101 |
111 def run(outdir, resultfile, indir, infiles, read_only, cmd): | 102 def MODEcheck(outdir, indir, dictfiles, read_only, cmd): |
112 """Implements the 'run' mode.""" | 103 """No-op.""" |
113 if not cmd: | 104 return 0 |
114 print >> sys.stderr, 'Using first input %s as executable' % infiles[0] | 105 |
115 cmd = [infiles[0]] | 106 |
107 def MODEhashtable(outdir, indir, dictfiles, read_only, cmd): | |
108 """Ignores cmd and read_only.""" | |
109 for relfile, properties in dictfiles.iteritems(): | |
110 infile = os.path.join(indir, relfile) | |
111 outfile = os.path.join(outdir, properties['sha-1']) | |
112 if os.path.isfile(outfile): | |
113 # Just do a quick check that the file size matches. | |
114 if os.stat(infile).st_size == os.stat(outfile).st_size: | |
115 continue | |
116 # Otherwise, an exception will be raised. | |
csharp
2012/03/08 21:53:39
Where is this exception raised?
M-A Ruel
2012/03/08 21:56:41
Inside link_file() since outfile exists.
| |
117 tree_creator.link_file(outfile, infile, tree_creator.HARDLINK) | |
118 return 0 | |
119 | |
120 | |
121 def MODEremap(outdir, indir, dictfiles, read_only, cmd): | |
122 """Ignores cmd.""" | |
123 if not outdir: | |
124 outdir = tempfile.mkdtemp(prefix='isolate') | |
125 tree_creator.recreate_tree( | |
126 outdir, indir, dictfiles.keys(), tree_creator.HARDLINK) | |
127 if read_only: | |
128 tree_creator.make_writable(outdir, True) | |
129 return 0 | |
130 | |
131 | |
132 def MODErun(outdir, indir, dictfiles, read_only, cmd): | |
133 """Ignores outdir.""" | |
116 outdir = None | 134 outdir = None |
117 try: | 135 try: |
118 outdir = tempfile.mkdtemp(prefix='isolate') | 136 outdir = tempfile.mkdtemp(prefix='isolate') |
119 tree_creator.recreate_tree( | 137 tree_creator.recreate_tree( |
120 outdir, indir, infiles, tree_creator.HARDLINK) | 138 outdir, indir, dictfiles.keys(), tree_creator.HARDLINK) |
121 if read_only: | 139 if read_only: |
122 tree_creator.make_writable(outdir, True) | 140 tree_creator.make_writable(outdir, True) |
123 | 141 |
124 # Rebase the command to the right path. | 142 # Rebase the command to the right path. |
125 cwd = os.path.join(outdir, os.path.relpath(os.getcwd(), indir)) | 143 cwd = os.path.join(outdir, os.path.relpath(os.getcwd(), indir)) |
126 logging.info('Running %s, cwd=%s' % (cmd, cwd)) | 144 logging.info('Running %s, cwd=%s' % (cmd, cwd)) |
127 result = subprocess.call(cmd, cwd=cwd) | 145 return subprocess.call(cmd, cwd=cwd) |
128 if not result and resultfile: | |
129 # Signal the build tool that the test succeeded. | |
130 touch(resultfile) | |
131 return result | |
132 finally: | 146 finally: |
133 if read_only: | 147 if read_only: |
134 tree_creator.make_writable(outdir, False) | 148 tree_creator.make_writable(outdir, False) |
135 rmtree(outdir) | 149 tree_creator.rmtree(outdir) |
136 | |
137 | |
138 def hashtable(outdir, resultfile, indir, infiles): | |
139 """Implements the 'hashtable' mode.""" | |
140 results = {} | |
141 for relfile in infiles: | |
142 infile = os.path.join(indir, relfile) | |
143 h = hashlib.sha1() | |
144 with open(infile, 'rb') as f: | |
145 h.update(f.read()) | |
146 digest = h.hexdigest() | |
147 outfile = os.path.join(outdir, digest) | |
148 tree_creator.process_file(outfile, infile, tree_creator.HARDLINK) | |
149 results[relfile] = {'sha1': digest} | |
150 json.dump( | |
151 { | |
152 'files': results, | |
153 }, | |
154 open(resultfile, 'wb')) | |
155 | 150 |
156 | 151 |
157 def main(): | 152 def main(): |
158 parser = optparse.OptionParser( | 153 parser = optparse.OptionParser( |
159 usage='%prog [options] [inputs] -- [command line]', | 154 usage='%prog [options] [inputs] -- [command line]', |
160 description=sys.modules[__name__].__doc__) | 155 description=sys.modules[__name__].__doc__) |
161 parser.allow_interspersed_args = False | 156 parser.allow_interspersed_args = False |
162 parser.format_description = lambda *_: parser.description | 157 parser.format_description = lambda *_: parser.description |
163 parser.add_option( | 158 parser.add_option( |
164 '-v', '--verbose', action='count', default=0, help='Use multiple times') | 159 '-v', '--verbose', action='count', default=0, help='Use multiple times') |
165 parser.add_option( | 160 parser.add_option( |
166 '--mode', choices=VALID_MODES, | 161 '--mode', choices=VALID_MODES, |
167 help='Determines the action to be taken: %s' % ', '.join(VALID_MODES)) | 162 help='Determines the action to be taken: %s' % ', '.join(VALID_MODES)) |
168 parser.add_option( | 163 parser.add_option( |
169 '--result', metavar='FILE', | 164 '--result', metavar='FILE', |
170 help='File to be touched when the command succeeds') | 165 help='Output file containing the json information about inputs') |
171 parser.add_option( | 166 parser.add_option( |
172 '--root', metavar='DIR', help='Base directory to fetch files, required') | 167 '--root', metavar='DIR', help='Base directory to fetch files, required') |
173 parser.add_option( | 168 parser.add_option( |
174 '--outdir', metavar='DIR', | 169 '--outdir', metavar='DIR', |
175 help='Directory used to recreate the tree or store the hash table. ' | 170 help='Directory used to recreate the tree or store the hash table. ' |
176 'Defaults to a /tmp subdirectory for modes run and remap.') | 171 'For run and remap, uses a /tmp subdirectory. For the other modes, ' |
172 'defaults to the directory containing --result') | |
177 parser.add_option( | 173 parser.add_option( |
178 '--read-only', action='store_true', | 174 '--read-only', action='store_true', |
179 help='Make the temporary tree read-only') | 175 help='Make the temporary tree read-only') |
180 parser.add_option( | 176 parser.add_option( |
181 '--files', metavar='FILE', | 177 '--files', metavar='FILE', |
182 help='File to be read containing input files') | 178 help='File to be read containing input files') |
183 | 179 |
184 options, args = parser.parse_args() | 180 options, args = parser.parse_args() |
185 level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)] | 181 level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)] |
186 logging.basicConfig( | 182 logging.basicConfig( |
187 level=level, | 183 level=level, |
188 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s') | 184 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s') |
189 | 185 |
190 if not options.root: | 186 if not options.root: |
191 parser.error('--root is required.') | 187 parser.error('--root is required.') |
188 if not options.result: | |
189 parser.error('--result is required.') | |
192 | 190 |
193 if options.files: | 191 # Normalize the root input directory. |
194 args = [ | 192 indir = os.path.normpath(options.root) |
195 i.decode('utf-8') | 193 if not os.path.isdir(indir): |
196 for i in open(options.files, 'rb').read().splitlines() if i | 194 parser.error('%s is not a directory' % indir) |
197 ] + args | |
198 | 195 |
199 infiles, cmd = separate_inputs_command(args, options.root) | 196 # Do not call abspath until it was verified the directory exists. |
197 indir = os.path.abspath(indir) | |
198 | |
199 logging.info('sys.argv: %s' % sys.argv) | |
200 logging.info('cwd: %s' % os.getcwd()) | |
201 logging.info('Args: %s' % args) | |
202 infiles, cmd = separate_inputs_command(args, indir, options.files) | |
200 if not infiles: | 203 if not infiles: |
201 parser.error('Need at least one input file to map') | 204 parser.error('Need at least one input file to map') |
202 # Preprocess the input files. | 205 logging.info('infiles: %s' % infiles) |
206 | |
203 try: | 207 try: |
204 infiles, root = tree_creator.preprocess_inputs( | |
205 options.root, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x)) | |
206 return isolate( | 208 return isolate( |
207 options.outdir, | 209 options.outdir, |
208 options.result, | 210 os.path.abspath(options.result), |
209 root, | 211 indir, |
210 infiles, | 212 infiles, |
211 options.mode, | 213 options.mode, |
212 options.read_only, | 214 options.read_only, |
213 cmd) | 215 cmd) |
214 except tree_creator.MappingError, e: | 216 except tree_creator.MappingError, e: |
215 print >> sys.stderr, str(e) | 217 print >> sys.stderr, str(e) |
216 return 1 | 218 return 1 |
217 | 219 |
218 | 220 |
219 if __name__ == '__main__': | 221 if __name__ == '__main__': |
220 sys.exit(main()) | 222 sys.exit(main()) |
OLD | NEW |