Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(315)

Side by Side Diff: tools/isolate/isolate.py

Issue 9834090: Add tool to use the manifest, fetch and cache dependencies and run the test. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Rebase against 9835084 Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « no previous file | tools/isolate/tree_creator.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python 1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file. 4 # found in the LICENSE file.
5 5
6 """Does one of the following depending on the --mode argument: 6 """Does one of the following depending on the --mode argument:
7 check Verifies all the inputs exist, touches the file specified with 7 check Verifies all the inputs exist, touches the file specified with
8 --result and exits. 8 --result and exits.
9 hashtable Puts a manifest file and hard links each of the inputs into the 9 hashtable Puts a manifest file and hard links each of the inputs into the
10 output directory. 10 output directory.
11 remap Stores all the inputs files in a directory without running the 11 remap Stores all the inputs files in a directory without running the
12 executable. 12 executable.
13 run Recreates a tree with all the inputs files and run the executable 13 run Recreates a tree with all the inputs files and run the executable
14 in it. 14 in it.
15 trace Runs the executable without remapping it but traces all the files 15 trace Runs the executable without remapping it but traces all the files
16 it and its child processes access. 16 it and its child processes access.
17 17
18 See more information at 18 See more information at
19 http://dev.chromium.org/developers/testing/isolated-testing 19 http://dev.chromium.org/developers/testing/isolated-testing
20 """ 20 """
21 21
22 import hashlib
22 import json 23 import json
23 import logging 24 import logging
24 import optparse 25 import optparse
25 import os 26 import os
26 import re 27 import re
28 import stat
27 import subprocess 29 import subprocess
28 import sys 30 import sys
29 import tempfile 31 import tempfile
30 32
31 import trace_inputs 33 import trace_inputs
32 import tree_creator 34 import tree_creator
33 35
34 36
35 def relpath(path, root): 37 def relpath(path, root):
36 """os.path.relpath() that keeps trailing slash.""" 38 """os.path.relpath() that keeps trailing slash."""
37 out = os.path.relpath(path, root) 39 out = os.path.relpath(path, root)
38 if path.endswith('/'): 40 if path.endswith('/'):
39 out += '/' 41 out += '/'
40 return out 42 return out
41 43
42 44
43 def to_relative(path, root, relative): 45 def to_relative(path, root, relative):
44 """Converts any absolute path to a relative path, only if under root.""" 46 """Converts any absolute path to a relative path, only if under root."""
45 if path.startswith(root): 47 if path.startswith(root):
46 logging.info('%s starts with %s' % (path, root)) 48 logging.info('%s starts with %s' % (path, root))
47 path = os.path.relpath(path, relative) 49 path = os.path.relpath(path, relative)
48 else: 50 else:
49 logging.info('%s not under %s' % (path, root)) 51 logging.info('%s not under %s' % (path, root))
50 return path 52 return path
51 53
52 54
55 def expand_directories(indir, infiles, blacklist):
56 """Expands the directories, applies the blacklist and verifies files exist."""
57 logging.debug('expand_directories(%s, %s, %s)' % (indir, infiles, blacklist))
58 outfiles = []
59 for relfile in infiles:
60 if os.path.isabs(relfile):
61 raise tree_creator.MappingError('Can\'t map absolute path %s' % relfile)
62 infile = os.path.normpath(os.path.join(indir, relfile))
63 if not infile.startswith(indir):
64 raise tree_creator.MappingError(
65 'Can\'t map file %s outside %s' % (infile, indir))
66
67 if relfile.endswith('/'):
68 if not os.path.isdir(infile):
69 raise tree_creator.MappingError(
70 'Input directory %s must have a trailing slash' % infile)
71 for dirpath, dirnames, filenames in os.walk(infile):
72 # Convert the absolute path to subdir + relative subdirectory.
73 reldirpath = dirpath[len(indir)+1:]
74 outfiles.extend(os.path.join(reldirpath, f) for f in filenames)
75 for index, dirname in enumerate(dirnames):
76 # Do not process blacklisted directories.
77 if blacklist(os.path.join(reldirpath, dirname)):
78 del dirnames[index]
79 else:
80 if not os.path.isfile(infile):
81 raise tree_creator.MappingError('Input file %s doesn\'t exist' % infile)
82 outfiles.append(relfile)
83 return outfiles
84
85
86 def process_inputs(indir, infiles, need_hash, read_only):
87 """Returns a dictionary of input files, populated with the files' mode and
88 hash.
89
90 The file mode is manipulated if read_only is True. In practice, we only save
91 one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r).
92 """
93 outdict = {}
94 for infile in infiles:
95 filepath = os.path.join(indir, infile)
96 filemode = stat.S_IMODE(os.stat(filepath).st_mode)
97 # Remove write access for non-owner.
98 filemode &= ~(stat.S_IWGRP | stat.S_IWOTH)
99 if read_only:
100 filemode &= ~stat.S_IWUSR
101 if filemode & stat.S_IXUSR:
102 filemode |= (stat.S_IXGRP | stat.S_IXOTH)
103 else:
104 filemode &= ~(stat.S_IXGRP | stat.S_IXOTH)
105 outdict[infile] = {
106 'mode': filemode,
107 }
108 if need_hash:
109 h = hashlib.sha1()
110 with open(filepath, 'rb') as f:
111 h.update(f.read())
112 outdict[infile]['sha-1'] = h.hexdigest()
113 return outdict
114
115
116 def recreate_tree(outdir, indir, infiles, action):
117 """Creates a new tree with only the input files in it.
118
119 Arguments:
120 outdir: Output directory to create the files in.
121 indir: Root directory the infiles are based in.
122 infiles: List of files to map from |indir| to |outdir|.
123 action: See assert below.
124 """
125 logging.debug(
126 'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
127 logging.info('Mapping from %s to %s' % (indir, outdir))
128
129 assert action in (
130 tree_creator.HARDLINK, tree_creator.SYMLINK, tree_creator.COPY)
131 outdir = os.path.normpath(outdir)
132 if not os.path.isdir(outdir):
133 logging.info ('Creating %s' % outdir)
134 os.makedirs(outdir)
135 # Do not call abspath until the directory exists.
136 outdir = os.path.abspath(outdir)
137
138 for relfile in infiles:
139 infile = os.path.join(indir, relfile)
140 outfile = os.path.join(outdir, relfile)
141 outsubdir = os.path.dirname(outfile)
142 if not os.path.isdir(outsubdir):
143 os.makedirs(outsubdir)
144 tree_creator.link_file(outfile, infile, action)
145
146
53 def separate_inputs_command(args, root, files): 147 def separate_inputs_command(args, root, files):
54 """Strips off the command line from the inputs. 148 """Strips off the command line from the inputs.
55 149
56 gyp provides input paths relative to cwd. Convert them to be relative to root. 150 gyp provides input paths relative to cwd. Convert them to be relative to root.
57 OptionParser kindly strips off '--' from sys.argv if it's provided and that's 151 OptionParser kindly strips off '--' from sys.argv if it's provided and that's
58 the first non-arg value. Manually look up if it was present in sys.argv. 152 the first non-arg value. Manually look up if it was present in sys.argv.
59 """ 153 """
60 cmd = [] 154 cmd = []
61 if '--' in args: 155 if '--' in args:
62 i = args.index('--') 156 i = args.index('--')
(...skipping 24 matching lines...) Expand all
87 - cmd: Command to execute. 181 - cmd: Command to execute.
88 - no_save: If True, do not touch resultfile. 182 - no_save: If True, do not touch resultfile.
89 183
90 Some arguments are optional, dependending on |mode|. See the corresponding 184 Some arguments are optional, dependending on |mode|. See the corresponding
91 MODE<mode> function for the exact behavior. 185 MODE<mode> function for the exact behavior.
92 """ 186 """
93 mode_fn = getattr(sys.modules[__name__], 'MODE' + mode) 187 mode_fn = getattr(sys.modules[__name__], 'MODE' + mode)
94 assert mode_fn 188 assert mode_fn
95 assert os.path.isabs(resultfile) 189 assert os.path.isabs(resultfile)
96 190
97 infiles = tree_creator.expand_directories( 191 infiles = expand_directories(
98 indir, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x)) 192 indir, infiles, lambda x: re.match(r'.*\.(svn|pyc)$', x))
99 193
100 # Note the relative current directory. 194 # Note the relative current directory.
101 # In general, this path will be the path containing the gyp file where the 195 # In general, this path will be the path containing the gyp file where the
102 # target was defined. This relative directory may be created implicitely if a 196 # target was defined. This relative directory may be created implicitely if a
103 # file from this directory is needed to run the test. Otherwise it won't be 197 # file from this directory is needed to run the test. Otherwise it won't be
104 # created and the process creation will fail. It's up to the caller to create 198 # created and the process creation will fail. It's up to the caller to create
105 # this directory manually before starting the test. 199 # this directory manually before starting the test.
106 cwd = os.getcwd() 200 cwd = os.getcwd()
107 relative_cwd = os.path.relpath(cwd, indir) 201 relative_cwd = os.path.relpath(cwd, indir)
108 202
109 # Workaround make behavior of passing absolute paths. 203 # Workaround make behavior of passing absolute paths.
110 cmd = [to_relative(i, indir, cwd) for i in cmd] 204 cmd = [to_relative(i, indir, cwd) for i in cmd]
111 205
112 if not cmd: 206 if not cmd:
113 # Note that it is exactly the reverse of relative_cwd. 207 # Note that it is exactly the reverse of relative_cwd.
114 cmd = [os.path.join(os.path.relpath(indir, cwd), infiles[0])] 208 cmd = [os.path.join(os.path.relpath(indir, cwd), infiles[0])]
115 if cmd[0].endswith('.py'): 209 if cmd[0].endswith('.py'):
116 cmd.insert(0, sys.executable) 210 cmd.insert(0, sys.executable)
117 211
118 # Only hashtable mode really needs the sha-1. 212 # Only hashtable mode really needs the sha-1.
119 dictfiles = tree_creator.process_inputs( 213 dictfiles = process_inputs(indir, infiles, mode == 'hashtable', read_only)
120 indir, infiles, mode == 'hashtable', read_only)
121 214
122 result = mode_fn( 215 result = mode_fn(
123 outdir, indir, dictfiles, read_only, cmd, relative_cwd, resultfile) 216 outdir, indir, dictfiles, read_only, cmd, relative_cwd, resultfile)
124 217
125 if result == 0 and not no_save: 218 if result == 0 and not no_save:
126 # Saves the resulting file. 219 # Saves the resulting file.
127 out = { 220 out = {
128 'command': cmd, 221 'command': cmd,
129 'relative_cwd': relative_cwd, 222 'relative_cwd': relative_cwd,
130 'files': dictfiles, 223 'files': dictfiles,
(...skipping 26 matching lines...) Expand all
157 250
158 251
159 def MODEremap( 252 def MODEremap(
160 outdir, indir, dictfiles, read_only, _cmd, _relative_cwd, _resultfile): 253 outdir, indir, dictfiles, read_only, _cmd, _relative_cwd, _resultfile):
161 if not outdir: 254 if not outdir:
162 outdir = tempfile.mkdtemp(prefix='isolate') 255 outdir = tempfile.mkdtemp(prefix='isolate')
163 print 'Remapping into %s' % outdir 256 print 'Remapping into %s' % outdir
164 if len(os.listdir(outdir)): 257 if len(os.listdir(outdir)):
165 print 'Can\'t remap in a non-empty directory' 258 print 'Can\'t remap in a non-empty directory'
166 return 1 259 return 1
167 tree_creator.recreate_tree( 260 recreate_tree(outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
168 outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
169 if read_only: 261 if read_only:
170 tree_creator.make_writable(outdir, True) 262 tree_creator.make_writable(outdir, True)
171 return 0 263 return 0
172 264
173 265
174 def MODErun( 266 def MODErun(
175 _outdir, indir, dictfiles, read_only, cmd, relative_cwd, _resultfile): 267 _outdir, indir, dictfiles, read_only, cmd, relative_cwd, _resultfile):
176 """Always uses a temporary directory.""" 268 """Always uses a temporary directory."""
177 try: 269 try:
178 outdir = tempfile.mkdtemp(prefix='isolate') 270 outdir = tempfile.mkdtemp(prefix='isolate')
179 tree_creator.recreate_tree( 271 recreate_tree(outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
180 outdir, indir, dictfiles.keys(), tree_creator.HARDLINK)
181 cwd = os.path.join(outdir, relative_cwd) 272 cwd = os.path.join(outdir, relative_cwd)
182 if not os.path.isdir(cwd): 273 if not os.path.isdir(cwd):
183 os.makedirs(cwd) 274 os.makedirs(cwd)
184 if read_only: 275 if read_only:
185 tree_creator.make_writable(outdir, True) 276 tree_creator.make_writable(outdir, True)
186 277
187 logging.info('Running %s, cwd=%s' % (cmd, cwd)) 278 logging.info('Running %s, cwd=%s' % (cmd, cwd))
188 return subprocess.call(cmd, cwd=cwd) 279 return subprocess.call(cmd, cwd=cwd)
189 finally: 280 finally:
190 if read_only:
191 tree_creator.make_writable(outdir, False)
192 tree_creator.rmtree(outdir) 281 tree_creator.rmtree(outdir)
193 282
194 283
195 def MODEtrace( 284 def MODEtrace(
196 _outdir, indir, _dictfiles, _read_only, cmd, relative_cwd, resultfile): 285 _outdir, indir, _dictfiles, _read_only, cmd, relative_cwd, resultfile):
197 """Shortcut to use trace_inputs.py properly. 286 """Shortcut to use trace_inputs.py properly.
198 287
199 It constructs the equivalent of dictfiles. It is hardcoded to base the 288 It constructs the equivalent of dictfiles. It is hardcoded to base the
200 checkout at src/. 289 checkout at src/.
201 """ 290 """
(...skipping 102 matching lines...) Expand 10 before | Expand all | Expand 10 after
304 options.read_only, 393 options.read_only,
305 cmd, 394 cmd,
306 options.from_results) 395 options.from_results)
307 except tree_creator.MappingError, e: 396 except tree_creator.MappingError, e:
308 print >> sys.stderr, str(e) 397 print >> sys.stderr, str(e)
309 return 1 398 return 1
310 399
311 400
312 if __name__ == '__main__': 401 if __name__ == '__main__':
313 sys.exit(main()) 402 sys.exit(main())
OLDNEW
« no previous file with comments | « no previous file | tools/isolate/tree_creator.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698