Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(466)

Side by Side Diff: tools/isolate/tree_creator.py

Issue 9834090: Add tool to use the manifest, fetch and cache dependencies and run the test. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: Rebase against 9835084 Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « tools/isolate/isolate.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 4 # found in the LICENSE file.
4 5
5 """File related utility functions. 6 """Reads a manifest, creates a tree of hardlinks and runs the test.
6 7
7 Creates a tree of hardlinks, symlinks or copy the inputs files. Calculate files 8 Keeps a local cache.
8 hash.
9 """ 9 """
10 10
11 import ctypes 11 import ctypes
12 import hashlib 12 import json
13 import logging 13 import logging
14 import optparse
14 import os 15 import os
16 import re
15 import shutil 17 import shutil
16 import stat 18 import subprocess
17 import sys 19 import sys
20 import tempfile
18 import time 21 import time
22 import urllib
19 23
20 24
21 # Types of action accepted by recreate_tree(). 25 # Types of action accepted by recreate_tree().
22 HARDLINK, SYMLINK, COPY = range(4)[1:] 26 HARDLINK, SYMLINK, COPY = range(4)[1:]
23 27
24 28
25 class MappingError(OSError): 29 class MappingError(OSError):
26 """Failed to recreate the tree.""" 30 """Failed to recreate the tree."""
27 pass 31 pass
28 32
29 33
30 def os_link(source, link_name): 34 def os_link(source, link_name):
31 """Add support for os.link() on Windows.""" 35 """Add support for os.link() on Windows."""
32 if sys.platform == 'win32': 36 if sys.platform == 'win32':
33 if not ctypes.windll.kernel32.CreateHardLinkW( 37 if not ctypes.windll.kernel32.CreateHardLinkW(
34 unicode(link_name), unicode(source), 0): 38 unicode(link_name), unicode(source), 0):
35 raise OSError() 39 raise OSError()
36 else: 40 else:
37 os.link(source, link_name) 41 os.link(source, link_name)
38 42
39 43
40 def expand_directories(indir, infiles, blacklist):
41 """Expands the directories, applies the blacklist and verifies files exist."""
42 logging.debug('expand_directories(%s, %s, %s)' % (indir, infiles, blacklist))
43 outfiles = []
44 for relfile in infiles:
45 if os.path.isabs(relfile):
46 raise MappingError('Can\'t map absolute path %s' % relfile)
47 infile = os.path.normpath(os.path.join(indir, relfile))
48 if not infile.startswith(indir):
49 raise MappingError('Can\'t map file %s outside %s' % (infile, indir))
50
51 if relfile.endswith('/'):
52 if not os.path.isdir(infile):
53 raise MappingError(
54 'Input directory %s must have a trailing slash' % infile)
55 for dirpath, dirnames, filenames in os.walk(infile):
56 # Convert the absolute path to subdir + relative subdirectory.
57 relpath = dirpath[len(indir)+1:]
58 outfiles.extend(os.path.join(relpath, f) for f in filenames)
59 for index, dirname in enumerate(dirnames):
60 # Do not process blacklisted directories.
61 if blacklist(os.path.join(relpath, dirname)):
62 del dirnames[index]
63 else:
64 if not os.path.isfile(infile):
65 raise MappingError('Input file %s doesn\'t exist' % infile)
66 outfiles.append(relfile)
67 return outfiles
68
69
70 def process_inputs(indir, infiles, need_hash, read_only):
71 """Returns a dictionary of input files, populated with the files' mode and
72 hash.
73
74 The file mode is manipulated if read_only is True. In practice, we only save
75 one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r).
76 """
77 outdict = {}
78 for infile in infiles:
79 filepath = os.path.join(indir, infile)
80 filemode = stat.S_IMODE(os.stat(filepath).st_mode)
81 # Remove write access for non-owner.
82 filemode &= ~(stat.S_IWGRP | stat.S_IWOTH)
83 if read_only:
84 filemode &= ~stat.S_IWUSR
85 if filemode & stat.S_IXUSR:
86 filemode |= (stat.S_IXGRP | stat.S_IXOTH)
87 else:
88 filemode &= ~(stat.S_IXGRP | stat.S_IXOTH)
89 outdict[infile] = {
90 'mode': filemode,
91 }
92 if need_hash:
93 h = hashlib.sha1()
94 with open(filepath, 'rb') as f:
95 h.update(f.read())
96 outdict[infile]['sha-1'] = h.hexdigest()
97 return outdict
98
99
100 def link_file(outfile, infile, action): 44 def link_file(outfile, infile, action):
101 """Links a file. The type of link depends on |action|.""" 45 """Links a file. The type of link depends on |action|."""
102 logging.debug('Mapping %s to %s' % (infile, outfile)) 46 logging.debug('Mapping %s to %s' % (infile, outfile))
47 if action not in (HARDLINK, SYMLINK, COPY):
48 raise ValueError('Unknown mapping action %s' % action)
103 if os.path.isfile(outfile): 49 if os.path.isfile(outfile):
104 raise MappingError('%s already exist' % outfile) 50 raise MappingError('%s already exist' % outfile)
105 51
106 if action == COPY: 52 if action == COPY:
107 shutil.copy(infile, outfile) 53 shutil.copy(infile, outfile)
108 elif action == SYMLINK and sys.platform != 'win32': 54 elif action == SYMLINK and sys.platform != 'win32':
55 # On windows, symlink are converted to hardlink and fails over to copy.
109 os.symlink(infile, outfile) 56 os.symlink(infile, outfile)
110 elif action == HARDLINK: 57 else:
111 try: 58 try:
112 os_link(infile, outfile) 59 os_link(infile, outfile)
113 except OSError: 60 except OSError:
114 # Probably a different file system. 61 # Probably a different file system.
115 logging.warn( 62 logging.warn(
116 'Failed to hardlink, failing back to copy %s to %s' % ( 63 'Failed to hardlink, failing back to copy %s to %s' % (
117 infile, outfile)) 64 infile, outfile))
118 shutil.copy(infile, outfile) 65 shutil.copy(infile, outfile)
119 else:
120 raise ValueError('Unknown mapping action %s' % action)
121
122
123 def recreate_tree(outdir, indir, infiles, action):
124 """Creates a new tree with only the input files in it.
125
126 Arguments:
127 outdir: Output directory to create the files in.
128 indir: Root directory the infiles are based in.
129 infiles: List of files to map from |indir| to |outdir|.
130 action: See assert below.
131 """
132 logging.debug(
133 'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
134 logging.info('Mapping from %s to %s' % (indir, outdir))
135
136 assert action in (HARDLINK, SYMLINK, COPY)
137 outdir = os.path.normpath(outdir)
138 if not os.path.isdir(outdir):
139 logging.info ('Creating %s' % outdir)
140 os.makedirs(outdir)
141 # Do not call abspath until the directory exists.
142 outdir = os.path.abspath(outdir)
143
144 for relfile in infiles:
145 infile = os.path.join(indir, relfile)
146 outfile = os.path.join(outdir, relfile)
147 outsubdir = os.path.dirname(outfile)
148 if not os.path.isdir(outsubdir):
149 os.makedirs(outsubdir)
150 link_file(outfile, infile, action)
151 66
152 67
153 def _set_write_bit(path, read_only): 68 def _set_write_bit(path, read_only):
154 """Sets or resets the executable bit on a file or directory.""" 69 """Sets or resets the executable bit on a file or directory."""
155 mode = os.stat(path).st_mode 70 mode = os.stat(path).st_mode
156 if read_only: 71 if read_only:
157 mode = mode & 0500 72 mode = mode & 0500
158 else: 73 else:
159 mode = mode | 0200 74 mode = mode | 0200
160 if hasattr(os, 'lchmod'): 75 if hasattr(os, 'lchmod'):
161 os.lchmod(path, mode) # pylint: disable=E1101 76 os.lchmod(path, mode) # pylint: disable=E1101
162 else: 77 else:
163 # TODO(maruel): Implement proper DACL modification on Windows. 78 # TODO(maruel): Implement proper DACL modification on Windows.
164 os.chmod(path, mode) 79 os.chmod(path, mode)
165 80
166 81
167 def make_writable(root, read_only): 82 def make_writable(root, read_only):
168 """Toggle the writable bit on a directory tree.""" 83 """Toggle the writable bit on a directory tree."""
169 root = os.path.abspath(root) 84 root = os.path.abspath(root)
170 for dirpath, dirnames, filenames in os.walk(root, topdown=True): 85 for dirpath, dirnames, filenames in os.walk(root, topdown=True):
171 for filename in filenames: 86 for filename in filenames:
172 _set_write_bit(os.path.join(dirpath, filename), read_only) 87 _set_write_bit(os.path.join(dirpath, filename), read_only)
173 88
174 for dirname in dirnames: 89 for dirname in dirnames:
175 _set_write_bit(os.path.join(dirpath, dirname), read_only) 90 _set_write_bit(os.path.join(dirpath, dirname), read_only)
176 91
177 92
178 def rmtree(root): 93 def rmtree(root):
179 """Wrapper around shutil.rmtree() to retry automatically on Windows.""" 94 """Wrapper around shutil.rmtree() to retry automatically on Windows."""
95 make_writable(root, False)
180 if sys.platform == 'win32': 96 if sys.platform == 'win32':
181 for i in range(3): 97 for i in range(3):
182 try: 98 try:
183 shutil.rmtree(root) 99 shutil.rmtree(root)
184 break 100 break
185 except WindowsError: # pylint: disable=E0602 101 except WindowsError: # pylint: disable=E0602
186 delay = (i+1)*2 102 delay = (i+1)*2
187 print >> sys.stderr, ( 103 print >> sys.stderr, (
188 'The test has subprocess outliving it. Sleep %d seconds.' % delay) 104 'The test has subprocess outliving it. Sleep %d seconds.' % delay)
189 time.sleep(delay) 105 time.sleep(delay)
190 else: 106 else:
191 shutil.rmtree(root) 107 shutil.rmtree(root)
108
109
110 def open_remote(file_or_url):
111 """Reads a file or url."""
112 if re.match(r'^https?://.+$', file_or_url):
113 return urllib.urlopen(file_or_url)
114 return open(file_or_url, 'rb')
115
116
117 def download_or_copy(file_or_url, dest):
118 """Copies a file or download an url."""
119 if re.match(r'^https?://.+$', file_or_url):
120 urllib.urlretrieve(file_or_url, dest)
121 else:
122 shutil.copy(file_or_url, dest)
123
124
125 def get_free_space(path):
126 """Returns the number of free bytes."""
127 if sys.platform == 'win32':
128 free_bytes = ctypes.c_ulonglong(0)
129 ctypes.windll.kernel32.GetDiskFreeSpaceExW(
130 ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
131 return free_bytes.value
132 f = os.statvfs(path)
133 return f.f_bfree * f.f_frsize
134
135
136 class Cache(object):
137 """Stateful LRU cache.
138
139 Saves its state as json file.
140 """
141 STATE_FILE = 'state.json'
142
143 def __init__(self, cache_dir, remote, max_cache_size, min_free_space):
144 """
145 Arguments:
146 - cache_dir: Directory where to place the cache.
147 - remote: Remote directory (NFS, SMB, etc) or HTTP url to fetch the objects
148 from
149 - max_cache_size: Trim if the cache gets larger than this value. If 0, the
150 cache is effectively a leak.
151 - min_free_space: Trim if disk free space becomes lower than this value. If
152 0, it unconditionally fill the disk.
153 """
154 self.cache_dir = cache_dir
155 self.remote = remote
156 self.max_cache_size = max_cache_size
157 self.min_free_space = min_free_space
158 self.state_file = os.path.join(cache_dir, self.STATE_FILE)
159 # The files are kept as an array in a LRU style. E.g. self.state[0] is the
160 # oldest item.
161 self.state = []
162
163 if not os.path.isdir(self.cache_dir):
164 os.makedirs(self.cache_dir)
165 if os.path.isfile(self.state_file):
166 try:
167 self.state = json.load(open(self.state_file, 'rb'))
168 except ValueError:
169 # Too bad. The file will be overwritten and the cache cleared.
170 pass
171 self.trim()
172
173 def trim(self):
174 """Trims anything we don't know, make sure enough free space exists."""
175 for f in os.listdir(self.cache_dir):
176 if f == self.STATE_FILE or f in self.state:
177 continue
178 logging.warn('Unknown file %s from cache' % f)
179 # Insert as the oldest file. It will be deleted eventually if not
180 # accessed.
181 self.state.insert(0, f)
182
183 # Ensure enough free space.
184 while (
185 self.min_free_space and
186 self.state and
187 get_free_space(self.cache_dir) < self.min_free_space):
188 os.remove(self.path(self.state.pop(0)))
189
190 # Ensure maximum cache size.
191 if self.max_cache_size and self.state:
192 sizes = [os.stat(self.path(f)).st_size for f in self.state]
193 while sizes and sum(sizes) > self.max_cache_size:
194 # Delete the oldest item.
195 os.remove(self.path(self.state.pop(0)))
196 sizes.pop(0)
197
198 self.save()
199
200 def retrieve(self, item):
201 """Retrieves a file from the remote and add it to the cache."""
202 assert not '/' in item
203 try:
204 index = self.state.index(item)
205 # Was already in cache. Update it's LRU value.
206 self.state.pop(index)
207 self.state.append(item)
208 return False
209 except ValueError:
210 out = self.path(item)
211 download_or_copy(os.path.join(self.remote, item), out)
212 self.state.append(item)
213 return True
214 finally:
215 self.save()
216
217 def path(self, item):
218 """Returns the path to one item."""
219 return os.path.join(self.cache_dir, item)
220
221 def save(self):
222 """Saves the LRU ordering."""
223 json.dump(self.state, open(self.state_file, 'wb'))
224
225
226 def run_tha_test(manifest, cache_dir, remote, max_cache_size, min_free_space):
227 """Downloads the dependencies in the cache, hardlinks them into a temporary
228 directory and runs the executable.
229 """
230 cache = Cache(cache_dir, remote, max_cache_size, min_free_space)
231 outdir = tempfile.mkdtemp(prefix='run_tha_test')
232 try:
233 for filepath, properties in manifest['files'].iteritems():
234 infile = properties['sha-1']
235 outfile = os.path.join(outdir, filepath)
236 cache.retrieve(infile)
237 outfiledir = os.path.dirname(outfile)
238 if not os.path.isdir(outfiledir):
239 os.makedirs(outfiledir)
240 link_file(outfile, cache.path(infile), HARDLINK)
241 os.chmod(outfile, properties['mode'])
242
243 cwd = os.path.join(outdir, manifest['relative_cwd'])
244 if not os.path.isdir(cwd):
245 os.makedirs(cwd)
246 if manifest.get('read_only'):
247 make_writable(outdir, True)
248 cmd = manifest['command']
249 logging.info('Running %s, cwd=%s' % (cmd, cwd))
250 return subprocess.call(cmd, cwd=cwd)
251 finally:
252 # Save first, in case an exception occur in the following lines, then clean
253 # up.
254 cache.save()
255 rmtree(outdir)
256 cache.trim()
257
258
259 def main():
260 parser = optparse.OptionParser(
261 usage='%prog <options>', description=sys.modules[__name__].__doc__)
262 parser.add_option(
263 '-v', '--verbose', action='count', default=0, help='Use multiple times')
264 parser.add_option(
265 '-m', '--manifest',
266 metavar='FILE',
267 help='File/url describing what to map or run')
268 parser.add_option('--no-run', action='store_true', help='Skip the run part')
269 parser.add_option(
270 '--cache',
271 default='cache',
272 metavar='DIR',
273 help='Cache directory, default=%default')
274 parser.add_option(
275 '-r', '--remote', metavar='URL', help='Remote where to get the items')
276 parser.add_option(
277 '--max-cache-size',
278 type='int',
279 metavar='NNN',
280 default=20*1024*1024*1024,
281 help='Trim if the cache gets larger than this value, default=%default')
282 parser.add_option(
283 '--min-free-space',
284 type='int',
285 metavar='NNN',
286 default=1*1024*1024*1024,
287 help='Trim if disk free space becomes lower than this value, '
288 'default=%default')
289
290 options, args = parser.parse_args()
291 level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)]
292 logging.basicConfig(
293 level=level,
294 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s')
295
296 if not options.manifest:
297 parser.error('--manifest is required.')
298 if not options.remote:
299 parser.error('--remote is required.')
300 if args:
301 parser.error('Unsupported args %s' % ' '.join(args))
302
303 manifest = json.load(open_remote(options.manifest))
304 return run_tha_test(
305 manifest, os.path.abspath(options.cache), options.remote,
306 options.max_cache_size, options.min_free_space)
307
308
309 if __name__ == '__main__':
310 sys.exit(main())
OLDNEW
« no previous file with comments | « tools/isolate/isolate.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698