Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(700)

Side by Side Diff: tools/isolate/tree_creator.py

Issue 9834090: Add tool to use the manifest, fetch and cache dependencies and run the test. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/src
Patch Set: . Created 8 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« tools/isolate/isolate.py ('K') | « tools/isolate/isolate.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #!/usr/bin/env python
1 # Copyright (c) 2012 The Chromium Authors. All rights reserved. 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 3 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 4 # found in the LICENSE file.
4 5
5 """File related utility functions. 6 """Reads a manifest, creates a tree of hardlinks and runs the test.
6 7
7 Creates a tree of hardlinks, symlinks or copy the inputs files. Calculate files 8 Keeps a local cache.
8 hash.
9 """ 9 """
10 10
11 import ctypes 11 import ctypes
12 import hashlib 12 import json
13 import logging 13 import logging
14 import optparse
14 import os 15 import os
16 import re
15 import shutil 17 import shutil
16 import stat 18 import subprocess
17 import sys 19 import sys
20 import tempfile
18 import time 21 import time
22 import urllib
19 23
20 24
21 # Types of action accepted by recreate_tree(). 25 # Types of action accepted by recreate_tree().
22 HARDLINK, SYMLINK, COPY = range(4)[1:] 26 HARDLINK, SYMLINK, COPY = range(4)[1:]
23 27
24 28
25 class MappingError(OSError): 29 class MappingError(OSError):
26 """Failed to recreate the tree.""" 30 """Failed to recreate the tree."""
27 pass 31 pass
28 32
29 33
30 def os_link(source, link_name): 34 def os_link(source, link_name):
31 """Add support for os.link() on Windows.""" 35 """Add support for os.link() on Windows."""
32 if sys.platform == 'win32': 36 if sys.platform == 'win32':
33 if not ctypes.windll.kernel32.CreateHardLinkW( 37 if not ctypes.windll.kernel32.CreateHardLinkW(
34 unicode(link_name), unicode(source), 0): 38 unicode(link_name), unicode(source), 0):
35 raise OSError() 39 raise OSError()
36 else: 40 else:
37 os.link(source, link_name) 41 os.link(source, link_name)
38 42
39 43
40 def expand_directories(indir, infiles, blacklist):
41 """Expands the directories, applies the blacklist and verifies files exist."""
42 logging.debug('expand_directories(%s, %s, %s)' % (indir, infiles, blacklist))
43 outfiles = []
44 for relfile in infiles:
45 if os.path.isabs(relfile):
46 raise MappingError('Can\'t map absolute path %s' % relfile)
47 infile = os.path.normpath(os.path.join(indir, relfile))
48 if not infile.startswith(indir):
49 raise MappingError('Can\'t map file %s outside %s' % (infile, indir))
50
51 if relfile.endswith('/'):
52 if not os.path.isdir(infile):
53 raise MappingError(
54 'Input directory %s must have a trailing slash' % infile)
55 for dirpath, dirnames, filenames in os.walk(infile):
56 # Convert the absolute path to subdir + relative subdirectory.
57 relpath = dirpath[len(indir)+1:]
58 outfiles.extend(os.path.join(relpath, f) for f in filenames)
59 for index, dirname in enumerate(dirnames):
60 # Do not process blacklisted directories.
61 if blacklist(os.path.join(relpath, dirname)):
62 del dirnames[index]
63 else:
64 if not os.path.isfile(infile):
65 raise MappingError('Input file %s doesn\'t exist' % infile)
66 outfiles.append(relfile)
67 return outfiles
68
69
70 def process_inputs(indir, infiles, need_hash, read_only):
71 """Returns a dictionary of input files, populated with the files' mode and
72 hash.
73
74 The file mode is manipulated if read_only is True. In practice, we only save
75 one of 4 modes: 0755 (rwx), 0644 (rw), 0555 (rx), 0444 (r).
76 """
77 outdict = {}
78 for infile in infiles:
79 filepath = os.path.join(indir, infile)
80 filemode = stat.S_IMODE(os.stat(filepath).st_mode)
81 # Remove write access for non-owner.
82 filemode &= ~(stat.S_IWGRP | stat.S_IWOTH)
83 if read_only:
84 filemode &= ~stat.S_IWUSR
85 if filemode & stat.S_IXUSR:
86 filemode |= (stat.S_IXGRP | stat.S_IXOTH)
87 else:
88 filemode &= ~(stat.S_IXGRP | stat.S_IXOTH)
89 outdict[infile] = {
90 'mode': filemode,
91 }
92 if need_hash:
93 h = hashlib.sha1()
94 with open(filepath, 'rb') as f:
95 h.update(f.read())
96 outdict[infile]['sha-1'] = h.hexdigest()
97 return outdict
98
99
100 def link_file(outfile, infile, action): 44 def link_file(outfile, infile, action):
101 """Links a file. The type of link depends on |action|.""" 45 """Links a file. The type of link depends on |action|."""
102 logging.debug('Mapping %s to %s' % (infile, outfile)) 46 logging.debug('Mapping %s to %s' % (infile, outfile))
47 if action not in (HARDLINK, SYMLINK, COPY):
48 raise ValueError('Unknown mapping action %s' % action)
103 if os.path.isfile(outfile): 49 if os.path.isfile(outfile):
104 raise MappingError('%s already exist' % outfile) 50 raise MappingError('%s already exist' % outfile)
105 51
106 if action == COPY: 52 if action == COPY:
107 shutil.copy(infile, outfile) 53 shutil.copy(infile, outfile)
108 elif action == SYMLINK and sys.platform != 'win32': 54 elif action == SYMLINK and sys.platform != 'win32':
55 # On windows, symlink are converted to hardlink and fails over to copy.
109 os.symlink(infile, outfile) 56 os.symlink(infile, outfile)
110 elif action == HARDLINK: 57 else:
111 try: 58 try:
112 os_link(infile, outfile) 59 os_link(infile, outfile)
113 except OSError: 60 except OSError:
114 # Probably a different file system. 61 # Probably a different file system.
115 logging.warn( 62 logging.warn(
116 'Failed to hardlink, failing back to copy %s to %s' % ( 63 'Failed to hardlink, failing back to copy %s to %s' % (
117 infile, outfile)) 64 infile, outfile))
118 shutil.copy(infile, outfile) 65 shutil.copy(infile, outfile)
119 else:
120 raise ValueError('Unknown mapping action %s' % action)
121
122
123 def recreate_tree(outdir, indir, infiles, action):
124 """Creates a new tree with only the input files in it.
125
126 Arguments:
127 outdir: Output directory to create the files in.
128 indir: Root directory the infiles are based in.
129 infiles: List of files to map from |indir| to |outdir|.
130 action: See assert below.
131 """
132 logging.debug(
133 'recreate_tree(%s, %s, %s, %s)' % (outdir, indir, infiles, action))
134 logging.info('Mapping from %s to %s' % (indir, outdir))
135
136 assert action in (HARDLINK, SYMLINK, COPY)
137 outdir = os.path.normpath(outdir)
138 if not os.path.isdir(outdir):
139 logging.info ('Creating %s' % outdir)
140 os.makedirs(outdir)
141 # Do not call abspath until the directory exists.
142 outdir = os.path.abspath(outdir)
143
144 for relfile in infiles:
145 infile = os.path.join(indir, relfile)
146 outfile = os.path.join(outdir, relfile)
147 outsubdir = os.path.dirname(outfile)
148 if not os.path.isdir(outsubdir):
149 os.makedirs(outsubdir)
150 link_file(outfile, infile, action)
151 66
152 67
153 def _set_write_bit(path, read_only): 68 def _set_write_bit(path, read_only):
154 """Sets or resets the executable bit on a file or directory.""" 69 """Sets or resets the executable bit on a file or directory."""
155 mode = os.stat(path).st_mode 70 mode = os.stat(path).st_mode
156 if read_only: 71 if read_only:
157 mode = mode & 0500 72 mode = mode & 0500
158 else: 73 else:
159 mode = mode | 0200 74 mode = mode | 0200
160 if hasattr(os, 'lchmod'): 75 if hasattr(os, 'lchmod'):
161 os.lchmod(path, mode) # pylint: disable=E1101 76 os.lchmod(path, mode) # pylint: disable=E1101
162 else: 77 else:
163 # TODO(maruel): Implement proper DACL modification on Windows. 78 # TODO(maruel): Implement proper DACL modification on Windows.
164 os.chmod(path, mode) 79 os.chmod(path, mode)
165 80
166 81
167 def make_writable(root, read_only): 82 def make_writable(root, read_only):
168 """Toggle the writable bit on a directory tree.""" 83 """Toggle the writable bit on a directory tree."""
169 root = os.path.abspath(root) 84 root = os.path.abspath(root)
170 for dirpath, dirnames, filenames in os.walk(root, topdown=True): 85 for dirpath, dirnames, filenames in os.walk(root, topdown=True):
171 for filename in filenames: 86 for filename in filenames:
172 _set_write_bit(os.path.join(dirpath, filename), read_only) 87 _set_write_bit(os.path.join(dirpath, filename), read_only)
173 88
174 for dirname in dirnames: 89 for dirname in dirnames:
175 _set_write_bit(os.path.join(dirpath, dirname), read_only) 90 _set_write_bit(os.path.join(dirpath, dirname), read_only)
176 91
177 92
178 def rmtree(root): 93 def rmtree(root):
179 """Wrapper around shutil.rmtree() to retry automatically on Windows.""" 94 """Wrapper around shutil.rmtree() to retry automatically on Windows."""
95 make_writable(root, False)
180 if sys.platform == 'win32': 96 if sys.platform == 'win32':
181 for i in range(3): 97 for i in range(3):
182 try: 98 try:
183 shutil.rmtree(root) 99 shutil.rmtree(root)
184 break 100 break
185 except WindowsError: # pylint: disable=E0602 101 except WindowsError: # pylint: disable=E0602
186 delay = (i+1)*2 102 delay = (i+1)*2
187 print >> sys.stderr, ( 103 print >> sys.stderr, (
188 'The test has subprocess outliving it. Sleep %d seconds.' % delay) 104 'The test has subprocess outliving it. Sleep %d seconds.' % delay)
189 time.sleep(delay) 105 time.sleep(delay)
190 else: 106 else:
191 shutil.rmtree(root) 107 shutil.rmtree(root)
108
109
110 def open_remote(file_or_url):
111 """Reads a file or url."""
112 if re.match(r'^https?://.+$', file_or_url):
113 return urllib.urlopen(file_or_url)
114 return open(file_or_url, 'rb')
115
116
117 def download_or_copy(file_or_url, dest):
118 """Copies a file or download an url."""
119 if re.match(r'^https?://.+$', file_or_url):
120 urllib.urlretrieve(file_or_url, dest)
121 else:
122 shutil.copy(file_or_url, dest)
123
124
125 def get_free_space(path):
126 """Returns the number of free bytes."""
127 if sys.platform == 'win32':
128 free_bytes = ctypes.c_ulonglong(0)
129 ctypes.windll.kernel32.GetDiskFreeSpaceExW(
130 ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes))
131 return free_bytes.value
132 f = os.statvfs(path)
133 return f.f_bfree * f.f_frsize
134
135
136 class Cache(object):
137 """Stateful LRU cache.
138
139 Saves its state as json file.
140 """
141 # Trim if the cache gets larger than that.
142 MAX_SIZE = 20*1024*1024*1024
143 # Trim if disk free space becomes lower than that.
144 MIN_FREE_SPACE = 1*1024*1024*1024
145 STATE_FILE = 'state.json'
146
147 def __init__(self, cache_dir, remote):
148 self.cache_dir = cache_dir
149 self.remote = remote
150 self.state_file = os.path.join(cache_dir, self.STATE_FILE)
151 # The files are kept as an array in a LRU style. E.g. self.state[0] is the
152 # oldest item.
153 self.state = []
154
155 if not os.path.isdir(self.cache_dir):
156 os.makedirs(self.cache_dir)
157 if os.path.isfile(self.state_file):
158 try:
159 self.state = json.load(open(self.state_file, 'rb'))
160 except ValueError:
161 # Too bad. The file will be overwritten and the cache cleared.
162 pass
163 self.trim()
164
165 def trim(self):
166 """Trims anything we don't know, make sure enough free space exists."""
167 for f in os.listdir(self.cache_dir):
168 if f == self.STATE_FILE or f in self.state:
169 continue
170 logging.warn('Unknown file %s from cache' % f)
171 # Insert as the oldest file. It will be deleted eventually if not
172 # accessed.
173 self.state.insert(0, f)
174
175 # Ensure enough free space.
176 while (
177 self.MIN_FREE_SPACE and
Roger Tawa OOO till Jul 10th 2012/03/26 14:50:40 why is this in the condition?
M-A Ruel 2012/03/26 15:55:59 I explained in the __init__ docstring. 0 is a lega
178 self.state and
179 get_free_space(self.cache_dir) < self.MIN_FREE_SPACE):
180 os.remove(self.path(self.state.pop(0)))
181
182 # Ensure maximum cache size.
183 while self.MAX_SIZE and self.state and self.total_size() > self.MAX_SIZE:
Roger Tawa OOO till Jul 10th 2012/03/26 14:50:40 do you really want to call total_size() each time
M-A Ruel 2012/03/26 15:55:59 No.
184 os.remove(self.path(self.state.pop(0)))
185
186 self.save()
187
188 def retrieve(self, item):
189 """Retrieves a file from the remote and add it to the cache."""
190 assert not '/' in item
191 try:
192 index = self.state.index(item)
193 # Was already in cache. Update it's LRU value.
194 self.state.pop(index)
195 self.state.append(item)
196 return False
197 except ValueError:
198 out = self.path(item)
199 download_or_copy(os.path.join(self.remote, item), out)
200 self.state.append(item)
201 return True
202 finally:
203 self.save()
204
205 def path(self, item):
206 """Returns the path to one item."""
207 return os.path.join(self.cache_dir, item)
208
209 def total_size(self):
210 """Retrieves the current cache size."""
211 # TODO(maruel): Keep a cache!
212 return sum(os.stat(self.path(f)).st_size for f in self.state)
213
214 def save(self):
215 """Saves the LRU ordering."""
216 json.dump(self.state, open(self.state_file, 'wb'))
217
218
219 def run_tha_test(manifest, cache_dir, remote):
Roger Tawa OOO till Jul 10th 2012/03/26 14:50:40 i assume the typo is intentional :-)
M-A Ruel 2012/03/26 15:55:59 I didn't have enough imagination to figure out a g
220 """Downloads the dependencies in the cache, hardlinks them into a temporary
221 directory and runs the executable.
222 """
223 cache = Cache(cache_dir, remote)
224 outdir = tempfile.mkdtemp(prefix='run_tha_test')
225 try:
226 for filepath, properties in manifest['files'].iteritems():
227 infile = properties['sha-1']
228 outfile = os.path.join(outdir, filepath)
229 cache.retrieve(infile)
230 outfiledir = os.path.dirname(outfile)
231 if not os.path.isdir(outfiledir):
232 os.makedirs(outfiledir)
233 link_file(outfile, cache.path(infile), HARDLINK)
234 os.chmod(outfile, properties['mode'])
235
236 cwd = os.path.join(outdir, manifest['relative_cwd'])
237 if not os.path.isdir(cwd):
238 os.makedirs(cwd)
239 if manifest.get('read_only'):
240 make_writable(outdir, True)
241 cmd = manifest['command']
242 logging.info('Running %s, cwd=%s' % (cmd, cwd))
243 return subprocess.call(cmd, cwd=cwd)
244 finally:
245 cache.save()
246 rmtree(outdir)
247
248
249 def main():
250 parser = optparse.OptionParser(
251 usage='%prog <options>', description=sys.modules[__name__].__doc__)
252 parser.add_option(
253 '-v', '--verbose', action='count', default=0, help='Use multiple times')
254 parser.add_option(
255 '-m', '--manifest', help='File/url describing what to map or run')
256 parser.add_option('--no-run', action='store_true', help='Skip the run part')
257 parser.add_option('--cache', default='cache', help='Cache directory')
258 parser.add_option('-r', '--remote', help='Remote where to get the items')
259
260 options, args = parser.parse_args()
261 level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)]
262 logging.basicConfig(
263 level=level,
264 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s')
265
266 if not options.manifest:
267 parser.error('--manifest is required.')
268 if not options.remote:
269 parser.error('--remote is required.')
270 if args:
271 parser.error('Unsupported args %s' % ' '.join(args))
272
273 manifest = json.load(open_remote(options.manifest))
274 return run_tha_test(manifest, os.path.abspath(options.cache), options.remote)
275
276
277 if __name__ == '__main__':
278 sys.exit(main())
OLDNEW
« tools/isolate/isolate.py ('K') | « tools/isolate/isolate.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698