| OLD | NEW |
|---|
| (Empty) |
| 1 #!/usr/bin/env python | |
| 2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
| 3 # Use of this source code is governed by a BSD-style license that can be | |
| 4 # found in the LICENSE file. | |
| 5 | |
| 6 """Reads a manifest, creates a tree of hardlinks and runs the test. | |
| 7 | |
| 8 Keeps a local cache. | |
| 9 """ | |
| 10 | |
| 11 import ctypes | |
| 12 import json | |
| 13 import logging | |
| 14 import optparse | |
| 15 import os | |
| 16 import re | |
| 17 import shutil | |
| 18 import subprocess | |
| 19 import sys | |
| 20 import tempfile | |
| 21 import time | |
| 22 import urllib | |
| 23 | |
| 24 | |
| 25 # Types of action accepted by recreate_tree(). | |
| 26 HARDLINK, SYMLINK, COPY = range(4)[1:] | |
| 27 | |
| 28 | |
| 29 class MappingError(OSError): | |
| 30 """Failed to recreate the tree.""" | |
| 31 pass | |
| 32 | |
| 33 | |
| 34 def os_link(source, link_name): | |
| 35 """Add support for os.link() on Windows.""" | |
| 36 if sys.platform == 'win32': | |
| 37 if not ctypes.windll.kernel32.CreateHardLinkW( | |
| 38 unicode(link_name), unicode(source), 0): | |
| 39 raise OSError() | |
| 40 else: | |
| 41 os.link(source, link_name) | |
| 42 | |
| 43 | |
| 44 def link_file(outfile, infile, action): | |
| 45 """Links a file. The type of link depends on |action|.""" | |
| 46 logging.debug('Mapping %s to %s' % (infile, outfile)) | |
| 47 if action not in (HARDLINK, SYMLINK, COPY): | |
| 48 raise ValueError('Unknown mapping action %s' % action) | |
| 49 if os.path.isfile(outfile): | |
| 50 raise MappingError('%s already exist' % outfile) | |
| 51 | |
| 52 if action == COPY: | |
| 53 shutil.copy(infile, outfile) | |
| 54 elif action == SYMLINK and sys.platform != 'win32': | |
| 55 # On windows, symlink are converted to hardlink and fails over to copy. | |
| 56 os.symlink(infile, outfile) | |
| 57 else: | |
| 58 try: | |
| 59 os_link(infile, outfile) | |
| 60 except OSError: | |
| 61 # Probably a different file system. | |
| 62 logging.warn( | |
| 63 'Failed to hardlink, failing back to copy %s to %s' % ( | |
| 64 infile, outfile)) | |
| 65 shutil.copy(infile, outfile) | |
| 66 | |
| 67 | |
| 68 def _set_write_bit(path, read_only): | |
| 69 """Sets or resets the executable bit on a file or directory.""" | |
| 70 mode = os.stat(path).st_mode | |
| 71 if read_only: | |
| 72 mode = mode & 0500 | |
| 73 else: | |
| 74 mode = mode | 0200 | |
| 75 if hasattr(os, 'lchmod'): | |
| 76 os.lchmod(path, mode) # pylint: disable=E1101 | |
| 77 else: | |
| 78 # TODO(maruel): Implement proper DACL modification on Windows. | |
| 79 os.chmod(path, mode) | |
| 80 | |
| 81 | |
| 82 def make_writable(root, read_only): | |
| 83 """Toggle the writable bit on a directory tree.""" | |
| 84 root = os.path.abspath(root) | |
| 85 for dirpath, dirnames, filenames in os.walk(root, topdown=True): | |
| 86 for filename in filenames: | |
| 87 _set_write_bit(os.path.join(dirpath, filename), read_only) | |
| 88 | |
| 89 for dirname in dirnames: | |
| 90 _set_write_bit(os.path.join(dirpath, dirname), read_only) | |
| 91 | |
| 92 | |
| 93 def rmtree(root): | |
| 94 """Wrapper around shutil.rmtree() to retry automatically on Windows.""" | |
| 95 make_writable(root, False) | |
| 96 if sys.platform == 'win32': | |
| 97 for i in range(3): | |
| 98 try: | |
| 99 shutil.rmtree(root) | |
| 100 break | |
| 101 except WindowsError: # pylint: disable=E0602 | |
| 102 delay = (i+1)*2 | |
| 103 print >> sys.stderr, ( | |
| 104 'The test has subprocess outliving it. Sleep %d seconds.' % delay) | |
| 105 time.sleep(delay) | |
| 106 else: | |
| 107 shutil.rmtree(root) | |
| 108 | |
| 109 | |
| 110 def open_remote(file_or_url): | |
| 111 """Reads a file or url.""" | |
| 112 if re.match(r'^https?://.+$', file_or_url): | |
| 113 return urllib.urlopen(file_or_url) | |
| 114 return open(file_or_url, 'rb') | |
| 115 | |
| 116 | |
| 117 def download_or_copy(file_or_url, dest): | |
| 118 """Copies a file or download an url.""" | |
| 119 if re.match(r'^https?://.+$', file_or_url): | |
| 120 urllib.urlretrieve(file_or_url, dest) | |
| 121 else: | |
| 122 shutil.copy(file_or_url, dest) | |
| 123 | |
| 124 | |
| 125 def get_free_space(path): | |
| 126 """Returns the number of free bytes.""" | |
| 127 if sys.platform == 'win32': | |
| 128 free_bytes = ctypes.c_ulonglong(0) | |
| 129 ctypes.windll.kernel32.GetDiskFreeSpaceExW( | |
| 130 ctypes.c_wchar_p(path), None, None, ctypes.pointer(free_bytes)) | |
| 131 return free_bytes.value | |
| 132 f = os.statvfs(path) | |
| 133 return f.f_bfree * f.f_frsize | |
| 134 | |
| 135 | |
| 136 class Cache(object): | |
| 137 """Stateful LRU cache. | |
| 138 | |
| 139 Saves its state as json file. | |
| 140 """ | |
| 141 STATE_FILE = 'state.json' | |
| 142 | |
| 143 def __init__(self, cache_dir, remote, max_cache_size, min_free_space): | |
| 144 """ | |
| 145 Arguments: | |
| 146 - cache_dir: Directory where to place the cache. | |
| 147 - remote: Remote directory (NFS, SMB, etc) or HTTP url to fetch the objects | |
| 148 from | |
| 149 - max_cache_size: Trim if the cache gets larger than this value. If 0, the | |
| 150 cache is effectively a leak. | |
| 151 - min_free_space: Trim if disk free space becomes lower than this value. If | |
| 152 0, it unconditionally fill the disk. | |
| 153 """ | |
| 154 self.cache_dir = cache_dir | |
| 155 self.remote = remote | |
| 156 self.max_cache_size = max_cache_size | |
| 157 self.min_free_space = min_free_space | |
| 158 self.state_file = os.path.join(cache_dir, self.STATE_FILE) | |
| 159 # The files are kept as an array in a LRU style. E.g. self.state[0] is the | |
| 160 # oldest item. | |
| 161 self.state = [] | |
| 162 | |
| 163 if not os.path.isdir(self.cache_dir): | |
| 164 os.makedirs(self.cache_dir) | |
| 165 if os.path.isfile(self.state_file): | |
| 166 try: | |
| 167 self.state = json.load(open(self.state_file, 'rb')) | |
| 168 except ValueError: | |
| 169 # Too bad. The file will be overwritten and the cache cleared. | |
| 170 pass | |
| 171 self.trim() | |
| 172 | |
| 173 def trim(self): | |
| 174 """Trims anything we don't know, make sure enough free space exists.""" | |
| 175 for f in os.listdir(self.cache_dir): | |
| 176 if f == self.STATE_FILE or f in self.state: | |
| 177 continue | |
| 178 logging.warn('Unknown file %s from cache' % f) | |
| 179 # Insert as the oldest file. It will be deleted eventually if not | |
| 180 # accessed. | |
| 181 self.state.insert(0, f) | |
| 182 | |
| 183 # Ensure enough free space. | |
| 184 while ( | |
| 185 self.min_free_space and | |
| 186 self.state and | |
| 187 get_free_space(self.cache_dir) < self.min_free_space): | |
| 188 os.remove(self.path(self.state.pop(0))) | |
| 189 | |
| 190 # Ensure maximum cache size. | |
| 191 if self.max_cache_size and self.state: | |
| 192 sizes = [os.stat(self.path(f)).st_size for f in self.state] | |
| 193 while sizes and sum(sizes) > self.max_cache_size: | |
| 194 # Delete the oldest item. | |
| 195 os.remove(self.path(self.state.pop(0))) | |
| 196 sizes.pop(0) | |
| 197 | |
| 198 self.save() | |
| 199 | |
| 200 def retrieve(self, item): | |
| 201 """Retrieves a file from the remote and add it to the cache.""" | |
| 202 assert not '/' in item | |
| 203 try: | |
| 204 index = self.state.index(item) | |
| 205 # Was already in cache. Update it's LRU value. | |
| 206 self.state.pop(index) | |
| 207 self.state.append(item) | |
| 208 return False | |
| 209 except ValueError: | |
| 210 out = self.path(item) | |
| 211 download_or_copy(os.path.join(self.remote, item), out) | |
| 212 self.state.append(item) | |
| 213 return True | |
| 214 finally: | |
| 215 self.save() | |
| 216 | |
| 217 def path(self, item): | |
| 218 """Returns the path to one item.""" | |
| 219 return os.path.join(self.cache_dir, item) | |
| 220 | |
| 221 def save(self): | |
| 222 """Saves the LRU ordering.""" | |
| 223 json.dump(self.state, open(self.state_file, 'wb')) | |
| 224 | |
| 225 | |
| 226 def run_tha_test(manifest, cache_dir, remote, max_cache_size, min_free_space): | |
| 227 """Downloads the dependencies in the cache, hardlinks them into a temporary | |
| 228 directory and runs the executable. | |
| 229 """ | |
| 230 cache = Cache(cache_dir, remote, max_cache_size, min_free_space) | |
| 231 outdir = tempfile.mkdtemp(prefix='run_tha_test') | |
| 232 try: | |
| 233 for filepath, properties in manifest['files'].iteritems(): | |
| 234 infile = properties['sha-1'] | |
| 235 outfile = os.path.join(outdir, filepath) | |
| 236 cache.retrieve(infile) | |
| 237 outfiledir = os.path.dirname(outfile) | |
| 238 if not os.path.isdir(outfiledir): | |
| 239 os.makedirs(outfiledir) | |
| 240 link_file(outfile, cache.path(infile), HARDLINK) | |
| 241 os.chmod(outfile, properties['mode']) | |
| 242 | |
| 243 cwd = os.path.join(outdir, manifest['relative_cwd']) | |
| 244 if not os.path.isdir(cwd): | |
| 245 os.makedirs(cwd) | |
| 246 if manifest.get('read_only'): | |
| 247 make_writable(outdir, True) | |
| 248 cmd = manifest['command'] | |
| 249 logging.info('Running %s, cwd=%s' % (cmd, cwd)) | |
| 250 return subprocess.call(cmd, cwd=cwd) | |
| 251 finally: | |
| 252 # Save first, in case an exception occur in the following lines, then clean | |
| 253 # up. | |
| 254 cache.save() | |
| 255 rmtree(outdir) | |
| 256 cache.trim() | |
| 257 | |
| 258 | |
| 259 def main(): | |
| 260 parser = optparse.OptionParser( | |
| 261 usage='%prog <options>', description=sys.modules[__name__].__doc__) | |
| 262 parser.add_option( | |
| 263 '-v', '--verbose', action='count', default=0, help='Use multiple times') | |
| 264 parser.add_option( | |
| 265 '-m', '--manifest', | |
| 266 metavar='FILE', | |
| 267 help='File/url describing what to map or run') | |
| 268 parser.add_option('--no-run', action='store_true', help='Skip the run part') | |
| 269 parser.add_option( | |
| 270 '--cache', | |
| 271 default='cache', | |
| 272 metavar='DIR', | |
| 273 help='Cache directory, default=%default') | |
| 274 parser.add_option( | |
| 275 '-r', '--remote', metavar='URL', help='Remote where to get the items') | |
| 276 parser.add_option( | |
| 277 '--max-cache-size', | |
| 278 type='int', | |
| 279 metavar='NNN', | |
| 280 default=20*1024*1024*1024, | |
| 281 help='Trim if the cache gets larger than this value, default=%default') | |
| 282 parser.add_option( | |
| 283 '--min-free-space', | |
| 284 type='int', | |
| 285 metavar='NNN', | |
| 286 default=1*1024*1024*1024, | |
| 287 help='Trim if disk free space becomes lower than this value, ' | |
| 288 'default=%default') | |
| 289 | |
| 290 options, args = parser.parse_args() | |
| 291 level = [logging.ERROR, logging.INFO, logging.DEBUG][min(2, options.verbose)] | |
| 292 logging.basicConfig( | |
| 293 level=level, | |
| 294 format='%(levelname)5s %(module)15s(%(lineno)3d): %(message)s') | |
| 295 | |
| 296 if not options.manifest: | |
| 297 parser.error('--manifest is required.') | |
| 298 if not options.remote: | |
| 299 parser.error('--remote is required.') | |
| 300 if args: | |
| 301 parser.error('Unsupported args %s' % ' '.join(args)) | |
| 302 | |
| 303 manifest = json.load(open_remote(options.manifest)) | |
| 304 return run_tha_test( | |
| 305 manifest, os.path.abspath(options.cache), options.remote, | |
| 306 options.max_cache_size, options.min_free_space) | |
| 307 | |
| 308 | |
| 309 if __name__ == '__main__': | |
| 310 sys.exit(main()) | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 10027006:
Rename tree_creator.py to run_test_from_archive.py (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src