Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(171)

Side by Side Diff: bootstrap/bootstrap.py

Issue 1200843003: Added virtualenv for depot_tools (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Rebase Created 5 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « bootstrap/.gitignore ('k') | bootstrap/deps.pyl » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright 2014 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 import argparse
7 import contextlib
8 import glob
9 import logging
10 import os
11 import shutil
12 import subprocess
13 import sys
14 import tempfile
15
16 from util import STORAGE_URL, OBJECT_URL, LOCAL_STORAGE_PATH, LOCAL_OBJECT_URL
17 from util import read_deps, merge_deps, print_deps, platform_tag
18
19 LOGGER = logging.getLogger(__name__)
20
21 # /path/to/infra
22 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
23
24 PYTHON_BAT_WIN = '@%~dp0\\..\\Scripts\\python.exe %*'
25
26
27 class NoWheelException(Exception):
28 def __init__(self, name, version, build, source_sha):
29 super(NoWheelException, self).__init__(
30 'No matching wheel found for (%s==%s (build %s_%s))' %
31 (name, version, build, source_sha))
32
33
34 def check_pydistutils():
35 if os.path.exists(os.path.expanduser('~/.pydistutils.cfg')):
36 print >> sys.stderr, '\n'.join([
37 '',
38 '',
39 '=========== ERROR ===========',
40 'You have a ~/.pydistutils.cfg file, which interferes with the ',
41 'infra virtualenv environment. Please move it to the side and bootstrap ',
42 'again. Once infra has bootstrapped, you may move it back.',
43 '',
44 'Upstream bug: https://github.com/pypa/virtualenv/issues/88/',
45 ''
46 ])
47 sys.exit(1)
48
49
50 def ls(prefix):
51 from pip._vendor import requests # pylint: disable=E0611
52 data = requests.get(STORAGE_URL, params=dict(
53 prefix=prefix,
54 fields='items(name,md5Hash)'
55 )).json()
56 entries = data.get('items', [])
57 for entry in entries:
58 entry['md5Hash'] = entry['md5Hash'].decode('base64').encode('hex')
59 entry['local'] = False
60 # Also look in the local cache
61 entries.extend([
62 {'name': fname, 'md5Hash': None, 'local': True}
63 for fname in glob.glob(os.path.join(LOCAL_STORAGE_PATH,
64 prefix.split('/')[-1] + '*'))])
65 return entries
66
67
68 def sha_for(deps_entry):
69 if 'rev' in deps_entry:
70 return deps_entry['rev']
71 else:
72 return deps_entry['gs'].split('.')[0]
73
74
75 def get_links(deps):
76 import pip.wheel # pylint: disable=E0611
77 plat_tag = platform_tag()
78
79 links = []
80
81 for name, dep in deps.iteritems():
82 version, source_sha = dep['version'] , sha_for(dep)
83 prefix = 'wheels/{}-{}-{}_{}'.format(name, version, dep['build'],
84 source_sha)
85 generic_link = None
86 binary_link = None
87 local_link = None
88
89 for entry in ls(prefix):
90 fname = entry['name'].split('/')[-1]
91 md5hash = entry['md5Hash']
92 wheel_info = pip.wheel.Wheel.wheel_file_re.match(fname)
93 if not wheel_info:
94 LOGGER.warn('Skipping invalid wheel: %r', fname)
95 continue
96
97 if pip.wheel.Wheel(fname).supported():
98 if entry['local']:
99 link = LOCAL_OBJECT_URL.format(entry['name'])
100 local_link = link
101 continue
102 else:
103 link = OBJECT_URL.format(entry['name'], md5hash)
104 if fname.endswith('none-any.whl'):
105 if generic_link:
106 LOGGER.error(
107 'Found more than one generic matching wheel for %r: %r',
108 prefix, dep)
109 continue
110 generic_link = link
111 elif plat_tag in fname:
112 if binary_link:
113 LOGGER.error(
114 'Found more than one binary matching wheel for %r: %r',
115 prefix, dep)
116 continue
117 binary_link = link
118
119 if not binary_link and not generic_link and not local_link:
120 raise NoWheelException(name, version, dep['build'], source_sha)
121
122 links.append(local_link or binary_link or generic_link)
123
124 return links
125
126
127 @contextlib.contextmanager
128 def html_index(links):
129 tf = tempfile.mktemp('.html')
130 try:
131 with open(tf, 'w') as f:
132 print >> f, '<html><body>'
133 for link in links:
134 print >> f, '<a href="%s">wat</a>' % link
135 print >> f, '</body></html>'
136 yield tf
137 finally:
138 os.unlink(tf)
139
140
141 def install(deps):
142 bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
143 pip = os.path.join(sys.prefix, bin_dir, 'pip')
144
145 links = get_links(deps)
146 with html_index(links) as ipath:
147 requirements = []
148 # TODO(iannucci): Do this as a requirements.txt
149 for name, deps_entry in deps.iteritems():
150 if not deps_entry.get('implicit'):
151 requirements.append('%s==%s' % (name, deps_entry['version']))
152 subprocess.check_call(
153 [pip, 'install', '--no-index', '--download-cache',
154 os.path.join(ROOT, '.wheelcache'), '-f', ipath] + requirements)
155
156
157 def activate_env(env, deps):
158 if hasattr(sys, 'real_prefix'):
159 LOGGER.error('Already activated environment!')
160 return
161
162 print 'Activating environment: %r' % env
163 assert isinstance(deps, dict)
164
165 manifest_path = os.path.join(env, 'manifest.pyl')
166 cur_deps = read_deps(manifest_path)
167 if cur_deps != deps:
168 print ' Removing old environment: %r' % cur_deps
169 shutil.rmtree(env, ignore_errors=True)
170 cur_deps = None
171
172 if cur_deps is None:
173 check_pydistutils()
174
175 print ' Building new environment'
176 # Add in bundled virtualenv lib
177 sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'virtualenv'))
178 import virtualenv # pylint: disable=F0401
179 virtualenv.create_environment(
180 env, search_dirs=virtualenv.file_search_dirs())
181
182 print ' Activating environment'
183 # Ensure hermeticity during activation.
184 os.environ.pop('PYTHONPATH', None)
185 bin_dir = 'Scripts' if sys.platform.startswith('win') else 'bin'
186 activate_this = os.path.join(env, bin_dir, 'activate_this.py')
187 execfile(activate_this, dict(__file__=activate_this))
188
189 if cur_deps is None:
190 print ' Installing deps'
191 print_deps(deps, indent=2, with_implicit=False)
192 install(deps)
193 virtualenv.make_environment_relocatable(env)
194 with open(manifest_path, 'wb') as f:
195 f.write(repr(deps) + '\n')
196
197 # Create bin\python.bat on Windows to unify path where Python is found.
198 if sys.platform.startswith('win'):
199 bin_path = os.path.join(env, 'bin')
200 if not os.path.isdir(bin_path):
201 os.makedirs(bin_path)
202 python_bat_path = os.path.join(bin_path, 'python.bat')
203 if not os.path.isfile(python_bat_path):
204 with open(python_bat_path, 'w') as python_bat_file:
205 python_bat_file.write(PYTHON_BAT_WIN)
206
207 print 'Done creating environment'
208
209
210 def main(args):
211 parser = argparse.ArgumentParser()
212 parser.add_argument('--deps-file', '--deps_file', action='append',
213 help='Path to deps.pyl file (may be used multiple times)')
214 parser.add_argument('env_path',
215 help='Path to place environment (default: %(default)s)',
216 default='ENV')
217 opts = parser.parse_args(args)
218
219 deps = merge_deps(opts.deps_file)
220 activate_env(opts.env_path, deps)
221
222
223 if __name__ == '__main__':
224 logging.basicConfig()
225 LOGGER.setLevel(logging.DEBUG)
226 sys.exit(main(sys.argv[1:]))
OLDNEW
« no previous file with comments | « bootstrap/.gitignore ('k') | bootstrap/deps.pyl » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698