OLD | NEW |
| (Empty) |
1 #!/usr/bin/env python | |
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved. | |
3 # Use of this source code is governed by a BSD-style license that can be | |
4 # found in the LICENSE file. | |
5 | |
6 """Package Management | |
7 | |
8 This module is used to bring in external Python dependencies via eggs from the | |
9 PyPi repository. | |
10 | |
11 The approach is to create a site directory in depot_tools root which will | |
12 contain those packages that are listed as dependencies in the module variable | |
13 PACKAGES. Since we can't guarantee that setuptools is available in all | |
14 distributions this module also contains the ability to bootstrap the site | |
15 directory by manually downloading and installing setuptools. Once setuptools is | |
16 available it uses that to install the other packages in the traditional | |
17 manner. | |
18 | |
19 Use is simple: | |
20 | |
21 import package_management | |
22 | |
23 # Before any imports from the site directory, call this. This only needs | |
24 # to be called in one place near the beginning of the program. | |
25 package_management.SetupSiteDirectory() | |
26 | |
27 # If 'SetupSiteDirectory' fails it will complain with an error message but | |
28 # continue happily. Expect ImportErrors when trying to import any third | |
29 # party modules from the site directory. | |
30 | |
31 import some_third_party_module | |
32 | |
33 ... etc ... | |
34 """ | |
35 | |
36 import cStringIO | |
37 import os | |
38 import re | |
39 import shutil | |
40 import site | |
41 import subprocess | |
42 import sys | |
43 import tempfile | |
44 import urllib2 | |
45 | |
46 | |
47 # This is the version of setuptools that we will download if the local | |
48 # python distribution does not include one. | |
49 SETUPTOOLS = ('setuptools', '0.6c11') | |
50 | |
51 # These are the packages that are to be installed in the site directory. | |
52 # easy_install makes it so that the most recently installed version of a | |
53 # package is the one that takes precedence, even if a newer version exists | |
54 # in the site directory. This allows us to blindly install these one on top | |
55 # of the other without worrying about whats already installed. | |
56 # | |
57 # NOTE: If we are often rolling these dependencies then users' site | |
58 # directories will grow monotonically. We could be purging any orphaned | |
59 # packages using the tools provided by pkg_resources. | |
60 PACKAGES = (('logilab-common', '0.57.1'), | |
61 ('logilab-astng', '0.23.1'), | |
62 ('pylint', '0.25.1')) | |
63 | |
64 | |
65 # The Python version suffix used in generating the site directory and in | |
66 # requesting packages from PyPi. | |
67 VERSION_SUFFIX = "%d.%d" % sys.version_info[0:2] | |
68 | |
69 # This is the root directory of the depot_tools installation. | |
70 ROOT_DIR = os.path.abspath(os.path.dirname(__file__)) | |
71 | |
72 # This is the path of the site directory we will use. We make this | |
73 # python version specific so that this will continue to work even if the | |
74 # python version is rolled. | |
75 SITE_DIR = os.path.join(ROOT_DIR, 'site-packages-py%s' % VERSION_SUFFIX) | |
76 | |
77 # This status file is created the last time PACKAGES were rolled successfully. | |
78 # It is used to determine if packages need to be rolled by comparing against | |
79 # the age of __file__. | |
80 LAST_ROLLED = os.path.join(SITE_DIR, 'last_rolled.txt') | |
81 | |
82 | |
83 class Error(Exception): | |
84 """The base class for all module errors.""" | |
85 pass | |
86 | |
87 | |
88 class InstallError(Error): | |
89 """Thrown if an installation is unable to complete.""" | |
90 pass | |
91 | |
92 | |
93 class Package(object): | |
94 """A package represents a release of a project. | |
95 | |
96 We use this as a lightweight version of pkg_resources, allowing us to | |
97 perform an end-run around setuptools for the purpose of bootstrapping. Its | |
98 functionality is very limited. | |
99 | |
100 Attributes: | |
101 name: the name of the package. | |
102 version: the version of the package. | |
103 safe_name: the safe name of the package. | |
104 safe_version: the safe version string of the package. | |
105 file_name: the filename-safe name of the package. | |
106 file_version: the filename-safe version string of the package. | |
107 """ | |
108 | |
109 def __init__(self, name, version): | |
110 """Initialize this package. | |
111 | |
112 Args: | |
113 name: the name of the package. | |
114 version: the version of the package. | |
115 """ | |
116 self.name = name | |
117 self.version = version | |
118 self.safe_name = Package._MakeSafeName(self.name) | |
119 self.safe_version = Package._MakeSafeVersion(self.version) | |
120 self.file_name = Package._MakeSafeForFilename(self.safe_name) | |
121 self.file_version = Package._MakeSafeForFilename(self.safe_version) | |
122 | |
123 @staticmethod | |
124 def _MakeSafeName(name): | |
125 """Makes a safe package name, as per pkg_resources.""" | |
126 return re.sub('[^A-Za-z0-9]+', '-', name) | |
127 | |
128 @staticmethod | |
129 def _MakeSafeVersion(version): | |
130 """Makes a safe package version string, as per pkg_resources.""" | |
131 version = re.sub('\s+', '.', version) | |
132 return re.sub('[^A-Za-z0-9\.]+', '-', version) | |
133 | |
134 @staticmethod | |
135 def _MakeSafeForFilename(safe_name_or_version): | |
136 """Makes a safe name or safe version safe to use in a file name. | |
137 |safe_name_or_version| must be a safe name or version string as returned | |
138 by GetSafeName or GetSafeVersion. | |
139 """ | |
140 return re.sub('-', '_', safe_name_or_version) | |
141 | |
142 def GetAsRequirementString(self): | |
143 """Builds an easy_install requirements string representing this package.""" | |
144 return '%s==%s' % (self.name, self.version) | |
145 | |
146 def GetFilename(self, extension=None): | |
147 """Builds a filename for this package using the setuptools convention. | |
148 If |extensions| is provided it will be appended to the generated filename, | |
149 otherwise only the basename is returned. | |
150 | |
151 The following url discusses the filename format: | |
152 | |
153 http://svn.python.org/projects/sandbox/trunk/setuptools/doc/formats.txt | |
154 """ | |
155 filename = '%s-%s-py%s' % (self.file_name, self.file_version, | |
156 VERSION_SUFFIX) | |
157 | |
158 if extension: | |
159 if not extension.startswith('.'): | |
160 filename += '.' | |
161 filename += extension | |
162 | |
163 return filename | |
164 | |
165 def GetPyPiUrl(self, extension): | |
166 """Returns the URL where this package is hosted on PyPi.""" | |
167 return 'http://pypi.python.org/packages/%s/%c/%s/%s' % (VERSION_SUFFIX, | |
168 self.file_name[0], self.file_name, self.GetFilename(extension)) | |
169 | |
170 def DownloadEgg(self, dest_dir, overwrite=False): | |
171 """Downloads the EGG for this URL. | |
172 | |
173 Args: | |
174 dest_dir: The directory where the EGG should be written. If the EGG | |
175 has already been downloaded and cached the returned path may not | |
176 be in this directory. | |
177 overwite: If True the destination path will be overwritten even if | |
178 it already exists. Defaults to False. | |
179 | |
180 Returns: | |
181 The path to the written EGG. | |
182 | |
183 Raises: | |
184 Error: if dest_dir doesn't exist, the EGG is unable to be written, | |
185 the URL doesn't exist, or the server returned an error, or the | |
186 transmission was interrupted. | |
187 """ | |
188 if not os.path.exists(dest_dir): | |
189 raise Error('Path does not exist: %s' % dest_dir) | |
190 | |
191 if not os.path.isdir(dest_dir): | |
192 raise Error('Path is not a directory: %s' % dest_dir) | |
193 | |
194 filename = os.path.abspath(os.path.join(dest_dir, self.GetFilename('egg'))) | |
195 if os.path.exists(filename): | |
196 if os.path.isdir(filename): | |
197 raise Error('Path is a directory: %s' % filename) | |
198 if not overwrite: | |
199 return filename | |
200 | |
201 url = self.GetPyPiUrl('egg') | |
202 | |
203 try: | |
204 url_stream = urllib2.urlopen(url) | |
205 local_file = open(filename, 'wb') | |
206 local_file.write(url_stream.read()) | |
207 local_file.close() | |
208 return filename | |
209 except (IOError, urllib2.HTTPError, urllib2.URLError): | |
210 # Reraise with a new error type, keeping the original message and | |
211 # traceback. | |
212 raise Error, sys.exc_info()[1], sys.exc_info()[2] | |
213 | |
214 | |
215 def AddToPythonPath(path): | |
216 """Adds the provided path to the head of PYTHONPATH and sys.path.""" | |
217 path = os.path.abspath(path) | |
218 if path not in sys.path: | |
219 sys.path.insert(0, path) | |
220 | |
221 paths = os.environ.get('PYTHONPATH', '').split(os.pathsep) | |
222 if path not in paths: | |
223 paths.insert(0, path) | |
224 os.environ['PYTHONPATH'] = os.pathsep.join(paths) | |
225 | |
226 | |
227 def AddSiteDirectory(path): | |
228 """Adds the provided path to the runtime as a site directory. | |
229 | |
230 Any modules that are in the site directory will be available for importing | |
231 after this returns. If modules are added or deleted this must be called | |
232 again for the changes to be reflected in the runtime. | |
233 | |
234 This calls both AddToPythonPath and site.addsitedir. Both are needed to | |
235 convince easy_install to treat |path| as a site directory. | |
236 """ | |
237 AddToPythonPath(path) | |
238 site.addsitedir(path) # pylint: disable=E1101 | |
239 | |
240 def EnsureSiteDirectory(path): | |
241 """Creates and/or adds the provided path to the runtime as a site directory. | |
242 | |
243 This works like AddSiteDirectory but it will create the directory if it | |
244 does not yet exist. | |
245 | |
246 Raise: | |
247 Error: if the site directory is unable to be created, or if it exists and | |
248 is not a directory. | |
249 """ | |
250 if os.path.exists(path): | |
251 if not os.path.isdir(path): | |
252 raise Error('Path is not a directory: %s' % path) | |
253 else: | |
254 try: | |
255 os.mkdir(path) | |
256 except IOError: | |
257 raise Error('Unable to create directory: %s' % path) | |
258 | |
259 AddSiteDirectory(path) | |
260 | |
261 | |
262 def ModuleIsFromPackage(module, package_path): | |
263 """Determines if a module has been imported from a given package. | |
264 | |
265 Args: | |
266 module: the module to test. | |
267 package_path: the path to the package to test. | |
268 | |
269 Returns: | |
270 True if |module| has been imported from |package_path|, False otherwise. | |
271 """ | |
272 try: | |
273 m = os.path.abspath(module.__file__) | |
274 p = os.path.abspath(package_path) | |
275 if len(m) <= len(p): | |
276 return False | |
277 if m[0:len(p)] != p: | |
278 return False | |
279 return m[len(p)] == os.sep | |
280 except AttributeError: | |
281 return False | |
282 | |
283 | |
284 def _CaptureStdStreams(function, *args, **kwargs): | |
285 """Captures stdout and stderr while running the provided function. | |
286 | |
287 This only works if |function| only accesses sys.stdout and sys.stderr. If | |
288 we need more than this we'll have to use subprocess.Popen. | |
289 | |
290 Args: | |
291 function: the function to be called. | |
292 args: the arguments to pass to |function|. | |
293 kwargs: the keyword arguments to pass to |function|. | |
294 """ | |
295 orig_stdout = sys.stdout | |
296 orig_stderr = sys.stderr | |
297 sys.stdout = cStringIO.StringIO() | |
298 sys.stderr = cStringIO.StringIO() | |
299 try: | |
300 return function(*args, **kwargs) | |
301 finally: | |
302 sys.stdout = orig_stdout | |
303 sys.stderr = orig_stderr | |
304 | |
305 | |
306 def InstallPackage(url_or_req, site_dir): | |
307 """Installs a package to a site directory. | |
308 | |
309 |site_dir| must exist and already be an active site directory. setuptools | |
310 must in the path. Uses easy_install which may involve a download from | |
311 pypi.python.org, so this also requires network access. | |
312 | |
313 Args: | |
314 url_or_req: the package to install, expressed as an URL (may be local), | |
315 or a requirement string. | |
316 site_dir: the site directory in which to install it. | |
317 | |
318 Raises: | |
319 InstallError: if installation fails for any reason. | |
320 """ | |
321 args = ['--quiet', '--install-dir', site_dir, '--exclude-scripts', | |
322 '--always-unzip', '--no-deps', url_or_req] | |
323 | |
324 # The easy_install script only calls SystemExit if something goes wrong. | |
325 # Otherwise, it falls through returning None. | |
326 try: | |
327 import setuptools.command.easy_install | |
328 _CaptureStdStreams(setuptools.command.easy_install.main, args) | |
329 except (ImportError, SystemExit): | |
330 # Re-raise the error, preserving the stack trace and message. | |
331 raise InstallError, sys.exc_info()[1], sys.exc_info()[2] | |
332 | |
333 | |
334 def _RunInSubprocess(pycode): | |
335 """Launches a python subprocess with the provided code. | |
336 | |
337 The subprocess will be launched with the same stdout and stderr. The | |
338 subprocess will use the same instance of python as is currently running, | |
339 passing |pycode| as arguments to this script. |pycode| will be interpreted | |
340 as python code in the context of this module. | |
341 | |
342 Returns: | |
343 True if the subprocess returned 0, False if it returned an error. | |
344 """ | |
345 return not subprocess.call([sys.executable, __file__, pycode]) | |
346 | |
347 | |
348 def _LoadSetupToolsFromEggAndInstall(egg_path): | |
349 """Loads setuptools from the provided egg |egg_path|, and installs it to | |
350 SITE_DIR. | |
351 | |
352 This is intended to be run from a subprocess as it pollutes the running | |
353 instance of Python by importing a module and then forcibly deleting its | |
354 source. | |
355 | |
356 Returns: | |
357 True on success, False on failure. | |
358 """ | |
359 AddToPythonPath(egg_path) | |
360 | |
361 try: | |
362 # Import setuptools and ensure it comes from the EGG. | |
363 import setuptools | |
364 if not ModuleIsFromPackage(setuptools, egg_path): | |
365 raise ImportError() | |
366 except ImportError: | |
367 print ' Unable to import downloaded package!' | |
368 return False | |
369 | |
370 try: | |
371 print ' Using setuptools to install itself ...' | |
372 InstallPackage(egg_path, SITE_DIR) | |
373 except InstallError: | |
374 print ' Unable to install setuptools!' | |
375 return False | |
376 | |
377 return True | |
378 | |
379 | |
380 def BootstrapSetupTools(): | |
381 """Bootstraps the runtime with setuptools. | |
382 | |
383 Will try to import setuptools directly. If not found it will attempt to | |
384 download it and load it from there. If the download is successful it will | |
385 then use setuptools to install itself in the site directory. | |
386 | |
387 This is meant to be run from a child process as it modifies the running | |
388 instance of Python by importing modules and then physically deleting them | |
389 from disk. | |
390 | |
391 Returns: | |
392 Returns True if 'import setuptools' will succeed, False otherwise. | |
393 """ | |
394 AddSiteDirectory(SITE_DIR) | |
395 | |
396 # Check if setuptools is already available. If so, we're done. | |
397 try: | |
398 import setuptools # pylint: disable=W0612 | |
399 return True | |
400 except ImportError: | |
401 pass | |
402 | |
403 print 'Bootstrapping setuptools ...' | |
404 | |
405 EnsureSiteDirectory(SITE_DIR) | |
406 | |
407 # Download the egg to a temp directory. | |
408 dest_dir = tempfile.mkdtemp('depot_tools') | |
409 path = None | |
410 try: | |
411 package = Package(*SETUPTOOLS) | |
412 print ' Downloading %s ...' % package.GetFilename() | |
413 path = package.DownloadEgg(dest_dir) | |
414 except Error: | |
415 print ' Download failed!' | |
416 shutil.rmtree(dest_dir) | |
417 return False | |
418 | |
419 try: | |
420 # Load the downloaded egg, and install it to the site directory. Do this | |
421 # in a subprocess so as not to pollute this runtime. | |
422 pycode = '_LoadSetupToolsFromEggAndInstall(%s)' % repr(path) | |
423 if not _RunInSubprocess(pycode): | |
424 raise Error() | |
425 | |
426 # Reload our site directory, which should now contain setuptools. | |
427 AddSiteDirectory(SITE_DIR) | |
428 | |
429 # Try to import setuptools | |
430 import setuptools | |
431 except ImportError: | |
432 print ' Unable to import setuptools!' | |
433 return False | |
434 except Error: | |
435 # This happens if RunInSubProcess fails, and the appropriate error has | |
436 # already been written to stdout. | |
437 return False | |
438 finally: | |
439 # Delete the temp directory. | |
440 shutil.rmtree(dest_dir) | |
441 | |
442 return True | |
443 | |
444 | |
445 def _GetModTime(path): | |
446 """Gets the last modification time associated with |path| in seconds since | |
447 epoch, returning 0 if |path| does not exist. | |
448 """ | |
449 try: | |
450 return os.stat(path).st_mtime | |
451 except: # pylint: disable=W0702 | |
452 # This error is different depending on the OS, hence no specified type. | |
453 return 0 | |
454 | |
455 | |
456 def _SiteDirectoryIsUpToDate(): | |
457 return _GetModTime(LAST_ROLLED) > _GetModTime(__file__) | |
458 | |
459 | |
460 def UpdateSiteDirectory(): | |
461 """Installs the packages from PACKAGES if they are not already installed. | |
462 At this point we must have setuptools in the site directory. | |
463 | |
464 This is intended to be run in a subprocess *prior* to the site directory | |
465 having been added to the parent process as it may cause packages to be | |
466 added and/or removed. | |
467 | |
468 Returns: | |
469 True on success, False otherwise. | |
470 """ | |
471 if _SiteDirectoryIsUpToDate(): | |
472 return True | |
473 | |
474 try: | |
475 EnsureSiteDirectory(SITE_DIR) | |
476 import pkg_resources | |
477 | |
478 # Determine if any packages actually need installing. | |
479 missing_packages = [] | |
480 for package in [SETUPTOOLS] + list(PACKAGES): | |
481 pkg = Package(*package) | |
482 req = pkg.GetAsRequirementString() | |
483 | |
484 # It may be that this package is already available in the site | |
485 # directory. If so, we can skip past it without trying to install it. | |
486 pkg_req = pkg_resources.Requirement.parse(req) | |
487 try: | |
488 dist = pkg_resources.working_set.find(pkg_req) | |
489 if dist: | |
490 continue | |
491 except pkg_resources.VersionConflict: | |
492 # This happens if another version of the package is already | |
493 # installed in another site directory (ie: the system site directory). | |
494 pass | |
495 | |
496 missing_packages.append(pkg) | |
497 | |
498 # Install the missing packages. | |
499 if missing_packages: | |
500 print 'Updating python packages ...' | |
501 for pkg in missing_packages: | |
502 print ' Installing %s ...' % pkg.GetFilename() | |
503 InstallPackage(pkg.GetAsRequirementString(), SITE_DIR) | |
504 | |
505 # Touch the status file so we know that we're up to date next time. | |
506 open(LAST_ROLLED, 'wb') | |
507 except InstallError, e: | |
508 print ' Installation failed: %s' % str(e) | |
509 return False | |
510 | |
511 return True | |
512 | |
513 | |
514 def SetupSiteDirectory(): | |
515 """Sets up the site directory, bootstrapping setuptools if necessary. | |
516 | |
517 If this finishes successfully then SITE_DIR will exist and will contain | |
518 the appropriate version of setuptools and all of the packages listed in | |
519 PACKAGES. | |
520 | |
521 This is the main workhorse of this module. Calling this will do everything | |
522 necessary to ensure that you have the desired packages installed in the | |
523 site directory, and the site directory enabled in this process. | |
524 | |
525 Returns: | |
526 True on success, False on failure. | |
527 """ | |
528 if _SiteDirectoryIsUpToDate(): | |
529 AddSiteDirectory(SITE_DIR) | |
530 return True | |
531 | |
532 if not _RunInSubprocess('BootstrapSetupTools()'): | |
533 return False | |
534 | |
535 if not _RunInSubprocess('UpdateSiteDirectory()'): | |
536 return False | |
537 | |
538 # Process the site directory so that the packages within it are available | |
539 # for import. | |
540 AddSiteDirectory(SITE_DIR) | |
541 | |
542 return True | |
543 | |
544 | |
545 def CanImportFromSiteDirectory(package_name): | |
546 """Determines if the given package can be imported from the site directory. | |
547 | |
548 Args: | |
549 package_name: the name of the package to import. | |
550 | |
551 Returns: | |
552 True if 'import package_name' will succeed and return a module from the | |
553 site directory, False otherwise. | |
554 """ | |
555 try: | |
556 return ModuleIsFromPackage(__import__(package_name), SITE_DIR) | |
557 except ImportError: | |
558 return False | |
559 | |
560 | |
561 def Test(): | |
562 """Runs SetupSiteDirectory and then tries to load pylint, ensuring that it | |
563 comes from the site directory just created. This is an end-to-end unittest | |
564 and allows for simple testing from the command-line by running | |
565 | |
566 ./package_management.py 'Test()' | |
567 """ | |
568 print 'Testing package_management.' | |
569 if not SetupSiteDirectory(): | |
570 print 'SetupSiteDirectory failed.' | |
571 return False | |
572 if not CanImportFromSiteDirectory('pylint'): | |
573 print 'CanImportFromSiteDirectory failed.' | |
574 return False | |
575 print 'Success!' | |
576 return True | |
577 | |
578 | |
579 def Main(): | |
580 """The main entry for the package management script. | |
581 | |
582 If no arguments are provided simply runs SetupSiteDirectory. If arguments | |
583 have been passed we execute the first argument as python code in the | |
584 context of this module. This mechanism is used during the bootstrap | |
585 process so that the main instance of Python does not have its runtime | |
586 polluted by various intermediate packages and imports. | |
587 | |
588 Returns: | |
589 0 on success, 1 otherwise. | |
590 """ | |
591 if len(sys.argv) == 2: | |
592 result = False | |
593 exec('result = %s' % sys.argv[1]) | |
594 | |
595 # Translate the success state to a return code. | |
596 return not result | |
597 else: | |
598 return not SetupSiteDirectory() | |
599 | |
600 | |
601 if __name__ == '__main__': | |
602 sys.exit(Main()) | |
OLD | NEW |