Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(138)

Side by Side Diff: package_management.py

Issue 10447014: Add pylint to depot_tools. (Closed) Base URL: svn://svn.chromium.org/chrome/trunk/tools/depot_tools
Patch Set: Fix unittests. Created 8 years, 6 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
« no previous file with comments | « bootstrap/win/win_tools.bat ('k') | presubmit_canned_checks.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
(Empty)
1 #!/usr/bin/env python
2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
3 # Use of this source code is governed by a BSD-style license that can be
4 # found in the LICENSE file.
5
6 """Package Management
7
8 This module is used to bring in external Python dependencies via eggs from the
9 PyPi repository.
10
11 The approach is to create a site directory in depot_tools root which will
12 contain those packages that are listed as dependencies in the module variable
13 PACKAGES. Since we can't guarantee that setuptools is available in all
14 distributions this module also contains the ability to bootstrap the site
15 directory by manually downloading and installing setuptools. Once setuptools is
16 available it uses that to install the other packages in the traditional
17 manner.
18
19 Use is simple:
20
21 import package_management
22
23 # Before any imports from the site directory, call this. This only needs
24 # to be called in one place near the beginning of the program.
25 package_management.SetupSiteDirectory()
26
27 # If 'SetupSiteDirectory' fails it will complain with an error message but
28 # continue happily. Expect ImportErrors when trying to import any third
29 # party modules from the site directory.
30
31 import some_third_party_module
32
33 ... etc ...
34 """
35
36 import cStringIO
37 import os
38 import re
39 import shutil
40 import site
41 import subprocess
42 import sys
43 import tempfile
44 import urllib2
45
46
47 # This is the version of setuptools that we will download if the local
48 # python distribution does not include one.
49 SETUPTOOLS = ('setuptools', '0.6c11')
50
51 # These are the packages that are to be installed in the site directory.
52 # easy_install makes it so that the most recently installed version of a
53 # package is the one that takes precedence, even if a newer version exists
54 # in the site directory. This allows us to blindly install these one on top
55 # of the other without worrying about whats already installed.
56 #
57 # NOTE: If we are often rolling these dependencies then users' site
58 # directories will grow monotonically. We could be purging any orphaned
59 # packages using the tools provided by pkg_resources.
60 PACKAGES = (('logilab-common', '0.57.1'),
61 ('logilab-astng', '0.23.1'),
62 ('pylint', '0.25.1'))
63
64
65 # The Python version suffix used in generating the site directory and in
66 # requesting packages from PyPi.
67 VERSION_SUFFIX = "%d.%d" % sys.version_info[0:2]
68
69 # This is the root directory of the depot_tools installation.
70 ROOT_DIR = os.path.abspath(os.path.dirname(__file__))
71
72 # This is the path of the site directory we will use. We make this
73 # python version specific so that this will continue to work even if the
74 # python version is rolled.
75 SITE_DIR = os.path.join(ROOT_DIR, 'site-packages-py%s' % VERSION_SUFFIX)
76
77 # This status file is created the last time PACKAGES were rolled successfully.
78 # It is used to determine if packages need to be rolled by comparing against
79 # the age of __file__.
80 LAST_ROLLED = os.path.join(SITE_DIR, 'last_rolled.txt')
81
82
83 class Error(Exception):
84 """The base class for all module errors."""
85 pass
86
87
88 class InstallError(Error):
89 """Thrown if an installation is unable to complete."""
90 pass
91
92
93 class Package(object):
94 """A package represents a release of a project.
95
96 We use this as a lightweight version of pkg_resources, allowing us to
97 perform an end-run around setuptools for the purpose of bootstrapping. Its
98 functionality is very limited.
99
100 Attributes:
101 name: the name of the package.
102 version: the version of the package.
103 safe_name: the safe name of the package.
104 safe_version: the safe version string of the package.
105 file_name: the filename-safe name of the package.
106 file_version: the filename-safe version string of the package.
107 """
108
109 def __init__(self, name, version):
110 """Initialize this package.
111
112 Args:
113 name: the name of the package.
114 version: the version of the package.
115 """
116 self.name = name
117 self.version = version
118 self.safe_name = Package._MakeSafeName(self.name)
119 self.safe_version = Package._MakeSafeVersion(self.version)
120 self.file_name = Package._MakeSafeForFilename(self.safe_name)
121 self.file_version = Package._MakeSafeForFilename(self.safe_version)
122
123 @staticmethod
124 def _MakeSafeName(name):
125 """Makes a safe package name, as per pkg_resources."""
126 return re.sub('[^A-Za-z0-9]+', '-', name)
127
128 @staticmethod
129 def _MakeSafeVersion(version):
130 """Makes a safe package version string, as per pkg_resources."""
131 version = re.sub('\s+', '.', version)
132 return re.sub('[^A-Za-z0-9\.]+', '-', version)
133
134 @staticmethod
135 def _MakeSafeForFilename(safe_name_or_version):
136 """Makes a safe name or safe version safe to use in a file name.
137 |safe_name_or_version| must be a safe name or version string as returned
138 by GetSafeName or GetSafeVersion.
139 """
140 return re.sub('-', '_', safe_name_or_version)
141
142 def GetAsRequirementString(self):
143 """Builds an easy_install requirements string representing this package."""
144 return '%s==%s' % (self.name, self.version)
145
146 def GetFilename(self, extension=None):
147 """Builds a filename for this package using the setuptools convention.
148 If |extensions| is provided it will be appended to the generated filename,
149 otherwise only the basename is returned.
150
151 The following url discusses the filename format:
152
153 http://svn.python.org/projects/sandbox/trunk/setuptools/doc/formats.txt
154 """
155 filename = '%s-%s-py%s' % (self.file_name, self.file_version,
156 VERSION_SUFFIX)
157
158 if extension:
159 if not extension.startswith('.'):
160 filename += '.'
161 filename += extension
162
163 return filename
164
165 def GetPyPiUrl(self, extension):
166 """Returns the URL where this package is hosted on PyPi."""
167 return 'http://pypi.python.org/packages/%s/%c/%s/%s' % (VERSION_SUFFIX,
168 self.file_name[0], self.file_name, self.GetFilename(extension))
169
170 def DownloadEgg(self, dest_dir, overwrite=False):
171 """Downloads the EGG for this URL.
172
173 Args:
174 dest_dir: The directory where the EGG should be written. If the EGG
175 has already been downloaded and cached the returned path may not
176 be in this directory.
177 overwite: If True the destination path will be overwritten even if
178 it already exists. Defaults to False.
179
180 Returns:
181 The path to the written EGG.
182
183 Raises:
184 Error: if dest_dir doesn't exist, the EGG is unable to be written,
185 the URL doesn't exist, or the server returned an error, or the
186 transmission was interrupted.
187 """
188 if not os.path.exists(dest_dir):
189 raise Error('Path does not exist: %s' % dest_dir)
190
191 if not os.path.isdir(dest_dir):
192 raise Error('Path is not a directory: %s' % dest_dir)
193
194 filename = os.path.abspath(os.path.join(dest_dir, self.GetFilename('egg')))
195 if os.path.exists(filename):
196 if os.path.isdir(filename):
197 raise Error('Path is a directory: %s' % filename)
198 if not overwrite:
199 return filename
200
201 url = self.GetPyPiUrl('egg')
202
203 try:
204 url_stream = urllib2.urlopen(url)
205 local_file = open(filename, 'wb')
206 local_file.write(url_stream.read())
207 local_file.close()
208 return filename
209 except (IOError, urllib2.HTTPError, urllib2.URLError):
210 # Reraise with a new error type, keeping the original message and
211 # traceback.
212 raise Error, sys.exc_info()[1], sys.exc_info()[2]
213
214
215 def AddToPythonPath(path):
216 """Adds the provided path to the head of PYTHONPATH and sys.path."""
217 path = os.path.abspath(path)
218 if path not in sys.path:
219 sys.path.insert(0, path)
220
221 paths = os.environ.get('PYTHONPATH', '').split(os.pathsep)
222 if path not in paths:
223 paths.insert(0, path)
224 os.environ['PYTHONPATH'] = os.pathsep.join(paths)
225
226
227 def AddSiteDirectory(path):
228 """Adds the provided path to the runtime as a site directory.
229
230 Any modules that are in the site directory will be available for importing
231 after this returns. If modules are added or deleted this must be called
232 again for the changes to be reflected in the runtime.
233
234 This calls both AddToPythonPath and site.addsitedir. Both are needed to
235 convince easy_install to treat |path| as a site directory.
236 """
237 AddToPythonPath(path)
238 site.addsitedir(path) # pylint: disable=E1101
239
240 def EnsureSiteDirectory(path):
241 """Creates and/or adds the provided path to the runtime as a site directory.
242
243 This works like AddSiteDirectory but it will create the directory if it
244 does not yet exist.
245
246 Raise:
247 Error: if the site directory is unable to be created, or if it exists and
248 is not a directory.
249 """
250 if os.path.exists(path):
251 if not os.path.isdir(path):
252 raise Error('Path is not a directory: %s' % path)
253 else:
254 try:
255 os.mkdir(path)
256 except IOError:
257 raise Error('Unable to create directory: %s' % path)
258
259 AddSiteDirectory(path)
260
261
262 def ModuleIsFromPackage(module, package_path):
263 """Determines if a module has been imported from a given package.
264
265 Args:
266 module: the module to test.
267 package_path: the path to the package to test.
268
269 Returns:
270 True if |module| has been imported from |package_path|, False otherwise.
271 """
272 try:
273 m = os.path.abspath(module.__file__)
274 p = os.path.abspath(package_path)
275 if len(m) <= len(p):
276 return False
277 if m[0:len(p)] != p:
278 return False
279 return m[len(p)] == os.sep
280 except AttributeError:
281 return False
282
283
284 def _CaptureStdStreams(function, *args, **kwargs):
285 """Captures stdout and stderr while running the provided function.
286
287 This only works if |function| only accesses sys.stdout and sys.stderr. If
288 we need more than this we'll have to use subprocess.Popen.
289
290 Args:
291 function: the function to be called.
292 args: the arguments to pass to |function|.
293 kwargs: the keyword arguments to pass to |function|.
294 """
295 orig_stdout = sys.stdout
296 orig_stderr = sys.stderr
297 sys.stdout = cStringIO.StringIO()
298 sys.stderr = cStringIO.StringIO()
299 try:
300 return function(*args, **kwargs)
301 finally:
302 sys.stdout = orig_stdout
303 sys.stderr = orig_stderr
304
305
306 def InstallPackage(url_or_req, site_dir):
307 """Installs a package to a site directory.
308
309 |site_dir| must exist and already be an active site directory. setuptools
310 must in the path. Uses easy_install which may involve a download from
311 pypi.python.org, so this also requires network access.
312
313 Args:
314 url_or_req: the package to install, expressed as an URL (may be local),
315 or a requirement string.
316 site_dir: the site directory in which to install it.
317
318 Raises:
319 InstallError: if installation fails for any reason.
320 """
321 args = ['--quiet', '--install-dir', site_dir, '--exclude-scripts',
322 '--always-unzip', '--no-deps', url_or_req]
323
324 # The easy_install script only calls SystemExit if something goes wrong.
325 # Otherwise, it falls through returning None.
326 try:
327 import setuptools.command.easy_install
328 _CaptureStdStreams(setuptools.command.easy_install.main, args)
329 except (ImportError, SystemExit):
330 # Re-raise the error, preserving the stack trace and message.
331 raise InstallError, sys.exc_info()[1], sys.exc_info()[2]
332
333
334 def _RunInSubprocess(pycode):
335 """Launches a python subprocess with the provided code.
336
337 The subprocess will be launched with the same stdout and stderr. The
338 subprocess will use the same instance of python as is currently running,
339 passing |pycode| as arguments to this script. |pycode| will be interpreted
340 as python code in the context of this module.
341
342 Returns:
343 True if the subprocess returned 0, False if it returned an error.
344 """
345 return not subprocess.call([sys.executable, __file__, pycode])
346
347
348 def _LoadSetupToolsFromEggAndInstall(egg_path):
349 """Loads setuptools from the provided egg |egg_path|, and installs it to
350 SITE_DIR.
351
352 This is intended to be run from a subprocess as it pollutes the running
353 instance of Python by importing a module and then forcibly deleting its
354 source.
355
356 Returns:
357 True on success, False on failure.
358 """
359 AddToPythonPath(egg_path)
360
361 try:
362 # Import setuptools and ensure it comes from the EGG.
363 import setuptools
364 if not ModuleIsFromPackage(setuptools, egg_path):
365 raise ImportError()
366 except ImportError:
367 print ' Unable to import downloaded package!'
368 return False
369
370 try:
371 print ' Using setuptools to install itself ...'
372 InstallPackage(egg_path, SITE_DIR)
373 except InstallError:
374 print ' Unable to install setuptools!'
375 return False
376
377 return True
378
379
380 def BootstrapSetupTools():
381 """Bootstraps the runtime with setuptools.
382
383 Will try to import setuptools directly. If not found it will attempt to
384 download it and load it from there. If the download is successful it will
385 then use setuptools to install itself in the site directory.
386
387 This is meant to be run from a child process as it modifies the running
388 instance of Python by importing modules and then physically deleting them
389 from disk.
390
391 Returns:
392 Returns True if 'import setuptools' will succeed, False otherwise.
393 """
394 AddSiteDirectory(SITE_DIR)
395
396 # Check if setuptools is already available. If so, we're done.
397 try:
398 import setuptools # pylint: disable=W0612
399 return True
400 except ImportError:
401 pass
402
403 print 'Bootstrapping setuptools ...'
404
405 EnsureSiteDirectory(SITE_DIR)
406
407 # Download the egg to a temp directory.
408 dest_dir = tempfile.mkdtemp('depot_tools')
409 path = None
410 try:
411 package = Package(*SETUPTOOLS)
412 print ' Downloading %s ...' % package.GetFilename()
413 path = package.DownloadEgg(dest_dir)
414 except Error:
415 print ' Download failed!'
416 shutil.rmtree(dest_dir)
417 return False
418
419 try:
420 # Load the downloaded egg, and install it to the site directory. Do this
421 # in a subprocess so as not to pollute this runtime.
422 pycode = '_LoadSetupToolsFromEggAndInstall(%s)' % repr(path)
423 if not _RunInSubprocess(pycode):
424 raise Error()
425
426 # Reload our site directory, which should now contain setuptools.
427 AddSiteDirectory(SITE_DIR)
428
429 # Try to import setuptools
430 import setuptools
431 except ImportError:
432 print ' Unable to import setuptools!'
433 return False
434 except Error:
435 # This happens if RunInSubProcess fails, and the appropriate error has
436 # already been written to stdout.
437 return False
438 finally:
439 # Delete the temp directory.
440 shutil.rmtree(dest_dir)
441
442 return True
443
444
445 def _GetModTime(path):
446 """Gets the last modification time associated with |path| in seconds since
447 epoch, returning 0 if |path| does not exist.
448 """
449 try:
450 return os.stat(path).st_mtime
451 except: # pylint: disable=W0702
452 # This error is different depending on the OS, hence no specified type.
453 return 0
454
455
456 def _SiteDirectoryIsUpToDate():
457 return _GetModTime(LAST_ROLLED) > _GetModTime(__file__)
458
459
460 def UpdateSiteDirectory():
461 """Installs the packages from PACKAGES if they are not already installed.
462 At this point we must have setuptools in the site directory.
463
464 This is intended to be run in a subprocess *prior* to the site directory
465 having been added to the parent process as it may cause packages to be
466 added and/or removed.
467
468 Returns:
469 True on success, False otherwise.
470 """
471 if _SiteDirectoryIsUpToDate():
472 return True
473
474 try:
475 EnsureSiteDirectory(SITE_DIR)
476 import pkg_resources
477
478 # Determine if any packages actually need installing.
479 missing_packages = []
480 for package in [SETUPTOOLS] + list(PACKAGES):
481 pkg = Package(*package)
482 req = pkg.GetAsRequirementString()
483
484 # It may be that this package is already available in the site
485 # directory. If so, we can skip past it without trying to install it.
486 pkg_req = pkg_resources.Requirement.parse(req)
487 try:
488 dist = pkg_resources.working_set.find(pkg_req)
489 if dist:
490 continue
491 except pkg_resources.VersionConflict:
492 # This happens if another version of the package is already
493 # installed in another site directory (ie: the system site directory).
494 pass
495
496 missing_packages.append(pkg)
497
498 # Install the missing packages.
499 if missing_packages:
500 print 'Updating python packages ...'
501 for pkg in missing_packages:
502 print ' Installing %s ...' % pkg.GetFilename()
503 InstallPackage(pkg.GetAsRequirementString(), SITE_DIR)
504
505 # Touch the status file so we know that we're up to date next time.
506 open(LAST_ROLLED, 'wb')
507 except InstallError, e:
508 print ' Installation failed: %s' % str(e)
509 return False
510
511 return True
512
513
514 def SetupSiteDirectory():
515 """Sets up the site directory, bootstrapping setuptools if necessary.
516
517 If this finishes successfully then SITE_DIR will exist and will contain
518 the appropriate version of setuptools and all of the packages listed in
519 PACKAGES.
520
521 This is the main workhorse of this module. Calling this will do everything
522 necessary to ensure that you have the desired packages installed in the
523 site directory, and the site directory enabled in this process.
524
525 Returns:
526 True on success, False on failure.
527 """
528 if _SiteDirectoryIsUpToDate():
529 AddSiteDirectory(SITE_DIR)
530 return True
531
532 if not _RunInSubprocess('BootstrapSetupTools()'):
533 return False
534
535 if not _RunInSubprocess('UpdateSiteDirectory()'):
536 return False
537
538 # Process the site directory so that the packages within it are available
539 # for import.
540 AddSiteDirectory(SITE_DIR)
541
542 return True
543
544
545 def CanImportFromSiteDirectory(package_name):
546 """Determines if the given package can be imported from the site directory.
547
548 Args:
549 package_name: the name of the package to import.
550
551 Returns:
552 True if 'import package_name' will succeed and return a module from the
553 site directory, False otherwise.
554 """
555 try:
556 return ModuleIsFromPackage(__import__(package_name), SITE_DIR)
557 except ImportError:
558 return False
559
560
561 def Test():
562 """Runs SetupSiteDirectory and then tries to load pylint, ensuring that it
563 comes from the site directory just created. This is an end-to-end unittest
564 and allows for simple testing from the command-line by running
565
566 ./package_management.py 'Test()'
567 """
568 print 'Testing package_management.'
569 if not SetupSiteDirectory():
570 print 'SetupSiteDirectory failed.'
571 return False
572 if not CanImportFromSiteDirectory('pylint'):
573 print 'CanImportFromSiteDirectory failed.'
574 return False
575 print 'Success!'
576 return True
577
578
579 def Main():
580 """The main entry for the package management script.
581
582 If no arguments are provided simply runs SetupSiteDirectory. If arguments
583 have been passed we execute the first argument as python code in the
584 context of this module. This mechanism is used during the bootstrap
585 process so that the main instance of Python does not have its runtime
586 polluted by various intermediate packages and imports.
587
588 Returns:
589 0 on success, 1 otherwise.
590 """
591 if len(sys.argv) == 2:
592 result = False
593 exec('result = %s' % sys.argv[1])
594
595 # Translate the success state to a return code.
596 return not result
597 else:
598 return not SetupSiteDirectory()
599
600
601 if __name__ == '__main__':
602 sys.exit(Main())
OLDNEW
« no previous file with comments | « bootstrap/win/win_tools.bat ('k') | presubmit_canned_checks.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698