OLD | NEW |
1 #!/usr/bin/env python | 1 #!/usr/bin/env python |
2 # Copyright 2012 The LUCI Authors. All rights reserved. | 2 # Copyright 2012 The LUCI Authors. All rights reserved. |
3 # Use of this source code is governed under the Apache License, Version 2.0 | 3 # Use of this source code is governed under the Apache License, Version 2.0 |
4 # that can be found in the LICENSE file. | 4 # that can be found in the LICENSE file. |
5 | 5 |
6 """Runs a command with optional isolated input/output. | 6 """Runs a command with optional isolated input/output. |
7 | 7 |
8 Despite name "run_isolated", can run a generic non-isolated command specified as | 8 Despite name "run_isolated", can run a generic non-isolated command specified as |
9 args. | 9 args. |
10 | 10 |
11 If input isolated hash is provided, fetches it, creates a tree of hard links, | 11 If input isolated hash is provided, fetches it, creates a tree of hard links, |
12 appends args to the command in the fetched isolated and runs it. | 12 appends args to the command in the fetched isolated and runs it. |
13 To improve performance, keeps a local cache. | 13 To improve performance, keeps a local cache. |
14 The local cache can safely be deleted. | 14 The local cache can safely be deleted. |
15 | 15 |
16 Any ${EXECUTABLE_SUFFIX} on the command line will be replaced with ".exe" string | 16 Any ${EXECUTABLE_SUFFIX} on the command line will be replaced with ".exe" string |
17 on Windows and "" on other platforms. | 17 on Windows and "" on other platforms. |
18 | 18 |
19 Any ${ISOLATED_OUTDIR} on the command line will be replaced by the location of a | 19 Any ${ISOLATED_OUTDIR} on the command line will be replaced by the location of a |
20 temporary directory upon execution of the command specified in the .isolated | 20 temporary directory upon execution of the command specified in the .isolated |
21 file. All content written to this directory will be uploaded upon termination | 21 file. All content written to this directory will be uploaded upon termination |
22 and the .isolated file describing this directory will be printed to stdout. | 22 and the .isolated file describing this directory will be printed to stdout. |
23 | 23 |
24 Any ${SWARMING_BOT_FILE} on the command line will be replaced by the value of | 24 Any ${SWARMING_BOT_FILE} on the command line will be replaced by the value of |
25 the --bot-file parameter. This file is used by a swarming bot to communicate | 25 the --bot-file parameter. This file is used by a swarming bot to communicate |
26 state of the host to tasks. It is written to by the swarming bot's | 26 state of the host to tasks. It is written to by the swarming bot's |
27 on_before_task() hook in the swarming server's custom bot_config.py. | 27 on_before_task() hook in the swarming server's custom bot_config.py. |
28 """ | 28 """ |
29 | 29 |
30 __version__ = '0.8.4' | 30 __version__ = '0.8.5' |
31 | 31 |
32 import base64 | 32 import base64 |
| 33 import collections |
33 import logging | 34 import logging |
34 import optparse | 35 import optparse |
35 import os | 36 import os |
36 import sys | 37 import sys |
37 import tempfile | 38 import tempfile |
38 import time | 39 import time |
39 | 40 |
40 from third_party.depot_tools import fix_encoding | 41 from third_party.depot_tools import fix_encoding |
41 | 42 |
42 from utils import file_path | 43 from utils import file_path |
(...skipping 333 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
376 # 'items_cold': '<large.pack()>', | 377 # 'items_cold': '<large.pack()>', |
377 # 'items_hot': '<large.pack()>', | 378 # 'items_hot': '<large.pack()>', |
378 # }, | 379 # }, |
379 # 'upload': { | 380 # 'upload': { |
380 # 'duration': 0., | 381 # 'duration': 0., |
381 # 'items_cold': '<large.pack()>', | 382 # 'items_cold': '<large.pack()>', |
382 # 'items_hot': '<large.pack()>', | 383 # 'items_hot': '<large.pack()>', |
383 # }, | 384 # }, |
384 # }, | 385 # }, |
385 }, | 386 }, |
| 387 # 'cipd_pins': { |
| 388 # 'packages': [ |
| 389 # {'package_name': ..., 'version': ..., 'path': ...}, |
| 390 # ... |
| 391 # ], |
| 392 # 'client_package': {'package_name': ..., 'version': ...}, |
| 393 # }, |
386 'outputs_ref': None, | 394 'outputs_ref': None, |
387 'version': 5, | 395 'version': 5, |
388 } | 396 } |
389 | 397 |
390 if root_dir: | 398 if root_dir: |
391 file_path.ensure_tree(root_dir, 0700) | 399 file_path.ensure_tree(root_dir, 0700) |
392 else: | 400 else: |
393 root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None | 401 root_dir = os.path.dirname(cache.cache_dir) if cache.cache_dir else None |
394 # See comment for these constants. | 402 # See comment for these constants. |
395 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) | 403 run_dir = make_temp_dir(ISOLATED_RUN_DIR, root_dir) |
396 # storage should be normally set but don't crash if it is not. This can happen | 404 # storage should be normally set but don't crash if it is not. This can happen |
397 # as Swarming task can run without an isolate server. | 405 # as Swarming task can run without an isolate server. |
398 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None | 406 out_dir = make_temp_dir(ISOLATED_OUT_DIR, root_dir) if storage else None |
399 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) | 407 tmp_dir = make_temp_dir(ISOLATED_TMP_DIR, root_dir) |
400 cwd = run_dir | 408 cwd = run_dir |
401 | 409 |
402 try: | 410 try: |
403 cipd_stats = install_packages_fn(run_dir) | 411 cipd_info = install_packages_fn(run_dir) |
404 if cipd_stats: | 412 if cipd_info: |
405 result['stats']['cipd'] = cipd_stats | 413 result['stats']['cipd'] = cipd_info['stats'] |
| 414 result['cipd_pins'] = cipd_info['cipd_pins'] |
406 | 415 |
407 if isolated_hash: | 416 if isolated_hash: |
408 isolated_stats = result['stats'].setdefault('isolated', {}) | 417 isolated_stats = result['stats'].setdefault('isolated', {}) |
409 bundle, isolated_stats['download'] = fetch_and_map( | 418 bundle, isolated_stats['download'] = fetch_and_map( |
410 isolated_hash=isolated_hash, | 419 isolated_hash=isolated_hash, |
411 storage=storage, | 420 storage=storage, |
412 cache=cache, | 421 cache=cache, |
413 outdir=run_dir, | 422 outdir=run_dir, |
414 use_symlinks=use_symlinks) | 423 use_symlinks=use_symlinks) |
415 if not bundle.command: | 424 if not bundle.command: |
(...skipping 116 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
532 for later examination. | 541 for later examination. |
533 result_json: file path to dump result metadata into. If set, the process | 542 result_json: file path to dump result metadata into. If set, the process |
534 exit code is always 0 unless an internal error occurred. | 543 exit code is always 0 unless an internal error occurred. |
535 root_dir: path to the directory to use to create the temporary directory. If | 544 root_dir: path to the directory to use to create the temporary directory. If |
536 not specified, a random temporary directory is created. | 545 not specified, a random temporary directory is created. |
537 hard_timeout: kills the process if it lasts more than this amount of | 546 hard_timeout: kills the process if it lasts more than this amount of |
538 seconds. | 547 seconds. |
539 grace_period: number of seconds to wait between SIGTERM and SIGKILL. | 548 grace_period: number of seconds to wait between SIGTERM and SIGKILL. |
540 extra_args: optional arguments to add to the command stated in the .isolate | 549 extra_args: optional arguments to add to the command stated in the .isolate |
541 file. Ignored if isolate_hash is empty. | 550 file. Ignored if isolate_hash is empty. |
542 install_packages_fn: function (dir) => cipd_stats. Installs packages. | 551 install_packages_fn: function (dir) => {"stats": cipd_stats, "pins": |
| 552 cipd_pins}. Installs packages. |
543 use_symlinks: create tree with symlinks instead of hardlinks. | 553 use_symlinks: create tree with symlinks instead of hardlinks. |
544 | 554 |
545 Returns: | 555 Returns: |
546 Process exit code that should be used. | 556 Process exit code that should be used. |
547 """ | 557 """ |
548 assert bool(command) ^ bool(isolated_hash) | 558 assert bool(command) ^ bool(isolated_hash) |
549 extra_args = extra_args or [] | 559 extra_args = extra_args or [] |
550 | 560 |
551 if any(ISOLATED_OUTDIR_PARAMETER in a for a in (command or extra_args)): | 561 if any(ISOLATED_OUTDIR_PARAMETER in a for a in (command or extra_args)): |
552 assert storage is not None, 'storage is None although outdir is specified' | 562 assert storage is not None, 'storage is None although outdir is specified' |
(...skipping 35 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
588 print( | 598 print( |
589 '[run_isolated_out_hack]%s[/run_isolated_out_hack]' % | 599 '[run_isolated_out_hack]%s[/run_isolated_out_hack]' % |
590 tools.format_json(data, dense=True)) | 600 tools.format_json(data, dense=True)) |
591 sys.stdout.flush() | 601 sys.stdout.flush() |
592 return result['exit_code'] or int(bool(result['internal_failure'])) | 602 return result['exit_code'] or int(bool(result['internal_failure'])) |
593 | 603 |
594 | 604 |
595 def install_packages( | 605 def install_packages( |
596 run_dir, packages, service_url, client_package_name, | 606 run_dir, packages, service_url, client_package_name, |
597 client_version, cache_dir=None, timeout=None): | 607 client_version, cache_dir=None, timeout=None): |
598 """Installs packages. Returns stats. | 608 """Installs packages. Returns stats, cipd client info and pins. |
| 609 |
| 610 pins and the cipd client info are in the form of: |
| 611 [ |
| 612 { |
| 613 "path": path, "package_name": package_name, "version": version, |
| 614 }, |
| 615 ... |
| 616 ] |
| 617 (the cipd client info is a single dictionary instead of a list) |
| 618 |
| 619 such that they correspond 1:1 to all input package arguments from the command |
| 620 line. These dictionaries make their all the way back to swarming, where they |
| 621 become the arguments of CipdPackage. |
599 | 622 |
600 Args: | 623 Args: |
601 run_dir (str): root of installation. | 624 run_dir (str): root of installation. |
602 packages: packages to install, dict {path: [(package_name, version)]. | 625 packages: packages to install, list [(path, package_name, version), ...] |
603 service_url (str): CIPD server url, e.g. | 626 service_url (str): CIPD server url, e.g. |
604 "https://chrome-infra-packages.appspot.com." | 627 "https://chrome-infra-packages.appspot.com." |
605 client_package_name (str): CIPD package name of CIPD client. | 628 client_package_name (str): CIPD package name of CIPD client. |
606 client_version (str): Version of CIPD client. | 629 client_version (str): Version of CIPD client. |
607 cache_dir (str): where to keep cache of cipd clients, packages and tags. | 630 cache_dir (str): where to keep cache of cipd clients, packages and tags. |
608 timeout: max duration in seconds that this function can take. | 631 timeout: max duration in seconds that this function can take. |
609 """ | 632 """ |
610 assert cache_dir | 633 assert cache_dir |
611 if not packages: | 634 if not packages: |
612 return None | 635 return None |
613 | 636 |
614 timeoutfn = tools.sliding_timeout(timeout) | 637 timeoutfn = tools.sliding_timeout(timeout) |
615 start = time.time() | 638 start = time.time() |
616 cache_dir = os.path.abspath(cache_dir) | 639 cache_dir = os.path.abspath(cache_dir) |
617 | 640 |
618 run_dir = os.path.abspath(run_dir) | 641 run_dir = os.path.abspath(run_dir) |
619 | 642 |
| 643 package_pins = [None]*len(packages) |
| 644 def insert_pin(path, name, version, idx): |
| 645 path = path.replace(os.path.sep, '/') |
| 646 package_pins[idx] = { |
| 647 'package_name': name, |
| 648 'path': path, |
| 649 'version': version, |
| 650 } |
| 651 |
620 get_client_start = time.time() | 652 get_client_start = time.time() |
621 client_manager = cipd.get_client( | 653 client_manager = cipd.get_client( |
622 service_url, client_package_name, client_version, cache_dir, | 654 service_url, client_package_name, client_version, cache_dir, |
623 timeout=timeoutfn()) | 655 timeout=timeoutfn()) |
| 656 |
| 657 by_path = collections.defaultdict(list) |
| 658 for i, (path, name, version) in enumerate(packages): |
| 659 path = path.replace('/', os.path.sep) |
| 660 by_path[path].append((name, version, i)) |
| 661 |
624 with client_manager as client: | 662 with client_manager as client: |
| 663 client_package = { |
| 664 'package_name': client.package_name, |
| 665 'version': client.instance_id, |
| 666 } |
625 get_client_duration = time.time() - get_client_start | 667 get_client_duration = time.time() - get_client_start |
626 for path, packages in sorted(packages.iteritems()): | 668 for path, pkgs in sorted(by_path.iteritems()): |
627 site_root = os.path.abspath(os.path.join(run_dir, path)) | 669 site_root = os.path.abspath(os.path.join(run_dir, path)) |
628 if not site_root.startswith(run_dir): | 670 if not site_root.startswith(run_dir): |
629 raise cipd.Error('Invalid CIPD package path "%s"' % path) | 671 raise cipd.Error('Invalid CIPD package path "%s"' % path) |
630 | 672 |
631 # Do not clean site_root before installation because it may contain other | 673 # Do not clean site_root before installation because it may contain other |
632 # site roots. | 674 # site roots. |
633 file_path.ensure_tree(site_root, 0770) | 675 file_path.ensure_tree(site_root, 0770) |
634 client.ensure( | 676 pins = client.ensure( |
635 site_root, packages, | 677 site_root, [(name, vers) for name, vers, _ in pkgs], |
636 cache_dir=os.path.join(cache_dir, 'cipd_internal'), | 678 cache_dir=os.path.join(cache_dir, 'cipd_internal'), |
637 timeout=timeoutfn()) | 679 timeout=timeoutfn()) |
| 680 for i, pin in enumerate(pins): |
| 681 insert_pin(path, pin[0], pin[1], pkgs[i][2]) |
638 file_path.make_tree_files_read_only(site_root) | 682 file_path.make_tree_files_read_only(site_root) |
639 | 683 |
640 total_duration = time.time() - start | 684 total_duration = time.time() - start |
641 logging.info( | 685 logging.info( |
642 'Installing CIPD client and packages took %d seconds', total_duration) | 686 'Installing CIPD client and packages took %d seconds', total_duration) |
643 | 687 |
| 688 assert None not in package_pins |
| 689 |
644 return { | 690 return { |
645 'duration': total_duration, | 691 'stats': { |
646 'get_client_duration': get_client_duration, | 692 'duration': total_duration, |
| 693 'get_client_duration': get_client_duration, |
| 694 }, |
| 695 'cipd_pins': { |
| 696 'client_package': client_package, |
| 697 'packages': package_pins, |
| 698 } |
647 } | 699 } |
648 | 700 |
649 | 701 |
650 def create_option_parser(): | 702 def create_option_parser(): |
651 parser = logging_utils.OptionParserWithLogging( | 703 parser = logging_utils.OptionParserWithLogging( |
652 usage='%prog <options> [command to run or extra args]', | 704 usage='%prog <options> [command to run or extra args]', |
653 version=__version__, | 705 version=__version__, |
654 log_file=RUN_ISOLATED_LOG_FILE) | 706 log_file=RUN_ISOLATED_LOG_FILE) |
655 parser.add_option( | 707 parser.add_option( |
656 '--clean', action='store_true', | 708 '--clean', action='store_true', |
(...skipping 113 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
770 print >> sys.stderr, ex.message | 822 print >> sys.stderr, ex.message |
771 return 1 | 823 return 1 |
772 | 824 |
773 | 825 |
774 if __name__ == '__main__': | 826 if __name__ == '__main__': |
775 subprocess42.inhibit_os_error_reporting() | 827 subprocess42.inhibit_os_error_reporting() |
776 # Ensure that we are always running with the correct encoding. | 828 # Ensure that we are always running with the correct encoding. |
777 fix_encoding.fix_encoding() | 829 fix_encoding.fix_encoding() |
778 file_path.enable_symlink() | 830 file_path.enable_symlink() |
779 sys.exit(main(sys.argv[1:])) | 831 sys.exit(main(sys.argv[1:])) |
OLD | NEW |