Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(41)

Side by Side Diff: recipe_engine/step_runner.py

Issue 1773273003: Make output placeholders like json.output index-able by name. (Closed) Base URL: https://chromium.googlesource.com/external/github.com/luci/recipes-py@master
Patch Set: Rebase. Created 4 years, 9 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « recipe_engine/recipe_test_api.py ('k') | recipe_engine/util.py » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright (c) 2013-2015 The Chromium Authors. All rights reserved. 1 # Copyright (c) 2013-2015 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import cStringIO 5 import cStringIO
6 import collections 6 import collections
7 import contextlib 7 import contextlib
8 import datetime 8 import datetime
9 import json 9 import json
10 import os 10 import os
(...skipping 416 matching lines...) Expand 10 before | Expand all | Expand 10 after
427 @property 427 @property
428 def steps_ran(self): 428 def steps_ran(self):
429 return self._step_history.values() 429 return self._step_history.values()
430 430
431 431
432 # Result of 'render_step'. 432 # Result of 'render_step'.
433 Placeholders = collections.namedtuple( 433 Placeholders = collections.namedtuple(
434 'Placeholders', ['inputs_cmd', 'outputs_cmd', 'stdout', 'stderr', 'stdin']) 434 'Placeholders', ['inputs_cmd', 'outputs_cmd', 'stdout', 'stderr', 'stdin'])
435 435
436 436
437 # Singleton object to indicate a value is not set.
438 UNSET_VALUE = object()
439
440
437 def render_step(step, step_test): 441 def render_step(step, step_test):
438 """Renders a step so that it can be fed to annotator.py. 442 """Renders a step so that it can be fed to annotator.py.
439 443
440 Args: 444 Args:
441 step: The step to render. 445 step: The step to render.
442 step_test: The test data json dictionary for this step, if any. 446 step_test: The test data json dictionary for this step, if any.
443 Passed through unaltered to each placeholder. 447 Passed through unaltered to each placeholder.
444 448
445 Returns the rendered step and a Placeholders object representing any 449 Returns the rendered step and a Placeholders object representing any
446 placeholder instances that were found while rendering. 450 placeholder instances that were found while rendering.
447 """ 451 """
448 rendered_step = dict(step) 452 rendered_step = dict(step)
449 453
450 # Process 'cmd', rendering placeholders there. 454 # Process 'cmd', rendering placeholders there.
451 input_phs = collections.defaultdict(lambda: collections.defaultdict(list)) 455 input_phs = collections.defaultdict(lambda: collections.defaultdict(list))
452 output_phs = collections.defaultdict(lambda: collections.defaultdict(list)) 456 output_phs = collections.defaultdict(
457 lambda: collections.defaultdict(collections.OrderedDict))
453 new_cmd = [] 458 new_cmd = []
454 for item in step.get('cmd', []): 459 for item in step.get('cmd', []):
455 if isinstance(item, util.Placeholder): 460 if isinstance(item, util.Placeholder):
456 module_name, placeholder_name = item.name_pieces 461 module_name, placeholder_name = item.namespaces
457 tdata = step_test.pop_placeholder(item.name_pieces) 462 tdata = step_test.pop_placeholder(
463 module_name, placeholder_name, item.name)
458 new_cmd.extend(item.render(tdata)) 464 new_cmd.extend(item.render(tdata))
459 if isinstance(item, util.InputPlaceholder): 465 if isinstance(item, util.InputPlaceholder):
460 input_phs[module_name][placeholder_name].append((item, tdata)) 466 input_phs[module_name][placeholder_name].append((item, tdata))
461 else: 467 else:
462 assert isinstance(item, util.OutputPlaceholder), ( 468 assert isinstance(item, util.OutputPlaceholder), (
463 'Not an OutputPlaceholder: %r' % item) 469 'Not an OutputPlaceholder: %r' % item)
464 output_phs[module_name][placeholder_name].append((item, tdata)) 470 # This assert ensures that:
471 # no two placeholders have the same name
472 # at most one placeholder has the default name
473 assert item.name not in output_phs[module_name][placeholder_name], (
474 'Step "%s" has multiple output placeholders of %s.%s. Please '
475 'specify explicit and different names for them.' % (
476 step['name'], module_name, placeholder_name))
477 output_phs[module_name][placeholder_name][item.name] = (item, tdata)
465 else: 478 else:
466 new_cmd.append(item) 479 new_cmd.append(item)
467 rendered_step['cmd'] = new_cmd 480 rendered_step['cmd'] = new_cmd
468 481
469 # Process 'stdout', 'stderr' and 'stdin' placeholders, if given. 482 # Process 'stdout', 'stderr' and 'stdin' placeholders, if given.
470 stdio_placeholders = {} 483 stdio_placeholders = {}
471 for key in ('stdout', 'stderr', 'stdin'): 484 for key in ('stdout', 'stderr', 'stdin'):
472 placeholder = step.get(key) 485 placeholder = step.get(key)
473 tdata = None 486 tdata = None
474 if placeholder: 487 if placeholder:
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
506 for _, items in pholders.iteritems(): 519 for _, items in pholders.iteritems():
507 for ph, td in items: 520 for ph, td in items:
508 ph.cleanup(td.enabled) 521 ph.cleanup(td.enabled)
509 522
510 # Output placeholders inside step |cmd|. 523 # Output placeholders inside step |cmd|.
511 for module_name, pholders in placeholders.outputs_cmd.iteritems(): 524 for module_name, pholders in placeholders.outputs_cmd.iteritems():
512 assert not hasattr(step_result, module_name) 525 assert not hasattr(step_result, module_name)
513 o = BlankObject() 526 o = BlankObject()
514 setattr(step_result, module_name, o) 527 setattr(step_result, module_name, o)
515 528
516 for placeholder_name, items in pholders.iteritems(): 529 for placeholder_name, instances in pholders.iteritems():
517 lst = [ph.result(step_result.presentation, td) for ph, td in items] 530 named_results = {}
518 setattr(o, placeholder_name+"_all", lst) 531 default_result = UNSET_VALUE
519 setattr(o, placeholder_name, lst[0]) 532 for _, (ph, td) in instances.iteritems():
533 result = ph.result(step_result.presentation, td)
534 if ph.name is None:
535 default_result = result
536 else:
537 named_results[ph.name] = result
538 setattr(o, placeholder_name + "s", named_results)
539
540 if default_result is UNSET_VALUE and len(named_results) == 1:
541 # If only 1 output placeholder with an explicit name, we set the default
542 # output.
543 default_result = named_results.values()[0]
544
545 # If 2+ placeholders have explicit names, we don't set the default output.
546 if default_result is not UNSET_VALUE:
547 setattr(o, placeholder_name, default_result)
520 548
521 # Placeholders that are used with IO redirection. 549 # Placeholders that are used with IO redirection.
522 for key in ('stdout', 'stderr', 'stdin'): 550 for key in ('stdout', 'stderr', 'stdin'):
523 assert not hasattr(step_result, key) 551 assert not hasattr(step_result, key)
524 ph, td = getattr(placeholders, key) 552 ph, td = getattr(placeholders, key)
525 if ph: 553 if ph:
526 if isinstance(ph, util.OutputPlaceholder): 554 if isinstance(ph, util.OutputPlaceholder):
527 setattr(step_result, key, ph.result(step_result.presentation, td)) 555 setattr(step_result, key, ph.result(step_result.presentation, td))
528 else: 556 else:
529 assert isinstance(ph, util.InputPlaceholder), ( 557 assert isinstance(ph, util.InputPlaceholder), (
(...skipping 52 matching lines...) Expand 10 before | Expand all | Expand 10 after
582 supplied command, and only uses the |env| kwarg for modifying the environment 610 supplied command, and only uses the |env| kwarg for modifying the environment
583 of the child process. 611 of the child process.
584 """ 612 """
585 saved_path = os.environ['PATH'] 613 saved_path = os.environ['PATH']
586 try: 614 try:
587 if path is not None: 615 if path is not None:
588 os.environ['PATH'] = path 616 os.environ['PATH'] = path
589 yield 617 yield
590 finally: 618 finally:
591 os.environ['PATH'] = saved_path 619 os.environ['PATH'] = saved_path
OLDNEW
« no previous file with comments | « recipe_engine/recipe_test_api.py ('k') | recipe_engine/util.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698