OLD | NEW |
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 """CIPD-specific code is concentrated here.""" | 5 """CIPD-specific code is concentrated here.""" |
6 | 6 |
| 7 import contextlib |
| 8 import logging |
7 import re | 9 import re |
8 | 10 |
9 # Regular expressions below are copied from | 11 # Regular expressions below are copied from |
10 # https://chromium.googlesource.com/infra/infra/+/468bb43/appengine/chrome_infra
_packages/cipd/impl.py | 12 # https://chromium.googlesource.com/infra/infra/+/468bb43/appengine/chrome_infra
_packages/cipd/impl.py |
11 # https://chromium.googlesource.com/infra/infra/+/468bb43/appengine/chrome_infra
_packages/cas/impl.py | 13 # https://chromium.googlesource.com/infra/infra/+/468bb43/appengine/chrome_infra
_packages/cas/impl.py |
12 | 14 |
13 PACKAGE_NAME_RE = re.compile(r'^([a-z0-9_\-]+/)*[a-z0-9_\-]+$') | 15 PACKAGE_NAME_RE = re.compile(r'^([a-z0-9_\-]+/)*[a-z0-9_\-]+$') |
14 INSTANCE_ID_RE = re.compile(r'^[0-9a-f]{40}$') | 16 INSTANCE_ID_RE = re.compile(r'^[0-9a-f]{40}$') |
15 TAG_KEY_RE = re.compile(r'^[a-z0-9_\-]+$') | 17 TAG_KEY_RE = re.compile(r'^[a-z0-9_\-]+$') |
16 REF_RE = re.compile(r'^[a-z0-9_\-]{1,100}$') | 18 REF_RE = re.compile(r'^[a-z0-9_\-]{1,100}$') |
17 TAG_MAX_LEN = 400 | 19 TAG_MAX_LEN = 400 |
18 | 20 |
19 | 21 |
20 # CIPD package name template parameters allow a user to reference different | 22 # CIPD package name template parameters allow a user to reference different |
21 # packages for different enviroments. Inspired by | 23 # packages for different enviroments. Inspired by |
22 # https://chromium.googlesource.com/infra/infra/+/f1072a132c68532b548458392c5444
f04386d684/build/README.md | 24 # https://chromium.googlesource.com/infra/infra/+/f1072a132c68532b548458392c5444
f04386d684/build/README.md |
23 # The values of the parameters are computed on the bot. | 25 # The values of the parameters are computed on the bot. |
24 # | 26 # |
25 # Platform parameter value is "<os>-<arch>" string, where | 27 # Platform parameter value is "<os>-<arch>" string, where |
26 # os can be "linux", "mac" or "windows" and arch can be "386", "amd64" or | 28 # os can be "linux", "mac" or "windows" and arch can be "386", "amd64" or |
27 # "armv6l". | 29 # "armv6l". |
28 PARAM_PLATFORM = '${platform}' | 30 PARAM_PLATFORM = '${platform}' |
| 31 PARAM_PLATFORM_ESC = re.escape(PARAM_PLATFORM) |
29 # OS version parameter defines major and minor version of the OS distribution. | 32 # OS version parameter defines major and minor version of the OS distribution. |
30 # It is useful if package depends on .dll/.so libraries provided by the OS. | 33 # It is useful if package depends on .dll/.so libraries provided by the OS. |
31 # Example values: "ubuntu14_04", "mac10_9", "win6_1". | 34 # Example values: "ubuntu14_04", "mac10_9", "win6_1". |
32 PARAM_OS_VER = '${os_ver}' | 35 PARAM_OS_VER = '${os_ver}' |
| 36 PARAM_OS_VER_ESC = re.escape(PARAM_OS_VER) |
33 ALL_PARAMS = (PARAM_PLATFORM, PARAM_OS_VER) | 37 ALL_PARAMS = (PARAM_PLATFORM, PARAM_OS_VER) |
34 | 38 |
35 | 39 |
| 40 @contextlib.contextmanager |
| 41 def pin_check_fn(platform, os_ver): |
| 42 """Yields a function that verifies that an input CipdPackage could have been |
| 43 plausibly expanded, via pinning, to another CipdPackage. Repeated invocations |
| 44 of the function will retain knowledge of any resolved name template paramters |
| 45 like ${platform} and ${os_ver}. |
| 46 |
| 47 Args: |
| 48 platform - a pre-defined expansion of ${platform}, or None to learn from the |
| 49 first valid checked CipdPackage containing ${platform}. |
| 50 os_ver - a pre-defined expansion of ${os_ver}, or None to learn from the |
| 51 first valid checked CipdPackage containing ${os_ver}. |
| 52 |
| 53 Args of yielded function: |
| 54 original - a CipdPackage which may contain template params like ${platform} |
| 55 expanded - a CipdPackage which is nominally an expansion of original. |
| 56 |
| 57 CipdPackage is a duck-typed object which has three string properties: |
| 58 'package_name', 'path' and 'version'. |
| 59 |
| 60 Yielded function raises: |
| 61 ValueError if expanded is not a valid derivation of original. |
| 62 |
| 63 Example: |
| 64 with pin_check_fn(None, None) as check: |
| 65 check(CipdPackage('', '${platform}', 'ref'), |
| 66 CipdPackage('', 'windows-amd64', 'deadbeef'*5)) |
| 67 check(CipdPackage('', '${platform}', 'ref'), |
| 68 CipdPackage('', 'linux-amd64', 'deadbeef'*5)) ## will raise ValueError |
| 69 """ |
| 70 plat_ref = [platform] |
| 71 os_ver_ref = [os_ver] |
| 72 def _check_fn(original, expanded): |
| 73 if original.path != expanded.path: |
| 74 logging.warn('Mismatched path: %r v %r', original.path, expanded.path) |
| 75 raise ValueError('Mismatched path') |
| 76 |
| 77 def sub_param(regex, param_esc, param_re, param_const): |
| 78 # This is validated at task creation time as well, but just to make sure. |
| 79 if regex.count(param_esc) > 1: |
| 80 logging.warn('Duplicate template param %r: %r', param_esc, regex) |
| 81 raise ValueError('%s occurs more than once in name.' % param_esc) |
| 82 |
| 83 ret = False |
| 84 if param_const is None: |
| 85 ret = param_esc in regex |
| 86 if ret: |
| 87 regex = regex.replace(param_esc, param_re, 1) |
| 88 else: |
| 89 regex = regex.replace(param_esc, param_const, 1) |
| 90 return regex, ret |
| 91 |
| 92 name_regex = re.escape(original.package_name) |
| 93 name_regex, scan_plat = sub_param( |
| 94 name_regex, PARAM_PLATFORM_ESC, r'(?P<platform>\w+-[a-z0-9]+)', |
| 95 plat_ref[0]) |
| 96 name_regex, scan_os_ver = sub_param( |
| 97 name_regex, PARAM_OS_VER_ESC, r'(?P<os_ver>[_a-z0-9]+)', |
| 98 os_ver_ref[0]) |
| 99 |
| 100 match = re.match(name_regex, expanded.package_name) |
| 101 if not match: |
| 102 logging.warn('Mismatched package_name: %r | %r v %r', |
| 103 original.package_name, name_regex, expanded.package_name) |
| 104 raise ValueError('Mismatched package_name') |
| 105 |
| 106 if is_valid_instance_id(original.version): |
| 107 if original.version != expanded.version: |
| 108 logging.warn('Mismatched pins: %r v %r', original.version, |
| 109 expanded.version) |
| 110 raise ValueError('Mismatched pins') |
| 111 else: |
| 112 if not is_valid_instance_id(expanded.version): |
| 113 logging.warn('Pin not a pin: %r', expanded.version) |
| 114 raise ValueError('Pin value is not a pin') |
| 115 |
| 116 if scan_plat: |
| 117 plat_ref[0] = re.escape(match.group('platform')) |
| 118 if scan_os_ver: |
| 119 os_ver_ref[0] = re.escape(match.group('os_ver')) |
| 120 |
| 121 yield _check_fn |
| 122 |
| 123 |
36 def is_valid_package_name(package_name): | 124 def is_valid_package_name(package_name): |
37 """Returns True if |package_name| is a valid CIPD package name.""" | 125 """Returns True if |package_name| is a valid CIPD package name.""" |
38 return bool(PACKAGE_NAME_RE.match(package_name)) | 126 return bool(PACKAGE_NAME_RE.match(package_name)) |
39 | 127 |
40 | 128 |
41 def is_valid_package_name_template(template): | 129 def is_valid_package_name_template(template): |
42 """Returns True if |package_name| is a valid CIPD package name template.""" | 130 """Returns True if |package_name| is a valid CIPD package name template.""" |
43 # Render known parameters first. | 131 # Render known parameters first. |
44 for p in ALL_PARAMS: | 132 for p in ALL_PARAMS: |
45 template = template.replace(p, 'x') | 133 template = template.replace(p, 'x') |
| 134 if template.count(p) > 1: |
| 135 return False |
46 return is_valid_package_name(template) | 136 return is_valid_package_name(template) |
47 | 137 |
48 | 138 |
49 def is_valid_version(version): | 139 def is_valid_version(version): |
50 """Returns True if |version| is a valid CIPD package version.""" | 140 """Returns True if |version| is a valid CIPD package version.""" |
51 return bool( | 141 return bool( |
52 INSTANCE_ID_RE.match(version) or | 142 is_valid_instance_id(version) or |
53 is_valid_tag(version) or | 143 is_valid_tag(version) or |
54 REF_RE.match(version) | 144 REF_RE.match(version) |
55 ) | 145 ) |
56 | 146 |
57 | 147 |
58 def is_valid_tag(tag): | 148 def is_valid_tag(tag): |
59 """True if string looks like a valid package instance tag.""" | 149 """True if string looks like a valid package instance tag.""" |
60 if not tag or ':' not in tag or len(tag) > TAG_MAX_LEN: | 150 if not tag or ':' not in tag or len(tag) > TAG_MAX_LEN: |
61 return False | 151 return False |
62 # Care only about the key. Value can be anything (including empty string). | 152 # Care only about the key. Value can be anything (including empty string). |
63 return bool(TAG_KEY_RE.match(tag.split(':', 1)[0])) | 153 return bool(TAG_KEY_RE.match(tag.split(':', 1)[0])) |
64 | 154 |
65 | 155 |
| 156 def is_valid_instance_id(version): |
| 157 """Returns True if |version| is an insance_id.""" |
| 158 return bool(INSTANCE_ID_RE.match(version)) |
| 159 |
| 160 |
66 def is_pinned_version(version): | 161 def is_pinned_version(version): |
67 """Returns True if |version| is pinned.""" | 162 """Returns True if |version| is pinned.""" |
68 return bool(INSTANCE_ID_RE.match(version)) or is_valid_tag(version) | 163 return is_valid_instance_id(version) or is_valid_tag(version) |
OLD | NEW |