OLD | NEW |
1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file | 1 // Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file |
2 // for details. All rights reserved. Use of this source code is governed by a | 2 // for details. All rights reserved. Use of this source code is governed by a |
3 // BSD-style license that can be found in the LICENSE file. | 3 // BSD-style license that can be found in the LICENSE file. |
4 | 4 |
5 library barback.phase; | 5 library barback.phase; |
6 | 6 |
7 import 'dart:async'; | 7 import 'dart:async'; |
8 | 8 |
9 import 'asset.dart'; | 9 import 'asset.dart'; |
10 import 'asset_cascade.dart'; | 10 import 'asset_cascade.dart'; |
11 import 'asset_id.dart'; | 11 import 'asset_id.dart'; |
12 import 'asset_node.dart'; | 12 import 'asset_node.dart'; |
13 import 'asset_set.dart'; | 13 import 'asset_set.dart'; |
14 import 'errors.dart'; | 14 import 'errors.dart'; |
15 import 'transform_node.dart'; | 15 import 'transform_node.dart'; |
16 import 'transformer.dart'; | 16 import 'transformer.dart'; |
| 17 import 'utils.dart'; |
17 | 18 |
18 /// One phase in the ordered series of transformations in an [AssetCascade]. | 19 /// One phase in the ordered series of transformations in an [AssetCascade]. |
19 /// | 20 /// |
20 /// Each phase can access outputs from previous phases and can in turn pass | 21 /// Each phase can access outputs from previous phases and can in turn pass |
21 /// outputs to later phases. Phases are processed strictly serially. All | 22 /// outputs to later phases. Phases are processed strictly serially. All |
22 /// transforms in a phase will be complete before moving on to the next phase. | 23 /// transforms in a phase will be complete before moving on to the next phase. |
23 /// Within a single phase, all transforms will be run in parallel. | 24 /// Within a single phase, all transforms will be run in parallel. |
24 /// | 25 /// |
25 /// Building can be interrupted between phases. For example, a source is added | 26 /// Building can be interrupted between phases. For example, a source is added |
26 /// which starts the background process. Sometime during, say, phase 2 (which | 27 /// which starts the background process. Sometime during, say, phase 2 (which |
(...skipping 11 matching lines...) Expand all Loading... |
38 /// | 39 /// |
39 /// Their outputs will be available to the next phase. | 40 /// Their outputs will be available to the next phase. |
40 final List<Transformer> _transformers; | 41 final List<Transformer> _transformers; |
41 | 42 |
42 /// The inputs that are available for transforms in this phase to consume. | 43 /// The inputs that are available for transforms in this phase to consume. |
43 /// | 44 /// |
44 /// For the first phase, these will be the source assets. For all other | 45 /// For the first phase, these will be the source assets. For all other |
45 /// phases, they will be the outputs from the previous phase. | 46 /// phases, they will be the outputs from the previous phase. |
46 final inputs = new Map<AssetId, AssetNode>(); | 47 final inputs = new Map<AssetId, AssetNode>(); |
47 | 48 |
48 /// The transforms currently applicable to assets in [inputs]. | 49 /// The transforms currently applicable to assets in [inputs], indexed by |
| 50 /// the ids of their primary inputs. |
49 /// | 51 /// |
50 /// These are the transforms that have been "wired up": they represent a | 52 /// These are the transforms that have been "wired up": they represent a |
51 /// repeatable transformation of a single concrete set of inputs. "dart2js" | 53 /// repeatable transformation of a single concrete set of inputs. "dart2js" |
52 /// is a transformer. "dart2js on web/main.dart" is a transform. | 54 /// is a transformer. "dart2js on web/main.dart" is a transform. |
53 final _transforms = new Set<TransformNode>(); | 55 final _transforms = new Map<AssetId, Set<TransformNode>>(); |
54 | 56 |
55 /// The nodes that are new in this phase since the last time [process] was | 57 /// Futures that will complete once the transformers that can consume a given |
56 /// called. | 58 /// asset are determined. |
57 /// | 59 /// |
58 /// When we process, we'll check these to see if we can hang new transforms | 60 /// Whenever an asset is added or modified, we need to asynchronously |
59 /// off them. | 61 /// determine which transformers can use it as their primary input. We can't |
60 final _newInputs = new Set<AssetNode>(); | 62 /// start processing until we know which transformers to run, and this allows |
| 63 /// us to wait until we do. |
| 64 var _adjustTransformersFutures = new Map<AssetId, Future>(); |
| 65 |
| 66 /// New asset nodes that were added while [_adjustTransformers] was still |
| 67 /// being run on an old version of that asset. |
| 68 var _pendingNewInputs = new Map<AssetId, AssetNode>(); |
| 69 |
| 70 /// The ids of assets that are emitted by transforms in this phase. |
| 71 /// |
| 72 /// This is used to detect collisions where multiple transforms emit the same |
| 73 /// output. |
| 74 final _outputs = new Set<AssetId>(); |
61 | 75 |
62 /// The phase after this one. | 76 /// The phase after this one. |
63 /// | 77 /// |
64 /// Outputs from this phase will be passed to it. | 78 /// Outputs from this phase will be passed to it. |
65 final Phase _next; | 79 final Phase _next; |
66 | 80 |
67 Phase(this.cascade, this._index, this._transformers, this._next); | 81 Phase(this.cascade, this._index, this._transformers, this._next); |
68 | 82 |
69 /// Updates the phase's inputs with [updated] and removes [removed]. | 83 /// Adds a new asset as an input for this phase. |
70 /// | 84 /// |
71 /// This marks any affected [transforms] as dirty or discards them if their | 85 /// [node] doesn't have to be [AssetState.AVAILABLE]. Once it is, the phase |
72 /// inputs are removed. | 86 /// will automatically begin determining which transforms can consume it as a |
73 void updateInputs(AssetSet updated, Set<AssetId> removed) { | 87 /// primary input. The transforms themselves won't be applied until [process] |
74 // Remove any nodes that are no longer being output. Handle removals first | 88 /// is called, however. |
75 // in case there are assets that were removed by one transform but updated | 89 /// |
76 // by another. In that case, the update should win. | 90 /// This should only be used for brand-new assets or assets that have been |
77 for (var id in removed) { | 91 /// removed and re-created. The phase will automatically handle updated assets |
78 var node = inputs.remove(id); | 92 /// using the [AssetNode.onStateChange] stream. |
79 | 93 void addInput(AssetNode node) { |
80 // Every transform that was using it is dirty now. | 94 // We remove [node.id] from [inputs] as soon as the node is removed rather |
81 if (node != null) { | 95 // than at the same time [node.id] is removed from [_transforms] so we don't |
82 node.consumers.forEach((consumer) => consumer.dirty()); | 96 // have to wait on [_adjustTransformers]. It's important that [inputs] is |
83 } | 97 // always up-to-date so that the [AssetCascade] can look there for available |
| 98 // assets. |
| 99 inputs[node.id] = node; |
| 100 node.whenRemoved.then((_) => inputs.remove(node.id)); |
| 101 |
| 102 if (!_adjustTransformersFutures.containsKey(node.id)) { |
| 103 _transforms[node.id] = new Set<TransformNode>(); |
| 104 _adjustTransformers(node); |
| 105 return; |
84 } | 106 } |
85 | 107 |
86 // Update and new or modified assets. | 108 // If an input is added while the same input is still being processed, |
87 for (var asset in updated) { | 109 // that means that the asset was removed and recreated while |
88 var node = inputs[asset.id]; | 110 // [_adjustTransformers] was being run on the old value. We have to wait |
89 if (node == null) { | 111 // until that finishes, then run it again on whatever the newest version |
90 // It's a new node. Add it and remember it so we can see if any new | 112 // of that asset is. |
91 // transforms will consume it. | 113 |
92 node = new AssetNode(asset); | 114 // We may already be waiting for the existing [_adjustTransformers] call to |
93 inputs[asset.id] = node; | 115 // finish. If so, all we need to do is change the node that will be loaded |
94 _newInputs.add(node); | 116 // after it completes. |
95 } else { | 117 var containedKey = _pendingNewInputs.containsKey(node.id); |
96 node.updateAsset(asset); | 118 _pendingNewInputs[node.id] = node; |
97 } | 119 if (containedKey) return; |
98 } | 120 |
| 121 // If we aren't already waiting, start doing so. |
| 122 _adjustTransformersFutures[node.id].then((_) { |
| 123 assert(!_adjustTransformersFutures.containsKey(node.id)); |
| 124 assert(_pendingNewInputs.containsKey(node.id)); |
| 125 _transforms[node.id] = new Set<TransformNode>(); |
| 126 _adjustTransformers(_pendingNewInputs.remove(node.id)); |
| 127 }, onError: (_) { |
| 128 // If there was a programmatic error while processing the old input, |
| 129 // we don't want to just ignore it; it may have left the system in an |
| 130 // inconsistent state. We also don't want to top-level it, so we |
| 131 // ignore it here but don't start processing the new input. That way |
| 132 // when [process] is called, the error will be piped through its |
| 133 // return value. |
| 134 }).catchError((e) { |
| 135 // If our code above has a programmatic error, ensure it will be piped |
| 136 // through [process] by putting it into [_adjustTransformersFutures]. |
| 137 _adjustTransformersFutures[node.id] = new Future.error(e); |
| 138 }); |
| 139 } |
| 140 |
| 141 /// Returns the input for this phase with the given [id], but only if that |
| 142 /// input is known not to be consumed as a transformer's primary input. |
| 143 /// |
| 144 /// If the input is unavailable, or if the phase hasn't determined whether or |
| 145 /// not any transformers will consume it as a primary input, null will be |
| 146 /// returned instead. This means that the return value is guaranteed to always |
| 147 /// be [AssetState.AVAILABLE]. |
| 148 AssetNode getUnconsumedInput(AssetId id) { |
| 149 if (!inputs.containsKey(id)) return null; |
| 150 |
| 151 // If the asset has transforms, it's not unconsumed. |
| 152 if (!_transforms[id].isEmpty) return null; |
| 153 |
| 154 // If we're working on figuring out if the asset has transforms, we can't |
| 155 // prove that it's unconsumed. |
| 156 if (_adjustTransformersFutures.containsKey(id)) return null; |
| 157 |
| 158 // The asset should be available. If it were removed, it wouldn't be in |
| 159 // _inputs, and if it were dirty, it'd be in _adjustTransformersFutures. |
| 160 assert(inputs[id].state.isAvailable); |
| 161 return inputs[id]; |
| 162 } |
| 163 |
| 164 /// Asynchronously determines which transformers can consume [node] as a |
| 165 /// primary input and creates transforms for them. |
| 166 /// |
| 167 /// This ensures that if [node] is modified or removed during or after the |
| 168 /// time it takes to adjust its transformers, they're appropriately |
| 169 /// re-adjusted. Its progress can be tracked in [_adjustTransformersFutures]. |
| 170 void _adjustTransformers(AssetNode node) { |
| 171 // Once the input is available, hook up transformers for it. If it changes |
| 172 // while that's happening, try again. |
| 173 _adjustTransformersFutures[node.id] = node.tryUntilStable((asset) { |
| 174 var oldTransformers = _transforms[node.id] |
| 175 .map((transform) => transform.transformer).toSet(); |
| 176 |
| 177 return _removeStaleTransforms(asset) |
| 178 .then((_) => _addFreshTransforms(node, oldTransformers)); |
| 179 }).then((_) { |
| 180 // Now all the transforms are set up correctly and the asset is available |
| 181 // for the time being. Set up handlers for when the asset changes in the |
| 182 // future. |
| 183 node.onStateChange.first.then((state) { |
| 184 if (state.isRemoved) { |
| 185 _transforms.remove(node.id); |
| 186 } else { |
| 187 _adjustTransformers(node); |
| 188 } |
| 189 }).catchError((e) { |
| 190 _adjustTransformersFutures[node.id] = new Future.error(e); |
| 191 }); |
| 192 }).catchError((error) { |
| 193 if (error is! AssetNotFoundException || error.id != node.id) throw error; |
| 194 |
| 195 // If the asset is removed, [tryUntilStable] will throw an |
| 196 // [AssetNotFoundException]. In that case, just remove all transforms for |
| 197 // the node. |
| 198 _transforms.remove(node.id); |
| 199 }).whenComplete(() { |
| 200 _adjustTransformersFutures.remove(node.id); |
| 201 }); |
| 202 |
| 203 // Don't top-level errors coming from the input processing. Any errors will |
| 204 // eventually be piped through [process]'s returned Future. |
| 205 _adjustTransformersFutures[node.id].catchError((_) {}); |
| 206 } |
| 207 |
| 208 // Remove any old transforms that used to have [asset] as a primary asset but |
| 209 // no longer apply to its new contents. |
| 210 Future _removeStaleTransforms(Asset asset) { |
| 211 return Future.wait(_transforms[asset.id].map((transform) { |
| 212 // TODO(rnystrom): Catch all errors from isPrimary() and redirect to |
| 213 // results. |
| 214 return transform.transformer.isPrimary(asset).then((isPrimary) { |
| 215 if (isPrimary) return; |
| 216 _transforms[asset.id].remove(transform); |
| 217 transform.remove(); |
| 218 }); |
| 219 })); |
| 220 } |
| 221 |
| 222 // Add new transforms for transformers that consider [node]'s asset to be a |
| 223 // primary input. |
| 224 // |
| 225 // [oldTransformers] is the set of transformers that had [node] as a primary |
| 226 // input prior to this. They don't need to be checked, since they were removed |
| 227 // or preserved in [_removeStaleTransforms]. |
| 228 Future _addFreshTransforms(AssetNode node, Set<Transformer> oldTransformers) { |
| 229 return Future.wait(_transformers.map((transformer) { |
| 230 if (oldTransformers.contains(transformer)) return new Future.value(); |
| 231 |
| 232 // If the asset is unavailable, the results of this [_adjustTransformers] |
| 233 // run will be discarded, so we can just short-circuit. |
| 234 if (node.asset == null) return new Future.value(); |
| 235 |
| 236 // We can safely access [node.asset] here even though it might have |
| 237 // changed since (as above) if it has, [_adjustTransformers] will just be |
| 238 // re-run. |
| 239 // TODO(rnystrom): Catch all errors from isPrimary() and redirect to |
| 240 // results. |
| 241 return transformer.isPrimary(node.asset).then((isPrimary) { |
| 242 if (!isPrimary) return; |
| 243 _transforms[node.id].add(new TransformNode(this, transformer, node)); |
| 244 }); |
| 245 })); |
99 } | 246 } |
100 | 247 |
101 /// Processes this phase. | 248 /// Processes this phase. |
102 /// | 249 /// |
103 /// For all new inputs, it tries to see if there are transformers that can | |
104 /// consume them. Then all applicable transforms are applied. | |
105 /// | |
106 /// Returns a future that completes when processing is done. If there is | 250 /// Returns a future that completes when processing is done. If there is |
107 /// nothing to process, returns `null`. | 251 /// nothing to process, returns `null`. |
108 Future process() { | 252 Future process() { |
109 var future = _processNewInputs(); | 253 if (_adjustTransformersFutures.isEmpty) return _processTransforms(); |
110 if (future == null) { | 254 return _waitForInputs().then((_) => _processTransforms()); |
111 return _processTransforms(); | 255 } |
112 } | 256 |
113 | 257 Future _waitForInputs() { |
114 return future.then((_) => _processTransforms()); | 258 if (_adjustTransformersFutures.isEmpty) return new Future.value(); |
115 } | 259 return Future.wait(_adjustTransformersFutures.values) |
116 | 260 .then((_) => _waitForInputs()); |
117 /// Creates new transforms for any new inputs that are applicable. | |
118 Future _processNewInputs() { | |
119 if (_newInputs.isEmpty) return null; | |
120 | |
121 var futures = []; | |
122 for (var node in _newInputs) { | |
123 for (var transformer in _transformers) { | |
124 // TODO(rnystrom): Catch all errors from isPrimary() and redirect | |
125 // to results. | |
126 futures.add(transformer.isPrimary(node.asset).then((isPrimary) { | |
127 if (!isPrimary) return; | |
128 var transform = new TransformNode(this, transformer, node); | |
129 node.consumers.add(transform); | |
130 _transforms.add(transform); | |
131 })); | |
132 } | |
133 } | |
134 | |
135 _newInputs.clear(); | |
136 | |
137 return Future.wait(futures); | |
138 } | 261 } |
139 | 262 |
140 /// Applies all currently wired up and dirty transforms. | 263 /// Applies all currently wired up and dirty transforms. |
141 /// | |
142 /// Passes their outputs to the next phase. | |
143 Future _processTransforms() { | 264 Future _processTransforms() { |
144 // Convert this to a list so we can safely modify _transforms while | 265 // Convert this to a list so we can safely modify _transforms while |
145 // iterating over it. | 266 // iterating over it. |
146 var dirtyTransforms = _transforms.where((transform) => transform.isDirty) | 267 var dirtyTransforms = |
147 .toList(); | 268 flatten(_transforms.values.map((transforms) => transforms.toList())) |
| 269 .where((transform) => transform.isDirty).toList(); |
148 if (dirtyTransforms.isEmpty) return null; | 270 if (dirtyTransforms.isEmpty) return null; |
149 | 271 |
150 return Future.wait(dirtyTransforms.map((transform) { | 272 return Future.wait(dirtyTransforms.map((transform) => transform.apply())) |
151 if (inputs.containsKey(transform.primary.id)) return transform.apply(); | 273 .then((allNewOutputs) { |
152 | 274 var newOutputs = allNewOutputs.reduce((set1, set2) => set1.union(set2)); |
153 // If the primary input for the transform has been removed, get rid of it | 275 |
154 // and all its outputs. | |
155 _transforms.remove(transform); | |
156 return new Future.value( | |
157 new TransformOutputs(new AssetSet(), transform.outputs)); | |
158 })).then((transformOutputs) { | |
159 // Collect all of the outputs. Since the transforms are run in parallel, | |
160 // we have to be careful here to ensure that the result is deterministic | |
161 // and not influenced by the order that transforms complete. | |
162 var updated = new AssetSet(); | |
163 var removed = new Set<AssetId>(); | |
164 var collisions = new Set<AssetId>(); | 276 var collisions = new Set<AssetId>(); |
165 | 277 for (var newOutput in newOutputs) { |
166 // Handle the generated outputs of all transforms first. | 278 if (_outputs.contains(newOutput.id)) { |
167 for (var outputs in transformOutputs) { | 279 collisions.add(newOutput.id); |
168 // Collect the outputs of all transformers together. | 280 } else { |
169 for (var asset in outputs.updated) { | 281 _next.addInput(newOutput); |
170 if (updated.containsId(asset.id)) { | 282 _outputs.add(newOutput.id); |
171 // Report a collision. | 283 newOutput.whenRemoved.then((_) => _outputs.remove(newOutput.id)); |
172 collisions.add(asset.id); | |
173 } else { | |
174 // TODO(rnystrom): In the case of a collision, the asset that | |
175 // "wins" is chosen non-deterministically. Do something better. | |
176 updated.add(asset); | |
177 } | |
178 } | 284 } |
179 | |
180 // Track any assets no longer output by this transform. We don't | |
181 // handle the case where *another* transform generates the asset | |
182 // no longer generated by this one. updateInputs() handles that. | |
183 removed.addAll(outputs.removed); | |
184 } | 285 } |
185 | 286 |
186 // Report any collisions in deterministic order. | 287 // Report collisions in a deterministic order. |
187 collisions = collisions.toList(); | 288 collisions = collisions.toList(); |
188 collisions.sort((a, b) => a.toString().compareTo(b.toString())); | 289 collisions.sort((a, b) => a.toString().compareTo(b.toString())); |
189 for (var collision in collisions) { | 290 for (var collision in collisions) { |
190 cascade.reportError(new AssetCollisionException(collision)); | 291 cascade.reportError(new AssetCollisionException(collision)); |
191 // TODO(rnystrom): Define what happens after a collision occurs. | 292 // TODO(rnystrom): Define what happens after a collision occurs. |
192 } | 293 } |
193 | |
194 // Pass the outputs to the next phase. | |
195 _next.updateInputs(updated, removed); | |
196 }); | 294 }); |
197 } | 295 } |
198 } | 296 } |
OLD | NEW |