OLD | NEW |
1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
4 | 4 |
5 package deps | 5 package deps |
6 | 6 |
7 import ( | 7 import ( |
8 "errors" | |
9 "fmt" | 8 "fmt" |
10 | 9 |
11 "golang.org/x/net/context" | 10 "golang.org/x/net/context" |
12 "google.golang.org/grpc/codes" | 11 "google.golang.org/grpc/codes" |
13 | 12 |
14 "github.com/luci/gae/service/datastore" | 13 "github.com/luci/gae/service/datastore" |
15 | 14 |
16 "github.com/luci/luci-go/common/grpcutil" | 15 "github.com/luci/luci-go/common/grpcutil" |
17 "github.com/luci/luci-go/common/logging" | 16 "github.com/luci/luci-go/common/logging" |
18 "github.com/luci/luci-go/common/parallel" | 17 "github.com/luci/luci-go/common/parallel" |
19 "github.com/luci/luci-go/common/stringset" | 18 "github.com/luci/luci-go/common/stringset" |
20 | 19 |
21 » "github.com/luci/luci-go/common/api/dm/service/v1" | 20 » dm "github.com/luci/luci-go/common/api/dm/service/v1" |
22 "github.com/luci/luci-go/common/api/dm/template" | 21 "github.com/luci/luci-go/common/api/dm/template" |
23 | 22 |
24 "github.com/luci/luci-go/appengine/tumble" | 23 "github.com/luci/luci-go/appengine/tumble" |
25 | 24 |
| 25 "github.com/luci/luci-go/appengine/cmd/dm/distributor" |
26 "github.com/luci/luci-go/appengine/cmd/dm/model" | 26 "github.com/luci/luci-go/appengine/cmd/dm/model" |
27 "github.com/luci/luci-go/appengine/cmd/dm/mutate" | 27 "github.com/luci/luci-go/appengine/cmd/dm/mutate" |
28 ) | 28 ) |
29 | 29 |
30 func (d *deps) runEnsureGraphDepsWalk(c context.Context, req *dm.EnsureGraphData
Req, newAttempts *dm.AttemptList) (*dm.GraphData, error) { | 30 func (d *deps) runEnsureGraphDepsWalk(c context.Context, req *dm.EnsureGraphData
Req, newAttempts *dm.AttemptList) (*dm.GraphData, error) { |
31 // first lets run a query to load all of the proposed attempts. | 31 // first lets run a query to load all of the proposed attempts. |
32 wgreq := &dm.WalkGraphReq{ | 32 wgreq := &dm.WalkGraphReq{ |
33 Query: dm.AttemptListQuery(newAttempts), | 33 Query: dm.AttemptListQuery(newAttempts), |
34 Limit: &dm.WalkGraphReq_Limit{ | 34 Limit: &dm.WalkGraphReq_Limit{ |
35 MaxDepth: 1, | 35 MaxDepth: 1, |
(...skipping 180 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
216 } | 216 } |
217 | 217 |
218 // not all of the attemps exist/are finished, we have to block. | 218 // not all of the attemps exist/are finished, we have to block. |
219 rsp.Result = nil | 219 rsp.Result = nil |
220 rsp.ShouldHalt = true | 220 rsp.ShouldHalt = true |
221 | 221 |
222 return tumbleNow(c, &mutate.AddDeps{ | 222 return tumbleNow(c, &mutate.AddDeps{ |
223 Auth: req.ForExecution, | 223 Auth: req.ForExecution, |
224 Quests: newQuests, | 224 Quests: newQuests, |
225 // Attempts we think are missing | 225 // Attempts we think are missing |
226 » » Atmpts: newAttempts, | 226 » » Attempts: newAttempts, |
227 // Deps we think are missing (>= newAttempts) | 227 // Deps we think are missing (>= newAttempts) |
228 Deps: missingDeps, | 228 Deps: missingDeps, |
229 }) | 229 }) |
230 } | 230 } |
231 | 231 |
232 type templateFileKey struct { | 232 type templateFileKey struct { |
233 project, ref string | 233 project, ref string |
234 } | 234 } |
235 | 235 |
236 type templateFile struct { | 236 type templateFile struct { |
(...skipping 26 matching lines...) Expand all Loading... |
263 if err == nil { | 263 if err == nil { |
264 return false | 264 return false |
265 } | 265 } |
266 if rsp.TemplateError == nil { | 266 if rsp.TemplateError == nil { |
267 rsp.TemplateError = make([]string, len(req.TemplateQuest
)) | 267 rsp.TemplateError = make([]string, len(req.TemplateQuest
)) |
268 } | 268 } |
269 rsp.TemplateError[i] = err.Error() | 269 rsp.TemplateError[i] = err.Error() |
270 return true | 270 return true |
271 } | 271 } |
272 | 272 |
| 273 dists := map[string]distributor.D{} |
| 274 |
273 newQuests = make(map[string]*model.Quest, len(req.Quest)+len(req.Templat
eQuest)) | 275 newQuests = make(map[string]*model.Quest, len(req.Quest)+len(req.Templat
eQuest)) |
274 newAttempts = dm.NewAttemptList(nil) | 276 newAttempts = dm.NewAttemptList(nil) |
275 | 277 |
| 278 reg := distributor.GetRegistry(c) |
| 279 |
276 // render all quest descriptions | 280 // render all quest descriptions |
277 for i, qDesc := range req.Quest { | 281 for i, qDesc := range req.Quest { |
278 » » var q *model.Quest | 282 » » q := model.NewQuest(c, qDesc) |
279 » » if q, err = model.NewQuest(c, qDesc); err != nil { | 283 |
280 » » » err = grpcutil.MaybeLogErr(c, err, codes.InvalidArgument
, "bad quest description") | 284 » » d, ok := dists[qDesc.DistributorConfigName] |
| 285 » » if !ok { |
| 286 » » » if d, _, err = reg.MakeDistributor(c, qDesc.DistributorC
onfigName); err != nil { |
| 287 » » » » return |
| 288 » » » } |
| 289 » » » dists[qDesc.DistributorConfigName] = d |
| 290 » » } |
| 291 |
| 292 » » if err = d.Validate(qDesc.JsonPayload); err != nil { |
| 293 » » » err = grpcutil.MaybeLogErr(c, err, codes.InvalidArgument
, |
| 294 » » » » "JSON payload is invalid for this distributor co
nfiguration.") |
281 return | 295 return |
282 } | 296 } |
283 | 297 |
284 // all provided quest descriptions MUST include at least one att
empt | 298 // all provided quest descriptions MUST include at least one att
empt |
285 if _, ok := req.Attempts.To[q.ID]; !ok { | 299 if _, ok := req.Attempts.To[q.ID]; !ok { |
286 c = logging.SetFields(c, logging.Fields{"id": q.ID, "idx
": i}) | 300 c = logging.SetFields(c, logging.Fields{"id": q.ID, "idx
": i}) |
287 err = grpcutil.MaybeLogErr(c, | 301 err = grpcutil.MaybeLogErr(c, |
288 » » » » errors.New("Quest entries must have a matching A
ttempts entry"), | 302 » » » » fmt.Errorf("Quest %d:%q must have a matching Att
empts entry", i, q.ID), |
289 codes.InvalidArgument, "no matches") | 303 codes.InvalidArgument, "no matches") |
290 return | 304 return |
291 } | 305 } |
292 | 306 |
293 if _, ok := newQuests[q.ID]; !ok { | 307 if _, ok := newQuests[q.ID]; !ok { |
294 newQuests[q.ID] = q | 308 newQuests[q.ID] = q |
295 } | 309 } |
296 } | 310 } |
297 | 311 |
298 // copy all normal attempt descriptions | 312 // copy all normal attempt descriptions |
299 for qid, nums := range req.Attempts.To { | 313 for qid, nums := range req.Attempts.To { |
300 newNums := &dm.AttemptList_Nums{Nums: make([]uint32, len(nums.Nu
ms))} | 314 newNums := &dm.AttemptList_Nums{Nums: make([]uint32, len(nums.Nu
ms))} |
301 copy(newNums.Nums, nums.Nums) | 315 copy(newNums.Nums, nums.Nums) |
302 newAttempts.To[qid] = newNums | 316 newAttempts.To[qid] = newNums |
303 } | 317 } |
304 | 318 |
305 // render all templates and template attempts into newQuests | 319 // render all templates and template attempts into newQuests |
306 templateFiles := templateFileCache{} | 320 templateFiles := templateFileCache{} |
307 for i := 0; i < len(req.TemplateQuest); i++ { | 321 for i := 0; i < len(req.TemplateQuest); i++ { |
308 inst := req.TemplateQuest[i] | 322 inst := req.TemplateQuest[i] |
309 | 323 |
310 var vers string | 324 var vers string |
311 var desc *dm.Quest_Desc | 325 var desc *dm.Quest_Desc |
312 if desc, vers, err = templateFiles.render(c, inst); setTemplateE
rr(i, err) { | 326 if desc, vers, err = templateFiles.render(c, inst); setTemplateE
rr(i, err) { |
313 continue | 327 continue |
314 } | 328 } |
315 | 329 » » if setTemplateErr(i, desc.Normalize()) { |
316 » » var q *model.Quest | |
317 » » q, err = model.NewQuest(c, desc) | |
318 » » if setTemplateErr(i, err) { | |
319 continue | 330 continue |
320 } | 331 } |
321 | 332 |
| 333 q := model.NewQuest(c, desc) |
| 334 |
322 rsp.TemplateIds = append(rsp.TemplateIds, dm.NewQuestID(q.ID)) | 335 rsp.TemplateIds = append(rsp.TemplateIds, dm.NewQuestID(q.ID)) |
323 | 336 |
324 // if we have any errors going on, might as well skip the rest | 337 // if we have any errors going on, might as well skip the rest |
325 if len(rsp.TemplateError) > 0 { | 338 if len(rsp.TemplateError) > 0 { |
326 continue | 339 continue |
327 } | 340 } |
328 | 341 |
329 anums := newAttempts.To[q.ID] | 342 anums := newAttempts.To[q.ID] |
330 anums.Nums = append(anums.Nums, req.TemplateAttempt[i].Nums...) | 343 anums.Nums = append(anums.Nums, req.TemplateAttempt[i].Nums...) |
331 if err := anums.Normalize(); err != nil { | 344 if err := anums.Normalize(); err != nil { |
(...skipping 10 matching lines...) Expand all Loading... |
342 Project: inst.Project, Ref: inst.Ref, Version: vers, | 355 Project: inst.Project, Ref: inst.Ref, Version: vers, |
343 Name: inst.Specifier.TemplateName}) | 356 Name: inst.Specifier.TemplateName}) |
344 } | 357 } |
345 | 358 |
346 return | 359 return |
347 } | 360 } |
348 | 361 |
349 func (d *deps) EnsureGraphData(c context.Context, req *dm.EnsureGraphDataReq) (r
sp *dm.EnsureGraphDataRsp, err error) { | 362 func (d *deps) EnsureGraphData(c context.Context, req *dm.EnsureGraphDataReq) (r
sp *dm.EnsureGraphDataRsp, err error) { |
350 // TODO(riannucci): real non-execution authentication | 363 // TODO(riannucci): real non-execution authentication |
351 if req.ForExecution != nil { | 364 if req.ForExecution != nil { |
| 365 logging.Fields{"execution": req.ForExecution.Id}.Infof(c, "on be
half of") |
352 _, _, err := model.AuthenticateExecution(c, req.ForExecution) | 366 _, _, err := model.AuthenticateExecution(c, req.ForExecution) |
353 if err != nil { | 367 if err != nil { |
354 return nil, grpcutil.MaybeLogErr(c, err, codes.Unauthent
icated, "bad execution auth") | 368 return nil, grpcutil.MaybeLogErr(c, err, codes.Unauthent
icated, "bad execution auth") |
355 } | 369 } |
356 } | 370 } |
357 | 371 |
358 // render any quest descirptions, templates and template attempts into | 372 // render any quest descirptions, templates and template attempts into |
359 // a single merged set of new quests and new attempts | 373 // a single merged set of new quests and new attempts |
360 rsp, newQuests, newAttempts, err := renderRequest(c, req) | 374 rsp, newQuests, newAttempts, err := renderRequest(c, req) |
361 if err != nil || len(rsp.TemplateError) > 0 { | 375 if err != nil || len(rsp.TemplateError) > 0 { |
362 return | 376 return |
363 } | 377 } |
364 | 378 |
365 newQuestList := make([]*model.Quest, 0, len(newQuests)) | 379 newQuestList := make([]*model.Quest, 0, len(newQuests)) |
366 for _, q := range newQuests { | 380 for _, q := range newQuests { |
367 newQuestList = append(newQuestList, q) | 381 newQuestList = append(newQuestList, q) |
368 } | 382 } |
369 | 383 |
370 err = d.ensureGraphData(c, req, newQuestList, newAttempts, rsp) | 384 err = d.ensureGraphData(c, req, newQuestList, newAttempts, rsp) |
371 | 385 |
372 return | 386 return |
373 } | 387 } |
OLD | NEW |