| OLD | NEW |
|---|
| 1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
| 2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
| 3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
| 4 | 4 |
| 5 package datastore | 5 package datastore |
| 6 | 6 |
| 7 import ( | 7 import ( |
| 8 "fmt" | 8 "fmt" |
| 9 "io" | 9 "io" |
| 10 "io/ioutil" | 10 "io/ioutil" |
| 11 "os" | 11 "os" |
| 12 "path/filepath" | 12 "path/filepath" |
| 13 "reflect" | |
| 14 "runtime" | 13 "runtime" |
| 15 "strings" | 14 "strings" |
| 16 | 15 |
| 17 "github.com/luci/luci-go/common/errors" | |
| 18 | |
| 19 "gopkg.in/yaml.v2" | 16 "gopkg.in/yaml.v2" |
| 20 ) | 17 ) |
| 21 | 18 |
| 22 type datastoreImpl struct { | |
| 23 RawInterface | |
| 24 | |
| 25 aid string | |
| 26 ns string | |
| 27 } | |
| 28 | |
| 29 var _ Interface = (*datastoreImpl)(nil) | |
| 30 | |
| 31 func (d *datastoreImpl) KeyForObj(src interface{}) *Key { | |
| 32 ret, err := d.KeyForObjErr(src) | |
| 33 if err != nil { | |
| 34 panic(err) | |
| 35 } | |
| 36 return ret | |
| 37 } | |
| 38 | |
| 39 func (d *datastoreImpl) KeyForObjErr(src interface{}) (*Key, error) { | |
| 40 return newKeyObjErr(d.aid, d.ns, getMGS(src)) | |
| 41 } | |
| 42 | |
| 43 func (d *datastoreImpl) MakeKey(elems ...interface{}) *Key { | |
| 44 return MakeKey(d.aid, d.ns, elems...) | |
| 45 } | |
| 46 | |
| 47 func (d *datastoreImpl) NewKey(kind, stringID string, intID int64, parent *Key)
*Key { | |
| 48 return NewKey(d.aid, d.ns, kind, stringID, intID, parent) | |
| 49 } | |
| 50 | |
| 51 func (d *datastoreImpl) NewIncompleteKeys(count int, kind string, parent *Key) (
keys []*Key) { | |
| 52 if count > 0 { | |
| 53 keys = make([]*Key, count) | |
| 54 for i := range keys { | |
| 55 keys[i] = d.NewKey(kind, "", 0, parent) | |
| 56 } | |
| 57 } | |
| 58 return | |
| 59 } | |
| 60 | |
| 61 func (d *datastoreImpl) NewKeyToks(toks []KeyTok) *Key { | |
| 62 return NewKeyToks(d.aid, d.ns, toks) | |
| 63 } | |
| 64 | |
| 65 // PopulateKey loads key into obj. | |
| 66 // | |
| 67 // obj is any object that Interface.Get is able to accept. | |
| 68 // | |
| 69 // Upon successful application, this method will return true. If the key could | |
| 70 // not be applied to the object, this method will return false. It will panic if | |
| 71 // obj is an invalid datastore model. | |
| 72 // | |
| 73 // This method will panic if obj is an invalid datastore model. If the key could | |
| 74 // not be applied to the object, nothing will happen. | |
| 75 func PopulateKey(obj interface{}, key *Key) bool { | |
| 76 return populateKeyMGS(getMGS(obj), key) | |
| 77 } | |
| 78 | |
| 79 func populateKeyMGS(mgs MetaGetterSetter, key *Key) bool { | |
| 80 if mgs.SetMeta("key", key) { | |
| 81 return true | |
| 82 } | |
| 83 | |
| 84 lst := key.LastTok() | |
| 85 if lst.StringID != "" { | |
| 86 if !mgs.SetMeta("id", lst.StringID) { | |
| 87 return false | |
| 88 } | |
| 89 } else { | |
| 90 if !mgs.SetMeta("id", lst.IntID) { | |
| 91 return false | |
| 92 } | |
| 93 } | |
| 94 | |
| 95 mgs.SetMeta("kind", lst.Kind) | |
| 96 mgs.SetMeta("parent", key.Parent()) | |
| 97 return true | |
| 98 } | |
| 99 | |
| 100 func checkMultiSliceType(v interface{}) error { | |
| 101 if reflect.TypeOf(v).Kind() == reflect.Slice { | |
| 102 return nil | |
| 103 } | |
| 104 return fmt.Errorf("argument must be a slice, not %T", v) | |
| 105 | |
| 106 } | |
| 107 | |
| 108 func runParseCallback(cbIface interface{}) (isKey, hasErr, hasCursorCB bool, mat
*multiArgType) { | |
| 109 badSig := func() { | |
| 110 panic(fmt.Errorf( | |
| 111 "cb does not match the required callback signature: `%T`
!= `func(TYPE, [CursorCB]) [error]`", | |
| 112 cbIface)) | |
| 113 } | |
| 114 | |
| 115 if cbIface == nil { | |
| 116 badSig() | |
| 117 } | |
| 118 | |
| 119 // TODO(riannucci): Profile and determine if any of this is causing a re
al | |
| 120 // slowdown. Could potentially cache reflection stuff by cbTyp? | |
| 121 cbTyp := reflect.TypeOf(cbIface) | |
| 122 | |
| 123 if cbTyp.Kind() != reflect.Func { | |
| 124 badSig() | |
| 125 } | |
| 126 | |
| 127 numIn := cbTyp.NumIn() | |
| 128 if numIn != 1 && numIn != 2 { | |
| 129 badSig() | |
| 130 } | |
| 131 | |
| 132 firstArg := cbTyp.In(0) | |
| 133 if firstArg == typeOfKey { | |
| 134 isKey = true | |
| 135 } else { | |
| 136 mat = mustParseArg(firstArg, false) | |
| 137 if mat.newElem == nil { | |
| 138 badSig() | |
| 139 } | |
| 140 } | |
| 141 | |
| 142 hasCursorCB = numIn == 2 | |
| 143 if hasCursorCB && cbTyp.In(1) != typeOfCursorCB { | |
| 144 badSig() | |
| 145 } | |
| 146 | |
| 147 if cbTyp.NumOut() > 1 { | |
| 148 badSig() | |
| 149 } else if cbTyp.NumOut() == 1 && cbTyp.Out(0) != typeOfError { | |
| 150 badSig() | |
| 151 } | |
| 152 hasErr = cbTyp.NumOut() == 1 | |
| 153 | |
| 154 return | |
| 155 } | |
| 156 | |
| 157 func (d *datastoreImpl) AllocateIDs(ent ...interface{}) error { | |
| 158 if len(ent) == 0 { | |
| 159 return nil | |
| 160 } | |
| 161 | |
| 162 mma, err := makeMetaMultiArg(ent, mmaWriteKeys) | |
| 163 if err != nil { | |
| 164 panic(err) | |
| 165 } | |
| 166 | |
| 167 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false) | |
| 168 if err != nil { | |
| 169 if len(ent) == 1 { | |
| 170 // Single-argument Exists will return a single error. | |
| 171 err = errors.SingleError(err) | |
| 172 } | |
| 173 return err | |
| 174 } | |
| 175 if len(keys) == 0 { | |
| 176 return nil | |
| 177 } | |
| 178 | |
| 179 // Convert each key to be partial valid, assigning an integer ID of 0. C
onfirm | |
| 180 // that each object can be populated with such a key. | |
| 181 for i, key := range keys { | |
| 182 keys[i] = key.Incomplete() | |
| 183 } | |
| 184 | |
| 185 var et errorTracker | |
| 186 it := mma.iterator(et.init(mma)) | |
| 187 err = filterStop(d.RawInterface.AllocateIDs(keys, func(key *Key, err err
or) error { | |
| 188 it.next(func(mat *multiArgType, v reflect.Value) error { | |
| 189 if err != nil { | |
| 190 return err | |
| 191 } | |
| 192 | |
| 193 if !mat.setKey(v, key) { | |
| 194 return ErrInvalidKey | |
| 195 } | |
| 196 return nil | |
| 197 }) | |
| 198 | |
| 199 return nil | |
| 200 })) | |
| 201 if err == nil { | |
| 202 err = et.error() | |
| 203 } | |
| 204 | |
| 205 if err != nil && len(ent) == 1 { | |
| 206 // Single-argument Exists will return a single error. | |
| 207 err = errors.SingleError(err) | |
| 208 } | |
| 209 return err | |
| 210 } | |
| 211 | |
| 212 func (d *datastoreImpl) Run(q *Query, cbIface interface{}) error { | |
| 213 isKey, hasErr, hasCursorCB, mat := runParseCallback(cbIface) | |
| 214 | |
| 215 if isKey { | |
| 216 q = q.KeysOnly(true) | |
| 217 } | |
| 218 fq, err := q.Finalize() | |
| 219 if err != nil { | |
| 220 return err | |
| 221 } | |
| 222 | |
| 223 cbVal := reflect.ValueOf(cbIface) | |
| 224 var cb func(reflect.Value, CursorCB) error | |
| 225 switch { | |
| 226 case hasErr && hasCursorCB: | |
| 227 cb = func(v reflect.Value, cb CursorCB) error { | |
| 228 err := cbVal.Call([]reflect.Value{v, reflect.ValueOf(cb)
})[0].Interface() | |
| 229 if err != nil { | |
| 230 return err.(error) | |
| 231 } | |
| 232 return nil | |
| 233 } | |
| 234 | |
| 235 case hasErr && !hasCursorCB: | |
| 236 cb = func(v reflect.Value, _ CursorCB) error { | |
| 237 err := cbVal.Call([]reflect.Value{v})[0].Interface() | |
| 238 if err != nil { | |
| 239 return err.(error) | |
| 240 } | |
| 241 return nil | |
| 242 } | |
| 243 | |
| 244 case !hasErr && hasCursorCB: | |
| 245 cb = func(v reflect.Value, cb CursorCB) error { | |
| 246 cbVal.Call([]reflect.Value{v, reflect.ValueOf(cb)}) | |
| 247 return nil | |
| 248 } | |
| 249 | |
| 250 case !hasErr && !hasCursorCB: | |
| 251 cb = func(v reflect.Value, _ CursorCB) error { | |
| 252 cbVal.Call([]reflect.Value{v}) | |
| 253 return nil | |
| 254 } | |
| 255 } | |
| 256 | |
| 257 if isKey { | |
| 258 err = d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, gc Curs
orCB) error { | |
| 259 return cb(reflect.ValueOf(k), gc) | |
| 260 }) | |
| 261 } else { | |
| 262 err = d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, gc Cur
sorCB) error { | |
| 263 itm := mat.newElem() | |
| 264 if err := mat.setPM(itm, pm); err != nil { | |
| 265 return err | |
| 266 } | |
| 267 mat.setKey(itm, k) | |
| 268 return cb(itm, gc) | |
| 269 }) | |
| 270 } | |
| 271 return filterStop(err) | |
| 272 } | |
| 273 | |
| 274 func (d *datastoreImpl) Count(q *Query) (int64, error) { | |
| 275 fq, err := q.Finalize() | |
| 276 if err != nil { | |
| 277 return 0, err | |
| 278 } | |
| 279 v, err := d.RawInterface.Count(fq) | |
| 280 return v, filterStop(err) | |
| 281 } | |
| 282 | |
| 283 func (d *datastoreImpl) GetAll(q *Query, dst interface{}) error { | |
| 284 v := reflect.ValueOf(dst) | |
| 285 if v.Kind() != reflect.Ptr { | |
| 286 panic(fmt.Errorf("invalid GetAll dst: must have a ptr-to-slice:
%T", dst)) | |
| 287 } | |
| 288 if !v.IsValid() || v.IsNil() { | |
| 289 panic(errors.New("invalid GetAll dst: <nil>")) | |
| 290 } | |
| 291 | |
| 292 if keys, ok := dst.(*[]*Key); ok { | |
| 293 fq, err := q.KeysOnly(true).Finalize() | |
| 294 if err != nil { | |
| 295 return err | |
| 296 } | |
| 297 | |
| 298 return d.RawInterface.Run(fq, func(k *Key, _ PropertyMap, _ Curs
orCB) error { | |
| 299 *keys = append(*keys, k) | |
| 300 return nil | |
| 301 }) | |
| 302 } | |
| 303 fq, err := q.Finalize() | |
| 304 if err != nil { | |
| 305 return err | |
| 306 } | |
| 307 | |
| 308 slice := v.Elem() | |
| 309 mat := mustParseMultiArg(slice.Type()) | |
| 310 if mat.newElem == nil { | |
| 311 panic(fmt.Errorf("invalid GetAll dst (non-concrete element type)
: %T", dst)) | |
| 312 } | |
| 313 | |
| 314 errs := map[int]error{} | |
| 315 i := 0 | |
| 316 err = filterStop(d.RawInterface.Run(fq, func(k *Key, pm PropertyMap, _ C
ursorCB) error { | |
| 317 slice.Set(reflect.Append(slice, mat.newElem())) | |
| 318 itm := slice.Index(i) | |
| 319 mat.setKey(itm, k) | |
| 320 err := mat.setPM(itm, pm) | |
| 321 if err != nil { | |
| 322 errs[i] = err | |
| 323 } | |
| 324 i++ | |
| 325 return nil | |
| 326 })) | |
| 327 if err == nil { | |
| 328 if len(errs) > 0 { | |
| 329 me := make(errors.MultiError, slice.Len()) | |
| 330 for i, e := range errs { | |
| 331 me[i] = e | |
| 332 } | |
| 333 err = me | |
| 334 } | |
| 335 } | |
| 336 return err | |
| 337 } | |
| 338 | |
| 339 func (d *datastoreImpl) Exists(ent ...interface{}) (*ExistsResult, error) { | |
| 340 if len(ent) == 0 { | |
| 341 return nil, nil | |
| 342 } | |
| 343 | |
| 344 mma, err := makeMetaMultiArg(ent, mmaKeysOnly) | |
| 345 if err != nil { | |
| 346 panic(err) | |
| 347 } | |
| 348 | |
| 349 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false) | |
| 350 if err != nil { | |
| 351 if len(ent) == 1 { | |
| 352 // Single-argument Exists will return a single error. | |
| 353 err = errors.SingleError(err) | |
| 354 } | |
| 355 return nil, err | |
| 356 } | |
| 357 if len(keys) == 0 { | |
| 358 return nil, nil | |
| 359 } | |
| 360 | |
| 361 var bt boolTracker | |
| 362 it := mma.iterator(bt.init(mma)) | |
| 363 err = filterStop(d.RawInterface.GetMulti(keys, nil, func(_ PropertyMap,
err error) error { | |
| 364 it.next(func(*multiArgType, reflect.Value) error { | |
| 365 return err | |
| 366 }) | |
| 367 return nil | |
| 368 })) | |
| 369 if err == nil { | |
| 370 err = bt.error() | |
| 371 } | |
| 372 | |
| 373 if err != nil && len(ent) == 1 { | |
| 374 // Single-argument Exists will return a single error. | |
| 375 err = errors.SingleError(err) | |
| 376 } | |
| 377 return bt.result(), err | |
| 378 } | |
| 379 | |
| 380 func (d *datastoreImpl) ExistsMulti(keys []*Key) (BoolList, error) { | |
| 381 v, err := d.Exists(keys) | |
| 382 if err != nil { | |
| 383 return nil, err | |
| 384 } | |
| 385 return v.List(0), nil | |
| 386 } | |
| 387 | |
| 388 func (d *datastoreImpl) Get(dst ...interface{}) (err error) { | |
| 389 if len(dst) == 0 { | |
| 390 return nil | |
| 391 } | |
| 392 | |
| 393 mma, err := makeMetaMultiArg(dst, mmaReadWrite) | |
| 394 if err != nil { | |
| 395 panic(err) | |
| 396 } | |
| 397 | |
| 398 keys, pms, err := mma.getKeysPMs(d.aid, d.ns, true) | |
| 399 if err != nil { | |
| 400 if len(dst) == 1 { | |
| 401 // Single-argument Get will return a single error. | |
| 402 err = errors.SingleError(err) | |
| 403 } | |
| 404 return err | |
| 405 } | |
| 406 if len(keys) == 0 { | |
| 407 return nil | |
| 408 } | |
| 409 | |
| 410 var et errorTracker | |
| 411 it := mma.iterator(et.init(mma)) | |
| 412 meta := NewMultiMetaGetter(pms) | |
| 413 err = filterStop(d.RawInterface.GetMulti(keys, meta, func(pm PropertyMap
, err error) error { | |
| 414 it.next(func(mat *multiArgType, slot reflect.Value) error { | |
| 415 if err != nil { | |
| 416 return err | |
| 417 } | |
| 418 return mat.setPM(slot, pm) | |
| 419 }) | |
| 420 return nil | |
| 421 })) | |
| 422 | |
| 423 if err == nil { | |
| 424 err = et.error() | |
| 425 } | |
| 426 | |
| 427 if err != nil && len(dst) == 1 { | |
| 428 // Single-argument Get will return a single error. | |
| 429 err = errors.SingleError(err) | |
| 430 } | |
| 431 return err | |
| 432 } | |
| 433 | |
| 434 func (d *datastoreImpl) GetMulti(dst interface{}) error { | |
| 435 if err := checkMultiSliceType(dst); err != nil { | |
| 436 panic(err) | |
| 437 } | |
| 438 return d.Get(dst) | |
| 439 } | |
| 440 | |
| 441 func (d *datastoreImpl) Put(src ...interface{}) (err error) { | |
| 442 if len(src) == 0 { | |
| 443 return nil | |
| 444 } | |
| 445 | |
| 446 mma, err := makeMetaMultiArg(src, mmaReadWrite) | |
| 447 if err != nil { | |
| 448 panic(err) | |
| 449 } | |
| 450 | |
| 451 keys, vals, err := mma.getKeysPMs(d.aid, d.ns, false) | |
| 452 if err != nil { | |
| 453 if len(src) == 1 { | |
| 454 // Single-argument Put will return a single error. | |
| 455 err = errors.SingleError(err) | |
| 456 } | |
| 457 return err | |
| 458 } | |
| 459 if len(keys) == 0 { | |
| 460 return nil | |
| 461 } | |
| 462 | |
| 463 i := 0 | |
| 464 var et errorTracker | |
| 465 it := mma.iterator(et.init(mma)) | |
| 466 err = filterStop(d.RawInterface.PutMulti(keys, vals, func(key *Key, err
error) error { | |
| 467 it.next(func(mat *multiArgType, slot reflect.Value) error { | |
| 468 if err != nil { | |
| 469 return err | |
| 470 } | |
| 471 if key != keys[i] { | |
| 472 mat.setKey(slot, key) | |
| 473 } | |
| 474 return nil | |
| 475 }) | |
| 476 | |
| 477 i++ | |
| 478 return nil | |
| 479 })) | |
| 480 | |
| 481 if err == nil { | |
| 482 err = et.error() | |
| 483 } | |
| 484 | |
| 485 if err != nil && len(src) == 1 { | |
| 486 // Single-argument Put will return a single error. | |
| 487 err = errors.SingleError(err) | |
| 488 } | |
| 489 return err | |
| 490 } | |
| 491 | |
| 492 func (d *datastoreImpl) PutMulti(src interface{}) error { | |
| 493 if err := checkMultiSliceType(src); err != nil { | |
| 494 panic(err) | |
| 495 } | |
| 496 return d.Put(src) | |
| 497 } | |
| 498 | |
| 499 func (d *datastoreImpl) Delete(ent ...interface{}) error { | |
| 500 if len(ent) == 0 { | |
| 501 return nil | |
| 502 } | |
| 503 | |
| 504 mma, err := makeMetaMultiArg(ent, mmaKeysOnly) | |
| 505 if err != nil { | |
| 506 panic(err) | |
| 507 } | |
| 508 | |
| 509 keys, _, err := mma.getKeysPMs(d.aid, d.ns, false) | |
| 510 if err != nil { | |
| 511 if len(ent) == 1 { | |
| 512 // Single-argument Delete will return a single error. | |
| 513 err = errors.SingleError(err) | |
| 514 } | |
| 515 return err | |
| 516 } | |
| 517 if len(keys) == 0 { | |
| 518 return nil | |
| 519 } | |
| 520 | |
| 521 var et errorTracker | |
| 522 it := mma.iterator(et.init(mma)) | |
| 523 err = filterStop(d.RawInterface.DeleteMulti(keys, func(err error) error
{ | |
| 524 it.next(func(*multiArgType, reflect.Value) error { | |
| 525 return err | |
| 526 }) | |
| 527 | |
| 528 return nil | |
| 529 })) | |
| 530 if err == nil { | |
| 531 err = et.error() | |
| 532 } | |
| 533 | |
| 534 if err != nil && len(ent) == 1 { | |
| 535 // Single-argument Delete will return a single error. | |
| 536 err = errors.SingleError(err) | |
| 537 } | |
| 538 return err | |
| 539 } | |
| 540 | |
| 541 func (d *datastoreImpl) DeleteMulti(keys []*Key) error { | |
| 542 return d.Delete(keys) | |
| 543 } | |
| 544 | |
| 545 func (d *datastoreImpl) Raw() RawInterface { | |
| 546 return d.RawInterface | |
| 547 } | |
| 548 | |
| 549 // ParseIndexYAML parses the contents of a index YAML file into a list of | 19 // ParseIndexYAML parses the contents of a index YAML file into a list of |
| 550 // IndexDefinitions. | 20 // IndexDefinitions. |
| 551 func ParseIndexYAML(content io.Reader) ([]*IndexDefinition, error) { | 21 func ParseIndexYAML(content io.Reader) ([]*IndexDefinition, error) { |
| 552 serialized, err := ioutil.ReadAll(content) | 22 serialized, err := ioutil.ReadAll(content) |
| 553 if err != nil { | 23 if err != nil { |
| 554 return nil, err | 24 return nil, err |
| 555 } | 25 } |
| 556 | 26 |
| 557 var m map[string][]*IndexDefinition | 27 var m map[string][]*IndexDefinition |
| 558 if err := yaml.Unmarshal(serialized, &m); err != nil { | 28 if err := yaml.Unmarshal(serialized, &m); err != nil { |
| (...skipping 62 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 621 } | 91 } |
| 622 } | 92 } |
| 623 | 93 |
| 624 if isRoot(currentDir) { | 94 if isRoot(currentDir) { |
| 625 return nil, fmt.Errorf("datastore: failed to find index
YAML file") | 95 return nil, fmt.Errorf("datastore: failed to find index
YAML file") |
| 626 } | 96 } |
| 627 | 97 |
| 628 currentDir = filepath.Dir(currentDir) | 98 currentDir = filepath.Dir(currentDir) |
| 629 } | 99 } |
| 630 } | 100 } |
| 631 | |
| 632 func filterStop(err error) error { | |
| 633 if err == Stop { | |
| 634 err = nil | |
| 635 } | |
| 636 return err | |
| 637 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 2302743002:
Interface update, per-method Contexts. (Closed)
