OLD | NEW |
1 // Copyright 2015 The LUCI Authors. All rights reserved. | 1 // Copyright 2015 The LUCI Authors. All rights reserved. |
2 // Use of this source code is governed under the Apache License, Version 2.0 | 2 // Use of this source code is governed under the Apache License, Version 2.0 |
3 // that can be found in the LICENSE file. | 3 // that can be found in the LICENSE file. |
4 | 4 |
5 package serialize | 5 package serialize |
6 | 6 |
7 import ( | 7 import ( |
8 "bytes" | 8 "bytes" |
9 "errors" | 9 "errors" |
10 "fmt" | 10 "fmt" |
(...skipping 319 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by | 330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by |
331 // property name before they're serialized to buf (mostly useful for testing, | 331 // property name before they're serialized to buf (mostly useful for testing, |
332 // but also potentially useful if you need to make a hash of the property data). | 332 // but also potentially useful if you need to make a hash of the property data). |
333 // | 333 // |
334 // Write skips metadata keys. | 334 // Write skips metadata keys. |
335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er
ror) { | 335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er
ror) { |
336 defer recoverTo(&err) | 336 defer recoverTo(&err) |
337 rows := make(sort.StringSlice, 0, len(pm)) | 337 rows := make(sort.StringSlice, 0, len(pm)) |
338 tmpBuf := &bytes.Buffer{} | 338 tmpBuf := &bytes.Buffer{} |
339 pm, _ = pm.Save(false) | 339 pm, _ = pm.Save(false) |
340 » for name, vals := range pm { | 340 » for name, pdata := range pm { |
341 tmpBuf.Reset() | 341 tmpBuf.Reset() |
342 _, e := cmpbin.WriteString(tmpBuf, name) | 342 _, e := cmpbin.WriteString(tmpBuf, name) |
343 panicIf(e) | 343 panicIf(e) |
344 » » _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | 344 |
345 » » panicIf(e) | 345 » » switch t := pdata.(type) { |
346 » » for _, p := range vals { | 346 » » case ds.Property: |
347 » » » panicIf(WriteProperty(tmpBuf, context, p)) | 347 » » » _, e = cmpbin.WriteInt(tmpBuf, -1) |
| 348 » » » panicIf(e) |
| 349 » » » panicIf(WriteProperty(tmpBuf, context, t)) |
| 350 |
| 351 » » case ds.PropertySlice: |
| 352 » » » _, e = cmpbin.WriteInt(tmpBuf, int64(len(t))) |
| 353 » » » panicIf(e) |
| 354 » » » for _, p := range t { |
| 355 » » » » panicIf(WriteProperty(tmpBuf, context, p)) |
| 356 » » » } |
| 357 |
| 358 » » default: |
| 359 » » » return fmt.Errorf("unknown PropertyData type %T", t) |
348 } | 360 } |
349 rows = append(rows, tmpBuf.String()) | 361 rows = append(rows, tmpBuf.String()) |
350 } | 362 } |
351 | 363 |
352 if WritePropertyMapDeterministic { | 364 if WritePropertyMapDeterministic { |
353 rows.Sort() | 365 rows.Sort() |
354 } | 366 } |
355 | 367 |
356 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | 368 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) |
357 panicIf(e) | 369 panicIf(e) |
(...skipping 17 matching lines...) Expand all Loading... |
375 return | 387 return |
376 } | 388 } |
377 | 389 |
378 pm = make(ds.PropertyMap, numRows) | 390 pm = make(ds.PropertyMap, numRows) |
379 | 391 |
380 name, prop := "", ds.Property{} | 392 name, prop := "", ds.Property{} |
381 for i := uint64(0); i < numRows; i++ { | 393 for i := uint64(0); i < numRows; i++ { |
382 name, _, e = cmpbin.ReadString(buf) | 394 name, _, e = cmpbin.ReadString(buf) |
383 panicIf(e) | 395 panicIf(e) |
384 | 396 |
385 » » numProps, _, e := cmpbin.ReadUint(buf) | 397 » » numProps, _, e := cmpbin.ReadInt(buf) |
386 panicIf(e) | 398 panicIf(e) |
387 » » if numProps > ReadPropertyMapReasonableLimit { | 399 » » switch { |
| 400 » » case numProps < 0: |
| 401 » » » // Single property. |
| 402 » » » prop, err = ReadProperty(buf, context, appid, namespace) |
| 403 » » » panicIf(err) |
| 404 » » » pm[name] = prop |
| 405 |
| 406 » » case uint64(numProps) > ReadPropertyMapReasonableLimit: |
388 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | 407 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) |
389 return | 408 return |
| 409 |
| 410 default: |
| 411 props := make(ds.PropertySlice, 0, numProps) |
| 412 for j := int64(0); j < numProps; j++ { |
| 413 prop, err = ReadProperty(buf, context, appid, na
mespace) |
| 414 panicIf(err) |
| 415 props = append(props, prop) |
| 416 } |
| 417 pm[name] = props |
390 } | 418 } |
391 props := make([]ds.Property, 0, numProps) | |
392 for j := uint64(0); j < numProps; j++ { | |
393 prop, err = ReadProperty(buf, context, appid, namespace) | |
394 panicIf(err) | |
395 props = append(props, prop) | |
396 } | |
397 pm[name] = props | |
398 } | 419 } |
399 return | 420 return |
400 } | 421 } |
401 | 422 |
402 // WriteIndexColumn writes an IndexColumn to the buffer. | 423 // WriteIndexColumn writes an IndexColumn to the buffer. |
403 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) { | 424 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) { |
404 defer recoverTo(&err) | 425 defer recoverTo(&err) |
405 | 426 |
406 if !c.Descending { | 427 if !c.Descending { |
407 panicIf(buf.WriteByte(0)) | 428 panicIf(buf.WriteByte(0)) |
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
468 | 489 |
469 sb, err := ReadIndexColumn(buf) | 490 sb, err := ReadIndexColumn(buf) |
470 panicIf(err) | 491 panicIf(err) |
471 | 492 |
472 i.SortBy = append(i.SortBy, sb) | 493 i.SortBy = append(i.SortBy, sb) |
473 } | 494 } |
474 | 495 |
475 return | 496 return |
476 } | 497 } |
477 | 498 |
478 // SerializedPslice is all of the serialized DSProperty values in qASC order. | 499 // SerializedPslice is all of the serialized DSProperty values in ASC order. |
479 type SerializedPslice [][]byte | 500 type SerializedPslice [][]byte |
480 | 501 |
481 func (s SerializedPslice) Len() int { return len(s) } | 502 func (s SerializedPslice) Len() int { return len(s) } |
482 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } | 503 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } |
483 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j])
< 0 } | 504 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j])
< 0 } |
484 | 505 |
485 // PropertySlice serializes a single row of a DSProperty map. | 506 // PropertySlice serializes a single row of a DSProperty map. |
| 507 // |
| 508 // It does not differentiate between single- and multi- properties. |
486 func PropertySlice(vals ds.PropertySlice) SerializedPslice { | 509 func PropertySlice(vals ds.PropertySlice) SerializedPslice { |
487 dups := stringset.New(0) | 510 dups := stringset.New(0) |
488 ret := make(SerializedPslice, 0, len(vals)) | 511 ret := make(SerializedPslice, 0, len(vals)) |
489 for _, v := range vals { | 512 for _, v := range vals { |
490 if v.IndexSetting() == ds.NoIndex { | 513 if v.IndexSetting() == ds.NoIndex { |
491 continue | 514 continue |
492 } | 515 } |
493 | 516 |
494 data := ToBytes(v) | 517 data := ToBytes(v) |
495 dataS := string(data) | 518 dataS := string(data) |
(...skipping 16 matching lines...) Expand all Loading... |
512 // datastore/serialize's encodings. | 535 // datastore/serialize's encodings. |
513 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) { | 536 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) { |
514 ret = make(SerializedPmap, len(pm)+2) | 537 ret = make(SerializedPmap, len(pm)+2) |
515 if k != nil { | 538 if k != nil { |
516 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))} | 539 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))} |
517 for k != nil { | 540 for k != nil { |
518 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte
s(ds.MkProperty(k))) | 541 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte
s(ds.MkProperty(k))) |
519 k = k.Parent() | 542 k = k.Parent() |
520 } | 543 } |
521 } | 544 } |
522 » for k, vals := range pm { | 545 » for k := range pm { |
523 » » newVals := PropertySlice(vals) | 546 » » newVals := PropertySlice(pm.Slice(k)) |
524 if len(newVals) > 0 { | 547 if len(newVals) > 0 { |
525 ret[k] = newVals | 548 ret[k] = newVals |
526 } | 549 } |
527 } | 550 } |
528 return | 551 return |
529 } | 552 } |
530 | 553 |
531 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) { | 554 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) { |
532 buf := bytes.Buffer{} | 555 buf := bytes.Buffer{} |
533 | 556 |
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
612 } | 635 } |
613 } | 636 } |
614 | 637 |
615 func recoverTo(err *error) { | 638 func recoverTo(err *error) { |
616 if r := recover(); r != nil { | 639 if r := recover(); r != nil { |
617 if rerr := r.(parseError); rerr != nil { | 640 if rerr := r.(parseError); rerr != nil { |
618 *err = error(rerr) | 641 *err = error(rerr) |
619 } | 642 } |
620 } | 643 } |
621 } | 644 } |
OLD | NEW |