Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(999)

Side by Side Diff: service/datastore/serialize/serialize.go

Issue 2342063003: Differentiate between single- and multi- props. (Closed)
Patch Set: Slice is now always a clone. This is marginally worse performance, but a much safer UI. Created 4 years, 3 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « service/datastore/properties_test.go ('k') | service/datastore/serialize/serialize_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2015 The LUCI Authors. All rights reserved. 1 // Copyright 2015 The LUCI Authors. All rights reserved.
2 // Use of this source code is governed under the Apache License, Version 2.0 2 // Use of this source code is governed under the Apache License, Version 2.0
3 // that can be found in the LICENSE file. 3 // that can be found in the LICENSE file.
4 4
5 package serialize 5 package serialize
6 6
7 import ( 7 import (
8 "bytes" 8 "bytes"
9 "errors" 9 "errors"
10 "fmt" 10 "fmt"
(...skipping 319 matching lines...) Expand 10 before | Expand all | Expand 10 after
330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by 330 // If WritePropertyMapDeterministic is true, then the rows will be sorted by
331 // property name before they're serialized to buf (mostly useful for testing, 331 // property name before they're serialized to buf (mostly useful for testing,
332 // but also potentially useful if you need to make a hash of the property data). 332 // but also potentially useful if you need to make a hash of the property data).
333 // 333 //
334 // Write skips metadata keys. 334 // Write skips metadata keys.
335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er ror) { 335 func WritePropertyMap(buf Buffer, context KeyContext, pm ds.PropertyMap) (err er ror) {
336 defer recoverTo(&err) 336 defer recoverTo(&err)
337 rows := make(sort.StringSlice, 0, len(pm)) 337 rows := make(sort.StringSlice, 0, len(pm))
338 tmpBuf := &bytes.Buffer{} 338 tmpBuf := &bytes.Buffer{}
339 pm, _ = pm.Save(false) 339 pm, _ = pm.Save(false)
340 » for name, vals := range pm { 340 » for name, pdata := range pm {
341 tmpBuf.Reset() 341 tmpBuf.Reset()
342 _, e := cmpbin.WriteString(tmpBuf, name) 342 _, e := cmpbin.WriteString(tmpBuf, name)
343 panicIf(e) 343 panicIf(e)
344 » » _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) 344
345 » » panicIf(e) 345 » » switch t := pdata.(type) {
346 » » for _, p := range vals { 346 » » case ds.Property:
347 » » » panicIf(WriteProperty(tmpBuf, context, p)) 347 » » » _, e = cmpbin.WriteInt(tmpBuf, -1)
348 » » » panicIf(e)
349 » » » panicIf(WriteProperty(tmpBuf, context, t))
350
351 » » case ds.PropertySlice:
352 » » » _, e = cmpbin.WriteInt(tmpBuf, int64(len(t)))
353 » » » panicIf(e)
354 » » » for _, p := range t {
355 » » » » panicIf(WriteProperty(tmpBuf, context, p))
356 » » » }
357
358 » » default:
359 » » » return fmt.Errorf("unknown PropertyData type %T", t)
348 } 360 }
349 rows = append(rows, tmpBuf.String()) 361 rows = append(rows, tmpBuf.String())
350 } 362 }
351 363
352 if WritePropertyMapDeterministic { 364 if WritePropertyMapDeterministic {
353 rows.Sort() 365 rows.Sort()
354 } 366 }
355 367
356 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) 368 _, e := cmpbin.WriteUint(buf, uint64(len(pm)))
357 panicIf(e) 369 panicIf(e)
(...skipping 17 matching lines...) Expand all
375 return 387 return
376 } 388 }
377 389
378 pm = make(ds.PropertyMap, numRows) 390 pm = make(ds.PropertyMap, numRows)
379 391
380 name, prop := "", ds.Property{} 392 name, prop := "", ds.Property{}
381 for i := uint64(0); i < numRows; i++ { 393 for i := uint64(0); i < numRows; i++ {
382 name, _, e = cmpbin.ReadString(buf) 394 name, _, e = cmpbin.ReadString(buf)
383 panicIf(e) 395 panicIf(e)
384 396
385 » » numProps, _, e := cmpbin.ReadUint(buf) 397 » » numProps, _, e := cmpbin.ReadInt(buf)
386 panicIf(e) 398 panicIf(e)
387 » » if numProps > ReadPropertyMapReasonableLimit { 399 » » switch {
400 » » case numProps < 0:
401 » » » // Single property.
402 » » » prop, err = ReadProperty(buf, context, appid, namespace)
403 » » » panicIf(err)
404 » » » pm[name] = prop
405
406 » » case uint64(numProps) > ReadPropertyMapReasonableLimit:
388 err = fmt.Errorf("helper: tried to decode map with huge number of properties %d", numProps) 407 err = fmt.Errorf("helper: tried to decode map with huge number of properties %d", numProps)
389 return 408 return
409
410 default:
411 props := make(ds.PropertySlice, 0, numProps)
412 for j := int64(0); j < numProps; j++ {
413 prop, err = ReadProperty(buf, context, appid, na mespace)
414 panicIf(err)
415 props = append(props, prop)
416 }
417 pm[name] = props
390 } 418 }
391 props := make([]ds.Property, 0, numProps)
392 for j := uint64(0); j < numProps; j++ {
393 prop, err = ReadProperty(buf, context, appid, namespace)
394 panicIf(err)
395 props = append(props, prop)
396 }
397 pm[name] = props
398 } 419 }
399 return 420 return
400 } 421 }
401 422
402 // WriteIndexColumn writes an IndexColumn to the buffer. 423 // WriteIndexColumn writes an IndexColumn to the buffer.
403 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) { 424 func WriteIndexColumn(buf Buffer, c ds.IndexColumn) (err error) {
404 defer recoverTo(&err) 425 defer recoverTo(&err)
405 426
406 if !c.Descending { 427 if !c.Descending {
407 panicIf(buf.WriteByte(0)) 428 panicIf(buf.WriteByte(0))
(...skipping 60 matching lines...) Expand 10 before | Expand all | Expand 10 after
468 489
469 sb, err := ReadIndexColumn(buf) 490 sb, err := ReadIndexColumn(buf)
470 panicIf(err) 491 panicIf(err)
471 492
472 i.SortBy = append(i.SortBy, sb) 493 i.SortBy = append(i.SortBy, sb)
473 } 494 }
474 495
475 return 496 return
476 } 497 }
477 498
478 // SerializedPslice is all of the serialized DSProperty values in qASC order. 499 // SerializedPslice is all of the serialized DSProperty values in ASC order.
479 type SerializedPslice [][]byte 500 type SerializedPslice [][]byte
480 501
481 func (s SerializedPslice) Len() int { return len(s) } 502 func (s SerializedPslice) Len() int { return len(s) }
482 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } 503 func (s SerializedPslice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
483 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j]) < 0 } 504 func (s SerializedPslice) Less(i, j int) bool { return bytes.Compare(s[i], s[j]) < 0 }
484 505
485 // PropertySlice serializes a single row of a DSProperty map. 506 // PropertySlice serializes a single row of a DSProperty map.
507 //
508 // It does not differentiate between single- and multi- properties.
486 func PropertySlice(vals ds.PropertySlice) SerializedPslice { 509 func PropertySlice(vals ds.PropertySlice) SerializedPslice {
487 dups := stringset.New(0) 510 dups := stringset.New(0)
488 ret := make(SerializedPslice, 0, len(vals)) 511 ret := make(SerializedPslice, 0, len(vals))
489 for _, v := range vals { 512 for _, v := range vals {
490 if v.IndexSetting() == ds.NoIndex { 513 if v.IndexSetting() == ds.NoIndex {
491 continue 514 continue
492 } 515 }
493 516
494 data := ToBytes(v) 517 data := ToBytes(v)
495 dataS := string(data) 518 dataS := string(data)
(...skipping 16 matching lines...) Expand all
512 // datastore/serialize's encodings. 535 // datastore/serialize's encodings.
513 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) { 536 func PropertyMapPartially(k *ds.Key, pm ds.PropertyMap) (ret SerializedPmap) {
514 ret = make(SerializedPmap, len(pm)+2) 537 ret = make(SerializedPmap, len(pm)+2)
515 if k != nil { 538 if k != nil {
516 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))} 539 ret["__key__"] = [][]byte{ToBytes(ds.MkProperty(k))}
517 for k != nil { 540 for k != nil {
518 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte s(ds.MkProperty(k))) 541 ret["__ancestor__"] = append(ret["__ancestor__"], ToByte s(ds.MkProperty(k)))
519 k = k.Parent() 542 k = k.Parent()
520 } 543 }
521 } 544 }
522 » for k, vals := range pm { 545 » for k := range pm {
523 » » newVals := PropertySlice(vals) 546 » » newVals := PropertySlice(pm.Slice(k))
524 if len(newVals) > 0 { 547 if len(newVals) > 0 {
525 ret[k] = newVals 548 ret[k] = newVals
526 } 549 }
527 } 550 }
528 return 551 return
529 } 552 }
530 553
531 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) { 554 func toBytesErr(i interface{}, ctx KeyContext) (ret []byte, err error) {
532 buf := bytes.Buffer{} 555 buf := bytes.Buffer{}
533 556
(...skipping 78 matching lines...) Expand 10 before | Expand all | Expand 10 after
612 } 635 }
613 } 636 }
614 637
615 func recoverTo(err *error) { 638 func recoverTo(err *error) {
616 if r := recover(); r != nil { 639 if r := recover(); r != nil {
617 if rerr := r.(parseError); rerr != nil { 640 if rerr := r.(parseError); rerr != nil {
618 *err = error(rerr) 641 *err = error(rerr)
619 } 642 }
620 } 643 }
621 } 644 }
OLDNEW
« no previous file with comments | « service/datastore/properties_test.go ('k') | service/datastore/serialize/serialize_test.go » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698