| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 // HEAVILY adapted from github.com/golang/appengine/datastore | |
| 6 | |
| 7 package rawdatastore | |
| 8 | |
| 9 import ( | |
| 10 "fmt" | |
| 11 "reflect" | |
| 12 "strconv" | |
| 13 "strings" | |
| 14 "sync" | |
| 15 "time" | |
| 16 "unicode" | |
| 17 | |
| 18 "github.com/luci/gae/service/blobstore" | |
| 19 "github.com/luci/luci-go/common/errors" | |
| 20 ) | |
| 21 | |
| 22 // Entities with more than this many indexed properties will not be saved. | |
| 23 const maxIndexedProperties = 20000 | |
| 24 | |
| 25 type structTag struct { | |
| 26 name string | |
| 27 idxSetting IndexSetting | |
| 28 isSlice bool | |
| 29 substructCodec *structCodec | |
| 30 convert bool | |
| 31 metaVal interface{} | |
| 32 canSet bool | |
| 33 } | |
| 34 | |
| 35 type structCodec struct { | |
| 36 byMeta map[string]int | |
| 37 byName map[string]int | |
| 38 byIndex []structTag | |
| 39 hasSlice bool | |
| 40 problem error | |
| 41 } | |
| 42 | |
| 43 type structPLS struct { | |
| 44 o reflect.Value | |
| 45 c *structCodec | |
| 46 } | |
| 47 | |
| 48 var _ PropertyLoadSaver = (*structPLS)(nil) | |
| 49 | |
| 50 // typeMismatchReason returns a string explaining why the property p could not | |
| 51 // be stored in an entity field of type v.Type(). | |
| 52 func typeMismatchReason(val interface{}, v reflect.Value) string { | |
| 53 entityType := reflect.TypeOf(val) | |
| 54 return fmt.Sprintf("type mismatch: %s versus %v", entityType, v.Type()) | |
| 55 } | |
| 56 | |
| 57 func (p *structPLS) Load(propMap PropertyMap) error { | |
| 58 if err := p.Problem(); err != nil { | |
| 59 return err | |
| 60 } | |
| 61 | |
| 62 convFailures := errors.MultiError(nil) | |
| 63 | |
| 64 t := reflect.Type(nil) | |
| 65 for name, props := range propMap { | |
| 66 multiple := len(props) > 1 | |
| 67 for i, prop := range props { | |
| 68 if reason := loadInner(p.c, p.o, i, name, prop, multiple
); reason != "" { | |
| 69 if t == nil { | |
| 70 t = p.o.Type() | |
| 71 } | |
| 72 convFailures = append(convFailures, &ErrFieldMis
match{ | |
| 73 StructType: t, | |
| 74 FieldName: name, | |
| 75 Reason: reason, | |
| 76 }) | |
| 77 } | |
| 78 } | |
| 79 } | |
| 80 | |
| 81 if len(convFailures) > 0 { | |
| 82 return convFailures | |
| 83 } | |
| 84 | |
| 85 return nil | |
| 86 } | |
| 87 | |
| 88 func loadInner(codec *structCodec, structValue reflect.Value, index int, name st
ring, p Property, requireSlice bool) string { | |
| 89 var v reflect.Value | |
| 90 // Traverse a struct's struct-typed fields. | |
| 91 for { | |
| 92 fieldIndex, ok := codec.byName[name] | |
| 93 if !ok { | |
| 94 return "no such struct field" | |
| 95 } | |
| 96 v = structValue.Field(fieldIndex) | |
| 97 | |
| 98 st := codec.byIndex[fieldIndex] | |
| 99 if st.substructCodec == nil { | |
| 100 break | |
| 101 } | |
| 102 | |
| 103 if v.Kind() == reflect.Slice { | |
| 104 for v.Len() <= index { | |
| 105 v.Set(reflect.Append(v, reflect.New(v.Type().Ele
m()).Elem())) | |
| 106 } | |
| 107 structValue = v.Index(index) | |
| 108 requireSlice = false | |
| 109 } else { | |
| 110 structValue = v | |
| 111 } | |
| 112 // Strip the "I." from "I.X". | |
| 113 name = name[len(st.name):] | |
| 114 codec = st.substructCodec | |
| 115 } | |
| 116 | |
| 117 doConversion := func(v reflect.Value) (string, bool) { | |
| 118 a := v.Addr() | |
| 119 if conv, ok := a.Interface().(PropertyConverter); ok { | |
| 120 err := conv.FromProperty(p) | |
| 121 if err != nil { | |
| 122 return err.Error(), true | |
| 123 } | |
| 124 return "", true | |
| 125 } | |
| 126 return "", false | |
| 127 } | |
| 128 | |
| 129 if ret, ok := doConversion(v); ok { | |
| 130 return ret | |
| 131 } | |
| 132 | |
| 133 var slice reflect.Value | |
| 134 if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8
{ | |
| 135 slice = v | |
| 136 v = reflect.New(v.Type().Elem()).Elem() | |
| 137 } else if requireSlice { | |
| 138 return "multiple-valued property requires a slice field type" | |
| 139 } | |
| 140 | |
| 141 pVal := p.Value() | |
| 142 | |
| 143 if ret, ok := doConversion(v); ok { | |
| 144 if ret != "" { | |
| 145 return ret | |
| 146 } | |
| 147 } else { | |
| 148 knd := v.Kind() | |
| 149 if v.Type().Implements(typeOfKey) { | |
| 150 knd = reflect.Interface | |
| 151 } | |
| 152 switch knd { | |
| 153 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, re
flect.Int64: | |
| 154 x, ok := pVal.(int64) | |
| 155 if !ok && pVal != nil { | |
| 156 return typeMismatchReason(pVal, v) | |
| 157 } | |
| 158 if v.OverflowInt(x) { | |
| 159 return fmt.Sprintf("value %v overflows struct fi
eld of type %v", x, v.Type()) | |
| 160 } | |
| 161 v.SetInt(x) | |
| 162 case reflect.Bool: | |
| 163 x, ok := pVal.(bool) | |
| 164 if !ok && pVal != nil { | |
| 165 return typeMismatchReason(pVal, v) | |
| 166 } | |
| 167 v.SetBool(x) | |
| 168 case reflect.String: | |
| 169 switch x := pVal.(type) { | |
| 170 case blobstore.Key: | |
| 171 v.SetString(string(x)) | |
| 172 case string: | |
| 173 v.SetString(x) | |
| 174 default: | |
| 175 if pVal != nil { | |
| 176 return typeMismatchReason(pVal, v) | |
| 177 } | |
| 178 } | |
| 179 case reflect.Float32, reflect.Float64: | |
| 180 x, ok := pVal.(float64) | |
| 181 if !ok && pVal != nil { | |
| 182 return typeMismatchReason(pVal, v) | |
| 183 } | |
| 184 if v.OverflowFloat(x) { | |
| 185 return fmt.Sprintf("value %v overflows struct fi
eld of type %v", x, v.Type()) | |
| 186 } | |
| 187 v.SetFloat(x) | |
| 188 case reflect.Interface: | |
| 189 x, ok := pVal.(Key) | |
| 190 if !ok && pVal != nil { | |
| 191 return typeMismatchReason(pVal, v) | |
| 192 } | |
| 193 if x != nil { | |
| 194 v.Set(reflect.ValueOf(x)) | |
| 195 } | |
| 196 case reflect.Struct: | |
| 197 switch v.Type() { | |
| 198 case typeOfTime: | |
| 199 x, ok := pVal.(time.Time) | |
| 200 if !ok && pVal != nil { | |
| 201 return typeMismatchReason(pVal, v) | |
| 202 } | |
| 203 v.Set(reflect.ValueOf(x)) | |
| 204 case typeOfGeoPoint: | |
| 205 x, ok := pVal.(GeoPoint) | |
| 206 if !ok && pVal != nil { | |
| 207 return typeMismatchReason(pVal, v) | |
| 208 } | |
| 209 v.Set(reflect.ValueOf(x)) | |
| 210 default: | |
| 211 panic(fmt.Errorf("helper: impossible: %s", typeM
ismatchReason(pVal, v))) | |
| 212 } | |
| 213 case reflect.Slice: | |
| 214 switch x := pVal.(type) { | |
| 215 case []byte: | |
| 216 v.SetBytes(x) | |
| 217 case ByteString: | |
| 218 v.SetBytes([]byte(x)) | |
| 219 default: | |
| 220 panic(fmt.Errorf("helper: impossible: %s", typeM
ismatchReason(pVal, v))) | |
| 221 } | |
| 222 default: | |
| 223 panic(fmt.Errorf("helper: impossible: %s", typeMismatchR
eason(pVal, v))) | |
| 224 } | |
| 225 } | |
| 226 if slice.IsValid() { | |
| 227 slice.Set(reflect.Append(slice, v)) | |
| 228 } | |
| 229 return "" | |
| 230 } | |
| 231 | |
| 232 func (p *structPLS) Save(withMeta bool) (PropertyMap, error) { | |
| 233 size := len(p.c.byName) | |
| 234 if withMeta { | |
| 235 size += len(p.c.byMeta) | |
| 236 } | |
| 237 ret := make(PropertyMap, size) | |
| 238 if _, err := p.save(ret, "", ShouldIndex); err != nil { | |
| 239 return nil, err | |
| 240 } | |
| 241 if withMeta { | |
| 242 for k := range p.c.byMeta { | |
| 243 val, err := p.GetMeta(k) | |
| 244 if err != nil { | |
| 245 return nil, err // TODO(riannucci): should these
be ignored? | |
| 246 } | |
| 247 p := Property{} | |
| 248 if err = p.SetValue(val, NoIndex); err != nil { | |
| 249 return nil, err | |
| 250 } | |
| 251 ret["$"+k] = []Property{p} | |
| 252 } | |
| 253 } | |
| 254 return ret, nil | |
| 255 } | |
| 256 | |
| 257 func (p *structPLS) save(propMap PropertyMap, prefix string, is IndexSetting) (i
dxCount int, err error) { | |
| 258 if err = p.Problem(); err != nil { | |
| 259 return | |
| 260 } | |
| 261 | |
| 262 saveProp := func(name string, si IndexSetting, v reflect.Value, st *stru
ctTag) (err error) { | |
| 263 if st.substructCodec != nil { | |
| 264 count, err := (&structPLS{v, st.substructCodec}).save(pr
opMap, name, si) | |
| 265 if err == nil { | |
| 266 idxCount += count | |
| 267 if idxCount > maxIndexedProperties { | |
| 268 err = errors.New("gae: too many indexed
properties") | |
| 269 } | |
| 270 } | |
| 271 return err | |
| 272 } | |
| 273 | |
| 274 prop := Property{} | |
| 275 if st.convert { | |
| 276 prop, err = v.Addr().Interface().(PropertyConverter).ToP
roperty() | |
| 277 } else { | |
| 278 err = prop.SetValue(v.Interface(), si) | |
| 279 } | |
| 280 if err != nil { | |
| 281 return err | |
| 282 } | |
| 283 propMap[name] = append(propMap[name], prop) | |
| 284 if prop.IndexSetting() == ShouldIndex { | |
| 285 idxCount++ | |
| 286 if idxCount > maxIndexedProperties { | |
| 287 return errors.New("gae: too many indexed propert
ies") | |
| 288 } | |
| 289 } | |
| 290 return nil | |
| 291 } | |
| 292 | |
| 293 for i, st := range p.c.byIndex { | |
| 294 if st.name == "-" { | |
| 295 continue | |
| 296 } | |
| 297 name := st.name | |
| 298 if prefix != "" { | |
| 299 name = prefix + name | |
| 300 } | |
| 301 v := p.o.Field(i) | |
| 302 is1 := is | |
| 303 if st.idxSetting == NoIndex { | |
| 304 is1 = NoIndex | |
| 305 } | |
| 306 if st.isSlice { | |
| 307 for j := 0; j < v.Len(); j++ { | |
| 308 if err = saveProp(name, is1, v.Index(j), &st); e
rr != nil { | |
| 309 return | |
| 310 } | |
| 311 } | |
| 312 } else { | |
| 313 if err = saveProp(name, is1, v, &st); err != nil { | |
| 314 return | |
| 315 } | |
| 316 } | |
| 317 } | |
| 318 return | |
| 319 } | |
| 320 | |
| 321 func (p *structPLS) GetMeta(key string) (interface{}, error) { | |
| 322 if err := p.Problem(); err != nil { | |
| 323 return nil, err | |
| 324 } | |
| 325 idx, ok := p.c.byMeta[key] | |
| 326 if !ok { | |
| 327 return nil, ErrMetaFieldUnset | |
| 328 } | |
| 329 st := p.c.byIndex[idx] | |
| 330 val := st.metaVal | |
| 331 f := p.o.Field(idx) | |
| 332 if st.canSet { | |
| 333 if !reflect.DeepEqual(reflect.Zero(f.Type()).Interface(), f.Inte
rface()) { | |
| 334 val = f.Interface() | |
| 335 if bf, ok := val.(Toggle); ok { | |
| 336 val = bf == On // true if On, otherwise false | |
| 337 } | |
| 338 } | |
| 339 } | |
| 340 return val, nil | |
| 341 } | |
| 342 | |
| 343 func (p *structPLS) SetMeta(key string, val interface{}) (err error) { | |
| 344 if err = p.Problem(); err != nil { | |
| 345 return | |
| 346 } | |
| 347 idx, ok := p.c.byMeta[key] | |
| 348 if !ok { | |
| 349 return ErrMetaFieldUnset | |
| 350 } | |
| 351 if !p.c.byIndex[idx].canSet { | |
| 352 return fmt.Errorf("gae/helper: cannot set meta %q: unexported fi
eld", key) | |
| 353 } | |
| 354 // setting a BoolField | |
| 355 if b, ok := val.(bool); ok { | |
| 356 if b { | |
| 357 val = On | |
| 358 } else { | |
| 359 val = Off | |
| 360 } | |
| 361 } | |
| 362 p.o.Field(idx).Set(reflect.ValueOf(val)) | |
| 363 return nil | |
| 364 } | |
| 365 | |
| 366 func (p *structPLS) Problem() error { return p.c.problem } | |
| 367 | |
| 368 var ( | |
| 369 // The RWMutex is chosen intentionally, as the majority of access to the | |
| 370 // structCodecs map will be in parallel and will be to read an existing
codec. | |
| 371 // There's no reason to serialize goroutines on every | |
| 372 // gae.RawDatastore.{Get,Put}{,Multi} call. | |
| 373 structCodecsMutex sync.RWMutex | |
| 374 structCodecs = map[reflect.Type]*structCodec{} | |
| 375 ) | |
| 376 | |
| 377 // validPropertyName returns whether name consists of one or more valid Go | |
| 378 // identifiers joined by ".". | |
| 379 func validPropertyName(name string) bool { | |
| 380 if name == "" { | |
| 381 return false | |
| 382 } | |
| 383 for _, s := range strings.Split(name, ".") { | |
| 384 if s == "" { | |
| 385 return false | |
| 386 } | |
| 387 first := true | |
| 388 for _, c := range s { | |
| 389 if first { | |
| 390 first = false | |
| 391 if c != '_' && !unicode.IsLetter(c) { | |
| 392 return false | |
| 393 } | |
| 394 } else { | |
| 395 if c != '_' && !unicode.IsLetter(c) && !unicode.
IsDigit(c) { | |
| 396 return false | |
| 397 } | |
| 398 } | |
| 399 } | |
| 400 } | |
| 401 return true | |
| 402 } | |
| 403 | |
| 404 var ( | |
| 405 errRecursiveStruct = fmt.Errorf("(internal): struct type is recursively
defined") | |
| 406 ) | |
| 407 | |
| 408 func getStructCodecLocked(t reflect.Type) (c *structCodec) { | |
| 409 if c, ok := structCodecs[t]; ok { | |
| 410 return c | |
| 411 } | |
| 412 | |
| 413 me := func(fmtStr string, args ...interface{}) error { | |
| 414 return fmt.Errorf(fmtStr, args...) | |
| 415 } | |
| 416 | |
| 417 c = &structCodec{ | |
| 418 byIndex: make([]structTag, t.NumField()), | |
| 419 byName: make(map[string]int, t.NumField()), | |
| 420 byMeta: make(map[string]int, t.NumField()), | |
| 421 problem: errRecursiveStruct, // we'll clear this later if it's n
ot recursive | |
| 422 } | |
| 423 defer func() { | |
| 424 // If the codec has a problem, free up the indexes | |
| 425 if c.problem != nil { | |
| 426 c.byIndex = nil | |
| 427 c.byName = nil | |
| 428 c.byMeta = nil | |
| 429 } | |
| 430 }() | |
| 431 structCodecs[t] = c | |
| 432 | |
| 433 for i := range c.byIndex { | |
| 434 st := &c.byIndex[i] | |
| 435 f := t.Field(i) | |
| 436 name := f.Tag.Get("gae") | |
| 437 opts := "" | |
| 438 if i := strings.Index(name, ","); i != -1 { | |
| 439 name, opts = name[:i], name[i+1:] | |
| 440 } | |
| 441 st.canSet = f.PkgPath == "" // blank == exported | |
| 442 switch { | |
| 443 case name == "": | |
| 444 if !f.Anonymous { | |
| 445 name = f.Name | |
| 446 } | |
| 447 case name[0] == '$': | |
| 448 name = name[1:] | |
| 449 if _, ok := c.byMeta[name]; ok { | |
| 450 c.problem = me("meta field %q set multiple times
", "$"+name) | |
| 451 return | |
| 452 } | |
| 453 c.byMeta[name] = i | |
| 454 mv, err := convertMeta(opts, f.Type) | |
| 455 if err != nil { | |
| 456 c.problem = me("meta field %q has bad type: %s",
"$"+name, err) | |
| 457 return | |
| 458 } | |
| 459 st.metaVal = mv | |
| 460 fallthrough | |
| 461 case name == "-": | |
| 462 st.name = "-" | |
| 463 continue | |
| 464 default: | |
| 465 if !validPropertyName(name) { | |
| 466 c.problem = me("struct tag has invalid property
name: %q", name) | |
| 467 return | |
| 468 } | |
| 469 } | |
| 470 if !st.canSet { | |
| 471 st.name = "-" | |
| 472 continue | |
| 473 } | |
| 474 | |
| 475 substructType := reflect.Type(nil) | |
| 476 ft := f.Type | |
| 477 if reflect.PtrTo(ft).Implements(typeOfPropertyConverter) { | |
| 478 st.convert = true | |
| 479 } else { | |
| 480 switch f.Type.Kind() { | |
| 481 case reflect.Struct: | |
| 482 if ft != typeOfTime && ft != typeOfGeoPoint { | |
| 483 substructType = ft | |
| 484 } | |
| 485 case reflect.Slice: | |
| 486 if reflect.PtrTo(ft.Elem()).Implements(typeOfPro
pertyConverter) { | |
| 487 st.convert = true | |
| 488 } else if ft.Elem().Kind() == reflect.Struct { | |
| 489 substructType = ft.Elem() | |
| 490 } | |
| 491 st.isSlice = ft.Elem().Kind() != reflect.Uint8 | |
| 492 c.hasSlice = c.hasSlice || st.isSlice | |
| 493 case reflect.Interface: | |
| 494 if ft != typeOfKey { | |
| 495 c.problem = me("field %q has non-concret
e interface type %s", | |
| 496 f.Name, f.Type) | |
| 497 return | |
| 498 } | |
| 499 } | |
| 500 } | |
| 501 | |
| 502 if substructType != nil { | |
| 503 sub := getStructCodecLocked(substructType) | |
| 504 if sub.problem != nil { | |
| 505 if sub.problem == errRecursiveStruct { | |
| 506 c.problem = me("field %q is recursively
defined", f.Name) | |
| 507 } else { | |
| 508 c.problem = me("field %q has problem: %s
", f.Name, sub.problem) | |
| 509 } | |
| 510 return | |
| 511 } | |
| 512 st.substructCodec = sub | |
| 513 if st.isSlice && sub.hasSlice { | |
| 514 c.problem = me( | |
| 515 "flattening nested structs leads to a sl
ice of slices: field %q", | |
| 516 f.Name) | |
| 517 return | |
| 518 } | |
| 519 c.hasSlice = c.hasSlice || sub.hasSlice | |
| 520 if name != "" { | |
| 521 name += "." | |
| 522 } | |
| 523 for relName := range sub.byName { | |
| 524 absName := name + relName | |
| 525 if _, ok := c.byName[absName]; ok { | |
| 526 c.problem = me("struct tag has repeated
property name: %q", absName) | |
| 527 return | |
| 528 } | |
| 529 c.byName[absName] = i | |
| 530 } | |
| 531 } else { | |
| 532 if !st.convert { // check the underlying static type of
the field | |
| 533 t := ft | |
| 534 if st.isSlice { | |
| 535 t = t.Elem() | |
| 536 } | |
| 537 v := reflect.New(t).Elem().Interface() | |
| 538 v, _ = UpconvertUnderlyingType(v, t) | |
| 539 if _, err := PropertyTypeOf(v, false); err != ni
l { | |
| 540 c.problem = me("field %q has invalid typ
e: %s", name, ft) | |
| 541 return | |
| 542 } | |
| 543 } | |
| 544 | |
| 545 if _, ok := c.byName[name]; ok { | |
| 546 c.problem = me("struct tag has repeated property
name: %q", name) | |
| 547 return | |
| 548 } | |
| 549 c.byName[name] = i | |
| 550 } | |
| 551 st.name = name | |
| 552 if opts == "noindex" { | |
| 553 st.idxSetting = NoIndex | |
| 554 } | |
| 555 } | |
| 556 if c.problem == errRecursiveStruct { | |
| 557 c.problem = nil | |
| 558 } | |
| 559 return | |
| 560 } | |
| 561 | |
| 562 func convertMeta(val string, t reflect.Type) (interface{}, error) { | |
| 563 switch t { | |
| 564 case typeOfString: | |
| 565 return val, nil | |
| 566 case typeOfInt64: | |
| 567 if val == "" { | |
| 568 return int64(0), nil | |
| 569 } | |
| 570 return strconv.ParseInt(val, 10, 64) | |
| 571 case typeOfToggle: | |
| 572 switch val { | |
| 573 case "on", "On", "true": | |
| 574 return true, nil | |
| 575 case "off", "Off", "false": | |
| 576 return false, nil | |
| 577 } | |
| 578 return nil, fmt.Errorf("Toggle field has bad/missing default, go
t %q", val) | |
| 579 } | |
| 580 return nil, fmt.Errorf("helper: meta field with bad type/value %s/%q", t
, val) | |
| 581 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1259593005:
Add 'user friendly' datastore API. (Closed)
Base URL: https://github.com/luci/gae.git@master