| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package rawdatastore | |
| 6 | |
| 7 import ( | |
| 8 "errors" | |
| 9 "fmt" | |
| 10 "math" | |
| 11 "reflect" | |
| 12 "time" | |
| 13 | |
| 14 "github.com/luci/gae/service/blobstore" | |
| 15 ) | |
| 16 | |
| 17 var ( | |
| 18 minTime = time.Unix(int64(math.MinInt64)/1e6, (int64(math.MinInt64)%1e6)
*1e3) | |
| 19 maxTime = time.Unix(int64(math.MaxInt64)/1e6, (int64(math.MaxInt64)%1e6)
*1e3) | |
| 20 ) | |
| 21 | |
| 22 // IndexSetting indicates whether or not a Property should be indexed by the | |
| 23 // datastore. | |
| 24 type IndexSetting bool | |
| 25 | |
| 26 // ShouldIndex is the default, which is why it must assume the zero value, | |
| 27 // even though it's werid :(. | |
| 28 const ( | |
| 29 ShouldIndex IndexSetting = false | |
| 30 NoIndex IndexSetting = true | |
| 31 ) | |
| 32 | |
| 33 func (i IndexSetting) String() string { | |
| 34 if i { | |
| 35 return "NoIndex" | |
| 36 } | |
| 37 return "ShouldIndex" | |
| 38 } | |
| 39 | |
| 40 // Property is a value plus an indicator of whether the value should be | |
| 41 // indexed. Name and Multiple are stored in the PropertyMap object. | |
| 42 type Property struct { | |
| 43 value interface{} | |
| 44 indexSetting IndexSetting | |
| 45 propType PropertyType | |
| 46 } | |
| 47 | |
| 48 // MkProperty makes a new indexed* Property and returns it. If val is an | |
| 49 // invalid value, this panics (so don't do it). If you want to handle the error | |
| 50 // normally, use SetValue(..., ShouldIndex) instead. | |
| 51 // | |
| 52 // *indexed if val is not an unindexable type like []byte. | |
| 53 func MkProperty(val interface{}) Property { | |
| 54 ret := Property{} | |
| 55 if err := ret.SetValue(val, ShouldIndex); err != nil { | |
| 56 panic(err) | |
| 57 } | |
| 58 return ret | |
| 59 } | |
| 60 | |
| 61 // MkPropertyNI makes a new Property (with noindex set to true), and returns | |
| 62 // it. If val is an invalid value, this panics (so don't do it). If you want to | |
| 63 // handle the error normally, use SetValue(..., NoIndex) instead. | |
| 64 func MkPropertyNI(val interface{}) Property { | |
| 65 ret := Property{} | |
| 66 if err := ret.SetValue(val, NoIndex); err != nil { | |
| 67 panic(err) | |
| 68 } | |
| 69 return ret | |
| 70 } | |
| 71 | |
| 72 // PropertyConverter may be implemented by the pointer-to a struct field which | |
| 73 // is serialized by RawDatastore. Its ToProperty will be called on save, and | |
| 74 // it's FromProperty will be called on load (from datastore). The method may | |
| 75 // do arbitrary computation, and if it encounters an error, may return it. This | |
| 76 // error will be a fatal error (as defined by PropertyLoadSaver) for the | |
| 77 // struct conversion. | |
| 78 // | |
| 79 // Example: | |
| 80 // type Complex complex | |
| 81 // func (c *Complex) ToProperty() (ret Property, err error) { | |
| 82 // // something like: | |
| 83 // err = ret.SetValue(fmt.Sprint(*c), true) | |
| 84 // return | |
| 85 // } | |
| 86 // func (c *Complex) FromProperty(p Property) (err error) { | |
| 87 // ... load *c from p ... | |
| 88 // } | |
| 89 // | |
| 90 // type MyStruct struct { | |
| 91 // Complexity []Complex // acts like []complex, but can be serialized to DS | |
| 92 // } | |
| 93 type PropertyConverter interface { | |
| 94 // TODO(riannucci): Allow a convertable to return multiple values. This
is | |
| 95 // eminently doable (as long as the single-slice restriction is kept).
It | |
| 96 // could also cut down on the amount of reflection necessary when resolv
ing | |
| 97 // a path in a struct (in the struct loading routine in helper). | |
| 98 | |
| 99 ToProperty() (Property, error) | |
| 100 FromProperty(Property) error | |
| 101 } | |
| 102 | |
| 103 // PropertyType is a single-byte representation of the type of data contained | |
| 104 // in a Property. The specific values of this type information are chosen so | |
| 105 // that the types sort according to the order of types as sorted by the | |
| 106 // datastore. | |
| 107 type PropertyType byte | |
| 108 | |
| 109 // These constants are in the order described by | |
| 110 // https://cloud.google.com/appengine/docs/go/datastore/entities#Go_Value_type
_ordering | |
| 111 // with a slight divergence for the Int/Time split. | |
| 112 // NOTE: this enum can only occupy 7 bits, because we use the high bit to encode | |
| 113 // indexed/non-indexed. See typData.WriteBinary. | |
| 114 const ( | |
| 115 PTNull PropertyType = iota | |
| 116 PTInt | |
| 117 | |
| 118 // PTTime is a slight divergence from the way that datastore natively st
ores | |
| 119 // time. In datastore, times and integers actually sort together | |
| 120 // (apparently?). This is probably insane, and I don't want to add the | |
| 121 // complexity of field 'meaning' as a sparate concept from the field's '
type' | |
| 122 // (which is what datastore seems to do, judging from the protobufs). So
if | |
| 123 // you're here because you implemented an app which relies on time.Time
and | |
| 124 // int64 sorting together, then this is why your app acts differently in | |
| 125 // production. My advice is to NOT DO THAT. If you really want this (and
you | |
| 126 // probably don't), you should take care of the time.Time <-> int64 conv
ersion | |
| 127 // in your app and just use a property type of int64 (consider using | |
| 128 // PropertyConverter). | |
| 129 PTTime | |
| 130 | |
| 131 // PTBoolFalse and True are also a slight divergence, but not a semantic | |
| 132 // one. IIUC, in datastore 'bool' is actually the type and the value is
either | |
| 133 // 0 or 1 (taking another byte to store). Since we have plenty of space
in | |
| 134 // this type byte, I just merge the value into the type for booleans. If
this | |
| 135 // becomes problematic, consider changing this to just pvBool, and then | |
| 136 // encoding a 0 or 1 as a byte in the relevant marshalling routines. | |
| 137 PTBoolFalse | |
| 138 PTBoolTrue | |
| 139 | |
| 140 PTBytes // []byte or datastore.ByteString | |
| 141 PTString // string or string noindex | |
| 142 PTFloat | |
| 143 PTGeoPoint | |
| 144 PTKey | |
| 145 PTBlobKey | |
| 146 | |
| 147 PTUnknown | |
| 148 ) | |
| 149 | |
| 150 func (t PropertyType) String() string { | |
| 151 switch t { | |
| 152 case PTNull: | |
| 153 return "PTNull" | |
| 154 case PTInt: | |
| 155 return "PTInt" | |
| 156 case PTTime: | |
| 157 return "PTTime" | |
| 158 case PTBoolFalse: | |
| 159 return "PTBoolFalse" | |
| 160 case PTBoolTrue: | |
| 161 return "PTBoolTrue" | |
| 162 case PTBytes: | |
| 163 return "PTBytes" | |
| 164 case PTString: | |
| 165 return "PTString" | |
| 166 case PTFloat: | |
| 167 return "PTFloat" | |
| 168 case PTGeoPoint: | |
| 169 return "PTGeoPoint" | |
| 170 case PTKey: | |
| 171 return "PTKey" | |
| 172 case PTBlobKey: | |
| 173 return "PTBlobKey" | |
| 174 default: | |
| 175 return fmt.Sprintf("PTUnknown(%02x)", byte(t)) | |
| 176 } | |
| 177 } | |
| 178 | |
| 179 // PropertyTypeOf returns the PT* type of the given Property-compatible | |
| 180 // value v. If checkValid is true, this method will also ensure that time.Time | |
| 181 // and GeoPoint have valid values. | |
| 182 func PropertyTypeOf(v interface{}, checkValid bool) (PropertyType, error) { | |
| 183 switch x := v.(type) { | |
| 184 case nil: | |
| 185 return PTNull, nil | |
| 186 case int64: | |
| 187 return PTInt, nil | |
| 188 case float64: | |
| 189 return PTFloat, nil | |
| 190 case bool: | |
| 191 if x { | |
| 192 return PTBoolTrue, nil | |
| 193 } | |
| 194 return PTBoolFalse, nil | |
| 195 case []byte, ByteString: | |
| 196 return PTBytes, nil | |
| 197 case blobstore.Key: | |
| 198 return PTBlobKey, nil | |
| 199 case string: | |
| 200 return PTString, nil | |
| 201 case Key: | |
| 202 // TODO(riannucci): Check key for validity in its own namespace? | |
| 203 return PTKey, nil | |
| 204 case time.Time: | |
| 205 err := error(nil) | |
| 206 if checkValid && (x.Before(minTime) || x.After(maxTime)) { | |
| 207 err = errors.New("time value out of range") | |
| 208 } | |
| 209 if checkValid && x.Location() != time.UTC { | |
| 210 err = fmt.Errorf("time value has wrong Location: %s", x.
Location()) | |
| 211 } | |
| 212 return PTTime, err | |
| 213 case GeoPoint: | |
| 214 err := error(nil) | |
| 215 if checkValid && !x.Valid() { | |
| 216 err = errors.New("invalid GeoPoint value") | |
| 217 } | |
| 218 return PTGeoPoint, err | |
| 219 default: | |
| 220 return PTUnknown, fmt.Errorf("gae: Property has bad type %T", v) | |
| 221 } | |
| 222 } | |
| 223 | |
| 224 // UpconvertUnderlyingType takes an object o, and attempts to convert it to | |
| 225 // its native datastore-compatible type. e.g. int16 will convert to int64, and | |
| 226 // `type Foo string` will convert to `string`. | |
| 227 func UpconvertUnderlyingType(o interface{}, t reflect.Type) (interface{}, reflec
t.Type) { | |
| 228 v := reflect.ValueOf(o) | |
| 229 switch t.Kind() { | |
| 230 case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.In
t64: | |
| 231 o = v.Int() | |
| 232 t = typeOfInt64 | |
| 233 case reflect.Bool: | |
| 234 o = v.Bool() | |
| 235 t = typeOfBool | |
| 236 case reflect.String: | |
| 237 if t != typeOfBSKey { | |
| 238 o = v.String() | |
| 239 t = typeOfString | |
| 240 } | |
| 241 case reflect.Float32, reflect.Float64: | |
| 242 o = v.Float() | |
| 243 t = typeOfFloat64 | |
| 244 case reflect.Slice: | |
| 245 if t != typeOfByteString && t.Elem().Kind() == reflect.Uint8 { | |
| 246 o = v.Bytes() | |
| 247 t = typeOfByteSlice | |
| 248 } | |
| 249 case reflect.Struct: | |
| 250 if t == typeOfTime { | |
| 251 // time in a Property can only hold microseconds | |
| 252 o = v.Interface().(time.Time).Round(time.Microsecond) | |
| 253 } | |
| 254 } | |
| 255 return o, t | |
| 256 } | |
| 257 | |
| 258 // Value returns the current value held by this property. It's guaranteed to | |
| 259 // be a valid value type (i.e. `p.SetValue(p.Value(), true)` will never return | |
| 260 // an error). | |
| 261 func (p *Property) Value() interface{} { return p.value } | |
| 262 | |
| 263 // IndexSetting says weather or not the datastore should create indicies for | |
| 264 // this value. | |
| 265 func (p *Property) IndexSetting() IndexSetting { return p.indexSetting } | |
| 266 | |
| 267 // Type is the PT* type of the data contained in Value(). | |
| 268 func (p *Property) Type() PropertyType { return p.propType } | |
| 269 | |
| 270 // SetValue sets the Value field of a Property, and ensures that its value | |
| 271 // conforms to the permissible types. That way, you're guaranteed that if you | |
| 272 // have a Property, its value is valid. | |
| 273 // | |
| 274 // value is the property value. The valid types are: | |
| 275 // - int64 | |
| 276 // - bool | |
| 277 // - string | |
| 278 // - float64 | |
| 279 // - ByteString | |
| 280 // - Key | |
| 281 // - time.Time | |
| 282 // - blobstore.Key | |
| 283 // - GeoPoint | |
| 284 // - []byte (up to 1 megabyte in length) | |
| 285 // This set is smaller than the set of valid struct field types that the | |
| 286 // datastore can load and save. A Property Value cannot be a slice (apart | |
| 287 // from []byte); use multiple Properties instead. Also, a Value's type | |
| 288 // must be explicitly on the list above; it is not sufficient for the | |
| 289 // underlying type to be on that list. For example, a Value of "type | |
| 290 // myInt64 int64" is invalid. Smaller-width integers and floats are also | |
| 291 // invalid. Again, this is more restrictive than the set of valid struct | |
| 292 // field types. | |
| 293 // | |
| 294 // A value may also be the nil interface value; this is equivalent to | |
| 295 // Python's None but not directly representable by a Go struct. Loading | |
| 296 // a nil-valued property into a struct will set that field to the zero | |
| 297 // value. | |
| 298 func (p *Property) SetValue(value interface{}, is IndexSetting) (err error) { | |
| 299 t := reflect.Type(nil) | |
| 300 pt := PTNull | |
| 301 if value != nil { | |
| 302 t = reflect.TypeOf(value) | |
| 303 value, t = UpconvertUnderlyingType(value, t) | |
| 304 if pt, err = PropertyTypeOf(value, true); err != nil { | |
| 305 return | |
| 306 } | |
| 307 } | |
| 308 p.propType = pt | |
| 309 p.value = value | |
| 310 p.indexSetting = is | |
| 311 if t == typeOfByteSlice { | |
| 312 p.indexSetting = NoIndex | |
| 313 } | |
| 314 return | |
| 315 } | |
| 316 | |
| 317 // PropertyLoadSaver may be implemented by a user type, and RawDatastore will | |
| 318 // use this interface to serialize the type instead of trying to automatically | |
| 319 // create a serialization codec for it with helper.GetPLS. | |
| 320 type PropertyLoadSaver interface { | |
| 321 // Load takes the values from the given map and attempts to save them in
to | |
| 322 // the underlying object (usually a struct or a PropertyMap). If a fatal | |
| 323 // error occurs, it's returned via error. If non-fatal conversion errors | |
| 324 // occur, error will be a MultiError containing one or more ErrFieldMism
atch | |
| 325 // objects. | |
| 326 Load(PropertyMap) error | |
| 327 | |
| 328 // Save returns the current property as a PropertyMap. if withMeta is tr
ue, | |
| 329 // then the PropertyMap contains all the metadata (e.g. '$meta' fields) | |
| 330 // which was held by this PropertyLoadSaver. | |
| 331 Save(withMeta bool) (PropertyMap, error) | |
| 332 | |
| 333 // GetMeta will get information about the field which has the struct tag
in | |
| 334 // the form of `gae:"$<key>[,<value>]?"`. | |
| 335 // | |
| 336 // string and int64 fields will return the <value> in the struct tag, | |
| 337 // converted to the appropriate type, if the field has the zero value. | |
| 338 // | |
| 339 // Example: | |
| 340 // type MyStruct struct { | |
| 341 // CoolField int64 `gae:"$id,1"` | |
| 342 // } | |
| 343 // val, err := helper.GetPLS(&MyStruct{}).GetMeta("id") | |
| 344 // // val == 1 | |
| 345 // // err == nil | |
| 346 // | |
| 347 // val, err := helper.GetPLS(&MyStruct{10}).GetMeta("id") | |
| 348 // // val == 10 | |
| 349 // // err == nil | |
| 350 // | |
| 351 // Struct fields of type Toggle (which is an Auto/On/Off) allow you to | |
| 352 // specify a value of 'true' or 'false' for the default value of the str
uct | |
| 353 // tag, and GetMeta will return the combined value as a regular boolean
true | |
| 354 // or false value. If a field is Toggle, a <value> MUST be specified. | |
| 355 // | |
| 356 // Example: | |
| 357 // type MyStruct struct { | |
| 358 // TFlag Toggle `gae:"$flag1,true"` // defaults to true | |
| 359 // FFlag Toggle `gae:"$flag2,false"` // defaults to false | |
| 360 // // BadFlag Toggle `gae:"$flag3"` // ILLEGAL | |
| 361 // } | |
| 362 GetMeta(key string) (interface{}, error) | |
| 363 | |
| 364 // SetMeta allows you to set the current value of the meta-keyed field. | |
| 365 SetMeta(key string, val interface{}) error | |
| 366 | |
| 367 // Problem indicates that this PLS has a fatal problem. Usually this is | |
| 368 // set when the underlying struct has recursion, invalid field types, ne
sted | |
| 369 // slices, etc. | |
| 370 Problem() error | |
| 371 } | |
| 372 | |
| 373 // PropertyMap represents the contents of a datastore entity in a generic way. | |
| 374 // It maps from property name to a list of property values which correspond to | |
| 375 // that property name. It is the spiritual successor to PropertyList from the | |
| 376 // original SDK. | |
| 377 // | |
| 378 // PropertyMap may contain "meta" values, which are keyed with a '$' prefix. | |
| 379 // Technically the datastore allows arbitrary property names, but all of the | |
| 380 // SDKs go out of their way to try to make all property names valid programming | |
| 381 // language tokens. Special values must correspond to a single Property... | |
| 382 // corresponding to 0 is equivalent to unset, and corresponding to >1 is an | |
| 383 // error. So: | |
| 384 // | |
| 385 // { | |
| 386 // "$id": {MkProperty(1)}, // GetProperty("id") -> 1, nil | |
| 387 // "$foo": {}, // GetProperty("foo") -> nil, ErrMetaFieldUnset | |
| 388 // // GetProperty("bar") -> nil, ErrMetaFieldUnset | |
| 389 // "$meep": { | |
| 390 // MkProperty("hi"), | |
| 391 // MkProperty("there")}, // GetProperty("meep") -> nil, error! | |
| 392 // } | |
| 393 // | |
| 394 // Additionally, Save returns a copy of the map with the meta keys omitted (e.g. | |
| 395 // these keys are not going to be serialized to the datastore). | |
| 396 type PropertyMap map[string][]Property | |
| 397 | |
| 398 var _ PropertyLoadSaver = PropertyMap(nil) | |
| 399 | |
| 400 // Load implements PropertyLoadSaver.Load | |
| 401 func (pm PropertyMap) Load(props PropertyMap) error { | |
| 402 for k, v := range props { | |
| 403 pm[k] = append(pm[k], v...) | |
| 404 } | |
| 405 return nil | |
| 406 } | |
| 407 | |
| 408 // Save implements PropertyLoadSaver.Save by returning a copy of the | |
| 409 // current map data. | |
| 410 func (pm PropertyMap) Save(withMeta bool) (PropertyMap, error) { | |
| 411 if len(pm) == 0 { | |
| 412 return PropertyMap{}, nil | |
| 413 } | |
| 414 ret := make(PropertyMap, len(pm)) | |
| 415 for k, v := range pm { | |
| 416 if withMeta || len(k) == 0 || k[0] != '$' { | |
| 417 ret[k] = append(ret[k], v...) | |
| 418 } | |
| 419 } | |
| 420 return ret, nil | |
| 421 } | |
| 422 | |
| 423 // GetMeta implements PropertyLoadSaver.GetMeta, and returns the current value | |
| 424 // associated with the metadata key. It may return ErrMetaFieldUnset if the | |
| 425 // key doesn't exist. | |
| 426 func (pm PropertyMap) GetMeta(key string) (interface{}, error) { | |
| 427 v, ok := pm["$"+key] | |
| 428 if !ok || len(v) == 0 { | |
| 429 return nil, ErrMetaFieldUnset | |
| 430 } | |
| 431 if len(v) > 1 { | |
| 432 return nil, errors.New("gae: too many values for Meta key") | |
| 433 } | |
| 434 return v[0].Value(), nil | |
| 435 } | |
| 436 | |
| 437 // SetMeta implements PropertyLoadSaver.SetMeta. It will only return an error | |
| 438 // if `val` has an invalid type (e.g. not one supported by Property). | |
| 439 func (pm PropertyMap) SetMeta(key string, val interface{}) error { | |
| 440 prop := Property{} | |
| 441 if err := prop.SetValue(val, NoIndex); err != nil { | |
| 442 return err | |
| 443 } | |
| 444 pm["$"+key] = []Property{prop} | |
| 445 return nil | |
| 446 } | |
| 447 | |
| 448 // Problem implements PropertyLoadSaver.Problem. It ALWAYS returns nil. | |
| 449 func (pm PropertyMap) Problem() error { | |
| 450 return nil | |
| 451 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1259593005:
Add 'user friendly' datastore API. (Closed)
Base URL: https://github.com/luci/gae.git@master