| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package datastore | |
| 6 | |
| 7 import ( | |
| 8 "bytes" | |
| 9 "errors" | |
| 10 "fmt" | |
| 11 "sort" | |
| 12 "time" | |
| 13 | |
| 14 "github.com/luci/gae/service/blobstore" | |
| 15 "github.com/luci/luci-go/common/cmpbin" | |
| 16 ) | |
| 17 | |
| 18 // MaxIndexColumns is the maximum number of sort orders you may have on a | |
| 19 // single composite index. 64 was chosen as a likely-astronomical number. | |
| 20 const MaxIndexColumns = 64 | |
| 21 | |
| 22 // WritePropertyMapDeterministic allows tests to make WritePropertyMap | |
| 23 // deterministic. | |
| 24 var WritePropertyMapDeterministic = false | |
| 25 | |
| 26 // ReadPropertyMapReasonableLimit sets a limit on the number of rows and | |
| 27 // number of properties per row which can be read by ReadPropertyMap. The | |
| 28 // total number of Property objects readable by this method is this number | |
| 29 // squared (e.g. Limit rows * Limit properties) | |
| 30 const ReadPropertyMapReasonableLimit uint64 = 30000 | |
| 31 | |
| 32 // ReadKeyNumToksReasonableLimit is the maximum number of Key tokens that | |
| 33 // ReadKey is willing to read for a single key. | |
| 34 const ReadKeyNumToksReasonableLimit uint64 = 50 | |
| 35 | |
| 36 // KeyContext controls whether the various Write and Read serializtion | |
| 37 // routines should encode the context of Keys (read: the appid and namespace). | |
| 38 // Frequently the appid and namespace of keys are known in advance and so there'
s | |
| 39 // no reason to redundantly encode them. | |
| 40 type KeyContext bool | |
| 41 | |
| 42 // With- and WithoutContext indicate if the serialization method should include | |
| 43 // context for Keys. See KeyContext for more information. | |
| 44 const ( | |
| 45 WithContext KeyContext = true | |
| 46 WithoutContext = false | |
| 47 ) | |
| 48 | |
| 49 // WriteKey encodes a key to the buffer. If context is WithContext, then this | |
| 50 // encoded value will include the appid and namespace of the key. | |
| 51 func WriteKey(buf Buffer, context KeyContext, k Key) (err error) { | |
| 52 // [appid ++ namespace]? ++ #tokens ++ tokens* | |
| 53 defer recoverTo(&err) | |
| 54 appid, namespace, toks := KeySplit(k) | |
| 55 if context == WithContext { | |
| 56 panicIf(buf.WriteByte(1)) | |
| 57 _, e := cmpbin.WriteString(buf, appid) | |
| 58 panicIf(e) | |
| 59 _, e = cmpbin.WriteString(buf, namespace) | |
| 60 panicIf(e) | |
| 61 } else { | |
| 62 panicIf(buf.WriteByte(0)) | |
| 63 } | |
| 64 _, e := cmpbin.WriteUint(buf, uint64(len(toks))) | |
| 65 panicIf(e) | |
| 66 for _, tok := range toks { | |
| 67 panicIf(WriteKeyTok(buf, tok)) | |
| 68 } | |
| 69 return nil | |
| 70 } | |
| 71 | |
| 72 // ReadKey deserializes a key from the buffer. The value of context must match | |
| 73 // the value of context that was passed to WriteKey when the key was encoded. | |
| 74 // If context == WithoutContext, then the appid and namespace parameters are | |
| 75 // used in the decoded Key. Otherwise they're ignored. | |
| 76 func ReadKey(buf Buffer, context KeyContext, appid, namespace string) (ret Key,
err error) { | |
| 77 defer recoverTo(&err) | |
| 78 actualCtx, e := buf.ReadByte() | |
| 79 panicIf(e) | |
| 80 | |
| 81 actualAid, actualNS := "", "" | |
| 82 if actualCtx == 1 { | |
| 83 actualAid, _, e = cmpbin.ReadString(buf) | |
| 84 panicIf(e) | |
| 85 actualNS, _, e = cmpbin.ReadString(buf) | |
| 86 panicIf(e) | |
| 87 } else if actualCtx != 0 { | |
| 88 err = fmt.Errorf("helper: expected actualCtx to be 0 or 1, got %
d", actualCtx) | |
| 89 return | |
| 90 } | |
| 91 | |
| 92 if context == WithoutContext { | |
| 93 // overrwrite with the supplied ones | |
| 94 actualAid = appid | |
| 95 actualNS = namespace | |
| 96 } | |
| 97 | |
| 98 numToks, _, e := cmpbin.ReadUint(buf) | |
| 99 panicIf(e) | |
| 100 if numToks > ReadKeyNumToksReasonableLimit { | |
| 101 err = fmt.Errorf("helper: tried to decode huge key of length %d"
, numToks) | |
| 102 return | |
| 103 } | |
| 104 | |
| 105 toks := make([]KeyTok, numToks) | |
| 106 for i := uint64(0); i < numToks; i++ { | |
| 107 toks[i], e = ReadKeyTok(buf) | |
| 108 panicIf(e) | |
| 109 } | |
| 110 | |
| 111 return NewKeyToks(actualAid, actualNS, toks), nil | |
| 112 } | |
| 113 | |
| 114 // WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey | |
| 115 // instead of this. | |
| 116 func WriteKeyTok(buf Buffer, tok KeyTok) (err error) { | |
| 117 // tok.kind ++ typ ++ [tok.stringID || tok.intID] | |
| 118 defer recoverTo(&err) | |
| 119 _, e := cmpbin.WriteString(buf, tok.Kind) | |
| 120 panicIf(e) | |
| 121 if tok.StringID != "" { | |
| 122 panicIf(buf.WriteByte(byte(PTString))) | |
| 123 _, e := cmpbin.WriteString(buf, tok.StringID) | |
| 124 panicIf(e) | |
| 125 } else { | |
| 126 panicIf(buf.WriteByte(byte(PTInt))) | |
| 127 _, e := cmpbin.WriteInt(buf, tok.IntID) | |
| 128 panicIf(e) | |
| 129 } | |
| 130 return nil | |
| 131 } | |
| 132 | |
| 133 // ReadKeyTok reads a KeyTok from the buffer. You usually want ReadKey | |
| 134 // instead of this. | |
| 135 func ReadKeyTok(buf Buffer) (ret KeyTok, err error) { | |
| 136 defer recoverTo(&err) | |
| 137 e := error(nil) | |
| 138 ret.Kind, _, e = cmpbin.ReadString(buf) | |
| 139 panicIf(e) | |
| 140 | |
| 141 typ, e := buf.ReadByte() | |
| 142 panicIf(e) | |
| 143 | |
| 144 switch PropertyType(typ) { | |
| 145 case PTString: | |
| 146 ret.StringID, _, err = cmpbin.ReadString(buf) | |
| 147 case PTInt: | |
| 148 ret.IntID, _, err = cmpbin.ReadInt(buf) | |
| 149 if err == nil && ret.IntID <= 0 { | |
| 150 err = errors.New("helper: decoded key with empty stringI
D and zero/negative intID") | |
| 151 } | |
| 152 default: | |
| 153 err = fmt.Errorf("helper: invalid type %s", PropertyType(typ)) | |
| 154 } | |
| 155 return | |
| 156 } | |
| 157 | |
| 158 // Write writes a GeoPoint to the buffer. | |
| 159 func (gp GeoPoint) Write(buf Buffer) (err error) { | |
| 160 defer recoverTo(&err) | |
| 161 _, e := cmpbin.WriteFloat64(buf, gp.Lat) | |
| 162 panicIf(e) | |
| 163 _, e = cmpbin.WriteFloat64(buf, gp.Lng) | |
| 164 return e | |
| 165 } | |
| 166 | |
| 167 // Read reads a GeoPoint from the buffer. | |
| 168 func (gp *GeoPoint) Read(buf Buffer) (err error) { | |
| 169 defer recoverTo(&err) | |
| 170 e := error(nil) | |
| 171 gp.Lat, _, e = cmpbin.ReadFloat64(buf) | |
| 172 panicIf(e) | |
| 173 | |
| 174 gp.Lng, _, e = cmpbin.ReadFloat64(buf) | |
| 175 panicIf(e) | |
| 176 | |
| 177 if !gp.Valid() { | |
| 178 err = fmt.Errorf("helper: decoded invalid GeoPoint: %v", gp) | |
| 179 } | |
| 180 return | |
| 181 } | |
| 182 | |
| 183 // WriteTime writes a time.Time in a byte-sortable way. | |
| 184 // | |
| 185 // This method truncates the time to microseconds and drops the timezone, | |
| 186 // because that's the (undocumented) way that the appengine SDK does it. | |
| 187 func WriteTime(buf Buffer, t time.Time) error { | |
| 188 name, off := t.Zone() | |
| 189 if name != "UTC" || off != 0 { | |
| 190 panic(fmt.Errorf("helper: UTC OR DEATH: %s", t)) | |
| 191 } | |
| 192 _, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond
()/1e3)) | |
| 193 return err | |
| 194 } | |
| 195 | |
| 196 // ReadTime reads a time.Time from the buffer. | |
| 197 func ReadTime(buf Buffer) (time.Time, error) { | |
| 198 v, _, err := cmpbin.ReadUint(buf) | |
| 199 if err != nil { | |
| 200 return time.Time{}, err | |
| 201 } | |
| 202 return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil | |
| 203 } | |
| 204 | |
| 205 // Write writes a Property to the buffer. `context` behaves the same | |
| 206 // way that it does for WriteKey, but only has an effect if `p` contains a | |
| 207 // Key as its Value. | |
| 208 func (p *Property) Write(buf Buffer, context KeyContext) (err error) { | |
| 209 defer recoverTo(&err) | |
| 210 typb := byte(p.Type()) | |
| 211 if p.IndexSetting() == NoIndex { | |
| 212 typb |= 0x80 | |
| 213 } | |
| 214 panicIf(buf.WriteByte(typb)) | |
| 215 switch p.Type() { | |
| 216 case PTNull, PTBoolTrue, PTBoolFalse: | |
| 217 case PTInt: | |
| 218 _, err = cmpbin.WriteInt(buf, p.Value().(int64)) | |
| 219 case PTFloat: | |
| 220 _, err = cmpbin.WriteFloat64(buf, p.Value().(float64)) | |
| 221 case PTString: | |
| 222 _, err = cmpbin.WriteString(buf, p.Value().(string)) | |
| 223 case PTBytes: | |
| 224 if p.IndexSetting() == NoIndex { | |
| 225 _, err = cmpbin.WriteBytes(buf, p.Value().([]byte)) | |
| 226 } else { | |
| 227 _, err = cmpbin.WriteBytes(buf, p.Value().(ByteString)) | |
| 228 } | |
| 229 case PTTime: | |
| 230 err = WriteTime(buf, p.Value().(time.Time)) | |
| 231 case PTGeoPoint: | |
| 232 err = p.Value().(GeoPoint).Write(buf) | |
| 233 case PTKey: | |
| 234 err = WriteKey(buf, context, p.Value().(Key)) | |
| 235 case PTBlobKey: | |
| 236 _, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key
))) | |
| 237 } | |
| 238 return | |
| 239 } | |
| 240 | |
| 241 // Read reads a Property from the buffer. `context`, `appid`, and | |
| 242 // `namespace` behave the same way they do for ReadKey, but only have an | |
| 243 // effect if the decoded property has a Key value. | |
| 244 func (p *Property) Read(buf Buffer, context KeyContext, appid, namespace string)
(err error) { | |
| 245 val := interface{}(nil) | |
| 246 typb, err := buf.ReadByte() | |
| 247 if err != nil { | |
| 248 return | |
| 249 } | |
| 250 is := ShouldIndex | |
| 251 if (typb & 0x80) != 0 { | |
| 252 is = NoIndex | |
| 253 } | |
| 254 switch PropertyType(typb & 0x7f) { | |
| 255 case PTNull: | |
| 256 case PTBoolTrue: | |
| 257 val = true | |
| 258 case PTBoolFalse: | |
| 259 val = false | |
| 260 case PTInt: | |
| 261 val, _, err = cmpbin.ReadInt(buf) | |
| 262 case PTFloat: | |
| 263 val, _, err = cmpbin.ReadFloat64(buf) | |
| 264 case PTString: | |
| 265 val, _, err = cmpbin.ReadString(buf) | |
| 266 case PTBytes: | |
| 267 b := []byte(nil) | |
| 268 if b, _, err = cmpbin.ReadBytes(buf); err != nil { | |
| 269 break | |
| 270 } | |
| 271 if is == NoIndex { | |
| 272 val = b | |
| 273 } else { | |
| 274 val = ByteString(b) | |
| 275 } | |
| 276 case PTTime: | |
| 277 val, err = ReadTime(buf) | |
| 278 case PTGeoPoint: | |
| 279 gp := GeoPoint{} | |
| 280 err = gp.Read(buf) | |
| 281 val = gp | |
| 282 case PTKey: | |
| 283 val, err = ReadKey(buf, context, appid, namespace) | |
| 284 case PTBlobKey: | |
| 285 s := "" | |
| 286 if s, _, err = cmpbin.ReadString(buf); err != nil { | |
| 287 break | |
| 288 } | |
| 289 val = blobstore.Key(s) | |
| 290 default: | |
| 291 err = fmt.Errorf("read: unknown type! %v", typb) | |
| 292 } | |
| 293 if err == nil { | |
| 294 err = p.SetValue(val, is) | |
| 295 } | |
| 296 return | |
| 297 } | |
| 298 | |
| 299 // Write writes an entire PropertyMap to the buffer. `context` behaves the same | |
| 300 // way that it does for WriteKey. If WritePropertyMapDeterministic is true, then | |
| 301 // the rows will be sorted by property name before they're serialized to buf | |
| 302 // (mostly useful for testing, but also potentially useful if you need to make | |
| 303 // a hash of the property data). | |
| 304 // | |
| 305 // Write skips metadata keys. | |
| 306 func (pm PropertyMap) Write(buf Buffer, context KeyContext) (err error) { | |
| 307 defer recoverTo(&err) | |
| 308 rows := make(sort.StringSlice, 0, len(pm)) | |
| 309 tmpBuf := &bytes.Buffer{} | |
| 310 for name, vals := range pm { | |
| 311 if isMetaKey(name) { | |
| 312 continue | |
| 313 } | |
| 314 tmpBuf.Reset() | |
| 315 _, e := cmpbin.WriteString(tmpBuf, name) | |
| 316 panicIf(e) | |
| 317 _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | |
| 318 panicIf(e) | |
| 319 for _, p := range vals { | |
| 320 panicIf(p.Write(tmpBuf, context)) | |
| 321 } | |
| 322 rows = append(rows, tmpBuf.String()) | |
| 323 } | |
| 324 | |
| 325 if WritePropertyMapDeterministic { | |
| 326 rows.Sort() | |
| 327 } | |
| 328 | |
| 329 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | |
| 330 panicIf(e) | |
| 331 for _, r := range rows { | |
| 332 _, e := buf.WriteString(r) | |
| 333 panicIf(e) | |
| 334 } | |
| 335 return | |
| 336 } | |
| 337 | |
| 338 // Read reads a PropertyMap from the buffer. `context` and | |
| 339 // friends behave the same way that they do for ReadKey. | |
| 340 func (pm PropertyMap) Read(buf Buffer, context KeyContext, appid, namespace stri
ng) (err error) { | |
| 341 defer recoverTo(&err) | |
| 342 | |
| 343 numRows := uint64(0) | |
| 344 numRows, _, e := cmpbin.ReadUint(buf) | |
| 345 panicIf(e) | |
| 346 if numRows > ReadPropertyMapReasonableLimit { | |
| 347 err = fmt.Errorf("helper: tried to decode map with huge number o
f rows %d", numRows) | |
| 348 return | |
| 349 } | |
| 350 | |
| 351 name, prop := "", Property{} | |
| 352 for i := uint64(0); i < numRows; i++ { | |
| 353 name, _, e = cmpbin.ReadString(buf) | |
| 354 panicIf(e) | |
| 355 | |
| 356 numProps, _, e := cmpbin.ReadUint(buf) | |
| 357 panicIf(e) | |
| 358 if numProps > ReadPropertyMapReasonableLimit { | |
| 359 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | |
| 360 return | |
| 361 } | |
| 362 props := make([]Property, 0, numProps) | |
| 363 for j := uint64(0); j < numProps; j++ { | |
| 364 panicIf(prop.Read(buf, context, appid, namespace)) | |
| 365 props = append(props, prop) | |
| 366 } | |
| 367 pm[name] = props | |
| 368 } | |
| 369 return | |
| 370 } | |
| 371 | |
| 372 func (c *IndexColumn) Write(buf Buffer) (err error) { | |
| 373 defer recoverTo(&err) | |
| 374 | |
| 375 if c.Direction == ASCENDING { | |
| 376 panicIf(buf.WriteByte(0)) | |
| 377 } else { | |
| 378 panicIf(buf.WriteByte(1)) | |
| 379 } | |
| 380 _, err = cmpbin.WriteString(buf, c.Property) | |
| 381 return | |
| 382 } | |
| 383 | |
| 384 func (c *IndexColumn) Read(buf Buffer) (err error) { | |
| 385 defer recoverTo(&err) | |
| 386 | |
| 387 dir, err := buf.ReadByte() | |
| 388 panicIf(err) | |
| 389 | |
| 390 switch dir { | |
| 391 case 0: | |
| 392 c.Direction = ASCENDING | |
| 393 default: | |
| 394 c.Direction = DESCENDING | |
| 395 } | |
| 396 c.Property, _, err = cmpbin.ReadString(buf) | |
| 397 return err | |
| 398 } | |
| 399 | |
| 400 func (i *IndexDefinition) Write(buf Buffer) (err error) { | |
| 401 defer recoverTo(&err) | |
| 402 | |
| 403 if i.Builtin() { | |
| 404 panicIf(buf.WriteByte(0)) | |
| 405 } else { | |
| 406 panicIf(buf.WriteByte(1)) | |
| 407 } | |
| 408 _, err = cmpbin.WriteString(buf, i.Kind) | |
| 409 panicIf(err) | |
| 410 if !i.Ancestor { | |
| 411 panicIf(buf.WriteByte(0)) | |
| 412 } else { | |
| 413 panicIf(buf.WriteByte(1)) | |
| 414 } | |
| 415 _, err = cmpbin.WriteUint(buf, uint64(len(i.SortBy))) | |
| 416 panicIf(err) | |
| 417 for _, sb := range i.SortBy { | |
| 418 panicIf(sb.Write(buf)) | |
| 419 } | |
| 420 return | |
| 421 } | |
| 422 | |
| 423 func (i *IndexDefinition) Read(buf Buffer) (err error) { | |
| 424 defer recoverTo(&err) | |
| 425 | |
| 426 // discard builtin/complex byte | |
| 427 _, err = buf.ReadByte() | |
| 428 panicIf(err) | |
| 429 | |
| 430 i.Kind, _, err = cmpbin.ReadString(buf) | |
| 431 panicIf(err) | |
| 432 | |
| 433 anc, err := buf.ReadByte() | |
| 434 panicIf(err) | |
| 435 | |
| 436 i.Ancestor = anc == 1 | |
| 437 | |
| 438 numSorts, _, err := cmpbin.ReadUint(buf) | |
| 439 panicIf(err) | |
| 440 | |
| 441 if numSorts > MaxIndexColumns { | |
| 442 return fmt.Errorf("datastore: Got over %d sort orders: %d", | |
| 443 MaxIndexColumns, numSorts) | |
| 444 } | |
| 445 | |
| 446 if numSorts > 0 { | |
| 447 i.SortBy = make([]IndexColumn, numSorts) | |
| 448 for idx := range i.SortBy { | |
| 449 panicIf(i.SortBy[idx].Read(buf)) | |
| 450 } | |
| 451 } | |
| 452 | |
| 453 return | |
| 454 } | |
| 455 | |
| 456 type parseError error | |
| 457 | |
| 458 func panicIf(err error) { | |
| 459 if err != nil { | |
| 460 panic(parseError(err)) | |
| 461 } | |
| 462 } | |
| 463 | |
| 464 func recoverTo(err *error) { | |
| 465 if r := recover(); r != nil { | |
| 466 if rerr := r.(parseError); rerr != nil { | |
| 467 *err = error(rerr) | |
| 468 } | |
| 469 } | |
| 470 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1292913002:
Split off serialization and key functions to their own packages. (Closed)
Base URL: https://github.com/luci/gae.git@make_queries_better