| OLD | NEW |
|---|
| (Empty) |
| 1 // Copyright 2015 The Chromium Authors. All rights reserved. | |
| 2 // Use of this source code is governed by a BSD-style license that can be | |
| 3 // found in the LICENSE file. | |
| 4 | |
| 5 package rawdatastore | |
| 6 | |
| 7 import ( | |
| 8 "bytes" | |
| 9 "errors" | |
| 10 "fmt" | |
| 11 "sort" | |
| 12 "time" | |
| 13 | |
| 14 "github.com/luci/gae/service/blobstore" | |
| 15 "github.com/luci/luci-go/common/cmpbin" | |
| 16 ) | |
| 17 | |
| 18 // WritePropertyMapDeterministic allows tests to make WritePropertyMap | |
| 19 // deterministic. | |
| 20 var WritePropertyMapDeterministic = false | |
| 21 | |
| 22 // ReadPropertyMapReasonableLimit sets a limit on the number of rows and | |
| 23 // number of properties per row which can be read by ReadPropertyMap. The | |
| 24 // total number of Property objects readable by this method is this number | |
| 25 // squared (e.g. Limit rows * Limit properties) | |
| 26 const ReadPropertyMapReasonableLimit uint64 = 30000 | |
| 27 | |
| 28 // ReadKeyNumToksReasonableLimit is the maximum number of Key tokens that | |
| 29 // ReadKey is willing to read for a single key. | |
| 30 const ReadKeyNumToksReasonableLimit uint64 = 50 | |
| 31 | |
| 32 // KeyContext controls whether the various Write and Read serializtion | |
| 33 // routines should encode the context of Keys (read: the appid and namespace). | |
| 34 // Frequently the appid and namespace of keys are known in advance and so there'
s | |
| 35 // no reason to redundantly encode them. | |
| 36 type KeyContext bool | |
| 37 | |
| 38 // With- and WithoutContext indicate if the serialization method should include | |
| 39 // context for Keys. See KeyContext for more information. | |
| 40 const ( | |
| 41 WithContext KeyContext = true | |
| 42 WithoutContext = false | |
| 43 ) | |
| 44 | |
| 45 // WriteKey encodes a key to the buffer. If context is WithContext, then this | |
| 46 // encoded value will include the appid and namespace of the key. | |
| 47 func WriteKey(buf Buffer, context KeyContext, k Key) (err error) { | |
| 48 // [appid ++ namespace]? ++ #tokens ++ tokens* | |
| 49 defer recoverTo(&err) | |
| 50 appid, namespace, toks := KeySplit(k) | |
| 51 if context == WithContext { | |
| 52 panicIf(buf.WriteByte(1)) | |
| 53 _, e := cmpbin.WriteString(buf, appid) | |
| 54 panicIf(e) | |
| 55 _, e = cmpbin.WriteString(buf, namespace) | |
| 56 panicIf(e) | |
| 57 } else { | |
| 58 panicIf(buf.WriteByte(0)) | |
| 59 } | |
| 60 _, e := cmpbin.WriteUint(buf, uint64(len(toks))) | |
| 61 panicIf(e) | |
| 62 for _, tok := range toks { | |
| 63 panicIf(WriteKeyTok(buf, tok)) | |
| 64 } | |
| 65 return nil | |
| 66 } | |
| 67 | |
| 68 // ReadKey deserializes a key from the buffer. The value of context must match | |
| 69 // the value of context that was passed to WriteKey when the key was encoded. | |
| 70 // If context == WithoutContext, then the appid and namespace parameters are | |
| 71 // used in the decoded Key. Otherwise they're ignored. | |
| 72 func ReadKey(buf Buffer, context KeyContext, appid, namespace string) (ret Key,
err error) { | |
| 73 defer recoverTo(&err) | |
| 74 actualCtx, e := buf.ReadByte() | |
| 75 panicIf(e) | |
| 76 | |
| 77 actualAid, actualNS := "", "" | |
| 78 if actualCtx == 1 { | |
| 79 actualAid, _, e = cmpbin.ReadString(buf) | |
| 80 panicIf(e) | |
| 81 actualNS, _, e = cmpbin.ReadString(buf) | |
| 82 panicIf(e) | |
| 83 } else if actualCtx != 0 { | |
| 84 err = fmt.Errorf("helper: expected actualCtx to be 0 or 1, got %
d", actualCtx) | |
| 85 return | |
| 86 } | |
| 87 | |
| 88 if context == WithoutContext { | |
| 89 // overrwrite with the supplied ones | |
| 90 actualAid = appid | |
| 91 actualNS = namespace | |
| 92 } | |
| 93 | |
| 94 numToks, _, e := cmpbin.ReadUint(buf) | |
| 95 panicIf(e) | |
| 96 if numToks > ReadKeyNumToksReasonableLimit { | |
| 97 err = fmt.Errorf("helper: tried to decode huge key of length %d"
, numToks) | |
| 98 return | |
| 99 } | |
| 100 | |
| 101 toks := make([]KeyTok, numToks) | |
| 102 for i := uint64(0); i < numToks; i++ { | |
| 103 toks[i], e = ReadKeyTok(buf) | |
| 104 panicIf(e) | |
| 105 } | |
| 106 | |
| 107 return NewKeyToks(actualAid, actualNS, toks), nil | |
| 108 } | |
| 109 | |
| 110 // WriteKeyTok writes a KeyTok to the buffer. You usually want WriteKey | |
| 111 // instead of this. | |
| 112 func WriteKeyTok(buf Buffer, tok KeyTok) (err error) { | |
| 113 // tok.kind ++ typ ++ [tok.stringID || tok.intID] | |
| 114 defer recoverTo(&err) | |
| 115 _, e := cmpbin.WriteString(buf, tok.Kind) | |
| 116 panicIf(e) | |
| 117 if tok.StringID != "" { | |
| 118 panicIf(buf.WriteByte(byte(PTString))) | |
| 119 _, e := cmpbin.WriteString(buf, tok.StringID) | |
| 120 panicIf(e) | |
| 121 } else { | |
| 122 panicIf(buf.WriteByte(byte(PTInt))) | |
| 123 _, e := cmpbin.WriteInt(buf, tok.IntID) | |
| 124 panicIf(e) | |
| 125 } | |
| 126 return nil | |
| 127 } | |
| 128 | |
| 129 // ReadKeyTok reads a KeyTok from the buffer. You usually want ReadKey | |
| 130 // instead of this. | |
| 131 func ReadKeyTok(buf Buffer) (ret KeyTok, err error) { | |
| 132 defer recoverTo(&err) | |
| 133 e := error(nil) | |
| 134 ret.Kind, _, e = cmpbin.ReadString(buf) | |
| 135 panicIf(e) | |
| 136 | |
| 137 typ, e := buf.ReadByte() | |
| 138 panicIf(e) | |
| 139 | |
| 140 switch PropertyType(typ) { | |
| 141 case PTString: | |
| 142 ret.StringID, _, err = cmpbin.ReadString(buf) | |
| 143 case PTInt: | |
| 144 ret.IntID, _, err = cmpbin.ReadInt(buf) | |
| 145 if err == nil && ret.IntID <= 0 { | |
| 146 err = errors.New("helper: decoded key with empty stringI
D and zero/negative intID") | |
| 147 } | |
| 148 default: | |
| 149 err = fmt.Errorf("helper: invalid type %s", PropertyType(typ)) | |
| 150 } | |
| 151 return | |
| 152 } | |
| 153 | |
| 154 // Write writes a GeoPoint to the buffer. | |
| 155 func (gp GeoPoint) Write(buf Buffer) (err error) { | |
| 156 defer recoverTo(&err) | |
| 157 _, e := cmpbin.WriteFloat64(buf, gp.Lat) | |
| 158 panicIf(e) | |
| 159 _, e = cmpbin.WriteFloat64(buf, gp.Lng) | |
| 160 return e | |
| 161 } | |
| 162 | |
| 163 // Read reads a GeoPoint from the buffer. | |
| 164 func (gp *GeoPoint) Read(buf Buffer) (err error) { | |
| 165 defer recoverTo(&err) | |
| 166 e := error(nil) | |
| 167 gp.Lat, _, e = cmpbin.ReadFloat64(buf) | |
| 168 panicIf(e) | |
| 169 | |
| 170 gp.Lng, _, e = cmpbin.ReadFloat64(buf) | |
| 171 panicIf(e) | |
| 172 | |
| 173 if !gp.Valid() { | |
| 174 err = fmt.Errorf("helper: decoded invalid GeoPoint: %v", gp) | |
| 175 } | |
| 176 return | |
| 177 } | |
| 178 | |
| 179 // WriteTime writes a time.Time in a byte-sortable way. | |
| 180 // | |
| 181 // This method truncates the time to microseconds and drops the timezone, | |
| 182 // because that's the (undocumented) way that the appengine SDK does it. | |
| 183 func WriteTime(buf Buffer, t time.Time) error { | |
| 184 name, off := t.Zone() | |
| 185 if name != "UTC" || off != 0 { | |
| 186 panic(fmt.Errorf("helper: UTC OR DEATH: %s", t)) | |
| 187 } | |
| 188 _, err := cmpbin.WriteUint(buf, uint64(t.Unix())*1e6+uint64(t.Nanosecond
()/1e3)) | |
| 189 return err | |
| 190 } | |
| 191 | |
| 192 // ReadTime reads a time.Time from the buffer. | |
| 193 func ReadTime(buf Buffer) (time.Time, error) { | |
| 194 v, _, err := cmpbin.ReadUint(buf) | |
| 195 if err != nil { | |
| 196 return time.Time{}, err | |
| 197 } | |
| 198 return time.Unix(int64(v/1e6), int64((v%1e6)*1e3)).UTC(), nil | |
| 199 } | |
| 200 | |
| 201 // Write writes a Property to the buffer. `context` behaves the same | |
| 202 // way that it does for WriteKey, but only has an effect if `p` contains a | |
| 203 // Key as its Value. | |
| 204 func (p *Property) Write(buf Buffer, context KeyContext) (err error) { | |
| 205 defer recoverTo(&err) | |
| 206 typb := byte(p.Type()) | |
| 207 if p.IndexSetting() == NoIndex { | |
| 208 typb |= 0x80 | |
| 209 } | |
| 210 panicIf(buf.WriteByte(typb)) | |
| 211 switch p.Type() { | |
| 212 case PTNull, PTBoolTrue, PTBoolFalse: | |
| 213 case PTInt: | |
| 214 _, err = cmpbin.WriteInt(buf, p.Value().(int64)) | |
| 215 case PTFloat: | |
| 216 _, err = cmpbin.WriteFloat64(buf, p.Value().(float64)) | |
| 217 case PTString: | |
| 218 _, err = cmpbin.WriteString(buf, p.Value().(string)) | |
| 219 case PTBytes: | |
| 220 if p.IndexSetting() == NoIndex { | |
| 221 _, err = cmpbin.WriteBytes(buf, p.Value().([]byte)) | |
| 222 } else { | |
| 223 _, err = cmpbin.WriteBytes(buf, p.Value().(ByteString)) | |
| 224 } | |
| 225 case PTTime: | |
| 226 err = WriteTime(buf, p.Value().(time.Time)) | |
| 227 case PTGeoPoint: | |
| 228 err = p.Value().(GeoPoint).Write(buf) | |
| 229 case PTKey: | |
| 230 err = WriteKey(buf, context, p.Value().(Key)) | |
| 231 case PTBlobKey: | |
| 232 _, err = cmpbin.WriteString(buf, string(p.Value().(blobstore.Key
))) | |
| 233 } | |
| 234 return | |
| 235 } | |
| 236 | |
| 237 // Read reads a Property from the buffer. `context`, `appid`, and | |
| 238 // `namespace` behave the same way they do for ReadKey, but only have an | |
| 239 // effect if the decoded property has a Key value. | |
| 240 func (p *Property) Read(buf Buffer, context KeyContext, appid, namespace string)
(err error) { | |
| 241 val := interface{}(nil) | |
| 242 typb, err := buf.ReadByte() | |
| 243 if err != nil { | |
| 244 return | |
| 245 } | |
| 246 is := ShouldIndex | |
| 247 if (typb & 0x80) != 0 { | |
| 248 is = NoIndex | |
| 249 } | |
| 250 switch PropertyType(typb & 0x7f) { | |
| 251 case PTNull: | |
| 252 case PTBoolTrue: | |
| 253 val = true | |
| 254 case PTBoolFalse: | |
| 255 val = false | |
| 256 case PTInt: | |
| 257 val, _, err = cmpbin.ReadInt(buf) | |
| 258 case PTFloat: | |
| 259 val, _, err = cmpbin.ReadFloat64(buf) | |
| 260 case PTString: | |
| 261 val, _, err = cmpbin.ReadString(buf) | |
| 262 case PTBytes: | |
| 263 b := []byte(nil) | |
| 264 if b, _, err = cmpbin.ReadBytes(buf); err != nil { | |
| 265 break | |
| 266 } | |
| 267 if is == NoIndex { | |
| 268 val = b | |
| 269 } else { | |
| 270 val = ByteString(b) | |
| 271 } | |
| 272 case PTTime: | |
| 273 val, err = ReadTime(buf) | |
| 274 case PTGeoPoint: | |
| 275 gp := GeoPoint{} | |
| 276 err = gp.Read(buf) | |
| 277 val = gp | |
| 278 case PTKey: | |
| 279 val, err = ReadKey(buf, context, appid, namespace) | |
| 280 case PTBlobKey: | |
| 281 s := "" | |
| 282 if s, _, err = cmpbin.ReadString(buf); err != nil { | |
| 283 break | |
| 284 } | |
| 285 val = blobstore.Key(s) | |
| 286 default: | |
| 287 err = fmt.Errorf("read: unknown type! %v", typb) | |
| 288 } | |
| 289 if err == nil { | |
| 290 err = p.SetValue(val, is) | |
| 291 } | |
| 292 return | |
| 293 } | |
| 294 | |
| 295 // Write writes an entire PropertyMap to the buffer. `context` | |
| 296 // behaves the same way that it does for WriteKey. If | |
| 297 // WritePropertyMapDeterministic is true, then the rows will be sorted by | |
| 298 // property name before they're serialized to buf (mostly useful for testing, | |
| 299 // but also potentially useful if you need to make a hash of the property data). | |
| 300 func (pm PropertyMap) Write(buf Buffer, context KeyContext) (err error) { | |
| 301 defer recoverTo(&err) | |
| 302 rows := make(sort.StringSlice, 0, len(pm)) | |
| 303 tmpBuf := &bytes.Buffer{} | |
| 304 for name, vals := range pm { | |
| 305 tmpBuf.Reset() | |
| 306 _, e := cmpbin.WriteString(tmpBuf, name) | |
| 307 panicIf(e) | |
| 308 _, e = cmpbin.WriteUint(tmpBuf, uint64(len(vals))) | |
| 309 panicIf(e) | |
| 310 for _, p := range vals { | |
| 311 panicIf(p.Write(tmpBuf, context)) | |
| 312 } | |
| 313 rows = append(rows, tmpBuf.String()) | |
| 314 } | |
| 315 | |
| 316 if WritePropertyMapDeterministic { | |
| 317 rows.Sort() | |
| 318 } | |
| 319 | |
| 320 _, e := cmpbin.WriteUint(buf, uint64(len(pm))) | |
| 321 panicIf(e) | |
| 322 for _, r := range rows { | |
| 323 _, e := buf.WriteString(r) | |
| 324 panicIf(e) | |
| 325 } | |
| 326 return | |
| 327 } | |
| 328 | |
| 329 // Read reads a PropertyMap from the buffer. `context` and | |
| 330 // friends behave the same way that they do for ReadKey. | |
| 331 func (pm PropertyMap) Read(buf Buffer, context KeyContext, appid, namespace stri
ng) (err error) { | |
| 332 defer recoverTo(&err) | |
| 333 | |
| 334 numRows := uint64(0) | |
| 335 numRows, _, e := cmpbin.ReadUint(buf) | |
| 336 panicIf(e) | |
| 337 if numRows > ReadPropertyMapReasonableLimit { | |
| 338 err = fmt.Errorf("helper: tried to decode map with huge number o
f rows %d", numRows) | |
| 339 return | |
| 340 } | |
| 341 | |
| 342 name, prop := "", Property{} | |
| 343 for i := uint64(0); i < numRows; i++ { | |
| 344 name, _, e = cmpbin.ReadString(buf) | |
| 345 panicIf(e) | |
| 346 | |
| 347 numProps, _, e := cmpbin.ReadUint(buf) | |
| 348 panicIf(e) | |
| 349 if numProps > ReadPropertyMapReasonableLimit { | |
| 350 err = fmt.Errorf("helper: tried to decode map with huge
number of properties %d", numProps) | |
| 351 return | |
| 352 } | |
| 353 props := make([]Property, 0, numProps) | |
| 354 for j := uint64(0); j < numProps; j++ { | |
| 355 panicIf(prop.Read(buf, context, appid, namespace)) | |
| 356 props = append(props, prop) | |
| 357 } | |
| 358 pm[name] = props | |
| 359 } | |
| 360 return | |
| 361 } | |
| 362 | |
| 363 type parseError error | |
| 364 | |
| 365 func panicIf(err error) { | |
| 366 if err != nil { | |
| 367 panic(parseError(err)) | |
| 368 } | |
| 369 } | |
| 370 | |
| 371 func recoverTo(err *error) { | |
| 372 if r := recover(); r != nil { | |
| 373 if rerr := r.(parseError); rerr != nil { | |
| 374 *err = error(rerr) | |
| 375 } | |
| 376 } | |
| 377 } | |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 1259593005:
Add 'user friendly' datastore API. (Closed)
Base URL: https://github.com/luci/gae.git@master