GitHub #8: Seed indexes from index.yml

impl/memory/datastore.go

 // Copyright 2015 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 package memory
 
 import (
         "errors"
         "fmt"
+        "io"
+        "io/ioutil"
+        "os"
+        "path/filepath"
+        "runtime"
 
         "golang.org/x/net/context"
 
+        "gopkg.in/yaml.v2"
+
         ds "github.com/luci/gae/service/datastore"
         "github.com/luci/gae/service/info"
 )
 
+//////////////////////////////////// utility ///////////////////////////////////
+
+// fileExists returns true if the file f exists
+func fileExists(f string) bool {
+        if _, err := os.Stat(f); os.IsNotExist(err) {
+                return false
+        }
+        return true
+}
+
 //////////////////////////////////// public ////////////////////////////////////
 
 // useRDS adds a gae.Datastore implementation to context, accessible
 // by gae.GetDS(c)
 func useRDS(c context.Context) context.Context {
         return ds.SetRawFactory(c, func(ic context.Context, wantTxn bool) ds.RawInterface {
                 ns := curGID(ic).namespace
                 maybeTxnCtx := cur(ic)
 
                 needResetCtx := false
@@ skipping 131 matching lines @@
 }
 
 func (d *dsImpl) DisableSpecialEntities(enabled bool) {
         d.data.setDisableSpecialEntities(enabled)
 }
 
 func (d *dsImpl) Testable() ds.Testable {
         return d
 }
 
+func (d *dsImpl) ParseIndexYAML(content io.Reader) []*ds.IndexDefinition {
+        var m map[string][]*ds.IndexDefinition
+        serialized, err := ioutil.ReadAll(content)
+
+        if err != nil {
+                panic(fmt.Errorf("failed to parse index YAML: %v", content))
+        }
+
+        if err := yaml.Unmarshal(serialized, &m); err != nil {
+                panic(fmt.Errorf("failed to parse index YAML: %s", serialized))
+        }
+
+        if _, ok := m["indexes"]; !ok {
+                panic(fmt.Errorf("failed to parse index YAML: %s", serialized))
+        }
+
+        return m["indexes"]
+}
+
+func (d *dsImpl) FindAndAddIndexYAML() {
+        // get path of current source file
+        _, f, _, _ := runtime.Caller(1)
+        dir, err := filepath.Abs(filepath.Dir(f))
+
+        if err != nil {
+                panic(fmt.Errorf("failed to find index YAML file"))

[iannucci, 2016/01/13 19:12:20]

this error should be changed to be more meaningful ("failed to find source for
test")

In fact, the Caller(1) should probably be Caller(2)... We want to search from
the location of the /test/ file that called FindAndAddIndexYAML. If you want to
be fancy, you could walk up the call stack until you found the first file whose
name ends with _test.go.

[nishanths (utexas), 2016/01/14 21:12:56]

Done. Also implemented "look until you find  _test.go". As for the argument to
runtime.Caller, if I'm not mistaken, according to the docs, 0 identifies the
caller of runtime.Caller. https://golang.org/pkg/runtime/#Caller. So 1 would be
anything higher up. The latest commit starts the loop in the
`getCallingTestFilePath` function at 1, so that it's easier if/when
`getCallingTestFilePath` is exported in the future. 

+        }
+
+        for {
+                var filename string
+
+                if fileExists(filepath.Join(dir, "index.yml")) {

[dnj, 2016/01/13 16:16:40]

nit: Consider making this loop over a slice of candidate names:
for _, name := range []string{"index.yml", "index.yaml"}{
{

}

You also don't need "fileExists", since "os.Open" naturally tests for existence.
Just try to open each candidate file until one succeeds.

[iannucci, 2016/01/13 19:12:20]

Moreover, doing os.Open directly avoids an (admittedly unlikely) race condition
where the file could exist for the `fileExists` check, but not for the os.Open
call.

[nishanths (utexas), 2016/01/14 21:12:56]

Thanks for the tip!

[nishanths (utexas), 2016/01/14 21:12:56]

Done.

+                        filename = filepath.Join(dir, "index.yml")
+                } else if fileExists(filepath.Join(dir, "index.yaml")) {
+                        filename = filepath.Join(dir, "index.yaml")
+                }
+
+                if filename != "" {
+                        // found index file
+                        // read contents
+                        file, err := os.Open(filename)
+
+                        if err != nil {
+                                panic(fmt.Errorf("failed to open index YAML file: %s", filename))
+                        }
+
+                        defer file.Close()
+
+                        // parse the bytes that were read
+                        idxs := d.ParseIndexYAML(file)
+                        d.AddIndexes(idxs...)
+                        break
+                }
+
+                // already at root of drive
+                if dir == "/" && filepath.Dir(dir) == "/" {

[dnj, 2016/01/13 16:16:40]

This is not a cross-platform check, as on Windows "Dir" will terminate with
"X:\". You need to check if "dir" ends with os.PathSeparator.

[nishanths (utexas), 2016/01/14 21:12:56]

Done.

+                        panic(fmt.Errorf("failed to find index YAML file"))
+                }
+
+                // to parent directory
+                dir = filepath.Dir(dir)
+        }
+}
+
 ////////////////////////////////// txnDsImpl ///////////////////////////////////
 
 type txnDsImpl struct {
         data *txnDataStoreData
         ns   string
 }
 
 var _ ds.RawInterface = (*txnDsImpl)(nil)
 
 func (d *txnDsImpl) AllocateIDs(incomplete *ds.Key, n int) (int64, error) {
@@ skipping 40 matching lines @@
         return countQuery(fq, d.data.parent.aid, d.ns, true, d.data.snap, d.data.snap)
 }
 
 func (*txnDsImpl) RunInTransaction(func(c context.Context) error, *ds.TransactionOptions) error {
         return errors.New("datastore: nested transactions are not supported")
 }
 
 func (*txnDsImpl) Testable() ds.Testable {
         return nil
 }