GitHub #8: Seed indexes from index.yml

impl/memory/datastore.go

Index: impl/memory/datastore.go
diff --git a/impl/memory/datastore.go b/impl/memory/datastore.go
index 4ebfac0d4a647b1a7933613b4833006021a337a0..616ee575c4891c03e0dd3445f503ecdc9f78a298 100644
--- a/impl/memory/datastore.go
+++ b/impl/memory/datastore.go
@@ -7,13 +7,30 @@ package memory
 import (
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
 
 	"golang.org/x/net/context"
 
+	"gopkg.in/yaml.v2"
+
 	ds "github.com/luci/gae/service/datastore"
 	"github.com/luci/gae/service/info"
 )
 
+//////////////////////////////////// utility ///////////////////////////////////
+
+// fileExists returns true if the file f exists
+func fileExists(f string) bool {
+	if _, err := os.Stat(f); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
 //////////////////////////////////// public ////////////////////////////////////
 
 // useRDS adds a gae.Datastore implementation to context, accessible
@@ -165,6 +182,70 @@ func (d *dsImpl) Testable() ds.Testable {
 	return d
 }
 
+func (d *dsImpl) ParseIndexYAML(content io.Reader) []*ds.IndexDefinition {
+	var m map[string][]*ds.IndexDefinition
+	serialized, err := ioutil.ReadAll(content)
+
+	if err != nil {
+		panic(fmt.Errorf("failed to parse index YAML: %v", content))
+	}
+
+	if err := yaml.Unmarshal(serialized, &m); err != nil {
+		panic(fmt.Errorf("failed to parse index YAML: %s", serialized))
+	}
+
+	if _, ok := m["indexes"]; !ok {
+		panic(fmt.Errorf("failed to parse index YAML: %s", serialized))
+	}
+
+	return m["indexes"]
+}
+
+func (d *dsImpl) FindAndAddIndexYAML() {
+	// get path of current source file
+	_, f, _, _ := runtime.Caller(1)
+	dir, err := filepath.Abs(filepath.Dir(f))
+
+	if err != nil {
+		panic(fmt.Errorf("failed to find index YAML file"))

[iannucci, 2016/01/13 19:12:20]

this error should be changed to be more meaningful ("failed to find source for
test")

In fact, the Caller(1) should probably be Caller(2)... We want to search from
the location of the /test/ file that called FindAndAddIndexYAML. If you want to
be fancy, you could walk up the call stack until you found the first file whose
name ends with _test.go.

[nishanths (utexas), 2016/01/14 21:12:56]

Done. Also implemented "look until you find  _test.go". As for the argument to
runtime.Caller, if I'm not mistaken, according to the docs, 0 identifies the
caller of runtime.Caller. https://golang.org/pkg/runtime/#Caller. So 1 would be
anything higher up. The latest commit starts the loop in the
`getCallingTestFilePath` function at 1, so that it's easier if/when
`getCallingTestFilePath` is exported in the future. 

+	}
+
+	for {
+		var filename string
+
+		if fileExists(filepath.Join(dir, "index.yml")) {

[dnj, 2016/01/13 16:16:40]

nit: Consider making this loop over a slice of candidate names:
for _, name := range []string{"index.yml", "index.yaml"}{
{

}

You also don't need "fileExists", since "os.Open" naturally tests for existence.
Just try to open each candidate file until one succeeds.

[iannucci, 2016/01/13 19:12:20]

Moreover, doing os.Open directly avoids an (admittedly unlikely) race condition
where the file could exist for the `fileExists` check, but not for the os.Open
call.

[nishanths (utexas), 2016/01/14 21:12:56]

Thanks for the tip!

[nishanths (utexas), 2016/01/14 21:12:56]

Done.

+			filename = filepath.Join(dir, "index.yml")
+		} else if fileExists(filepath.Join(dir, "index.yaml")) {
+			filename = filepath.Join(dir, "index.yaml")
+		}
+
+		if filename != "" {
+			// found index file
+			// read contents
+			file, err := os.Open(filename)
+
+			if err != nil {
+				panic(fmt.Errorf("failed to open index YAML file: %s", filename))
+			}
+
+			defer file.Close()
+
+			// parse the bytes that were read
+			idxs := d.ParseIndexYAML(file)
+			d.AddIndexes(idxs...)
+			break
+		}
+
+		// already at root of drive
+		if dir == "/" && filepath.Dir(dir) == "/" {

[dnj, 2016/01/13 16:16:40]

This is not a cross-platform check, as on Windows "Dir" will terminate with
"X:\". You need to check if "dir" ends with os.PathSeparator.

[nishanths (utexas), 2016/01/14 21:12:56]

Done.

+			panic(fmt.Errorf("failed to find index YAML file"))
+		}
+
+		// to parent directory
+		dir = filepath.Dir(dir)
+	}
+}
+
 ////////////////////////////////// txnDsImpl ///////////////////////////////////
 
 type txnDsImpl struct {