Extensions Docs Server: Apps samples page

chrome/common/extensions/docs/server2/github_file_system.py

+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import json
+import os
+
+import appengine_memcache as memcache
+import file_system
+from io import BytesIO
+from future import Future
+from zipfile import ZipFile
+
+ZIP_KEY = 'zipball'
+

[not at google - send to devlin, 2012/08/09 07:52:36]

This is something you can do in a follow-up, but...

Downloading this stuff takes a little while, right? Like in the order of a few
seconds. Something we could do is put the downloading in a background thread,
and use the old version until it's finished.

It'd need to block if there's no old version (and we could use a warm-up request
to do that).

We'd also need to make sure not to spawn multiple gets etc etc. So yeah,
something to do in a follow-up.

[cduvall, 2012/08/09 19:20:08]

Cool, made a bug: http://crbug.com/141674

+class _AsyncFetchFutureZip(object):
+  def __init__(self, fetcher, blobstore, new_version, old_version):
+    self._fetch = fetcher.FetchAsync(ZIP_KEY)
+    self._blobstore = blobstore
+    self._new_version = new_version
+    self._old_version = old_version
+
+  def Get(self):
+    self._blobstore.Delete(ZIP_KEY, self._old_version)
+    blob = self._fetch.Get().content
+    self._blobstore.Set(ZIP_KEY, blob, self._new_version)

[not at google - send to devlin, 2012/08/09 07:52:36]

Yeah, so as I said, I don't see the difference between this and just replacing
the old version with the new.

[cduvall, 2012/08/09 19:20:08]

Ah, I just remembered why I versioned the zips. For the instances of the server
to share the zip file in the blobstore, they need to know what version it is. If
it is not keyed with a version, then instances have no way of knowing if the zip
stored in the blobstore is the old version or the new version.

+    return ZipFile(BytesIO(blob))
+
+class GithubFileSystem(file_system.FileSystem):
+  """FileSystem implementation which fetches resources from github.
+  """
+  def __init__(self, fetcher, memcache, blobstore):
+    self._fetcher = fetcher
+    self._memcache = memcache
+    self._blobstore = blobstore
+    self._version = self.Stat(ZIP_KEY).version
+    self._GetZip(self._version)
+
+  def _GetZip(self, version):
+    blob = self._blobstore.Get(ZIP_KEY, version)
+    if blob is not None:
+      self._zip_file = Future(value=ZipFile(BytesIO(blob)))
+    else:
+      self._zip_file = Future(delegate=_AsyncFetchFutureZip(self._fetcher,
+                                                            self._blobstore,
+                                                            version,
+                                                            self._version))
+    self._version = version
+
+  def _ReadFile(self, path):
+    zip_file = self._zip_file.Get()
+    prefix = zip_file.namelist()[0][:-1]
+    return zip_file.read(prefix + path)
+
+  def _ListDir(self, path):
+    filenames = self._zip_file.Get().namelist()
+    # Take out parent directory name (GoogleChrome-chrome-app-samples-c78a30f)
+    filenames = [f[len(filenames[0]) - 1:] for f in filenames]
+    # Remove the path of the directory we're listing from the filenames.
+    filenames = [f[len(path):] for f in filenames
+                 if f != path and f.startswith(path)]
+    # Remove all files not directly in this directory.
+    return [f for f in filenames if f[:-1].count('/') == 0]
+
+  def Read(self, paths, binary=False):
+    version = self.Stat(ZIP_KEY).version
+    if version != self._version:
+      self._GetZip(version)
+    result = {}
+    for path in paths:
+      if path.endswith('/'):
+        result[path] = self._ListDir(path)
+      else:
+        result[path] = self._ReadFile(path)
+    return Future(value=result)
+
+  def Stat(self, path):
+    version = self._memcache.Get(path, memcache.MEMCACHE_GITHUB_STAT)
+    if version is not None:
+      return self.StatInfo(version)
+    version = json.loads(
+        self._fetcher.Fetch('commits/HEAD').content)['commit']['tree']['sha']
+    self._memcache.Set(path, version, memcache.MEMCACHE_GITHUB_STAT)
+    return self.StatInfo(version)