serve api docs from cloud storage

utils/apidoc/app_engine/scripts/redirector.py

Index: utils/apidoc/app_engine/scripts/redirector.py
diff --git a/utils/apidoc/app_engine/scripts/redirector.py b/utils/apidoc/app_engine/scripts/redirector.py
index d1260901b0d53f6a58468cdd3cacf6262df6650b..d4b7cad926ea3879188a763816932e4faaeb98c3 100644
--- a/utils/apidoc/app_engine/scripts/redirector.py
+++ b/utils/apidoc/app_engine/scripts/redirector.py
@@ -2,18 +2,124 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
-import webapp2
+from __future__ import with_statement

[nweiz, 2012/07/25 18:50:49]

I think Python 2.7 supports with with statement natively.

[sethladd, 2012/07/25 20:48:45]

Done.

+import logging
+import re
+from webapp2 import *
+from datetime import datetime, timedelta
+from google.appengine.ext import blobstore
+from google.appengine.ext.webapp import blobstore_handlers
+from google.appengine.api import files
+from google.appengine.api import memcache

[nweiz, 2012/07/25 18:50:49]

from google.appengine.api import files, memcache

[sethladd, 2012/07/25 20:48:45]

Done.

+
+LATEST_DOC_VERSION = 0
+NEXT_DOC_VERSION_CHECK = None

[nweiz, 2012/07/25 18:50:49]

I believe only constants should be written in all caps. Since these change over
time, they should probably be class variables.

[sethladd, 2012/07/25 20:48:45]

Good call. Done.

+LATEST_DOC_VERSION_FILE = '/gs/dartlang-api-docs/latest.txt'
+ONE_HOUR = 60 * 60
+ONE_DAY = ONE_HOUR * 24
+ONE_WEEK = ONE_DAY * 7
+
+class ApiDocs(blobstore_handlers.BlobstoreDownloadHandler):
+
+  def load_latest_version(self):

[nweiz, 2012/07/25 18:50:49]

Calling this reload_latest_version would make it clearer what the difference is
between it and get_latest_version.

[sethladd, 2012/07/25 20:48:45]

I like it. Done.

+    global LATEST_DOC_VERSION
+    global NEXT_DOC_VERSION_CHECK
+    logging.info("Reloading the latest doc version pointer")
+    with files.open(LATEST_DOC_VERSION_FILE, 'r') as f:
+      data = f.read(100)
+      LATEST_DOC_VERSION = int(data.strip())
+      NEXT_DOC_VERSION_CHECK = datetime.now() + timedelta(days=1)
+
+  # TODO: put into memcache?
+  def get_latest_version(self):
+    forced_reload = self.request.get('force_reload')
+    if forced_reload != '' or \

[nweiz, 2012/07/25 18:50:49]

"!= ''" here is unnecessary, since '' is falsey in Python.

[sethladd, 2012/07/25 20:48:45]

I was trying to be more explicit, but I can be more pythony.

+        LATEST_DOC_VERSION == 0 or \
+        NEXT_DOC_VERSION_CHECK == None or \

[nweiz, 2012/07/25 18:50:49]

Similarly, I think it's better to use "not <varname>" here rather than
explicitly comparing them to a value.

[sethladd, 2012/07/25 20:48:45]

I talked to a Python guy here and he said to use "foo is None"

+        datetime.now() > NEXT_DOC_VERSION_CHECK:
+      self.load_latest_version()
+    return LATEST_DOC_VERSION
+
+  def get_cache_age(self, path):
+    if re.search(r'(png|jpg)$', path):
+      age = ONE_DAY

[nweiz, 2012/07/25 18:50:49]

Slightly cleaner to say "return ONE_DAY" rather than assigning to a variable and
returning that.

[sethladd, 2012/07/25 20:48:45]

I'm partial to how it is, all things being equal. is this bad python style?

[nweiz, 2012/07/25 20:58:03]

I dunno if it's bad style for Python in particular, but I tend to prefer
short-circuiting returns and fewer lines of code in general.

+    elif path.endswith('.ico'):
+      age = ONE_WEEK
+    else:
+      age = ONE_HOUR
+    return age
+
+  def resolve_doc_path(self):
+    if self.request.path.startswith('/docs/latest'):
+      version = self.get_latest_version()
+      path = self.request.path.replace('/docs/latest',
+          '/gs/dartlang-api-docs/' + str(version))
+    else:
+      path = self.request.path.replace('/docs', '/gs/dartlang-api-docs')
+    if path.endswith('/'):
+      path = path + 'index.html'
+    return path
 
-class DomRedirectPage(webapp2.RequestHandler):
   def get(self):
-    if self.request.path == '/dom.html':
-      self.redirect('/html.html', permanent=True)
-      return
+    path = self.resolve_doc_path()
+    gs_key = blobstore.create_gs_key(path)
+    age = self.get_cache_age(path)
+
+    self.response.headers['Cache-Control'] = 'max-age=' + \
+        str(age) + ',s-maxage=' + str(age)
+
+    # is there a better way to check if a file exists in cloud storage?
+    # AE will serve a 500 if the file doesn't exist, but that should
+    # be a 404
+
+    path_exists = memcache.get(path)
+    if path_exists is not None:
+      if path_exists == "1":
+        self.send_blob(gs_key)
+      else:
+        self.error(404)
+    else:
+      try:
+        with files.open(path, 'r') as f:

[nweiz, 2012/07/25 18:50:49]

Since you're just checking for an ExistenceError, files.open(...).close() would
be a little cleaner than using with.

[sethladd, 2012/07/25 20:48:45]

Yeah, that's nicer.

+          memcache.add(key=path, value="1", time=ONE_DAY)
+          self.send_blob(gs_key)
+      except files.file.ExistenceError:
+        memcache.add(key=path, value="0", time=ONE_DAY)
+        self.error(404)
+
+  # this doesn't get called, unfortunately.
+  # if this ever starts working, remove the try and files.open
+  # from get, above, and instead retroactively handle a missing file here
+  # def handle_exception(self, exception, debug_mode):
+  #   # awful hack for when file in cloud storage doesn't exist
+  #   if isinstance(exception, TypeError):
+  #     logging.debug('oh noes!')
+  #     path = self.resolve_path()
+  #     try:
+  #       with files.open(path, 'r') as f:
+  #         # the file really does exist, so 500 must be something else
+  #         self.error(500)
+  #     except files.file.ExistenceError:
+  #       # file does not exist
+  #       self.error(404)
+  #   else:
+  #     logging.exception(exception)
+
+def redir_to_latest(handler, *args, **kwargs):
+  path = kwargs['path']
+  if re.search(r'^(core|coreimpl|crypto|io|isolate|json|uri|utf|web)', path):
+    return '/docs/latest/dart_' + path
+  else:
+    return '/docs/latest/' + path
 
-    url = self.request.path[4:len(self.request.path)]
-    self.redirect('/html' + url, permanent=True)
+def redir_dom(handler, *args, **kwargs):
+  return '/docs/latest/dart_html' + kwargs['path']
 
-application = webapp2.WSGIApplication(
-                                     [('/dom.*', DomRedirectPage)],
-                                     debug=True)
+application = WSGIApplication(
+  [
+    Route('/dom<path:.*>', RedirectHandler, defaults={'_uri': redir_dom}),
+    ('/docs.*', ApiDocs),
+    Route('/<path:.*>', RedirectHandler, defaults={'_uri': redir_to_latest})
+  ],
+  debug=True)