Devserver: only populate data during cron jobs, meaning all data from instances

chrome/common/extensions/docs/server2/handler.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 from StringIO import StringIO
 
 from appengine_wrappers import webapp
 from appengine_wrappers import memcache
 from appengine_wrappers import urlfetch
 from branch_utility import BranchUtility
 from server_instance import ServerInstance
 import svn_constants
 import time
 
 # The default channel to serve docs for if no channel is specified.
 _DEFAULT_CHANNEL = 'stable'
 
 class Handler(webapp.RequestHandler):
+  # AppEngine instances should never need to call out to SVN. That should only
+  # ever be done by the cronjobs, which then write the result into DataStore,
+  # which is as far as instances look.
+  #
+  # Why? SVN is slow and a bit flaky. Cronjobs failing is annoying but
+  # temporary. Instances failing affects users, and is really bad.
+  #
+  # Anyway - to enforce this, we actually don't give instances access to SVN.
+  # If anything is missing from datastore, it'll be a 404. If the cronjobs
+  # don't manage to catch everything - uhoh. On the other hand, we'll figure it
+  # out pretty soon, and it also means that legitimate 404s are caught before a
+  # round trip to SVN.
+  #
+  # However, we can't expect users of preview.py to run a cronjob first, so,
+  # this is a hack allow that to be online all of the time.
+  # TODO(kalman): achieve this via proper dependency injection.
+  ALWAYS_ONLINE = False
+
   def __init__(self, request, response):
     super(Handler, self).__init__(request, response)
 
   def _HandleGet(self, path):
     channel_name, real_path = BranchUtility.SplitChannelNameFromPath(path)
 
     if channel_name == _DEFAULT_CHANNEL:
       self.redirect('/%s' % real_path)
       return
 
     if channel_name is None:
       channel_name = _DEFAULT_CHANNEL
 
     # TODO(kalman): Check if |path| is a directory and serve path/index.html
     # rather than special-casing apps/extensions.
     if real_path.strip('/') == 'apps':
       real_path = 'apps/index.html'
     if real_path.strip('/') == 'extensions':
       real_path = 'extensions/index.html'
 
-    server_instance = ServerInstance.GetOrCreate(channel_name)
+    constructor = (
+        ServerInstance.GetOrCreateOnline if Handler.ALWAYS_ONLINE else
+        ServerInstance.GetOrCreateOffline)
+    server_instance = constructor(channel_name)
 
     canonical_path = server_instance.path_canonicalizer.Canonicalize(real_path)
     if real_path != canonical_path:
       self.redirect(canonical_path)
       return
 
-    ServerInstance.GetOrCreate(channel_name).Get(real_path,
-                                                 self.request,
-                                                 self.response)
+    server_instance.Get(real_path, self.request, self.response)
 
   def _HandleCron(self, path):
     # Cron strategy:
     #
     # Find all public template files and static files, and render them. Most of
     # the time these won't have changed since the last cron run, so it's a
     # little wasteful, but hopefully rendering is really fast (if it isn't we
     # have a problem).
     class MockResponse(object):
       def __init__(self):
         self.status = 200
         self.out = StringIO()
         self.headers = {}
       def set_status(self, status):
         self.status = status
       def clear(self, *args):
         pass
 
     class MockRequest(object):
       def __init__(self, path):
         self.headers = {}
         self.path = path
         self.url = '//localhost/%s' % path
 
     channel = path.split('/')[-1]
     logging.info('cron/%s: starting' % channel)
 
-    server_instance = ServerInstance.GetOrCreate(channel)
+    server_instance = ServerInstance.GetOrCreateOnline(channel)
 
-    def run_cron_for_dir(d):
+    def run_cron_for_dir(d, path_prefix=''):
       error = None
       start_time = time.time()
       files = [f for f in server_instance.content_cache.GetFromFileListing(d)
                if not f.endswith('/')]
       for f in files:
+        path = '%s%s' % (path_prefix, f)
         try:
-          server_instance.Get(f, MockRequest(f), MockResponse())
+          response = MockResponse()
+          server_instance.Get(path, MockRequest(path), response)
+          if response.status != 200:
+            error = 'Got %s response' % response.status
         except error:
-          logging.error('cron/%s: error rendering %s/%s: %s' % (
-              channel, d, f, error))
-      logging.info('cron/%s: rendering %s files in %s took %s seconds' % (
-          channel, len(files), d, time.time() - start_time))
+          pass
+        if error:
+          logging.error('cron/%s: error rendering %s: %s' % (
+              channel, path, error))
+      logging.info('cron/%s: rendering %s files took %s seconds' % (
+          channel, len(files), time.time() - start_time))
       return error
 
     # Don't use "or" since we want to evaluate everything no matter what.
-    was_error = any((run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
-                     run_cron_for_dir(svn_constants.STATIC_PATH)))
+    was_error = any((
+        run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
+        run_cron_for_dir(svn_constants.STATIC_PATH, path_prefix='static/')))
 
     if was_error:
       self.response.status = 500
       self.response.out.write('Failure')
     else:
       self.response.status = 200
       self.response.out.write('Success')
 
     logging.info('cron/%s: finished' % channel)
 
@@ skipping 45 matching lines @@
     # file paths will know to treat this as a directory.
     if os.path.splitext(path)[1] == '' and path[-1] != '/':
       self.redirect(path + '/')
       return
 
     path = path.strip('/')
     if self._RedirectFromCodeDotGoogleDotCom(path):
       return
 
     self._HandleGet(path)