Devserver: only populate data during cron jobs, meaning all data from instances

chrome/common/extensions/docs/server2/handler.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 import logging
 import os
 from StringIO import StringIO
 
 from appengine_wrappers import webapp
 from appengine_wrappers import memcache
 from appengine_wrappers import urlfetch
 from branch_utility import BranchUtility
 from server_instance import ServerInstance
 import svn_constants
 import time
 
 # The default channel to serve docs for if no channel is specified.
 _DEFAULT_CHANNEL = 'stable'
 
 class Handler(webapp.RequestHandler):
+  # HACK: for preview.py.
+  # TODO(kalman): achieve this via proper dependency injection.
+  ALWAYS_ONLINE = False

[cduvall, 2013/04/17 23:30:37]

So this will have to be changed manually before uploading or using
dev_appserver?

[not at google - send to devlin, 2013/04/18 04:05:41]

Nope. Adding a comment here to explain the following:

What I'm going for is appengine instances never needing to call out to SVN,
since that has been flaky recently. If this is False we can catch errors in that
area ahead of time, and also, not bother asking SVN when we know it's going to
be a 404 anyway (e.g. when requesting a bogus URL). Only crons get to be online,
and they're allowed to fail.

+
   def __init__(self, request, response):
     super(Handler, self).__init__(request, response)
 
   def _HandleGet(self, path):
     channel_name, real_path = BranchUtility.SplitChannelNameFromPath(path)
 
     if channel_name == _DEFAULT_CHANNEL:
       self.redirect('/%s' % real_path)
       return
 
     if channel_name is None:
       channel_name = _DEFAULT_CHANNEL
 
     # TODO(kalman): Check if |path| is a directory and serve path/index.html
     # rather than special-casing apps/extensions.
     if real_path.strip('/') == 'apps':
       real_path = 'apps/index.html'
     if real_path.strip('/') == 'extensions':
       real_path = 'extensions/index.html'
 
-    server_instance = ServerInstance.GetOrCreate(channel_name)
+    constructor = (
+        ServerInstance.GetOrCreateOnline if Handler.ALWAYS_ONLINE else
+        ServerInstance.GetOrCreateOffline)
+    server_instance = constructor(channel_name)
 
     canonical_path = server_instance.path_canonicalizer.Canonicalize(real_path)
     if real_path != canonical_path:
       self.redirect(canonical_path)
       return
 
-    ServerInstance.GetOrCreate(channel_name).Get(real_path,
-                                                 self.request,
-                                                 self.response)
+    server_instance.Get(real_path, self.request, self.response)
 
   def _HandleCron(self, path):
     # Cron strategy:
     #
     # Find all public template files and static files, and render them. Most of
     # the time these won't have changed since the last cron run, so it's a
     # little wasteful, but hopefully rendering is really fast (if it isn't we
     # have a problem).
     class MockResponse(object):
       def __init__(self):
         self.status = 200
         self.out = StringIO()
         self.headers = {}
       def set_status(self, status):
         self.status = status
       def clear(self, *args):
         pass
 
     class MockRequest(object):
       def __init__(self, path):
         self.headers = {}
         self.path = path
         self.url = '//localhost/%s' % path
 
     channel = path.split('/')[-1]
     logging.info('cron/%s: starting' % channel)
 
-    server_instance = ServerInstance.GetOrCreate(channel)
+    server_instance = ServerInstance.GetOrCreateOnline(channel)
 
-    def run_cron_for_dir(d):
+    def run_cron_for_dir(d, path_prefix=''):
       error = None
       start_time = time.time()
       files = [f for f in server_instance.content_cache.GetFromFileListing(d)
                if not f.endswith('/')]
       for f in files:
+        path = '%s%s' % (path_prefix, f)
         try:
-          server_instance.Get(f, MockRequest(f), MockResponse())
+          response = MockResponse()
+          server_instance.Get(path, MockRequest(path), response)
+          if response.status != 200:
+            error = 'Got %s response' % response.status
         except error:
-          logging.error('cron/%s: error rendering %s/%s: %s' % (
-              channel, d, f, error))
-      logging.info('cron/%s: rendering %s files in %s took %s seconds' % (
-          channel, len(files), d, time.time() - start_time))
+          pass
+        if error:
+          logging.error('cron/%s: error rendering %s: %s' % (
+              channel, path, error))
+      logging.info('cron/%s: rendering %s files took %s seconds' % (
+          channel, len(files), time.time() - start_time))
       return error
 
     # Don't use "or" since we want to evaluate everything no matter what.
-    was_error = any((run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
-                     run_cron_for_dir(svn_constants.STATIC_PATH)))
+    was_error = any((
+        run_cron_for_dir(svn_constants.PUBLIC_TEMPLATE_PATH),
+        run_cron_for_dir(svn_constants.STATIC_PATH, path_prefix='static/')))
 
     if was_error:
       self.response.status = 500
       self.response.out.write('Failure')
     else:
       self.response.status = 200
       self.response.out.write('Success')
 
     logging.info('cron/%s: finished' % channel)
 
@@ skipping 45 matching lines @@
     # file paths will know to treat this as a directory.
     if os.path.splitext(path)[1] == '' and path[-1] != '/':
       self.redirect(path + '/')
       return
 
     path = path.strip('/')
     if self._RedirectFromCodeDotGoogleDotCom(path):
       return
 
     self._HandleGet(path)