Add refresh support to the console page.

app.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from __future__ import with_statement
 
 import datetime
+import json
 import logging
 import os
 import random
 import re
 import string
 import urllib
 
 from google.appengine.api import files, memcache, urlfetch
 from google.appengine.api.app_identity import get_application_id
 from google.appengine.ext import blobstore, db, deferred
@@ skipping 12 matching lines @@
 APP_NAME = get_application_id()
 
 # Deadline for fetching URLs (in seconds).
 URLFETCH_DEADLINE = 60*5  # 5 mins
 
 
 # Perform initial bootstrap for this module.
 console_template = ''
 def bootstrap():
   global console_template
-  with open('templates/console.html', 'r') as fh:
+  with open('templates/merger.html', 'r') as fh:
     console_template = fh.read()
 
 
-# Assumes localpath is already unquoted.
-def get_and_cache_page(localpath):
-  # E1101: 29,12:get_and_cache_page: Module 'google.appengine.api.memcache' has
-  # no 'get' member
-  # pylint: disable=E1101
-  content = memcache.get(localpath)
-  if content is not None:
-    logging.debug('content for %s found in memcache' % localpath)
-    return content
+def get_pagedata_from_cache(localpath):
+  memcache_data = memcache.get(localpath)
+  if not memcache_data:
+    return None
+  logging.debug('content for %s found in memcache' % localpath)
+  return json.loads(memcache_data)
 
+
+def put_pagedata_into_cache(localpath, page_data):
+  memcache_data = json.dumps(page_data)
+  if not memcache.set(key=localpath, value=memcache_data, time=2*60):
+    logging.error('put_pagedata_into_cache(\'%s\'): memcache.set() failed' % (
+        localpath))
+
+
+def get_and_cache_pagedata(localpath):
+  """Returns a page_data dict, optionally caching and looking up a blob.
+
+  get_and_cache_pagedata takes a localpath which is used to fetch data
+  from the cache.  If the data is present and there's no content blob,
+  then we have all of the data we need to return a page view to the user
+  and we return early.
+
+  Otherwise, we need to fetch the page object and set up the page data
+  for the page view.  If the page has a blob associated with it, then we
+  mark the page data as having a blob and cache it as-is without the blob.
+  If there's no blob, we associate the content with the page data and
+  cache that.  This is so the next time get_and_cache_pagedata is called
+  for either case, we'll get the same behavior (a page-lookup for blobful
+  content and a page cache hit for blobless content).
+
+  Here we assume localpath is already unquoted.
+  """
+  page_data = get_pagedata_from_cache(localpath)
+  if page_data and not page_data.get('content_blob'):
+    return page_data
   page = Page.all().filter('localpath =', localpath).get()
   if not page:
-    logging.error('get_and_cache_page(\'%s\'): no matching localpath in '
+    logging.error('get_and_cache_pagedata(\'%s\'): no matching localpath in '
         'datastore' % localpath)
-    return None
-  if page.content_blob is not None:
+    return {'content': None}
+  page_data = {
+    'body_class': page.body_class,
+    'offsite_base': page.offsite_base,
+    'title': page.title,
+  }
+  if page.content_blob:
     # Get the blob.
+    logging.debug('content for %s found in blobstore' % localpath)
     blob_reader = blobstore.BlobReader(page.content_blob)
-    content = blob_reader.read().decode('utf-8', 'replace')
-    logging.debug('content for %s found in blobstore' % localpath)
+    page_data['content_blob'] = True
+    put_pagedata_into_cache(localpath, page_data)
+    page_data['content'] = blob_reader.read().decode('utf-8', 'replace')
   else:
     logging.debug('content for %s found in datastore' % localpath)
-    content = page.content
-    # E1101: 39,11:get_and_cache_page: Module 'google.appengine.api.memcache'
-    # has no 'set' member
-    # pylint: disable=E1101
-    if not memcache.set(key=localpath, value=content, time=2*60):
-      logging.error('get_and_cache_page(\'%s\'): memcache.set() failed' %
-          localpath)
-  return content
+    page_data['content'] = page.content
+    put_pagedata_into_cache(localpath, page_data)
+  return page_data
 
 
 class ConsoleData(object):
   def __init__(self):
     self.row_orderedkeys = []
     self.row_data = {}
 
     # Retain order of observed masters.
     self.masters = []
 
@@ skipping 80 matching lines @@
   def Finish(self):
     self.row_orderedkeys = sorted(self.row_orderedkeys, key=int, reverse=True)
     # TODO(cmp): Look for row/master/categories that are unset.  If they are
     #            at the latest revisions, leave them unset.  If they are at
     #            the earliest revisions, set them to ''.
 
 
 # W0613:169,39:console_merger: Unused argument 'remoteurl'
 # W0613:169,19:console_merger: Unused argument 'unquoted_localpath'
 # pylint: disable=W0613
-def console_merger(unquoted_localpath, remote_url, content=None):
-  if content is None:
-    return None
+def console_merger(unquoted_localpath, remote_url, page_data=None):
+  page_data = page_data or {}
 
   masters = [
     'chromium.main',
     'chromium.chromiumos',
     'chromium.chrome',
     'chromium.memory',
   ]
   mergedconsole = ConsoleData()
   merged_page = None
   merged_tag = None
   fetch_timestamp = datetime.datetime.now()
   for master in masters:
-    master_content = get_and_cache_page('%s/console' % master)
+    page_data = get_and_cache_pagedata('%s/console' % master)
+    master_content = page_data['content']
     if master_content is None:
       continue
     master_content = master_content.encode('ascii', 'replace')
     this_page = BeautifulSoup(master_content)
     this_tag = this_page.find('table', {'class': 'ConsoleData'})
     # The first console is special, we reuse all of the console page.
     if not merged_page:
       merged_page = this_page
       merged_tag = this_tag
     mergedconsole.SawMaster(master)
@@ skipping 74 matching lines @@
   merged_content = re.sub(
       r'\'\<td\>\'', r"'<td ' + attributes + '>'", merged_content)
   merged_content = re.sub(
       r'\<iframe\>\</iframe\>',
       '<iframe \' + attributes + \' src="\' + url + \'"></iframe>',
       merged_content)
 
   # Update the merged console page.
   merged_page = get_or_create_page('chromium/console', None, maxage=30)
   logging.debug('console_merger: saving merged console')
-  save_page(merged_page, 'chromium/console', merged_content,
-            fetch_timestamp)
-  return merged_content
+  page_data['title'] = 'BuildBot: Chromium'
+  page_data['offsite_base'] = 'http://build.chromium.org/p/chromium'
+  page_data['body_class'] = 'interface'
+  page_data['content'] = merged_content
+  save_page(merged_page, 'chromium/console', fetch_timestamp, page_data)
+  return
 
 
-# W0613:284,20:console_handler: Unused argument 'unquoted_localpath'
-# pylint: disable=W0613
-def console_handler(unquoted_localpath, remoteurl, content=None):
-  if content is None:
-    return None
-  # TODO(cmp): Fix the LKGR link.
+def console_handler(_unquoted_localpath, remoteurl, page_data=None):
+  page_data = page_data or {}
+  content = page_data.get('content')
+  if not content:
+    return page_data
 
   # Decode content from utf-8 to unicode, replacing bad characters.
   content = content.decode('utf-8', 'replace')
 
   # Scrub in sheriff file content to console.
   sheriff_files = [
     'sheriff',
     'sheriff_android',
     'sheriff_cr_cros_gardeners',
     'sheriff_cros_mtv',
     'sheriff_cros_nonmtv',
     'sheriff_gpu',
     'sheriff_memory',
     'sheriff_nacl',
     'sheriff_perf',
     'sheriff_webkit',
   ]
   for sheriff_file in sheriff_files:
-    sheriff_content = get_and_cache_page('chromium/%s.js' % sheriff_file)
+    sheriff_page_data = get_and_cache_pagedata('chromium/%s.js' % sheriff_file)
+    sheriff_content = sheriff_page_data['content']
     console_re = (r'<script src=\'http://chromium-build.appspot.com/'
                    'p/chromium/%s.js\'></script>')
     content = re.sub(console_re % sheriff_file,
                      '<script>%s</script>' % sheriff_content, content)
 
   # Replace showBuildBox with direct links.
   content = re.sub(r'<a href=\'#\' onclick=\'showBuildBox\(\"./(.+)\", event\);'
                     ' return false;\'',
                    r"<a href='\1'", content)
 
@@ skipping 43 matching lines @@
   content = string.replace(content,
       "'/json/builders/Linux%20x64/builds/-1?as_text=1';",
       "'/json/builders/Linux%20x64/builds/-1/as_text=1.json';")
 
   # Fix up a reference to http chromium-build in BarUrl().
   content = string.replace(content,
       "return 'http://chromium-build.appspot.com/p/'",
       "return 'https://chromium-build.appspot.com/p/'")
 
   # Encode content from unicode to utf-8.
-  content = content.encode('utf-8')
-  return content
+  page_data['content'] = content.encode('utf-8')
+
+  # Last tweaks to HTML, plus extracting metadata about the page itself.
+  page_data['offsite_base'] = remoteurl + '/../'
+
+  # Extract the title from the page.
+  md = re.search(
+      r'^.*<title>([^\<]+)</title>',
+      page_data['content'],
+      re.MULTILINE|re.DOTALL)
+  if not md:
+    raise Exception('failed to locate title in page')
+  page_data['title'] = md.group(1)
+
+  # Remove the leading text up to the end of the opening body tag.  While
+  # there, extract the body_class from the page.
+  md = re.search(
+      r'^.*<body class="(\w+)\">(.*)$',
+      page_data['content'],
+      re.MULTILINE|re.DOTALL)
+  if not md:
+    raise Exception('failed to locate leading text up to body tag')
+  page_data['body_class'] = md.group(1)
+  page_data['content'] = md.group(2)
+
+  # Remove the leading div and hr tags.
+  md = re.search(
+      r'^.*?<hr/>(.*)$',
+      page_data['content'],
+      re.MULTILINE|re.DOTALL)
+  if not md:
+    raise Exception('failed to locate leading div and hr tags')
+  page_data['content'] = md.group(1)
+
+  # Strip the trailing body and html tags.
+  md = re.search(
+      r'^(.*)</body>.*$',
+      page_data['content'],
+      re.MULTILINE|re.DOTALL)
+  if not md:
+    raise Exception('failed to locate trailing body and html tags')
+  page_data['content'] = md.group(1)
+
+  return page_data
 
 
-def one_box_handler(unquoted_localpath, remoteurl, content=None):
+def one_box_handler(unquoted_localpath, remoteurl, page_data=None):
+  page_data = page_data or {}
+  content = page_data.get('content')
   if content is None:
-    return None
+    return page_data
   # Get the site name from the local path.
   md = re.match('^([^\/]+)/.*$', unquoted_localpath)
   if not md:
     logging.error('one_box_handler(\'%s\', \'%s\', \'%s\'): cannot get site '
-                  'from local path' % (unquoted_localpath, remoteurl, content))
-    return content
+                  'from local path' % (
+                      unquoted_localpath, remoteurl, page_data))
+    return page_data
   site = md.group(1)
   new_waterfall_url = 'http://build.chromium.org/p/%s/waterfall' % site
-  content = re.sub(r'waterfall', new_waterfall_url, content)
-  return content
+  page_data['content'] = re.sub(
+      r'waterfall',
+      new_waterfall_url,
+      page_data['content'])
+  return page_data
 
 
 
 # List of URLs to fetch.
 URLS = [
   # Console URLs.
   {
     'remoteurl': 'http://build.chromium.org/p/chromium/console',
     'localpath': 'chromium.main/console',
     'postfetch': console_handler,
@@ skipping 233 matching lines @@
     return urlfetch.fetch(url, deadline=URLFETCH_DEADLINE, *args, **kwargs)
   except urlfetch.DownloadError:
     logging.warn('urlfetch failed: %s' % url, exc_info=1)
     return None
 
 
 class Page(db.Model):
   fetch_timestamp = db.DateTimeProperty(required=True)
   localpath = db.StringProperty(required=True)
   content = db.TextProperty()
+  title = db.StringProperty()
+  offsite_base = db.StringProperty()
+  body_class = db.StringProperty()
   remoteurl = db.TextProperty()
   # Data updated separately, after creation.
   content_blob = blobstore.BlobReferenceProperty()
 
 
 def write_blob(data, mime_type):
   """Saves a Unicode string as a new blob, returns the blob's key."""
   file_name = files.blobstore.create(mime_type=mime_type)
   data = data.encode('utf-8')
   with files.open(file_name, 'a') as blob_file:
     blob_file.write(data)
   files.finalize(file_name)
   return files.blobstore.get_blob_key(file_name)
 
 
-def save_page(page, localpath, content, fetch_timestamp):
+def save_page(page, localpath, fetch_timestamp, page_data):
+  body_class = page_data.get('body_class', '')
+  content = page_data.get('content')
+  offsite_base = page_data.get('offsite_base', '')
+  title = page_data.get('title', '')
+
   content_blob_key = None
   try:
     content = content.decode('utf-8', 'replace')
   except UnicodeEncodeError:
     logging.debug('save_page: content was already in unicode')
   logging.debug('save_page: content size is %d' % len(content))
   if len(content.encode('utf-8')) >= 1024*1024:
     logging.debug('save_page: saving to blob')
     content_blob_key = write_blob(content, path_to_mime_type(localpath))
     content = None
   def tx_page(page_key):
     page = Page.get(page_key)
     # E1103:225,7:fetch_page.tx_page: Instance of 'list' has no
     # 'fetch_timestamp' member (but some types could not be inferred)
     # pylint: disable=E1103
     if page.fetch_timestamp > fetch_timestamp:
       return
     page.content = content
     page.content_blob = content_blob_key
     page.fetch_timestamp = fetch_timestamp
+    # title, offsite_base, body_class can all be empty strings for some
+    # content.  Where that's true, they're not used for displaying a console-
+    # like resource, and the content alone is returned to the web user.
+    page.title = title
+    page.offsite_base = offsite_base
+    page.body_class = body_class
     # E1103:231,4:fetch_page.tx_page: Instance of 'list' has no 'put' member
     # (but some types could not be inferred)
     # pylint: disable=E1103
     page.put()
   db.run_in_transaction(tx_page, page.key())
-  # E1101:232,11:fetch_page.tx_page: Module 'google.appengine.api.memcache'
-  # has no 'set' member
-  # pylint: disable=E1101
-  if page.content_blob is None:
-    if memcache.set(key=localpath, value=page.content, time=60):
-      logging.debug('tx_page(page key="%s"): memcache.set() succeeded' %
-          page.key())
-    else:
-      logging.error('tx_page(page key="%s"): memcache.set() failed' %
-          page.key())
+  page_data = {
+    'body_class': body_class,
+    'content': content,
+    'offsite_base': offsite_base,
+    'title': title,
+  }
+  if content_blob_key:
+    page_data['content_blob'] = True
+  put_pagedata_into_cache(localpath, page_data)
 
 
 def get_or_create_page(localpath, remoteurl, maxage):
   return Page.get_or_insert(
     key_name=localpath,
     localpath=localpath,
     remoteurl=remoteurl,
     maxage=maxage,
     fetch_timestamp=datetime.datetime.now() - datetime.timedelta(hours=24),
     content=None,
     content_blob=None)
 
 
-def fetch_page(localpath, remoteurl, maxage, postfetch=None, postsave=None):
+def fetch_page(localpath, remoteurl, maxage, postfetch=None, postsave=None,
+               fetch_url=nonfatal_fetch_url):
   """Fetches data about a set of pages."""
   unquoted_localpath = urllib.unquote(localpath)
   logging.debug('fetch_page("%s", "%s", "%s")' % (
       unquoted_localpath, remoteurl, maxage))
   page = get_or_create_page(unquoted_localpath, remoteurl, maxage)
 
   # Check if our copy of the page is younger than maxage.  If it is, we'll
   # skip the fetch.
   oldest_acceptable_timestamp = datetime.datetime.now() - datetime.timedelta(
       seconds=maxage)
   if (page.fetch_timestamp and
       page.fetch_timestamp > oldest_acceptable_timestamp):
     logging.debug('fetch_page: too recent, skipping')
     return
 
   # Perform the actual page fetch.
   fetch_timestamp = datetime.datetime.now()
-  response = nonfatal_fetch_url(remoteurl)
+  response = fetch_url(remoteurl)
   if not response:
     logging.warning('fetch_page: got empty response')
     return
   if response.status_code != 200:
     logging.warning('fetch_page: got non-empty response but code '
                     '%d' % response.status_code)
     return
 
   # We have actual content.  If there's one or more handlers, call them.
-  content = response.content
+  page_data = {}
+  page_data['content'] = response.content
   if postfetch:
     if not isinstance(postfetch, list):
       postfetch = [postfetch]
     for handler in postfetch:
       logging.debug('fetch_page: calling postfetch handler '
                     '%s' % handler.__name__)
-      content = handler(unquoted_localpath, remoteurl, content)
+      page_data = handler(unquoted_localpath, remoteurl, page_data)
 
   # Save the returned content into the DB and caching layers.
   logging.debug('fetch_page: saving page')
-  save_page(page, unquoted_localpath, content, fetch_timestamp)
+  save_page(page, unquoted_localpath, fetch_timestamp, page_data)
   if postsave:
     if not isinstance(postsave, list):
       postsave = [postsave]
     for handler in postsave:
       logging.debug('fetch_page: calling postsave handler '
                     '%s' % handler.__name__)
-      handler(unquoted_localpath, remoteurl, content)
+      handler(unquoted_localpath, remoteurl, page_data)
 
 
 EXT_TO_MIME = {
   '.css': 'text/css',
   '.js': 'text/javascript',
   '.json': 'application/json',
   '.html': 'text/html',
 }
 
 
 def path_to_mime_type(path):
   return EXT_TO_MIME.get(os.path.splitext(path)[1], 'text/html')
 
 
 def fetch_pages():
   """Starts a background fetch operation for pages that need it."""
   logging.debug('fetch_pages()')
   for url in URLS:
     deferred.defer(fetch_page, **url)