chromium-build app now renders console from stored rows.

app.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from __future__ import with_statement
 
 import datetime
 import json
 import logging
 import os
 import random
 import re
 import string
 import urllib
 
 from google.appengine.api import files, memcache, urlfetch
 from google.appengine.api.app_identity import get_application_id
 from google.appengine.ext import blobstore, db, deferred
 # F0401: 16,0: Unable to import 'webapp2_extras'
 # W0611: 16,0: Unused import jinja2
 # pylint: disable=F0401, W0611
 from webapp2_extras import jinja2
 # F0401:22,0: Unable to import 'jinja2'
 # pylint: disable=F0401
 from jinja2 import Environment, FileSystemLoader
 
-from third_party.BeautifulSoup.BeautifulSoup import BeautifulSoup
+from third_party.BeautifulSoup.BeautifulSoup import BeautifulSoup, Tag
 
 
 # Current application name.
 APP_NAME = get_application_id()
 
 # Deadline for fetching URLs (in seconds).
 URLFETCH_DEADLINE = 60*5  # 5 mins
 
 # Default masters to merge together.
 DEFAULT_MASTERS_TO_MERGE = [
   'chromium.main',
   'chromium.win',
   'chromium.mac',
   'chromium.linux',
   'chromium.chromiumos',
   'chromium.chrome',
   'chromium.memory',
 ]
 
 
 # Perform initial bootstrap for this module.
 console_template = ''
 def bootstrap():
   global console_template
   with open('templates/merger.html', 'r') as fh:
     console_template = fh.read()
 
 
-def get_pagedata_from_cache(localpath):
-  memcache_data = memcache.get(localpath)
-  if not memcache_data:
-    return None
-  logging.debug('content for %s found in memcache' % localpath)
-  return json.loads(memcache_data)
+##########
+# Page class definition and related functions.
+##########
+class Page(db.Model):
+  fetch_timestamp = db.DateTimeProperty(required=True)
+  localpath = db.StringProperty(required=True)
+  content = db.TextProperty()
+  title = db.StringProperty()
+  offsite_base = db.StringProperty()
+  body_class = db.StringProperty()
+  remoteurl = db.TextProperty()
+  # Data updated separately, after creation.
+  content_blob = blobstore.BlobReferenceProperty()
 
 
-def put_pagedata_into_cache(localpath, page_data):
-  memcache_data = json.dumps(page_data)
-  if not memcache.set(key=localpath, value=memcache_data, time=2*60):
-    logging.error('put_pagedata_into_cache(\'%s\'): memcache.set() failed' % (
-        localpath))
+def get_or_create_page(localpath, remoteurl, maxage):
+  return Page.get_or_insert(
+    key_name=localpath,
+    localpath=localpath,
+    remoteurl=remoteurl,
+    maxage=maxage,
+    fetch_timestamp=datetime.datetime.now() - datetime.timedelta(hours=24),
+    content=None,
+    content_blob=None)
 
 
 def get_and_cache_pagedata(localpath):
   """Returns a page_data dict, optionally caching and looking up a blob.
 
   get_and_cache_pagedata takes a localpath which is used to fetch data
   from the cache.  If the data is present and there's no content blob,
   then we have all of the data we need to return a page view to the user
   and we return early.
 
   Otherwise, we need to fetch the page object and set up the page data
   for the page view.  If the page has a blob associated with it, then we
   mark the page data as having a blob and cache it as-is without the blob.
   If there's no blob, we associate the content with the page data and
   cache that.  This is so the next time get_and_cache_pagedata is called
   for either case, we'll get the same behavior (a page-lookup for blobful
   content and a page cache hit for blobless content).
 
   Here we assume localpath is already unquoted.
   """
-  page_data = get_pagedata_from_cache(localpath)
+  page_data = get_data_from_cache(localpath)
   if page_data and not page_data.get('content_blob'):
     return page_data
   page = Page.all().filter('localpath =', localpath).get()
   if not page:
     logging.error('get_and_cache_pagedata(\'%s\'): no matching localpath in '
         'datastore' % localpath)
     return {'content': None}
   page_data = {
     'body_class': page.body_class,
     'offsite_base': page.offsite_base,
     'title': page.title,
   }
   if page.content_blob:
     # Get the blob.
     logging.debug('content for %s found in blobstore' % localpath)
     blob_reader = blobstore.BlobReader(page.content_blob)
     page_data['content_blob'] = True
-    put_pagedata_into_cache(localpath, page_data)
+    put_data_into_cache(localpath, page_data)
     page_data['content'] = blob_reader.read().decode('utf-8', 'replace')
   else:
     logging.debug('content for %s found in datastore' % localpath)
     page_data['content'] = page.content
-    put_pagedata_into_cache(localpath, page_data)
+    put_data_into_cache(localpath, page_data)
   return page_data
 
 
+def save_page(page, localpath, fetch_timestamp, page_data):
+  body_class = page_data.get('body_class', '')
+  content = page_data.get('content')
+  offsite_base = page_data.get('offsite_base', '')
+  title = page_data.get('title', '')
+
+  content_blob_key = None
+  try:
+    content = content.decode('utf-8', 'replace')
+  except UnicodeEncodeError:
+    logging.debug('save_page: content was already in unicode')
+  logging.debug('save_page: content size is %d' % len(content))
+  if len(content.encode('utf-8')) >= 10**6:
+    logging.debug('save_page: saving to blob')
+    content_blob_key = write_blob(content, path_to_mime_type(localpath))
+    content = None
+  def tx_page(page_key):
+    page = Page.get(page_key)
+    # E1103:225,7:fetch_page.tx_page: Instance of 'list' has no
+    # 'fetch_timestamp' member (but some types could not be inferred)
+    # pylint: disable=E1103
+    if page.fetch_timestamp > fetch_timestamp:
+      return
+    page.content = content
+    page.content_blob = content_blob_key
+    page.fetch_timestamp = fetch_timestamp
+    # title, offsite_base, body_class can all be empty strings for some
+    # content.  Where that's true, they're not used for displaying a console-
+    # like resource, and the content alone is returned to the web user.
+    page.title = title
+    page.offsite_base = offsite_base
+    page.body_class = body_class
+    # E1103:231,4:fetch_page.tx_page: Instance of 'list' has no 'put' member
+    # (but some types could not be inferred)
+    # pylint: disable=E1103
+    page.put()
+  db.run_in_transaction(tx_page, page.key())
+  page_data = {
+    'body_class': body_class,
+    'content': content,
+    'offsite_base': offsite_base,
+    'title': title,
+  }
+  if content_blob_key:
+    page_data['content_blob'] = True
+  put_data_into_cache(localpath, page_data)
+  logging.info('Saved and cached page with localpath %s' % localpath)
+
+
+##########
+# Row class definition and related functions.
+##########
+class Row(db.Model):
+  fetch_timestamp = db.DateTimeProperty(required=True)
+  rev_number = db.StringProperty(required=True)
+  localpath = db.StringProperty(required=True)
+  revision = db.TextProperty()
+  name = db.TextProperty()
+  status = db.TextProperty()
+  comment = db.TextProperty()
+  details = db.TextProperty()
+
+
+def get_or_create_row(localpath, revision):
+  return Row.get_or_insert(
+    key_name=localpath,
+    rev_number=revision,
+    localpath=localpath,
+    fetch_timestamp=datetime.datetime.now())
+
+
+def get_and_cache_rowdata(localpath):
+  """Returns a row_data dict.
+
+  get_and_cache_rowdata takes a localpath which is used to fetch data from the
+  cache. If the data is present, then we have all of the data we need and we
+  return early.
+
+  Otherwise, we need to fetch the row object and set up the row data.
+
+  Here we assume localpath is already unquoted.
+  """
+  row_data = get_data_from_cache(localpath)
+  if row_data:
+    return row_data
+  row = Row.all().filter('localpath =', localpath).get()
+  if not row:
+    logging.error('get_and_cache_rowdata(\'%s\'): no matching localpath in '
+        'datastore' % localpath)
+    return {}
+  row_data = {}
+  row_data['rev'] = row.revision
+  row_data['name'] = row.name
+  row_data['status'] = row.status
+  row_data['comment'] = row.comment
+  row_data['details'] = row.details
+  row_data['rev_number'] = row.rev_number
+  logging.debug('content for %s found in datastore' % localpath)
+  put_data_into_cache(localpath, row_data)
+  return row_data
+
+
+def save_row(row_data, localpath, timestamp):
+  rev_number = row_data['rev_number']
+  row = get_or_create_row(localpath, rev_number)
+  row_key = row.key()
+  def tx_row(row_key):
+    row = Row.get(row_key)
+    # E1103:959,7:save_row.tx_row: Instance of 'list' has no
+    # 'fetch_timestamp' member (but some types could not be inferred)
+    # pylint: disable=E1103
+    # if row.fetch_timestamp > timestamp:
+    #   return
+    row.fetch_timestamp = timestamp
+    row.revision = row_data['rev']
+    row.name = row_data['name']
+    row.status = row_data['status']
+    row.comment = row_data['comment']
+    row.details = row_data['details']
+    # E1103:967,4:save_row.tx_row: Instance of 'list' has no 'put' member
+    # (but some types could not be inferred)
+    # pylint: disable=E1103
+    row.put()
+  db.run_in_transaction(tx_row, row_key)
+  prev_rev = memcache.get(key='latest_rev')
+  if (rev_number > prev_rev):
+    memcache.set(key='latest_rev', value=rev_number)
+  put_data_into_cache(localpath, row_data)
+  logging.info('Saved and cached row with localpath %s' % localpath)
+
+
+##########
+# ConsoleData class definition and related functions.
+##########
 class ConsoleData(object):
   def __init__(self):
     self.row_orderedkeys = []
     self.row_data = {}
 
     # Retain order of observed masters.
     self.masters = []
 
     # Map(k,v): k=Master, v=List of categories
     self.category_order = {}
@@ skipping 45 matching lines @@
   def SetDetail(self, detail):
     self.last_row['detail'] = detail
 
   def AddCategory(self, category, builder_status):
     self.category_order[self.lastMasterSeen].append(category)
     # Map(k,v): k=Master/category, v=Dict of category data (last build status)
     self.category_data[self.lastMasterSeen].setdefault(category, {})
     self.category_data[self.lastMasterSeen][category] = builder_status
     self.category_count += 1
 
+  def AddRow(self, row):
+    revision = row['rev_number']
+    self.SawRevision(revision)
+    revlink = BeautifulSoup(row['rev']).td.a['href']
+    self.SetLink(revlink)
+    name = BeautifulSoup(row['name']).td.contents
+    self.SetName(self.ContentsToHtml(name))
+    status = BeautifulSoup(row['status']).findAll('table')
+    for i, stat in enumerate(status):
+      self.SetStatus(self.category_order[self.lastMasterSeen][i],
+                     unicode(stat))
+    comment = BeautifulSoup(row['comment']).td.contents
+    self.SetComment(self.ContentsToHtml(comment))
+    if row['details']:
+      details = BeautifulSoup(row['details']).td.contents
+      self.SetDetail(self.ContentsToHtml(details))
+
   def ParseRow(self, row):
     cells = row.findAll('td', recursive=False)
     # Figure out which row this is.
     for attrname, attrvalue in cells[0].attrs:
       if attrname != 'class':
         continue
       attrvalue = re.sub(r'^(\S+).*', r'\1', attrvalue)
       if attrvalue == 'DevRev':
         revision = cells[0]
         self.SawRevision(revision=revision.findAll('a')[0].contents[0])
@@ skipping 10 matching lines @@
       if attrvalue == 'DevDetails':
         self.SetDetail(detail=self.ContentsToHtml(cells[0].contents))
 
   def Finish(self):
     self.row_orderedkeys = sorted(self.row_orderedkeys, key=int, reverse=True)
     # TODO(cmp): Look for row/master/categories that are unset.  If they are
     #            at the latest revisions, leave them unset.  If they are at
     #            the earliest revisions, set them to ''.
 
 
-# W0613:169,39:console_merger: Unused argument 'remoteurl'
-# W0613:169,19:console_merger: Unused argument 'unquoted_localpath'
-# pylint: disable=W0613
-def console_merger(unquoted_localpath, remote_url, page_data=None,
-                   masters_to_merge=None):
-  page_data = page_data or {}
-
+##########
+# Heavy-lifting functions that do most of the console processing.
+# AKA postfetch and postsave functions/handlers.
+##########
+def console_merger(localpath, remoteurl, page_data,
+                   masters_to_merge=None, num_rows_to_merge=25):
   masters_to_merge = masters_to_merge or DEFAULT_MASTERS_TO_MERGE
   mergedconsole = ConsoleData()
-  merged_page = None
-  merged_tag = None
+  surroundings = get_and_cache_pagedata('surroundings')
+  merged_page = BeautifulSoup(surroundings['content'])
+  merged_tag = merged_page.find('table', 'ConsoleData')
+  latest_rev = int(memcache.get(key='latest_rev'))
+  if not latest_rev:
+    logging.error('console_merger(\'%s\', \'%s\', \'%s\'): cannot get latest '
+                  'revision number.' % (
+                      localpath, remoteurl, page_data))
+    return
   fetch_timestamp = datetime.datetime.now()
   for master in masters_to_merge:
-    page_data = get_and_cache_pagedata('%s/console' % master)
-    master_content = page_data['content']
-    if master_content is None:
+    # Fetch the summary one-box-per-builder for the master.
+    # If we don't get it, something is wrong, skip the master entirely.
+    master_summary = get_and_cache_pagedata('%s/console/summary' % master)
+    if not master_summary['content']:
       continue
-    master_content = master_content.encode('ascii', 'replace')
-    this_page = BeautifulSoup(master_content)
-    this_tag = this_page.find('table', {'class': 'ConsoleData'})
-    # The first console is special, we reuse all of the console page.
-    if not merged_page:
-      merged_page = this_page
-      merged_tag = this_tag
     mergedconsole.SawMaster(master)
+    # Get the categories for this builder. If the builder doesn't have any
+    # categories, just use the default empty-string category.
+    category_list = []
+    master_categories = get_and_cache_pagedata('%s/console/categories' % master)
+    if not master_categories['content']:
+      category_list.append('')
+    else:
+      category_row = BeautifulSoup(master_categories['content'])
+      category_list = map(lambda x: x.text,
+                          category_row.findAll('td', 'DevStatus'))
+    # Get the corresponding summary box(es).
+    summary_row = BeautifulSoup(master_summary['content'])
+    summary_list = summary_row.findAll('table')
+    for category, summary in zip(category_list, summary_list):
+      mergedconsole.AddCategory(category, summary)
 
-    # Parse each of the rows.
-    CATEGORY_ROW = 0
-    trs = this_tag.findAll('tr', recursive=False)
-
-    # Get the list of categories in |master|.
-    category_tds = trs[CATEGORY_ROW].findAll('td', recursive=False)[2:]
-    third_cell = category_tds[0]
-    third_cell_class = third_cell.attrs[0][1]
-    categories = []
-    if third_cell_class.startswith('DevStatus '):
-      BUILDER_STATUS_ROW = 2
-      FIRST_CL_ROW = 3
-      for index, category_td in enumerate(category_tds):
-        categories.append(category_td.contents[0].strip())
-    else:
-      # There's no categories + spacing row, the first row will be the builder
-      # status row.
-      categories.append('')
-      BUILDER_STATUS_ROW = 0
-      FIRST_CL_ROW = 1
-
-    # For each category in |master|, add the category plus its |builder_status|.
-    builder_tds = trs[BUILDER_STATUS_ROW].findAll('td', recursive=False)[2:]
-    for index, category in enumerate(categories):
-      builder_status = builder_tds[index].findAll('table', recursive=False)[0]
-      mergedconsole.AddCategory(category=category,
-                                builder_status=builder_status)
-
-    # For each of the remaining rows, add them to the console data.
-    for console_index in range(FIRST_CL_ROW, len(trs)):
-      console_row = trs[console_index]
-      mergedconsole.ParseRow(console_row)
-  # Add GC memory profiling.
-  # import gc
-  # gc.set_debug(gc.DEBUG_LEAK)
-  # logging.debug(gc.garbage)
-  # del gc.garbage[:]
-  mergedconsole.Finish()
+    # Fetch all of the rows that we need.
+    rows_fetched = 0
+    current_rev = latest_rev
+    while rows_fetched < num_rows_to_merge and current_rev >= 0:
+      row_data = get_and_cache_rowdata('%s/console/%s' % (master, current_rev))
+      if not row_data:
+        current_rev -= 1
+        continue
+      mergedconsole.AddRow(row_data)
+      current_rev -= 1
+      rows_fetched += 1
 
   # Convert the merged content into console content.
+  mergedconsole.Finish()
   template_environment = Environment()
   template_environment.loader = FileSystemLoader('.')
   def notstarted(builder_status):
     """Convert a BeautifulSoup Tag from builder status to a notstarted line."""
     builder_status = re.sub(r'DevSlaveBox', 'DevStatusBox', str(builder_status))
     builder_status = re.sub(r'class=\'([^\']*)\' target=',
                             'class=\'DevStatusBox notstarted\' target=',
                             builder_status)
     builder_status = re.sub(r'class="([^"]*)" target=',
                             'class="DevStatusBox notstarted" target=',
@@ skipping 22 matching lines @@
       r'\'\<div\>\'', r"'<div ' + attributes + '>'", merged_content)
   merged_content = re.sub(
       r'\'\<td\>\'', r"'<td ' + attributes + '>'", merged_content)
   merged_content = re.sub(
       r'\<iframe\>\</iframe\>',
       '<iframe \' + attributes + \' src="\' + url + \'"></iframe>',
       merged_content)
 
   # Update the merged console page.
   merged_page = get_or_create_page('chromium/console', None, maxage=30)
-  logging.debug('console_merger: saving merged console')
+  logging.info('console_merger: saving merged console')
+  page_data = get_and_cache_pagedata('chromium/console')
   page_data['title'] = 'BuildBot: Chromium'
   page_data['offsite_base'] = 'http://build.chromium.org/p/chromium'
   page_data['body_class'] = 'interface'
   page_data['content'] = merged_content
   save_page(merged_page, 'chromium/console', fetch_timestamp, page_data)
   return
 
 
-def console_handler(_unquoted_localpath, remoteurl, page_data=None):
+def console_handler(unquoted_localpath, remoteurl, page_data=None):
   page_data = page_data or {}
   content = page_data.get('content')
   if not content:
+    logging.error('console_handler(\'%s\', \'%s\', \'%s\'): cannot get site '
+                  'from local path' % (
+                      unquoted_localpath, remoteurl, page_data))
     return page_data
 
   # Decode content from utf-8 to unicode, replacing bad characters.
   content = content.decode('utf-8', 'replace')
 
   # Scrub in sheriff file content to console.
   sheriff_files = [
     'sheriff',
     'sheriff_android',
     'sheriff_cr_cros_gardeners',
@@ skipping 111 matching lines @@
   md = re.search(
       r'^(.*)</body>.*$',
       page_data['content'],
       re.MULTILINE|re.DOTALL)
   if not md:
     raise Exception('failed to locate trailing body and html tags')
   page_data['content'] = md.group(1)
 
   return page_data
 
-
-def get_or_create_row(localpath, revision):
-  return Row.get_or_insert(
-    key_name=revision + ' '+ localpath,
-    rev_number=revision,
-    localpath=localpath,
-    fetch_timestamp=datetime.datetime.now())
-
-
-def save_row(row_data, localpath, timestamp):
-  rev_number = row_data['rev_number']
-  row = get_or_create_row(localpath, rev_number)
-  row_key = row.key()
-  def tx_row(row_key):
-    row = Row.get(row_key)
-    # E1103:959,7:save_row.tx_row: Instance of 'list' has no
-    # 'fetch_timestamp' member (but some types could not be inferred)
-    # pylint: disable=E1103
-    # if row.fetch_timestamp > timestamp:
-    #   return
-    row.fetch_timestamp = timestamp
-    row.revision = row_data['rev']
-    row.name = row_data['name']
-    row.status = row_data['status']
-    row.comment = row_data['comment']
-    row.details = row_data['details']
-    # E1103:967,4:save_row.tx_row: Instance of 'list' has no 'put' member
-    # (but some types could not be inferred)
-    # pylint: disable=E1103
-    row.put()
-  db.run_in_transaction(tx_row, row_key)
-  memcache_data = json.dumps(row_data)
-  # A row should never be large enough to hit the blobstore, so we
-  # explicitly don't handle rows larger than 10^6 bytes.
-  if not memcache.set(key=str(row_key), value=memcache_data, time=2*60):
-    logging.error('save_row(\'%s\'): memcache.set() failed' % (row_key))
-
-
+# W0613:600,28:parse_master: Unused argument 'remoteurl'
+# pylint: disable=W0613
 def parse_master(localpath, remoteurl, page_data=None):
   """Part of the new pipeline to store individual rows rather than
   whole pages of html. Parses the master data into a set of rows,
   and writes them out to the datastore in an easily retrievable format.
 
-  Returns the same page_data as it was passed, so as to not interrupt
-  the current pipeline. This may change when we switch over.
+  Doesn't modify page_data dict.
   """
   ts = datetime.datetime.now()
   page_data = page_data or {}
   content = page_data.get('content')
   if not content:
     return page_data
   content = content.decode('utf-8', 'replace')
 
   # Split page into surroundings (announce, legend, footer) and data (rows).
   surroundings = BeautifulSoup(content)
   data = surroundings.find('table', 'ConsoleData')
-  data.extract()
+  new_data = Tag(surroundings, 'table', [('class', 'ConsoleData'),
+                                         ('width', '96%')])
+  data.replaceWith(new_data)
 
-  surroundings_page = get_or_create_page(localpath + '/surroundings',
+  surroundings_page = get_or_create_page('surroundings',
                                          None, maxage=30)
   surroundings_data = {}
-  surroundings_data['title'] = 'Surroundings for ' + localpath
+  surroundings_data['title'] = 'Surroundings'
   surroundings_data['content'] = unicode(surroundings)
-  save_page(surroundings_page, localpath + '/surroundings', ts,
+  save_page(surroundings_page, 'surroundings', ts,
             surroundings_data)
 
-  rows = data.tbody.findAll('tr', recursive=False)
+  rows = data.findAll('tr', recursive=False)
   # The first table row can be special: the list of categories.
   categories = None
   # If the first row contains a DevStatus cell...
   if rows[0].find('td', 'DevStatus') != None:
     # ...extract it into the categories...
     categories = rows[0]
     # ...and get rid of the next (spacer) row too.
     rows = rows[2:]
 
   if categories:
@@ skipping 20 matching lines @@
   # or a spacer row (in which case we finalize the row and save it).
   for row in rows:
     if row.find('td', 'DevComment'):
       curr_row['comment'] = unicode(row)
     elif row.find('td', 'DevDetails'):
       curr_row['details'] = unicode(row)
     elif row.find('td', 'DevStatus'):
       curr_row['rev'] = unicode(row.find('td', 'DevRev'))
       curr_row['rev_number'] = unicode(row.find('td', 'DevRev').a.string)
       curr_row['name'] = unicode(row.find('td', 'DevName'))
-      curr_row['status'] = unicode(row.findAll('td', 'DevStatus'))
+      curr_row['status'] = unicode(row.findAll('table'))
     else:
       if 'details' not in curr_row:
         curr_row['details'] = ''
-      save_row(curr_row, localpath, ts)
+      save_row(curr_row, localpath + '/' + curr_row['rev_number'], ts)
       curr_row = {}
 
   return page_data
 
 
 def one_box_handler(unquoted_localpath, remoteurl, page_data=None):
   page_data = page_data or {}
   content = page_data.get('content')
   if content is None:
     return page_data
   # Get the site name from the local path.
   md = re.match('^([^\/]+)/.*$', unquoted_localpath)
   if not md:
     logging.error('one_box_handler(\'%s\', \'%s\', \'%s\'): cannot get site '
                   'from local path' % (
                       unquoted_localpath, remoteurl, page_data))
     return page_data
   site = md.group(1)
   new_waterfall_url = 'http://build.chromium.org/p/%s/waterfall' % site
   page_data['content'] = re.sub(
       r'waterfall',
       new_waterfall_url,
       page_data['content'])
   return page_data
 
 
+##########
+# Utility functions for blobstore and memcache.
+##########
+def get_data_from_cache(localpath):
+  memcache_data = memcache.get(localpath)
+  if not memcache_data:
+    return None
+  logging.debug('content for %s found in memcache' % localpath)
+  return json.loads(memcache_data)
+
+
+def put_data_into_cache(localpath, data):
+  memcache_data = json.dumps(data)
+  if not memcache.set(key=localpath, value=memcache_data, time=2*60):
+    logging.error('put_data_into_cache(\'%s\'): memcache.set() failed' % (
+        localpath))
+
+
+def write_blob(data, mime_type):
+  """Saves a Unicode string as a new blob, returns the blob's key."""
+  file_name = files.blobstore.create(mime_type=mime_type)
+  data = data.encode('utf-8')
+  with files.open(file_name, 'a') as blob_file:
+    blob_file.write(data)
+  files.finalize(file_name)
+  return files.blobstore.get_blob_key(file_name)
+
+
+def path_to_mime_type(path):
+  return EXT_TO_MIME.get(os.path.splitext(path)[1], 'text/html')
+
+
+EXT_TO_MIME = {
+  '.css': 'text/css',
+  '.js': 'text/javascript',
+  '.json': 'application/json',
+  '.html': 'text/html',
+}
+
+
+##########
+# Functions for actually fetching original pages.
+##########
+def fetch_pages():
+  """Starts a background fetch operation for pages that need it."""
+  logging.debug('fetch_pages()')
+  for url in URLS:
+    deferred.defer(fetch_page, **url)
+
+
+def nonfatal_fetch_url(url, *args, **kwargs):
+  # Temporary workaround to disable AppEngine global cache of these pages.
+  if '?' in url:
+    url += '&' + str(random.random())
+  else:
+    url += '?' + str(random.random())
+
+  try:
+    return urlfetch.fetch(url, deadline=URLFETCH_DEADLINE, *args, **kwargs)
+  except urlfetch.DownloadError:
+    logging.warn('urlfetch failed: %s' % url, exc_info=1)
+    return None
+
+
+def fetch_page(localpath, remoteurl, maxage, postfetch=None, postsave=None,
+               fetch_url=nonfatal_fetch_url):
+  """Fetches data about a set of pages."""
+  if type(localpath) != type(''):
+    logging.error('fetch_page: localpath is %r, expected a string' % (
+        repr(localpath)))
+    return
+  unquoted_localpath = urllib.unquote(localpath)
+  logging.debug('fetch_page("%s", "%s", "%s")' % (
+      unquoted_localpath, remoteurl, maxage))
+  page = get_or_create_page(unquoted_localpath, remoteurl, maxage)
+
+  # Check if our copy of the page is younger than maxage.  If it is, we'll
+  # skip the fetch.
+  oldest_acceptable_timestamp = datetime.datetime.now() - datetime.timedelta(
+      seconds=maxage)
+  if (page.fetch_timestamp and
+      page.fetch_timestamp > oldest_acceptable_timestamp):
+    logging.debug('fetch_page: too recent, skipping')
+    return
+
+  # Perform the actual page fetch.
+  fetch_timestamp = datetime.datetime.now()
+  response = fetch_url(remoteurl)
+  if not response:
+    logging.warning('fetch_page: got empty response')
+    return
+  if response.status_code != 200:
+    logging.warning('fetch_page: got non-empty response but code '
+                    '%d' % response.status_code)
+    return
+
+  # We have actual content.  If there's one or more handlers, call them.
+  page_data = {}
+  page_data['content'] = response.content
+  if postfetch:
+    if not isinstance(postfetch, list):
+      postfetch = [postfetch]
+    for handler in postfetch:
+      logging.debug('fetch_page: calling postfetch handler '
+                    '%s' % handler.__name__)
+      page_data = handler(unquoted_localpath, remoteurl, page_data)
+
+  # Save the returned content into the DB and caching layers.
+  logging.debug('fetch_page: saving page')
+  save_page(page, unquoted_localpath, fetch_timestamp, page_data)
+  if postsave:
+    if not isinstance(postsave, list):
+      postsave = [postsave]
+    for handler in postsave:
+      logging.debug('fetch_page: calling postsave handler '
+                    '%s' % handler.__name__)
+      handler(unquoted_localpath, remoteurl, page_data)
+
 
 # List of URLs to fetch.
 URLS = [
   # Console URLs.
   {
     'remoteurl': 'http://build.chromium.org/p/chromium.chrome/console',
     'localpath': 'chromium.chrome/console',
     'postfetch': [console_handler, parse_master],
     'postsave': console_merger,
     'maxage': 30,  # 30 secs
@@ skipping 303 matching lines @@
   # LKGR JSON.
   {
     'remoteurl':
         ('http://build.chromium.org/p/chromium.lkgr/json/builders/Linux%20x64/'
          'builds/-1?as_text=1'),
     'localpath':
         'chromium.lkgr/json/builders/Linux%20x64/builds/-1/as_text=1.json',
     'maxage': 2*60,  # 2 mins
   },
 ]
-
-
-def nonfatal_fetch_url(url, *args, **kwargs):
-  # Temporary workaround to disable AppEngine global cache of these pages.
-  if '?' in url:
-    url += '&' + str(random.random())
-  else:
-    url += '?' + str(random.random())
-
-  try:
-    return urlfetch.fetch(url, deadline=URLFETCH_DEADLINE, *args, **kwargs)
-  except urlfetch.DownloadError:
-    logging.warn('urlfetch failed: %s' % url, exc_info=1)
-    return None
-
-
-class Row(db.Model):
-  fetch_timestamp = db.DateTimeProperty(required=True)
-  rev_number = db.StringProperty(required=True)
-  localpath = db.StringProperty(required=True)
-  revision = db.TextProperty()
-  name = db.TextProperty()
-  status = db.TextProperty()
-  comment = db.TextProperty()
-  details = db.TextProperty()
-
-
-class Page(db.Model):
-  fetch_timestamp = db.DateTimeProperty(required=True)
-  localpath = db.StringProperty(required=True)
-  content = db.TextProperty()
-  title = db.StringProperty()
-  offsite_base = db.StringProperty()
-  body_class = db.StringProperty()
-  remoteurl = db.TextProperty()
-  # Data updated separately, after creation.
-  content_blob = blobstore.BlobReferenceProperty()
-
-
-def write_blob(data, mime_type):
-  """Saves a Unicode string as a new blob, returns the blob's key."""
-  file_name = files.blobstore.create(mime_type=mime_type)
-  data = data.encode('utf-8')
-  with files.open(file_name, 'a') as blob_file:
-    blob_file.write(data)
-  files.finalize(file_name)
-  return files.blobstore.get_blob_key(file_name)
-
-
-def save_page(page, localpath, fetch_timestamp, page_data):
-  body_class = page_data.get('body_class', '')
-  content = page_data.get('content')
-  offsite_base = page_data.get('offsite_base', '')
-  title = page_data.get('title', '')
-
-  content_blob_key = None
-  try:
-    content = content.decode('utf-8', 'replace')
-  except UnicodeEncodeError:
-    logging.debug('save_page: content was already in unicode')
-  logging.debug('save_page: content size is %d' % len(content))
-  if len(content.encode('utf-8')) >= 10**6:
-    logging.debug('save_page: saving to blob')
-    content_blob_key = write_blob(content, path_to_mime_type(localpath))
-    content = None
-  def tx_page(page_key):
-    page = Page.get(page_key)
-    # E1103:225,7:fetch_page.tx_page: Instance of 'list' has no
-    # 'fetch_timestamp' member (but some types could not be inferred)
-    # pylint: disable=E1103
-    if page.fetch_timestamp > fetch_timestamp:
-      return
-    page.content = content
-    page.content_blob = content_blob_key
-    page.fetch_timestamp = fetch_timestamp
-    # title, offsite_base, body_class can all be empty strings for some
-    # content.  Where that's true, they're not used for displaying a console-
-    # like resource, and the content alone is returned to the web user.
-    page.title = title
-    page.offsite_base = offsite_base
-    page.body_class = body_class
-    # E1103:231,4:fetch_page.tx_page: Instance of 'list' has no 'put' member
-    # (but some types could not be inferred)
-    # pylint: disable=E1103
-    page.put()
-  db.run_in_transaction(tx_page, page.key())
-  page_data = {
-    'body_class': body_class,
-    'content': content,
-    'offsite_base': offsite_base,
-    'title': title,
-  }
-  if content_blob_key:
-    page_data['content_blob'] = True
-  put_pagedata_into_cache(localpath, page_data)
-
-
-def get_or_create_page(localpath, remoteurl, maxage):
-  return Page.get_or_insert(
-    key_name=localpath,
-    localpath=localpath,
-    remoteurl=remoteurl,
-    maxage=maxage,
-    fetch_timestamp=datetime.datetime.now() - datetime.timedelta(hours=24),
-    content=None,
-    content_blob=None)
-
-
-def fetch_page(localpath, remoteurl, maxage, postfetch=None, postsave=None,
-               fetch_url=nonfatal_fetch_url):
-  """Fetches data about a set of pages."""
-  if type(localpath) != type(''):
-    logging.error('fetch_page: localpath is %r, expected a string' % (
-        repr(localpath)))
-    return
-  unquoted_localpath = urllib.unquote(localpath)
-  logging.debug('fetch_page("%s", "%s", "%s")' % (
-      unquoted_localpath, remoteurl, maxage))
-  page = get_or_create_page(unquoted_localpath, remoteurl, maxage)
-
-  # Check if our copy of the page is younger than maxage.  If it is, we'll
-  # skip the fetch.
-  oldest_acceptable_timestamp = datetime.datetime.now() - datetime.timedelta(
-      seconds=maxage)
-  if (page.fetch_timestamp and
-      page.fetch_timestamp > oldest_acceptable_timestamp):
-    logging.debug('fetch_page: too recent, skipping')
-    return
-
-  # Perform the actual page fetch.
-  fetch_timestamp = datetime.datetime.now()
-  response = fetch_url(remoteurl)
-  if not response:
-    logging.warning('fetch_page: got empty response')
-    return
-  if response.status_code != 200:
-    logging.warning('fetch_page: got non-empty response but code '
-                    '%d' % response.status_code)
-    return
-
-  # We have actual content.  If there's one or more handlers, call them.
-  page_data = {}
-  page_data['content'] = response.content
-  if postfetch:
-    if not isinstance(postfetch, list):
-      postfetch = [postfetch]
-    for handler in postfetch:
-      logging.debug('fetch_page: calling postfetch handler '
-                    '%s' % handler.__name__)
-      page_data = handler(unquoted_localpath, remoteurl, page_data)
-
-  # Save the returned content into the DB and caching layers.
-  logging.debug('fetch_page: saving page')
-  save_page(page, unquoted_localpath, fetch_timestamp, page_data)
-  if postsave:
-    if not isinstance(postsave, list):
-      postsave = [postsave]
-    for handler in postsave:
-      logging.debug('fetch_page: calling postsave handler '
-                    '%s' % handler.__name__)
-      handler(unquoted_localpath, remoteurl, page_data)
-
-
-EXT_TO_MIME = {
-  '.css': 'text/css',
-  '.js': 'text/javascript',
-  '.json': 'application/json',
-  '.html': 'text/html',
-}
-
-
-def path_to_mime_type(path):
-  return EXT_TO_MIME.get(os.path.splitext(path)[1], 'text/html')
-
-
-def fetch_pages():
-  """Starts a background fetch operation for pages that need it."""
-  logging.debug('fetch_pages()')
-  for url in URLS:
-    deferred.defer(fetch_page, **url)