Put latest_rev data in datastore instead of memcache.

app.py

 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 from __future__ import with_statement
 
 import datetime
 import json
 import logging
 import os
@@ skipping 194 matching lines @@
 
   get_and_cache_rowdata takes a localpath which is used to fetch data from the
   cache. If the data is present, then we have all of the data we need and we
   return early.
 
   Otherwise, we need to fetch the row object and set up the row data.
 
   Here we assume localpath is already unquoted.
   """
   row_data = get_data_from_cache(localpath)
-  if row_data:
+  if row_data and type(row_data) == type({}):

[Ryan Tseng, 2013/01/15 22:05:32]

In what cases would it not be a dict?

[agable, 2013/01/15 22:09:41]

The latest_rev row used to be stored in memcache as a plain int. I want to make
sure it never picks up that int and tries to use it instead.

     return row_data
   row = Row.all().filter('localpath =', localpath).get()

[Ryan Tseng, 2013/01/15 22:05:32]

How many entires can we expect this to return?

[agable, 2013/01/15 22:09:41]

Exactly one. It is being replaced by row = Row.get_by_key_name(localpath) in a
different CL.

   if not row:
     logging.error('get_and_cache_rowdata(\'%s\'): no matching localpath in '
         'datastore' % localpath)
     return {}
   row_data = {}
   row_data['rev'] = row.revision
   row_data['name'] = row.name
   row_data['status'] = row.status
   row_data['comment'] = row.comment
   row_data['details'] = row.details
@@ skipping 19 matching lines @@
     row.revision = row_data['rev']
     row.name = row_data['name']
     row.status = row_data['status']
     row.comment = row_data['comment']
     row.details = row_data['details']
     # E1103:967,4:save_row.tx_row: Instance of 'list' has no 'put' member
     # (but some types could not be inferred)
     # pylint: disable=E1103
     row.put()
   db.run_in_transaction(tx_row, row_key)
-  prev_rev = memcache.get(key='latest_rev')
-  if (rev_number > prev_rev):
-    memcache.set(key='latest_rev', value=rev_number)
   put_data_into_cache(localpath, row_data)
   logging.info('Saved and cached row with localpath %s' % localpath)
+  # Update latest_rev in datastore & cache, or create it if it doesn't exist.
+  prev_rev = get_and_cache_rowdata('latest_rev')
+  if not prev_rev or rev_number > prev_rev['rev_number']:
+    latest_rev_row = {
+        'rev_number': rev_number,
+        'fetch_timestamp': datetime.datetime.now(),
+        'rev': None,
+        'name': None,
+        'status': None,
+        'comment': None,
+        'details': None,
+    }
+    save_row(latest_rev_row, 'latest_rev')
 
 
 ##########
 # ConsoleData class definition and related functions.
 ##########
 class ConsoleData(object):
   def __init__(self):
     self.row_orderedkeys = []
     self.row_data = {}
 
@@ skipping 109 matching lines @@
 # AKA postfetch and postsave functions/handlers.
 ##########
 def console_merger(localpath, remoteurl, page_data,
                    masters_to_merge=None, num_rows_to_merge=None):
   masters_to_merge = masters_to_merge or DEFAULT_MASTERS_TO_MERGE
   num_rows_to_merge = num_rows_to_merge or 25
   mergedconsole = ConsoleData()
   surroundings = get_and_cache_pagedata('surroundings')
   merged_page = BeautifulSoup(surroundings['content'])
   merged_tag = merged_page.find('table', 'ConsoleData')
-  latest_rev = int(memcache.get(key='latest_rev'))
+  latest_rev = int(get_and_cache_rowdata('latest_rev')['rev_number'])
   if not latest_rev:
     logging.error('console_merger(\'%s\', \'%s\', \'%s\'): cannot get latest '
                   'revision number.' % (
                       localpath, remoteurl, page_data))
     return
   fetch_timestamp = datetime.datetime.now()
   for master in masters_to_merge:
     # Fetch the summary one-box-per-builder for the master.
     # If we don't get it, something is wrong, skip the master entirely.
     master_summary = get_and_cache_pagedata('%s/console/summary' % master)
@@ skipping 772 matching lines @@
   # LKGR JSON.
   {
     'remoteurl':
         ('http://build.chromium.org/p/chromium.lkgr/json/builders/Linux%20x64/'
          'builds/-1?as_text=1'),
     'localpath':
         'chromium.lkgr/json/builders/Linux%20x64/builds/-1/as_text=1.json',
     'maxage': 2*60,  # 2 mins
   },
 ]