| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import logging | 5 import logging |
| 6 import traceback | 6 import traceback |
| 7 | 7 |
| 8 from app_yaml_helper import AppYamlHelper | 8 from app_yaml_helper import AppYamlHelper |
| 9 from appengine_wrappers import ( | 9 from appengine_wrappers import ( |
| 10 GetAppVersion, IsDeadlineExceededError, logservice) | 10 GetAppVersion, IsDeadlineExceededError, logservice) |
| 11 from branch_utility import BranchUtility | 11 from branch_utility import BranchUtility |
| 12 from compiled_file_system import CompiledFileSystem | 12 from compiled_file_system import CompiledFileSystem |
| 13 from data_source_registry import CreateDataSources | 13 from data_source_registry import CreateDataSources |
| 14 from empty_dir_file_system import EmptyDirFileSystem | 14 from empty_dir_file_system import EmptyDirFileSystem |
| 15 from environment import IsDevServer | 15 from environment import IsDevServer |
| 16 from extensions_paths import EXAMPLES, PUBLIC_TEMPLATES, SERVER2, STATIC_DOCS |
| 16 from file_system_util import CreateURLsFromPaths | 17 from file_system_util import CreateURLsFromPaths |
| 17 from future import Gettable, Future | 18 from future import Gettable, Future |
| 18 from github_file_system_provider import GithubFileSystemProvider | 19 from github_file_system_provider import GithubFileSystemProvider |
| 19 from host_file_system_provider import HostFileSystemProvider | 20 from host_file_system_provider import HostFileSystemProvider |
| 20 from object_store_creator import ObjectStoreCreator | 21 from object_store_creator import ObjectStoreCreator |
| 21 from render_servlet import RenderServlet | 22 from render_servlet import RenderServlet |
| 22 from server_instance import ServerInstance | 23 from server_instance import ServerInstance |
| 23 from servlet import Servlet, Request, Response | 24 from servlet import Servlet, Request, Response |
| 24 import svn_constants | |
| 25 from timer import Timer, TimerClosure | 25 from timer import Timer, TimerClosure |
| 26 | 26 |
| 27 |
| 27 class _SingletonRenderServletDelegate(RenderServlet.Delegate): | 28 class _SingletonRenderServletDelegate(RenderServlet.Delegate): |
| 28 def __init__(self, server_instance): | 29 def __init__(self, server_instance): |
| 29 self._server_instance = server_instance | 30 self._server_instance = server_instance |
| 30 | 31 |
| 31 def CreateServerInstance(self): | 32 def CreateServerInstance(self): |
| 32 return self._server_instance | 33 return self._server_instance |
| 33 | 34 |
| 34 class _CronLogger(object): | 35 class _CronLogger(object): |
| 35 '''Wraps the logging.* methods to prefix them with 'cron' and flush | 36 '''Wraps the logging.* methods to prefix them with 'cron' and flush |
| 36 immediately. The flushing is important because often these cron runs time | 37 immediately. The flushing is important because often these cron runs time |
| (...skipping 144 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 181 title = 'initializing %s parallel Cron targets' % len(targets) | 182 title = 'initializing %s parallel Cron targets' % len(targets) |
| 182 _cronlog.info(title) | 183 _cronlog.info(title) |
| 183 timer = Timer() | 184 timer = Timer() |
| 184 try: | 185 try: |
| 185 cron_futures = [run_cron_for_future(target) for target in targets] | 186 cron_futures = [run_cron_for_future(target) for target in targets] |
| 186 finally: | 187 finally: |
| 187 _cronlog.info('%s took %s' % (title, timer.Stop().FormatElapsed())) | 188 _cronlog.info('%s took %s' % (title, timer.Stop().FormatElapsed())) |
| 188 | 189 |
| 189 # Rendering the public templates will also pull in all of the private | 190 # Rendering the public templates will also pull in all of the private |
| 190 # templates. | 191 # templates. |
| 191 results.append(request_files_in_dir(svn_constants.PUBLIC_TEMPLATE_PATH)) | 192 results.append(request_files_in_dir(PUBLIC_TEMPLATES)) |
| 192 | 193 |
| 193 # Rendering the public templates will have pulled in the .js and | 194 # Rendering the public templates will have pulled in the .js and |
| 194 # manifest.json files (for listing examples on the API reference pages), | 195 # manifest.json files (for listing examples on the API reference pages), |
| 195 # but there are still images, CSS, etc. | 196 # but there are still images, CSS, etc. |
| 196 results.append(request_files_in_dir(svn_constants.STATIC_PATH, | 197 results.append(request_files_in_dir(STATIC_DOCS, prefix='static')) |
| 197 prefix='static/')) | |
| 198 | 198 |
| 199 # Samples are too expensive to run on the dev server, where there is no | 199 # Samples are too expensive to run on the dev server, where there is no |
| 200 # parallel fetch. | 200 # parallel fetch. |
| 201 if not IsDevServer(): | 201 if not IsDevServer(): |
| 202 # Fetch each individual sample file. | 202 # Fetch each individual sample file. |
| 203 results.append(request_files_in_dir(svn_constants.EXAMPLES_PATH, | 203 results.append(request_files_in_dir(EXAMPLES, |
| 204 prefix='extensions/examples/')) | 204 prefix='extensions/examples')) |
| 205 | 205 |
| 206 # Fetch the zip file of each example (contains all the individual | 206 # Fetch the zip file of each example (contains all the individual |
| 207 # files). | 207 # files). |
| 208 example_zips = [] | 208 example_zips = [] |
| 209 for root, _, files in trunk_fs.Walk(svn_constants.EXAMPLES_PATH): | 209 for root, _, files in trunk_fs.Walk(EXAMPLES): |
| 210 example_zips.extend( | 210 example_zips.extend( |
| 211 root + '.zip' for name in files if name == 'manifest.json') | 211 root + '.zip' for name in files if name == 'manifest.json') |
| 212 results.append(_RequestEachItem( | 212 results.append(_RequestEachItem( |
| 213 'example zips', | 213 'example zips', |
| 214 example_zips, | 214 example_zips, |
| 215 lambda path: render('extensions/examples/' + path))) | 215 lambda path: render('extensions/examples/' + path))) |
| 216 | 216 |
| 217 # Resolve the hand-written Cron method futures. | 217 # Resolve the hand-written Cron method futures. |
| 218 title = 'resolving %s parallel Cron targets' % len(targets) | 218 title = 'resolving %s parallel Cron targets' % len(targets) |
| 219 _cronlog.info(title) | 219 _cronlog.info(title) |
| (...skipping 25 matching lines...) Expand all Loading... |
| 245 | 245 |
| 246 # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not | 246 # IMPORTANT: Get a ServerInstance pinned to the most recent revision, not |
| 247 # HEAD. These cron jobs take a while and run very frequently such that | 247 # HEAD. These cron jobs take a while and run very frequently such that |
| 248 # there is usually one running at any given time, and eventually a file | 248 # there is usually one running at any given time, and eventually a file |
| 249 # that we're dealing with will change underneath it, putting the server in | 249 # that we're dealing with will change underneath it, putting the server in |
| 250 # an undefined state. | 250 # an undefined state. |
| 251 server_instance_near_head = self._CreateServerInstance( | 251 server_instance_near_head = self._CreateServerInstance( |
| 252 self._GetMostRecentRevision()) | 252 self._GetMostRecentRevision()) |
| 253 | 253 |
| 254 app_yaml_handler = AppYamlHelper( | 254 app_yaml_handler = AppYamlHelper( |
| 255 svn_constants.APP_YAML_PATH, | |
| 256 server_instance_near_head.object_store_creator, | 255 server_instance_near_head.object_store_creator, |
| 257 server_instance_near_head.host_file_system_provider) | 256 server_instance_near_head.host_file_system_provider) |
| 258 | 257 |
| 259 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): | 258 if app_yaml_handler.IsUpToDate(delegate.GetAppVersion()): |
| 260 return server_instance_near_head | 259 return server_instance_near_head |
| 261 | 260 |
| 262 # The version in app.yaml is greater than the currently running app's. | 261 # The version in app.yaml is greater than the currently running app's. |
| 263 # The safe version is the one before it changed. | 262 # The safe version is the one before it changed. |
| 264 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( | 263 safe_revision = app_yaml_handler.GetFirstRevisionGreaterThan( |
| 265 delegate.GetAppVersion()) - 1 | 264 delegate.GetAppVersion()) - 1 |
| 266 | 265 |
| 267 _cronlog.info('app version %s is out of date, safe is %s', | 266 _cronlog.info('app version %s is out of date, safe is %s', |
| 268 delegate.GetAppVersion(), safe_revision) | 267 delegate.GetAppVersion(), safe_revision) |
| 269 | 268 |
| 270 return self._CreateServerInstance(safe_revision) | 269 return self._CreateServerInstance(safe_revision) |
| 271 | 270 |
| 272 def _GetMostRecentRevision(self): | 271 def _GetMostRecentRevision(self): |
| 273 '''Gets the revision of the most recent patch submitted to the host file | 272 '''Gets the revision of the most recent patch submitted to the host file |
| 274 system. This is similar to HEAD but it's a concrete revision so won't | 273 system. This is similar to HEAD but it's a concrete revision so won't |
| 275 change as the cron runs. | 274 change as the cron runs. |
| 276 ''' | 275 ''' |
| 277 head_fs = ( | 276 head_fs = ( |
| 278 self._CreateServerInstance(None).host_file_system_provider.GetTrunk()) | 277 self._CreateServerInstance(None).host_file_system_provider.GetTrunk()) |
| 279 return head_fs.Stat('/').version | 278 return head_fs.Stat('').version |
| 280 | 279 |
| 281 def _CreateServerInstance(self, revision): | 280 def _CreateServerInstance(self, revision): |
| 282 '''Creates a ServerInstance pinned to |revision|, or HEAD if None. | 281 '''Creates a ServerInstance pinned to |revision|, or HEAD if None. |
| 283 NOTE: If passed None it's likely that during the cron run patches will be | 282 NOTE: If passed None it's likely that during the cron run patches will be |
| 284 submitted at HEAD, which may change data underneath the cron run. | 283 submitted at HEAD, which may change data underneath the cron run. |
| 285 ''' | 284 ''' |
| 286 object_store_creator = ObjectStoreCreator(start_empty=True) | 285 object_store_creator = ObjectStoreCreator(start_empty=True) |
| 287 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) | 286 branch_utility = self._delegate.CreateBranchUtility(object_store_creator) |
| 288 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( | 287 host_file_system_provider = self._delegate.CreateHostFileSystemProvider( |
| 289 object_store_creator, max_trunk_revision=revision) | 288 object_store_creator, max_trunk_revision=revision) |
| 290 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( | 289 github_file_system_provider = self._delegate.CreateGithubFileSystemProvider( |
| 291 object_store_creator) | 290 object_store_creator) |
| 292 return ServerInstance(object_store_creator, | 291 return ServerInstance(object_store_creator, |
| 293 CompiledFileSystem.Factory(object_store_creator), | 292 CompiledFileSystem.Factory(object_store_creator), |
| 294 branch_utility, | 293 branch_utility, |
| 295 host_file_system_provider, | 294 host_file_system_provider, |
| 296 github_file_system_provider) | 295 github_file_system_provider) |
| OLD | NEW |