Extensions Docs Server: Doc conversion script - SVN

chrome/common/extensions/docs/server2/converter.py

+#!/usr/bin/env python
+# Copyright (c) 2012 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Example run from the server2/ directory:
+# $ converter.py ../static/ ../../api templates/articles/ templates/intros/
+#   templates/public/ static/images/ -r
+
+import json
+import optparse
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+sys.path.append(os.path.join(sys.path[0], os.pardir, os.pardir, os.pardir,
+    os.pardir, os.pardir, 'tools'))
+import json_comment_eater
+
+from converter_html_parser import HandleDocFamily
+from docs_server_utils import SanitizeAPIName
+
+IGNORED_FILES = [
+  # These are custom files.
+  '404',
+  'apps_api_index',
+  'api_index',
+  'experimental',
+  'samples',
+  'index',
+  # These are APIs that should not have docs.
+  'test',
+  'experimental_idltest',
+]
+
+# These are mappings for APIs that have no intros. They are needed because the
+# names of the JSON files do not give enough information on the actual API name.
+CUSTOM_MAPPINGS = {
+  'experimental_input_virtual_keyboard': 'experimental_input_virtualKeyboard',
+  'input_ime': 'input_ime'
+}
+
+# These are the extension-only APIs that don't have explicit entries in
+# _permission_features.json, so we can't programmatically figure out that it's
+# extension-only.
+# From api_page_generator.js:548.
+EXTENSIONS_ONLY = [
+  'browserAction',
+  'extension',
+  'input.ime',
+  'omnibox',
+  'pageAction',
+  'scriptBadge',
+  'windows',
+  'experimental.devtools.audits',
+  'experimental.devtools.console',
+  'experimental.discovery',
+  'experimental.infobars',
+  'experimental.offscreenTabs',
+  'experimental.processes',
+  'experimental.speechInput'
+]
+
+def _ReadFile(filename):
+  with open(filename, 'r') as f:
+    return f.read()
+
+def _WriteFile(filename, data):
+  with open(filename, 'w+') as f:
+    f.write(data)
+
+def _UnixName(name):
+  """Returns the unix_style name for a given lowerCamelCase string.
+  Shamelessly stolen from json_schema_compiler/model.py.
+  """
+  name = os.path.splitext(name)[0]
+  s1 = re.sub('([a-z])([A-Z])', r'\1_\2', name)
+  s2 = re.sub('([A-Z]+)([A-Z][a-z])', r'\1_\2', s1)
+  return s2.replace('.', '_').lower()
+
+def _GetDestinations(api_name, api_dir):
+  if api_name in EXTENSIONS_ONLY:
+    return ['extensions']
+  if api_name.startswith('app'):
+    return ['apps']
+  with open(os.path.join(api_dir, '_permission_features.json')) as f:
+    permissions = json.loads(json_comment_eater.Nom(f.read()))
+  permissions_key = api_name.replace('experimental_', '').replace('_', '.')
+  if permissions_key in permissions:
+    return_list = []
+    types = permissions[permissions_key]['extension_types']
+    if ('packaged_app' in types or
+        'hosted_app' in types or
+        'platform_app' in types):
+      return_list.append('apps')
+    if 'extension' in types:
+      return_list.append('extensions')
+    return return_list
+  return ['extensions', 'apps']
+
+def _ListAllAPIs(dirname):
+  all_files = []
+  for path, dirs, files in os.walk(dirname):
+    if path == '.':
+      all_files.extend([f for f in files
+                        if f.endswith('.json') or f.endswith('.idl')])
+    else:
+      all_files.extend([os.path.join(path, f) for f in files
+                        if f.endswith('.json') or f.endswith('.idl')])
+  dirname = dirname.rstrip('/') + '/'
+  return [f[len(dirname):] for f in all_files
+          if os.path.splitext(f)[0].split('_')[-1] not in ['private',
+                                                           'internal']]
+
+def _MakeArticleTemplate(filename, path):
+  return ('{{+partials.standard_%s_article article:intros.%s}}' %
+          (path, filename))
+
+def _MakeAPITemplate(intro_name, path, api_name, has_intro):
+  if has_intro:
+    return ('{{+partials.standard_%s_api api:apis.%s intro:intros.%s}}' %
+            (path, api_name, intro_name))
+  else:
+    return ('{{+partials.standard_%s_api api:apis.%s}}' %
+            (path, api_name))
+
+def _GetAPIPath(name, api_dir):
+  api_files = _ListAllAPIs(api_dir)
+  for filename in api_files:
+    if name == _UnixName(SanitizeAPIName(filename)):
+      return _UnixName(filename)
+
+def _CleanAPIs(source_dir, api_dir, intros_dest, template_dest, exclude):
+  source_files = os.listdir(source_dir)
+  source_files_unix = set(_UnixName(f) for f in source_files)
+  api_files = set(_UnixName(SanitizeAPIName(f)) for f in _ListAllAPIs(api_dir))
+  intros_files = set(_UnixName(f) for f in os.listdir(intros_dest))
+  to_delete = api_files - source_files_unix - set(exclude)
+  for path, dirs, files in os.walk(template_dest):
+    for filename in files:
+      no_ext = os.path.splitext(filename)[0]
+      # Check for changes like appWindow -> app.window.
+      if (_UnixName(filename) in source_files_unix - set(exclude) and
+          no_ext not in source_files):
+        os.remove(os.path.join(path, filename))
+      if _UnixName(filename) in to_delete:
+        try:
+          os.remove(os.path.join(intros_dest, filename))
+        except OSError:
+          pass
+        os.remove(os.path.join(path, filename))
+
+def _FormatFile(contents, path, name, image_dest, replace, is_api):
+  # Copy all images referenced in the page.
+  for image in re.findall(r'="\.\./images/([^"]*)"', contents):
+    if not replace and os.path.exists(os.path.join(image_dest, image)):
+      continue
+    if '/' in image:
+      try:
+        os.makedirs(os.path.join(image_dest, image.rsplit('/', 1)[0]))
+      except:
+        pass
+    shutil.copy(
+        os.path.join(path, os.pardir, 'images', image),
+        os.path.join(image_dest, image))
+  contents = re.sub(r'<!--.*?(BEGIN|END).*?-->', r'', contents)
+  contents = re.sub(r'\.\./images', r'{{static}}/images', contents)
+  exp = re.compile(r'<div[^>.]*?id="pageData-showTOC"[^>.]*?>.*?</div>',
+                   flags=re.DOTALL)
+  contents = re.sub(exp, r'', contents)
+  exp = re.compile(r'<div[^>.]*?id="pageData-name"[^>.]*?>(.*?)</div>',
+                   flags=re.DOTALL)
+  if is_api:
+    contents = re.sub(exp, r'', contents)
+  else:
+    contents = re.sub(exp, r'<h1>\1</h1>', contents)
+  contents = contents.strip()
+  contents = HandleDocFamily(contents)
+
+  # Attempt to guess if the page has no title.
+  if '<h1' not in contents and not is_api:
+    title = _UnixName(name)
+    title = ' '.join([part[0].upper() + part[1:] for part in title.split('_')])
+    contents = ('<h1 class="page_title">%s</h1>' % title) + contents
+  return contents
+
+def _ProcessName(name):
+  processed_name = []
+  if name.startswith('experimental_'):
+    name = name[len('experimental_'):]
+    processed_name.append('experimental_')
+  parts = name.split('_')
+  processed_name.append(parts[0])
+  processed_name.extend([p[0].upper() + p[1:] for p in parts[1:]])
+  return ''.join(processed_name)
+
+def _MoveAllFiles(source_dir,
+                  api_dir,
+                  articles_dest,
+                  intros_dest,
+                  template_dest,
+                  image_dest,
+                  replace=False,
+                  exclude_dir=None,
+                  svn=False):
+  if exclude_dir is None:
+    exclude_files = []
+  else:
+    exclude_files = [_UnixName(f) for f in os.listdir(exclude_dir)]
+  exclude_files.extend(IGNORED_FILES)
+  api_files = _ListAllAPIs(api_dir)
+  original_files = []
+  for path, dirs, files in os.walk(template_dest):
+    original_files.extend(files)
+  if replace:
+    _CleanAPIs(source_dir, api_dir, intros_dest, template_dest, exclude_files)
+  files = set(os.listdir(source_dir))
+  unix_files = [_UnixName(f) for f in files]
+  for name in  [SanitizeAPIName(f)  for f in _ListAllAPIs(api_dir)]:
+    if _UnixName(name) not in unix_files:
+      files.add(name + '.html')
+  for file_ in files:
+    if (_UnixName(file_) in exclude_files or
+        file_.startswith('.') or
+        file_.startswith('_')):
+      continue
+    _MoveSingleFile(source_dir,
+                    file_,
+                    api_dir,
+                    articles_dest,
+                    intros_dest,
+                    template_dest,
+                    image_dest,
+                    replace=replace,
+                    svn=svn,
+                    original_files=original_files)
+
+def _MoveSingleFile(source_dir,
+                    source_file,
+                    api_dir,
+                    articles_dest,
+                    intros_dest,
+                    template_dest,
+                    image_dest,
+                    replace=False,
+                    svn=False,
+                    original_files=None):
+  unix_name = _UnixName(source_file)
+  is_api = unix_name in [_UnixName(SanitizeAPIName(f))
+                         for f in _ListAllAPIs(api_dir)]
+  if unix_name in CUSTOM_MAPPINGS:
+    processed_name = CUSTOM_MAPPINGS[unix_name]
+  else:
+    processed_name = os.path.splitext(source_file)[0].replace('.', '_')
+    if (is_api and
+        '_' in source_file.replace('experimental_', '') and
+        not os.path.exists(os.path.join(source_dir, source_file))):
+      processed_name = _ProcessName(processed_name)
+      if (original_files is None or
+          processed_name + '.html' not in original_files):
+        print 'WARNING: The correct name of this file was guessed:'
+        print
+        print '%s -> %s' % (os.path.splitext(source_file)[0], processed_name)
+        print
+        print ('If this is incorrect, change |CUSTOM_MAPPINGS| in chrome/'
+               'common/extensions/docs/server2/converter.py.')
+  try:
+    static_data = _FormatFile(_ReadFile(os.path.join(source_dir, source_file)),
+                              source_dir,
+                              source_file,
+                              image_dest,
+                              replace,
+                              is_api)
+  except IOError:
+    static_data = None
+  for path in _GetDestinations(processed_name, api_dir):
+    template_file = os.path.join(template_dest, path, processed_name + '.html')
+    if is_api:
+      template_data = _MakeAPITemplate(processed_name,
+                                       path,
+                                       _GetAPIPath(unix_name, api_dir),
+                                       static_data is not None)
+      static_file = os.path.join(intros_dest, processed_name + '.html')
+    else:
+      template_data = _MakeArticleTemplate(unix_name, path)
+      static_file = os.path.join(articles_dest, processed_name + '.html')
+    if replace or not os.path.exists(template_file):
+      _WriteFile(template_file, template_data)
+      if svn:
+        subprocess.call(['svn',
+                         'add',
+                         '--force',
+                         '--parents',
+                         '-q',
+                         template_file])
+  if static_data is not None and (replace or not os.path.exists(static_file)):
+    if svn:
+      if os.path.exists(static_file):
+        subprocess.call(['svn', 'rm', '--force', static_file])
+      subprocess.call(['svn',
+                       'cp',
+                       os.path.join(source_dir, source_file),
+                       static_file])
+    _WriteFile(static_file, static_data)
+
+if __name__ == '__main__':
+  parser = optparse.OptionParser(
+      description='Converts static files from the old documentation system to '
+                  'the new one. If run without -f, all the files in |src| will '
+                  'be converted.',
+      usage='usage: %prog [options] static_src_dir [-f static_src_file] '
+            'api_dir articles_dest intros_dest template_dest image_dest')
+  parser.add_option('-f',
+                    '--file',
+                    action='store_true',
+                    default=False,
+                    help='convert single file')
+  parser.add_option('-s',
+                    '--svn',
+                    action='store_true',
+                    default=False,
+                    help='use svn copy to copy intro files')
+  parser.add_option('-e',
+                    '--exclude',
+                    default=None,
+                    help='exclude files matching the names in this dir')
+  parser.add_option('-r',
+                    '--replace',
+                    action='store_true',
+                    default=False,
+                    help='replace existing files')
+  (opts, args) = parser.parse_args()
+  if (not opts.file and len(args) != 6) or (opts.file and len(args) != 7):
+    parser.error('incorrect number of arguments.')
+
+  if opts.file:
+    _MoveSingleFile(*args, replace=opts.replace, svn=opts.svn)
+  else:
+    _MoveAllFiles(*args,
+                  replace=opts.replace,
+                  exclude_dir=opts.exclude,
+                  svn=opts.svn)