Added gsutil to depot tools

third_party/gsutil/gslib/commands/chacl.py

Index: third_party/gsutil/gslib/commands/chacl.py
diff --git a/third_party/gsutil/gslib/commands/chacl.py b/third_party/gsutil/gslib/commands/chacl.py
new file mode 100644
index 0000000000000000000000000000000000000000..25d8d3e25fd4be83bcc707b7672c5064527bba26
--- /dev/null
+++ b/third_party/gsutil/gslib/commands/chacl.py
@@ -0,0 +1,523 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This module provides the chacl command to gsutil.
+
+This command allows users to easily specify changes to access control lists.
+"""
+
+import random
+import re
+import time
+from xml.dom import minidom
+from boto.exception import GSResponseError
+from boto.gs import acl
+from gslib import name_expansion
+from gslib.command import Command
+from gslib.command import COMMAND_NAME
+from gslib.command import COMMAND_NAME_ALIASES
+from gslib.command import CONFIG_REQUIRED
+from gslib.command import FILE_URIS_OK
+from gslib.command import MAX_ARGS
+from gslib.command import MIN_ARGS
+from gslib.command import PROVIDER_URIS_OK
+from gslib.command import SUPPORTED_SUB_ARGS
+from gslib.command import URIS_START_ARG
+from gslib.exception import CommandException
+from gslib.help_provider import HELP_NAME
+from gslib.help_provider import HELP_NAME_ALIASES
+from gslib.help_provider import HELP_ONE_LINE_SUMMARY
+from gslib.help_provider import HELP_TEXT
+from gslib.help_provider import HELP_TYPE
+from gslib.help_provider import HelpType
+from gslib.util import NO_MAX
+from gslib.util import Retry
+
+
+class ChangeType(object):
+  USER = 'User'
+  GROUP = 'Group'
+
+
+class AclChange(object):
+  """Represents a logical change to an access control list."""
+  public_scopes = ['AllAuthenticatedUsers', 'AllUsers']
+  id_scopes = ['UserById', 'GroupById']
+  email_scopes = ['UserByEmail', 'GroupByEmail']
+  domain_scopes = ['GroupByDomain']
+  scope_types = public_scopes + id_scopes + email_scopes + domain_scopes
+
+  permission_shorthand_mapping = {
+      'R': 'READ',
+      'W': 'WRITE',
+      'FC': 'FULL_CONTROL',
+      }
+
+  def __init__(self, acl_change_descriptor, scope_type, logger):
+    """Creates an AclChange object.
+
+    acl_change_descriptor: An acl change as described in chacl help.
+    scope_type: Either ChangeType.USER or ChangeType.GROUP, specifying the
+                extent of the scope.
+    logger: An instance of ThreadedLogger.
+    """
+    self.logger = logger
+    self.identifier = ''
+
+    self.raw_descriptor = acl_change_descriptor
+    self._Parse(acl_change_descriptor, scope_type)
+    self._Validate()
+
+  def __str__(self):
+    return 'AclChange<{0}|{1}|{2}>'.format(self.scope_type, self.perm,
+                                        self.identifier)
+
+  def _Parse(self, change_descriptor, scope_type):
+    """Parses an ACL Change descriptor."""
+
+    def _ClassifyScopeIdentifier(text):
+      re_map = {
+          'AllAuthenticatedUsers': r'^(AllAuthenticatedUsers|AllAuth)$',
+          'AllUsers': '^(AllUsers|All)$',
+          'Email': r'^.+@.+\..+$',
+          'Id': r'^[0-9A-Fa-f]{64}$',
+          'Domain': r'^[^@]+\..+$',
+          }
+      for type_string, regex in re_map.items():
+        if re.match(regex, text, re.IGNORECASE):
+          return type_string
+
+    if change_descriptor.count(':') != 1:
+      raise CommandException('{0} is an invalid change description.'
+                             .format(change_descriptor))
+
+    scope_string, perm_token = change_descriptor.split(':')
+
+    perm_token = perm_token.upper()
+    if perm_token in self.permission_shorthand_mapping:
+      self.perm = self.permission_shorthand_mapping[perm_token]
+    else:
+      self.perm = perm_token
+
+    scope_class = _ClassifyScopeIdentifier(scope_string)
+    if scope_class == 'Domain':
+      # This may produce an invalid UserByDomain scope,
+      # which is good because then validate can complain.
+      self.scope_type = '{0}ByDomain'.format(scope_type)
+      self.identifier = scope_string
+    elif scope_class in ['Email', 'Id']:
+      self.scope_type = '{0}By{1}'.format(scope_type, scope_class)
+      self.identifier = scope_string
+    elif scope_class == 'AllAuthenticatedUsers':
+      self.scope_type = 'AllAuthenticatedUsers'
+    elif scope_class == 'AllUsers':
+      self.scope_type = 'AllUsers'
+    else:
+      # This is just a fallback, so we set it to something
+      # and the validate step has something to go on.
+      self.scope_type = scope_string
+
+  def _Validate(self):
+    """Validates a parsed AclChange object."""
+
+    def _ThrowError(msg):
+      raise CommandException('{0} is not a valid ACL change\n{1}'
+                             .format(self.raw_descriptor, msg))
+
+    if self.scope_type not in self.scope_types:
+      _ThrowError('{0} is not a valid scope type'.format(self.scope_type))
+
+    if self.scope_type in self.public_scopes and self.identifier:
+      _ThrowError('{0} requires no arguments'.format(self.scope_type))
+
+    if self.scope_type in self.id_scopes and not self.identifier:
+      _ThrowError('{0} requires an id'.format(self.scope_type))
+
+    if self.scope_type in self.email_scopes and not self.identifier:
+      _ThrowError('{0} requires an email address'.format(self.scope_type))
+
+    if self.scope_type in self.domain_scopes and not self.identifier:
+      _ThrowError('{0} requires domain'.format(self.scope_type))
+
+    if self.perm not in self.permission_shorthand_mapping.values():
+      perms = ', '.join(self.permission_shorthand_mapping.values())
+      _ThrowError('Allowed permissions are {0}'.format(perms))
+
+  def _YieldMatchingEntries(self, current_acl):
+    """Generator that yields entries that match the change descriptor.
+
+    current_acl: An instance of bogo.gs.acl.ACL which will be searched
+                 for matching entries.
+    """
+    for entry in current_acl.entries.entry_list:
+      if entry.scope.type == self.scope_type:
+        if self.scope_type in ['UserById', 'GroupById']:
+          if self.identifier == entry.scope.id:
+            yield entry
+        elif self.scope_type in ['UserByEmail', 'GroupByEmail']:
+          if self.identifier == entry.scope.email_address:
+            yield entry
+        elif self.scope_type == 'GroupByDomain':
+          if self.identifier == entry.scope.domain:
+            yield entry
+        elif self.scope_type in ['AllUsers', 'AllAuthenticatedUsers']:
+          yield entry
+        else:
+          raise CommandException('Found an unrecognized ACL '
+                                 'entry type, aborting.')
+
+  def _AddEntry(self, current_acl):
+    """Adds an entry to an ACL."""
+    if self.scope_type in ['UserById', 'UserById', 'GroupById']:
+      entry = acl.Entry(type=self.scope_type, permission=self.perm,
+                        id=self.identifier)
+    elif self.scope_type in ['UserByEmail', 'GroupByEmail']:
+      entry = acl.Entry(type=self.scope_type, permission=self.perm,
+                        email_address=self.identifier)
+    elif self.scope_type == 'GroupByDomain':
+      entry = acl.Entry(type=self.scope_type, permission=self.perm,
+                        domain=self.identifier)
+    else:
+      entry = acl.Entry(type=self.scope_type, permission=self.perm)
+
+    current_acl.entries.entry_list.append(entry)
+
+  def Execute(self, uri, current_acl):
+    """Executes the described change on an ACL.
+
+    uri: The URI object to change.
+    current_acl: An instance of boto.gs.acl.ACL to permute.
+    """
+    self.logger.debug('Executing {0} on {1}'
+                      .format(self.raw_descriptor, uri))
+
+    if self.perm == 'WRITE' and uri.names_object():
+      self.logger.warn(
+          'Skipping {0} on {1}, as WRITE does not apply to objects'
+          .format(self.raw_descriptor, uri))
+      return 0
+
+    matching_entries = list(self._YieldMatchingEntries(current_acl))
+    change_count = 0
+    if matching_entries:
+      for entry in matching_entries:
+        if entry.permission != self.perm:
+          entry.permission = self.perm
+          change_count += 1
+    else:
+      self._AddEntry(current_acl)
+      change_count = 1
+
+    parsed_acl = minidom.parseString(current_acl.to_xml())
+    self.logger.debug('New Acl:\n{0}'.format(parsed_acl.toprettyxml()))
+    return change_count
+
+
+class AclDel(AclChange):
+  """Represents a logical change from an access control list."""
+  scope_regexes = {
+      r'All(Users)?': 'AllUsers',
+      r'AllAuth(enticatedUsers)?': 'AllAuthenticatedUsers',
+      }
+
+  def __init__(self, identifier, logger):
+    self.raw_descriptor = '-d {0}'.format(identifier)
+    self.logger = logger
+    self.identifier = identifier
+    for regex, scope in self.scope_regexes.items():
+      if re.match(regex, self.identifier, re.IGNORECASE):
+        self.identifier = scope
+    self.scope_type = 'Any'
+    self.perm = 'NONE'
+
+  def _YieldMatchingEntries(self, current_acl):
+    for entry in current_acl.entries.entry_list:
+      if self.identifier == entry.scope.id:
+        yield entry
+      elif self.identifier == entry.scope.email_address:
+        yield entry
+      elif self.identifier == entry.scope.domain:
+        yield entry
+      elif self.identifier == 'AllUsers' and entry.scope.type == 'AllUsers':
+        yield entry
+      elif (self.identifier == 'AllAuthenticatedUsers'
+            and entry.scope.type == 'AllAuthenticatedUsers'):
+        yield entry
+
+  def Execute(self, uri, current_acl):
+    self.logger.debug('Executing {0} on {1}'
+                      .format(self.raw_descriptor, uri))
+    matching_entries = list(self._YieldMatchingEntries(current_acl))
+    for entry in matching_entries:
+      current_acl.entries.entry_list.remove(entry)
+    parsed_acl = minidom.parseString(current_acl.to_xml())
+    self.logger.debug('New Acl:\n{0}'.format(parsed_acl.toprettyxml()))
+    return len(matching_entries)
+
+
+_detailed_help_text = ("""
+<B>SYNOPSIS</B>
+  gsutil chacl [-R] -u|-g|-d <grant>... uri...
+
+  where each <grant> is one of the following forms:
+  -u <id|email>:<perm>
+  -g <id|email|domain|All|AllAuth>:<perm>
+  -d <id|email|domain|All|AllAuth>
+
+<B>DESCRIPTION</B>
+  The chacl command updates access control lists, similar in spirit to the Linux
+  chmod command. You can specify multiple access grant additions and deletions
+  in a single command run; all changes will be made atomically to each object in
+  turn. For example, if the command requests deleting one grant and adding a
+  different grant, the ACLs being updated will never be left in an intermediate
+  state where one grant has been deleted but the second grant not yet added.
+  Each change specifies a user or group grant to add or delete, and for grant
+  additions, one of R, W, FC (for the permission to be granted). A more formal
+  description is provided in a later section; below we provide examples.
+
+  Note: If you want to set a simple "canned" ACL on each object (such as
+  project-private or public), or if you prefer to edit the XML representation
+  for ACLs, you can do that with the setacl command (see 'gsutil help setacl').
+
+
+<B>EXAMPLES</B>
+
+  Grant the user john.doe@example.com WRITE access to the bucket
+  example-bucket:
+
+    gsutil chacl -u john.doe@example.com:WRITE gs://example-bucket
+
+  Grant the group admins@example.com FULL_CONTROL access to all jpg files in
+  the top level of example-bucket:
+
+    gsutil chacl -g admins@example.com:FC gs://example-bucket/*.jpg
+
+  Grant the user with the specified canonical ID READ access to all objects in
+  example-bucket that begin with folder/:
+
+    gsutil chacl -R \\
+      -u 84fac329bceSAMPLE777d5d22b8SAMPLE77d85ac2SAMPLE2dfcf7c4adf34da46:R \\
+      gs://example-bucket/folder/
+
+  Grant all users from my-domain.org READ access to the bucket
+  gcs.my-domain.org:
+
+    gsutil chacl -g my-domain.org:R gs://gcs.my-domain.org
+
+  Remove any current access by john.doe@example.com from the bucket
+  example-bucket:
+
+    gsutil chacl -d john.doe@example.com gs://example-bucket
+
+  If you have a large number of objects to update, enabling multi-threading with
+  the gsutil -m flag can significantly improve performance. The following
+  command adds FULL_CONTROL for admin@example.org using multi-threading:
+
+    gsutil -m chacl -R -u admin@example.org:FC gs://example-bucket
+
+  Grant READ access to everyone from my-domain.org and to all authenticated
+  users, and grant FULL_CONTROL to admin@mydomain.org, for the buckets
+  my-bucket and my-other-bucket, with multi-threading enabled:
+
+    gsutil -m chacl -R -g my-domain.org:R -g AllAuth:R \\
+      -u admin@mydomain.org:FC gs://my-bucket/ gs://my-other-bucket
+
+
+<B>SCOPES</B>
+  There are four different scopes: Users, Groups, All Authenticated Users, and
+  All Users.
+
+  Users are added with -u and a plain ID or email address, as in
+  "-u john-doe@gmail.com:r"
+
+  Groups are like users, but specified with the -g flag, as in
+  "-g power-users@example.com:fc". Groups may also be specified as a full
+  domain, as in "-g my-company.com:r".
+
+  AllAuthenticatedUsers and AllUsers are specified directly, as
+  in "-g AllUsers:R" or "-g AllAuthenticatedUsers:FC". These are case
+  insensitive, and may be shortened to "all" and "allauth", respectively.
+
+  Removing permissions is specified with the -d flag and an ID, email
+  address, domain, or one of AllUsers or AllAuthenticatedUsers.
+
+  Many scopes can be specified on the same command line, allowing bundled
+  changes to be executed in a single run. This will reduce the number of
+  requests made to the server.
+
+
+<B>PERMISSIONS</B>
+  You may specify the following permissions with either their shorthand or
+  their full name:
+
+  R: READ
+  W: WRITE
+  FC: FULL_CONTROL
+
+
+<B>OPTIONS</B>
+  -R, -r      Performs chacl request recursively, to all objects under the
+              specified URI.
+
+  -u          Add or modify a user permission as specified in the SCOPES
+              and PERMISSIONS sections.
+
+  -g          Add or modify a group permission as specified in the SCOPES
+              and PERMISSIONS sections.
+
+  -d          Remove all permissions associated with the matching argument, as
+              specified in the SCOPES and PERMISSIONS sections.
+""")
+
+
+class ChAclCommand(Command):
+  """Implementation of gsutil chacl command."""
+
+  # Command specification (processed by parent class).
+  command_spec = {
+    # Name of command.
+    COMMAND_NAME : 'chacl',
+    # List of command name aliases.
+    COMMAND_NAME_ALIASES : [],
+    # Min number of args required by this command.
+    MIN_ARGS : 1,
+    # Max number of args required by this command, or NO_MAX.
+    MAX_ARGS : NO_MAX,
+    # Getopt-style string specifying acceptable sub args.
+    SUPPORTED_SUB_ARGS : 'Rrfg:u:d:',
+    # True if file URIs acceptable for this command.
+    FILE_URIS_OK : False,
+    # True if provider-only URIs acceptable for this command.
+    PROVIDER_URIS_OK : False,
+    # Index in args of first URI arg.
+    URIS_START_ARG : 1,
+    # True if must configure gsutil before running command.
+    CONFIG_REQUIRED : True,
+  }
+  help_spec = {
+    # Name of command or auxiliary help info for which this help applies.
+    HELP_NAME : 'chacl',
+    # List of help name aliases.
+    HELP_NAME_ALIASES : ['chmod'],
+    # Type of help:
+    HELP_TYPE : HelpType.COMMAND_HELP,
+    # One line summary of this help.
+    HELP_ONE_LINE_SUMMARY : 'Add / remove entries on bucket and/or object ACLs',
+    # The full help text.
+    HELP_TEXT : _detailed_help_text,
+  }
+
+  # Command entry point.
+  def RunCommand(self):
+    """This is the point of entry for the chacl command."""
+    self.parse_versions = True
+    self.changes = []
+
+    if self.sub_opts:
+      for o, a in self.sub_opts:
+        if o == '-g':
+          self.changes.append(AclChange(a, scope_type=ChangeType.GROUP,
+                                        logger=self.THREADED_LOGGER))
+        if o == '-u':
+          self.changes.append(AclChange(a, scope_type=ChangeType.USER,
+                                        logger=self.THREADED_LOGGER))
+        if o == '-d':
+          self.changes.append(AclDel(a, logger=self.THREADED_LOGGER))
+
+    if not self.changes:
+      raise CommandException(
+          'Please specify at least one access change '
+          'with the -g, -u, or -d flags')
+
+    storage_uri = self.UrisAreForSingleProvider(self.args)
+    if not (storage_uri and storage_uri.get_provider().name == 'google'):
+      raise CommandException('The "{0}" command can only be used with gs:// URIs'
+                             .format(self.command_name))
+
+    bulk_uris = set()
+    for uri_arg in self.args:
+      for result in self.WildcardIterator(uri_arg):
+        uri = result.uri
+        if uri.names_bucket():
+          if self.recursion_requested:
+            bulk_uris.add(uri.clone_replace_name('*').uri)
+          else:
+            # If applying to a bucket directly, the threading machinery will
+            # break, so we have to apply now, in the main thread.
+            self.ApplyAclChanges(uri)
+        else:
+          bulk_uris.add(uri_arg)
+
+    try:
+      name_expansion_iterator = name_expansion.NameExpansionIterator(
+          self.command_name, self.proj_id_handler, self.headers, self.debug,
+          self.bucket_storage_uri_class, bulk_uris, self.recursion_requested)
+    except CommandException as e:
+      # NameExpansionIterator will complain if there are no URIs, but we don't
+      # want to throw an error if we handled bucket URIs.
+      if e.reason == 'No URIs matched':
+        return 0
+      else:
+        raise e
+
+    self.everything_set_okay = True
+    self.Apply(self.ApplyAclChanges,
+               name_expansion_iterator,
+               self._ApplyExceptionHandler)
+    if not self.everything_set_okay:
+      raise CommandException('ACLs for some objects could not be set.')
+
+    return 0
+
+  def _ApplyExceptionHandler(self, exception):
+    self.THREADED_LOGGER.error('Encountered a problem: {0}'.format(exception))
+    self.everything_set_okay = False
+
+  @Retry(GSResponseError, tries=3, delay=1, backoff=2)
+  def ApplyAclChanges(self, uri_or_expansion_result):
+    """Applies the changes in self.changes to the provided URI."""
+    if isinstance(uri_or_expansion_result, name_expansion.NameExpansionResult):
+      uri = self.suri_builder.StorageUri(
+          uri_or_expansion_result.expanded_uri_str)
+    else:
+      uri = uri_or_expansion_result
+
+    try:
+      current_acl = uri.get_acl()
+    except GSResponseError as e:
+      self.THREADED_LOGGER.warning('Failed to set acl for {0}: {1}'
+                                   .format(uri, e.reason))
+      return
+
+    modification_count = 0
+    for change in self.changes:
+      modification_count += change.Execute(uri, current_acl)
+    if modification_count == 0:
+      self.THREADED_LOGGER.info('No changes to {0}'.format(uri))
+      return
+
+    # TODO: Remove the concept of forcing when boto provides access to
+    # bucket generation and meta_generation.
+    headers = dict(self.headers)
+    force = uri.names_bucket()
+    if not force:
+      key = uri.get_key()
+      headers['x-goog-if-generation-match'] = key.generation
+      headers['x-goog-if-metageneration-match'] = key.meta_generation
+    
+    # If this fails because of a precondition, it will raise a 
+    # GSResponseError for @Retry to handle.
+    uri.set_acl(current_acl, uri.object_name, False, headers)
+    self.THREADED_LOGGER.info('Updated ACL on {0}'.format(uri))
+