Added gsutil/boto to depot_tools/third_party

third_party/boto/boto/manage/volume.py

Index: third_party/boto/boto/manage/volume.py
diff --git a/third_party/boto/boto/manage/volume.py b/third_party/boto/boto/manage/volume.py
new file mode 100644
index 0000000000000000000000000000000000000000..49237d47eed2c324edb651bdf08a569394fb43cd
--- /dev/null
+++ b/third_party/boto/boto/manage/volume.py
@@ -0,0 +1,420 @@
+# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+from __future__ import with_statement
+from boto.sdb.db.model import Model
+from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty
+from boto.manage.server import Server
+from boto.manage import propget
+import boto.utils
+import boto.ec2
+import time
+import traceback
+from contextlib import closing
+import datetime
+
+
+class CommandLineGetter(object):
+    
+    def get_region(self, params):
+        if not params.get('region', None):
+            prop = self.cls.find_property('region_name')
+            params['region'] = propget.get(prop, choices=boto.ec2.regions)
+
+    def get_zone(self, params):
+        if not params.get('zone', None):
+            prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone',
+                                  choices=self.ec2.get_all_zones)
+            params['zone'] = propget.get(prop)
+            
+    def get_name(self, params):
+        if not params.get('name', None):
+            prop = self.cls.find_property('name')
+            params['name'] = propget.get(prop)
+
+    def get_size(self, params):
+        if not params.get('size', None):
+            prop = IntegerProperty(name='size', verbose_name='Size (GB)')
+            params['size'] = propget.get(prop)
+
+    def get_mount_point(self, params):
+        if not params.get('mount_point', None):
+            prop = self.cls.find_property('mount_point')
+            params['mount_point'] = propget.get(prop)
+
+    def get_device(self, params):
+        if not params.get('device', None):
+            prop = self.cls.find_property('device')
+            params['device'] = propget.get(prop)
+
+    def get(self, cls, params):
+        self.cls = cls
+        self.get_region(params)
+        self.ec2 = params['region'].connect()
+        self.get_zone(params)
+        self.get_name(params)
+        self.get_size(params)
+        self.get_mount_point(params)
+        self.get_device(params)
+
+class Volume(Model):
+
+    name = StringProperty(required=True, unique=True, verbose_name='Name')
+    region_name = StringProperty(required=True, verbose_name='EC2 Region')
+    zone_name = StringProperty(required=True, verbose_name='EC2 Zone')
+    mount_point = StringProperty(verbose_name='Mount Point')
+    device = StringProperty(verbose_name="Device Name", default='/dev/sdp')
+    volume_id = StringProperty(required=True)
+    past_volume_ids = ListProperty(item_type=str)
+    server = ReferenceProperty(Server, collection_name='volumes',
+                               verbose_name='Server Attached To')
+    volume_state = CalculatedProperty(verbose_name="Volume State",
+                                      calculated_type=str, use_method=True)
+    attachment_state = CalculatedProperty(verbose_name="Attachment State",
+                                          calculated_type=str, use_method=True)
+    size = CalculatedProperty(verbose_name="Size (GB)",
+                              calculated_type=int, use_method=True)
+
+    @classmethod
+    def create(cls, **params):
+        getter = CommandLineGetter()
+        getter.get(cls, params)
+        region = params.get('region')
+        ec2 = region.connect()
+        zone = params.get('zone')
+        size = params.get('size')
+        ebs_volume = ec2.create_volume(size, zone.name)
+        v = cls()
+        v.ec2 = ec2
+        v.volume_id = ebs_volume.id
+        v.name = params.get('name')
+        v.mount_point = params.get('mount_point')
+        v.device = params.get('device')
+        v.region_name = region.name
+        v.zone_name = zone.name
+        v.put()
+        return v
+
+    @classmethod
+    def create_from_volume_id(cls, region_name, volume_id, name):
+        vol = None
+        ec2 = boto.ec2.connect_to_region(region_name)
+        rs = ec2.get_all_volumes([volume_id])
+        if len(rs) == 1:
+            v = rs[0]
+            vol = cls()
+            vol.volume_id = v.id
+            vol.name = name
+            vol.region_name = v.region.name
+            vol.zone_name = v.zone
+            vol.put()
+        return vol
+
+    def create_from_latest_snapshot(self, name, size=None):
+        snapshot = self.get_snapshots()[-1]
+        return self.create_from_snapshot(name, snapshot, size)
+
+    def create_from_snapshot(self, name, snapshot, size=None):
+        if size < self.size:
+            size = self.size
+        ec2 = self.get_ec2_connection()
+        if self.zone_name == None or self.zone_name == '':
+            # deal with the migration case where the zone is not set in the logical volume:
+            current_volume = ec2.get_all_volumes([self.volume_id])[0]
+            self.zone_name = current_volume.zone
+        ebs_volume = ec2.create_volume(size, self.zone_name, snapshot)
+        v = Volume()
+        v.ec2 = self.ec2
+        v.volume_id = ebs_volume.id
+        v.name = name
+        v.mount_point = self.mount_point
+        v.device = self.device
+        v.region_name = self.region_name
+        v.zone_name = self.zone_name
+        v.put()
+        return v
+    
+    def get_ec2_connection(self):
+        if self.server:
+            return self.server.ec2
+        if not hasattr(self, 'ec2') or self.ec2 == None:
+            self.ec2 = boto.ec2.connect_to_region(self.region_name)
+        return self.ec2
+
+    def _volume_state(self):
+        ec2 = self.get_ec2_connection()
+        rs = ec2.get_all_volumes([self.volume_id])
+        return rs[0].volume_state()
+
+    def _attachment_state(self):
+        ec2 = self.get_ec2_connection()
+        rs = ec2.get_all_volumes([self.volume_id])
+        return rs[0].attachment_state()
+
+    def _size(self):
+        if not hasattr(self, '__size'):
+            ec2 = self.get_ec2_connection()
+            rs = ec2.get_all_volumes([self.volume_id])
+            self.__size = rs[0].size
+        return self.__size
+
+    def install_xfs(self):
+        if self.server:
+            self.server.install('xfsprogs xfsdump')
+
+    def get_snapshots(self):
+        """
+        Returns a list of all completed snapshots for this volume ID.
+        """
+        ec2 = self.get_ec2_connection()
+        rs = ec2.get_all_snapshots()
+        all_vols = [self.volume_id] + self.past_volume_ids
+        snaps = []
+        for snapshot in rs:
+            if snapshot.volume_id in all_vols:
+                if snapshot.progress == '100%':
+                    snapshot.date = boto.utils.parse_ts(snapshot.start_time)
+                    snapshot.keep = True
+                    snaps.append(snapshot)
+        snaps.sort(cmp=lambda x, y: cmp(x.date, y.date))
+        return snaps
+
+    def attach(self, server=None):
+        if self.attachment_state == 'attached':
+            print 'already attached'
+            return None
+        if server:
+            self.server = server
+            self.put()
+        ec2 = self.get_ec2_connection()
+        ec2.attach_volume(self.volume_id, self.server.instance_id, self.device)
+
+    def detach(self, force=False):
+        state = self.attachment_state
+        if state == 'available' or state == None or state == 'detaching':
+            print 'already detached'
+            return None
+        ec2 = self.get_ec2_connection()
+        ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force)
+        self.server = None
+        self.put()
+
+    def checkfs(self, use_cmd=None):
+        if self.server == None:
+            raise ValueError('server attribute must be set to run this command')
+        # detemine state of file system on volume, only works if attached
+        if use_cmd:
+            cmd = use_cmd
+        else:
+            cmd = self.server.get_cmdshell()
+        status = cmd.run('xfs_check %s' % self.device)
+        if not use_cmd:
+            cmd.close()
+        if status[1].startswith('bad superblock magic number 0'):
+            return False
+        return True
+
+    def wait(self):
+        if self.server == None:
+            raise ValueError('server attribute must be set to run this command')
+        with closing(self.server.get_cmdshell()) as cmd:
+            # wait for the volume device to appear
+            cmd = self.server.get_cmdshell()
+            while not cmd.exists(self.device):
+                boto.log.info('%s still does not exist, waiting 10 seconds' % self.device)
+                time.sleep(10)
+
+    def format(self):
+        if self.server == None:
+            raise ValueError('server attribute must be set to run this command')
+        status = None
+        with closing(self.server.get_cmdshell()) as cmd:
+            if not self.checkfs(cmd):
+                boto.log.info('make_fs...')
+                status = cmd.run('mkfs -t xfs %s' % self.device)
+        return status
+
+    def mount(self):
+        if self.server == None:
+            raise ValueError('server attribute must be set to run this command')
+        boto.log.info('handle_mount_point')
+        with closing(self.server.get_cmdshell()) as cmd:
+            cmd = self.server.get_cmdshell()
+            if not cmd.isdir(self.mount_point):
+                boto.log.info('making directory')
+                # mount directory doesn't exist so create it
+                cmd.run("mkdir %s" % self.mount_point)
+            else:
+                boto.log.info('directory exists already')
+                status = cmd.run('mount -l')
+                lines = status[1].split('\n')
+                for line in lines:
+                    t = line.split()
+                    if t and t[2] == self.mount_point:
+                        # something is already mounted at the mount point
+                        # unmount that and mount it as /tmp
+                        if t[0] != self.device:
+                            cmd.run('umount %s' % self.mount_point)
+                            cmd.run('mount %s /tmp' % t[0])
+                            cmd.run('chmod 777 /tmp')
+                            break
+            # Mount up our new EBS volume onto mount_point
+            cmd.run("mount %s %s" % (self.device, self.mount_point))
+            cmd.run('xfs_growfs %s' % self.mount_point)
+
+    def make_ready(self, server):
+        self.server = server
+        self.put()
+        self.install_xfs()
+        self.attach()
+        self.wait()
+        self.format()
+        self.mount()
+
+    def freeze(self):
+        if self.server:
+            return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point)
+
+    def unfreeze(self):
+        if self.server:
+            return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point)
+
+    def snapshot(self):
+        # if this volume is attached to a server
+        # we need to freeze the XFS file system
+        try:
+            self.freeze()
+            if self.server == None:
+                snapshot = self.get_ec2_connection().create_snapshot(self.volume_id)
+            else:
+                snapshot = self.server.ec2.create_snapshot(self.volume_id)
+            boto.log.info('Snapshot of Volume %s created: %s' %  (self.name, snapshot))
+        except Exception:
+            boto.log.info('Snapshot error')
+            boto.log.info(traceback.format_exc())
+        finally:
+            status = self.unfreeze()
+            return status
+
+    def get_snapshot_range(self, snaps, start_date=None, end_date=None):
+        l = []
+        for snap in snaps:
+            if start_date and end_date:
+                if snap.date >= start_date and snap.date <= end_date:
+                    l.append(snap)
+            elif start_date:
+                if snap.date >= start_date:
+                    l.append(snap)
+            elif end_date:
+                if snap.date <= end_date:
+                    l.append(snap)
+            else:
+                l.append(snap)
+        return l
+
+    def trim_snapshots(self, delete=False):
+        """
+        Trim the number of snapshots for this volume.  This method always
+        keeps the oldest snapshot.  It then uses the parameters passed in
+        to determine how many others should be kept.
+
+        The algorithm is to keep all snapshots from the current day.  Then
+        it will keep the first snapshot of the day for the previous seven days.
+        Then, it will keep the first snapshot of the week for the previous
+        four weeks.  After than, it will keep the first snapshot of the month
+        for as many months as there are.
+
+        """
+        snaps = self.get_snapshots()
+        # Always keep the oldest and the newest
+        if len(snaps) <= 2:
+            return snaps
+        snaps = snaps[1:-1]
+        now = datetime.datetime.now(snaps[0].date.tzinfo)
+        midnight = datetime.datetime(year=now.year, month=now.month,
+                                     day=now.day, tzinfo=now.tzinfo)
+        # Keep the first snapshot from each day of the previous week
+        one_week = datetime.timedelta(days=7, seconds=60*60)
+        print midnight-one_week, midnight
+        previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight)
+        print previous_week
+        if not previous_week:
+            return snaps
+        current_day = None
+        for snap in previous_week:
+            if current_day and current_day == snap.date.day:
+                snap.keep = False
+            else:
+                current_day = snap.date.day
+        # Get ourselves onto the next full week boundary
+        if previous_week:
+            week_boundary = previous_week[0].date
+            if week_boundary.weekday() != 0:
+                delta = datetime.timedelta(days=week_boundary.weekday())
+                week_boundary = week_boundary - delta
+        # Keep one within this partial week
+        partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date)
+        if len(partial_week) > 1:
+            for snap in partial_week[1:]:
+                snap.keep = False
+        # Keep the first snapshot of each week for the previous 4 weeks
+        for i in range(0, 4):
+            weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary)
+            if len(weeks_worth) > 1:
+                for snap in weeks_worth[1:]:
+                    snap.keep = False
+            week_boundary = week_boundary - one_week
+        # Now look through all remaining snaps and keep one per month
+        remainder = self.get_snapshot_range(snaps, end_date=week_boundary)
+        current_month = None
+        for snap in remainder:
+            if current_month and current_month == snap.date.month:
+                snap.keep = False
+            else:
+                current_month = snap.date.month
+        if delete:
+            for snap in snaps:
+                if not snap.keep:
+                    boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name))
+                    snap.delete()
+        return snaps
+                
+    def grow(self, size):
+        pass
+
+    def copy(self, snapshot):
+        pass
+
+    def get_snapshot_from_date(self, date):
+        pass
+
+    def delete(self, delete_ebs_volume=False):
+        if delete_ebs_volume:
+            self.detach()
+            ec2 = self.get_ec2_connection()
+            ec2.delete_volume(self.volume_id)
+        Model.delete(self)
+
+    def archive(self):
+        # snapshot volume, trim snaps, delete volume-id
+        pass
+    
+