Add pylint to depot_tools.

third_party/logilab/common/textutils.py

+# copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
+# contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
+#
+# This file is part of logilab-common.
+#
+# logilab-common is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Lesser General Public License as published by the Free
+# Software Foundation, either version 2.1 of the License, or (at your option) any
+# later version.
+#
+# logilab-common is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Lesser General Public License along
+# with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
+"""Some text manipulation utility functions.
+
+
+:group text formatting: normalize_text, normalize_paragraph, pretty_match,\
+unquote, colorize_ansi
+:group text manipulation: searchall, splitstrip
+:sort: text formatting, text manipulation
+
+:type ANSI_STYLES: dict(str)
+:var ANSI_STYLES: dictionary mapping style identifier to ANSI terminal code
+
+:type ANSI_COLORS: dict(str)
+:var ANSI_COLORS: dictionary mapping color identifier to ANSI terminal code
+
+:type ANSI_PREFIX: str
+:var ANSI_PREFIX:
+  ANSI terminal code notifying the start of an ANSI escape sequence
+
+:type ANSI_END: str
+:var ANSI_END:
+  ANSI terminal code notifying the end of an ANSI escape sequence
+
+:type ANSI_RESET: str
+:var ANSI_RESET:
+  ANSI terminal code resetting format defined by a previous ANSI escape sequence
+"""
+__docformat__ = "restructuredtext en"
+
+import sys
+import re
+import os.path as osp
+from warnings import warn
+from unicodedata import normalize as _uninormalize
+try:
+    from os import linesep
+except ImportError:
+    linesep = '\n' # gae
+
+from logilab.common.deprecation import deprecated
+
+MANUAL_UNICODE_MAP = {
+    u'\xa1': u'!',    # INVERTED EXCLAMATION MARK
+    u'\u0142': u'l',  # LATIN SMALL LETTER L WITH STROKE
+    u'\u2044': u'/',  # FRACTION SLASH
+    u'\xc6': u'AE',   # LATIN CAPITAL LETTER AE
+    u'\xa9': u'(c)',  # COPYRIGHT SIGN
+    u'\xab': u'"',    # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK
+    u'\xe6': u'ae',   # LATIN SMALL LETTER AE
+    u'\xae': u'(r)',  # REGISTERED SIGN
+    u'\u0153': u'oe', # LATIN SMALL LIGATURE OE
+    u'\u0152': u'OE', # LATIN CAPITAL LIGATURE OE
+    u'\xd8': u'O',    # LATIN CAPITAL LETTER O WITH STROKE
+    u'\xf8': u'o',    # LATIN SMALL LETTER O WITH STROKE
+    u'\xbb': u'"',    # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK
+    u'\xdf': u'ss',   # LATIN SMALL LETTER SHARP S
+    }
+
+def unormalize(ustring, ignorenonascii=None, substitute=None):
+    """replace diacritical characters with their corresponding ascii characters
+
+    Convert the unicode string to its long normalized form (unicode character
+    will be transform into several characters) and keep the first one only.
+    The normal form KD (NFKD) will apply the compatibility decomposition, i.e.
+    replace all compatibility characters with their equivalents.
+
+    :type substitute: str
+    :param substitute: replacement character to use if decomposition fails
+
+    :see: Another project about ASCII transliterations of Unicode text
+          http://pypi.python.org/pypi/Unidecode
+    """
+    # backward compatibility, ignorenonascii was a boolean
+    if ignorenonascii is not None:
+        warn("ignorenonascii is deprecated, use substitute named parameter instead",
+             DeprecationWarning, stacklevel=2)
+        if ignorenonascii:
+            substitute = ''
+    res = []
+    for letter in ustring[:]:
+        try:
+            replacement = MANUAL_UNICODE_MAP[letter]
+        except KeyError:
+            replacement = _uninormalize('NFKD', letter)[0]
+            if ord(replacement) >= 2 ** 7:
+                if substitute is None:
+                    raise ValueError("can't deal with non-ascii based characters")
+                replacement = substitute
+        res.append(replacement)
+    return u''.join(res)
+
+def unquote(string):
+    """remove optional quotes (simple or double) from the string
+
+    :type string: str or unicode
+    :param string: an optionally quoted string
+
+    :rtype: str or unicode
+    :return: the unquoted string (or the input string if it wasn't quoted)
+    """
+    if not string:
+        return string
+    if string[0] in '"\'':
+        string = string[1:]
+    if string[-1] in '"\'':
+        string = string[:-1]
+    return string
+
+
+_BLANKLINES_RGX = re.compile('\r?\n\r?\n')
+_NORM_SPACES_RGX = re.compile('\s+')
+
+def normalize_text(text, line_len=80, indent='', rest=False):
+    """normalize a text to display it with a maximum line size and
+    optionally arbitrary indentation. Line jumps are normalized but blank
+    lines are kept. The indentation string may be used to insert a
+    comment (#) or a quoting (>) mark  for instance.
+
+    :type text: str or unicode
+    :param text: the input text to normalize
+
+    :type line_len: int
+    :param line_len: expected maximum line's length, default to 80
+
+    :type indent: str or unicode
+    :param indent: optional string to use as indentation
+
+    :rtype: str or unicode
+    :return:
+      the input text normalized to fit on lines with a maximized size
+      inferior to `line_len`, and optionally prefixed by an
+      indentation string
+    """
+    if rest:
+        normp = normalize_rest_paragraph
+    else:
+        normp = normalize_paragraph
+    result = []
+    for text in _BLANKLINES_RGX.split(text):
+        result.append(normp(text, line_len, indent))
+    return ('%s%s%s' % (linesep, indent, linesep)).join(result)
+
+
+def normalize_paragraph(text, line_len=80, indent=''):
+    """normalize a text to display it with a maximum line size and
+    optionally arbitrary indentation. Line jumps are normalized. The
+    indentation string may be used top insert a comment mark for
+    instance.
+
+    :type text: str or unicode
+    :param text: the input text to normalize
+
+    :type line_len: int
+    :param line_len: expected maximum line's length, default to 80
+
+    :type indent: str or unicode
+    :param indent: optional string to use as indentation
+
+    :rtype: str or unicode
+    :return:
+      the input text normalized to fit on lines with a maximized size
+      inferior to `line_len`, and optionally prefixed by an
+      indentation string
+    """
+    text = _NORM_SPACES_RGX.sub(' ', text)
+    line_len = line_len - len(indent)
+    lines = []
+    while text:
+        aline, text = splittext(text.strip(), line_len)
+        lines.append(indent + aline)
+    return linesep.join(lines)
+
+def normalize_rest_paragraph(text, line_len=80, indent=''):
+    """normalize a ReST text to display it with a maximum line size and
+    optionally arbitrary indentation. Line jumps are normalized. The
+    indentation string may be used top insert a comment mark for
+    instance.
+
+    :type text: str or unicode
+    :param text: the input text to normalize
+
+    :type line_len: int
+    :param line_len: expected maximum line's length, default to 80
+
+    :type indent: str or unicode
+    :param indent: optional string to use as indentation
+
+    :rtype: str or unicode
+    :return:
+      the input text normalized to fit on lines with a maximized size
+      inferior to `line_len`, and optionally prefixed by an
+      indentation string
+    """
+    toreport = ''
+    lines = []
+    line_len = line_len - len(indent)
+    for line in text.splitlines():
+        line = toreport + _NORM_SPACES_RGX.sub(' ', line.strip())
+        toreport = ''
+        while len(line) > line_len:
+            # too long line, need split
+            line, toreport = splittext(line, line_len)
+            lines.append(indent + line)
+            if toreport:
+                line = toreport + ' '
+                toreport = ''
+            else:
+                line = ''
+        if line:
+            lines.append(indent + line.strip())
+    return linesep.join(lines)
+
+
+def splittext(text, line_len):
+    """split the given text on space according to the given max line size
+
+    return a 2-uple:
+    * a line <= line_len if possible
+    * the rest of the text which has to be reported on another line
+    """
+    if len(text) <= line_len:
+        return text, ''
+    pos = min(len(text)-1, line_len)
+    while pos > 0 and text[pos] != ' ':
+        pos -= 1
+    if pos == 0:
+        pos = min(len(text), line_len)
+        while len(text) > pos and text[pos] != ' ':
+            pos += 1
+    return text[:pos], text[pos+1:].strip()
+
+
+def splitstrip(string, sep=','):
+    """return a list of stripped string by splitting the string given as
+    argument on `sep` (',' by default). Empty string are discarded.
+
+    >>> splitstrip('a, b, c   ,  4,,')
+    ['a', 'b', 'c', '4']
+    >>> splitstrip('a')
+    ['a']
+    >>>
+
+    :type string: str or unicode
+    :param string: a csv line
+
+    :type sep: str or unicode
+    :param sep: field separator, default to the comma (',')
+
+    :rtype: str or unicode
+    :return: the unquoted string (or the input string if it wasn't quoted)
+    """
+    return [word.strip() for word in string.split(sep) if word.strip()]
+
+get_csv = deprecated('get_csv is deprecated, use splitstrip')(splitstrip)
+
+
+def split_url_or_path(url_or_path):
+    """return the latest component of a string containing either an url of the
+    form <scheme>://<path> or a local file system path
+    """
+    if '://' in url_or_path:
+        return url_or_path.rstrip('/').rsplit('/', 1)
+    return osp.split(url_or_path.rstrip(osp.sep))
+
+
+def text_to_dict(text):
+    """parse multilines text containing simple 'key=value' lines and return a
+    dict of {'key': 'value'}. When the same key is encountered multiple time,
+    value is turned into a list containing all values.
+
+    >>> text_to_dict('''multiple=1
+    ... multiple= 2
+    ... single =3
+    ... ''')
+    {'single': '3', 'multiple': ['1', '2']}
+
+    """
+    res = {}
+    if not text:
+        return res
+    for line in text.splitlines():
+        line = line.strip()
+        if line and not line.startswith('#'):
+            key, value = [w.strip() for w in line.split('=', 1)]
+            if key in res:
+                try:
+                    res[key].append(value)
+                except AttributeError:
+                    res[key] = [res[key], value]
+            else:
+                res[key] = value
+    return res
+
+
+_BLANK_URE = r'(\s|,)+'
+_BLANK_RE = re.compile(_BLANK_URE)
+__VALUE_URE = r'-?(([0-9]+\.[0-9]*)|((0x?)?[0-9]+))'
+__UNITS_URE = r'[a-zA-Z]+'
+_VALUE_RE = re.compile(r'(?P<value>%s)(?P<unit>%s)?'%(__VALUE_URE, __UNITS_URE))
+
+BYTE_UNITS = {
+    "b": 1,
+    "kb": 1024,
+    "mb": 1024 ** 2,
+    "gb": 1024 ** 3,
+    "tb": 1024 ** 4,
+}
+
+TIME_UNITS = {
+    "ms": 0.0001,
+    "s": 1,
+    "min": 60,
+    "h": 60 * 60,
+    "d": 60 * 60 *24,
+}
+
+def apply_units(string, units, inter=None, final=float, blank_reg=_BLANK_RE,
+                value_reg=_VALUE_RE):
+    """Parse the string applying the units defined in units
+    (e.g.: "1.5m",{'m',60} -> 80).
+
+    :type string: str or unicode
+    :param string: the string to parse
+
+    :type units: dict (or any object with __getitem__ using basestring key)
+    :param units: a dict mapping a unit string repr to its value
+
+    :type inter: type
+    :param inter: used to parse every intermediate value (need __sum__)
+
+    :type blank_reg: regexp
+    :param blank_reg: should match every blank char to ignore.
+
+    :type value_reg: regexp with "value" and optional "unit" group
+    :param value_reg: match a value and it's unit into the
+    """
+    if inter is None:
+        inter = final
+    string = _BLANK_RE.sub('', string)
+    values = []
+    for match in value_reg.finditer(string):
+        dic = match.groupdict()
+        #import sys
+        #print >> sys.stderr, dic
+        lit, unit = dic["value"], dic.get("unit")
+        value = inter(lit)
+        if unit is not None:
+            try:
+                value *= units[unit.lower()]
+            except KeyError:
+                raise KeyError('invalid unit %s. valid units are %s' %
+                               (unit, units.keys()))
+        values.append(value)
+    return final(sum(values))
+
+
+_LINE_RGX = re.compile('\r\n|\r+|\n')
+
+def pretty_match(match, string, underline_char='^'):
+    """return a string with the match location underlined:
+
+    >>> import re
+    >>> print(pretty_match(re.search('mange', 'il mange du bacon'), 'il mange du bacon'))
+    il mange du bacon
+       ^^^^^
+    >>>
+
+    :type match: _sre.SRE_match
+    :param match: object returned by re.match, re.search or re.finditer
+
+    :type string: str or unicode
+    :param string:
+      the string on which the regular expression has been applied to
+      obtain the `match` object
+
+    :type underline_char: str or unicode
+    :param underline_char:
+      character to use to underline the matched section, default to the
+      carret '^'
+
+    :rtype: str or unicode
+    :return:
+      the original string with an inserted line to underline the match
+      location
+    """
+    start = match.start()
+    end = match.end()
+    string = _LINE_RGX.sub(linesep, string)
+    start_line_pos = string.rfind(linesep, 0, start)
+    if start_line_pos == -1:
+        start_line_pos = 0
+        result = []
+    else:
+        result = [string[:start_line_pos]]
+        start_line_pos += len(linesep)
+    offset = start - start_line_pos
+    underline = ' ' * offset + underline_char * (end - start)
+    end_line_pos = string.find(linesep, end)
+    if end_line_pos == -1:
+        string = string[start_line_pos:]
+        result.append(string)
+        result.append(underline)
+    else:
+        end = string[end_line_pos + len(linesep):]
+        string = string[start_line_pos:end_line_pos]
+        result.append(string)
+        result.append(underline)
+        result.append(end)
+    return linesep.join(result).rstrip()
+
+
+# Ansi colorization ###########################################################
+
+ANSI_PREFIX = '\033['
+ANSI_END = 'm'
+ANSI_RESET = '\033[0m'
+ANSI_STYLES = {
+    'reset': "0",
+    'bold': "1",
+    'italic': "3",
+    'underline': "4",
+    'blink': "5",
+    'inverse': "7",
+    'strike': "9",
+}
+ANSI_COLORS = {
+    'reset': "0",
+    'black': "30",
+    'red': "31",
+    'green': "32",
+    'yellow': "33",
+    'blue': "34",
+    'magenta': "35",
+    'cyan': "36",
+    'white': "37",
+}
+
+def _get_ansi_code(color=None, style=None):
+    """return ansi escape code corresponding to color and style
+
+    :type color: str or None
+    :param color:
+      the color name (see `ANSI_COLORS` for available values)
+      or the color number when 256 colors are available
+
+    :type style: str or None
+    :param style:
+      style string (see `ANSI_COLORS` for available values). To get
+      several style effects at the same time, use a coma as separator.
+
+    :raise KeyError: if an unexistent color or style identifier is given
+
+    :rtype: str
+    :return: the built escape code
+    """
+    ansi_code = []
+    if style:
+        style_attrs = splitstrip(style)
+        for effect in style_attrs:
+            ansi_code.append(ANSI_STYLES[effect])
+    if color:
+        if color.isdigit():
+            ansi_code.extend(['38', '5'])
+            ansi_code.append(color)
+        else:
+            ansi_code.append(ANSI_COLORS[color])
+    if ansi_code:
+        return ANSI_PREFIX + ';'.join(ansi_code) + ANSI_END
+    return ''
+
+def colorize_ansi(msg, color=None, style=None):
+    """colorize message by wrapping it with ansi escape codes
+
+    :type msg: str or unicode
+    :param msg: the message string to colorize
+
+    :type color: str or None
+    :param color:
+      the color identifier (see `ANSI_COLORS` for available values)
+
+    :type style: str or None
+    :param style:
+      style string (see `ANSI_COLORS` for available values). To get
+      several style effects at the same time, use a coma as separator.
+
+    :raise KeyError: if an unexistent color or style identifier is given
+
+    :rtype: str or unicode
+    :return: the ansi escaped string
+    """
+    # If both color and style are not defined, then leave the text as is
+    if color is None and style is None:
+        return msg
+    escape_code = _get_ansi_code(color, style)
+    # If invalid (or unknown) color, don't wrap msg with ansi codes
+    if escape_code:
+        return '%s%s%s' % (escape_code, msg, ANSI_RESET)
+    return msg
+
+DIFF_STYLE = {'separator': 'cyan', 'remove': 'red', 'add': 'green'}
+
+def diff_colorize_ansi(lines, out=sys.stdout, style=DIFF_STYLE):
+    for line in lines:
+        if line[:4] in ('--- ', '+++ '):
+            out.write(colorize_ansi(line, style['separator']))
+        elif line[0] == '-':
+            out.write(colorize_ansi(line, style['remove']))
+        elif line[0] == '+':
+            out.write(colorize_ansi(line, style['add']))
+        elif line[:4] == '--- ':
+            out.write(colorize_ansi(line, style['separator']))
+        elif line[:4] == '+++ ':
+            out.write(colorize_ansi(line, style['separator']))
+        else:
+            out.write(line)
+