| OLD | NEW |
|---|
| (Empty) | |
| 1 # -*- coding: utf-8 -*- |
| 2 """ |
| 3 jinja2.lexer |
| 4 ~~~~~~~~~~~~ |
| 5 |
| 6 This module implements a Jinja / Python combination lexer. The |
| 7 `Lexer` class provided by this module is used to do some preprocessing |
| 8 for Jinja. |
| 9 |
| 10 On the one hand it filters out invalid operators like the bitshift |
| 11 operators we don't allow in templates. On the other hand it separates |
| 12 template code and python code in expressions. |
| 13 |
| 14 :copyright: (c) 2010 by the Jinja Team. |
| 15 :license: BSD, see LICENSE for more details. |
| 16 """ |
| 17 import re |
| 18 from operator import itemgetter |
| 19 from collections import deque |
| 20 from jinja2.exceptions import TemplateSyntaxError |
| 21 from jinja2.utils import LRUCache, next |
| 22 |
| 23 |
| 24 # cache for the lexers. Exists in order to be able to have multiple |
| 25 # environments with the same lexer |
| 26 _lexer_cache = LRUCache(50) |
| 27 |
| 28 # static regular expressions |
| 29 whitespace_re = re.compile(r'\s+', re.U) |
| 30 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" |
| 31 r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) |
| 32 integer_re = re.compile(r'\d+') |
| 33 |
| 34 # we use the unicode identifier rule if this python version is able |
| 35 # to handle unicode identifiers, otherwise the standard ASCII one. |
| 36 try: |
| 37 compile('föö', '<unknown>', 'eval') |
| 38 except SyntaxError: |
| 39 name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b') |
| 40 else: |
| 41 from jinja2 import _stringdefs |
| 42 name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start, |
| 43 _stringdefs.xid_continue)) |
| 44 |
| 45 float_re = re.compile(r'(?<!\.)\d+\.\d+') |
| 46 newline_re = re.compile(r'(\r\n|\r|\n)') |
| 47 |
| 48 # internal the tokens and keep references to them |
| 49 TOKEN_ADD = intern('add') |
| 50 TOKEN_ASSIGN = intern('assign') |
| 51 TOKEN_COLON = intern('colon') |
| 52 TOKEN_COMMA = intern('comma') |
| 53 TOKEN_DIV = intern('div') |
| 54 TOKEN_DOT = intern('dot') |
| 55 TOKEN_EQ = intern('eq') |
| 56 TOKEN_FLOORDIV = intern('floordiv') |
| 57 TOKEN_GT = intern('gt') |
| 58 TOKEN_GTEQ = intern('gteq') |
| 59 TOKEN_LBRACE = intern('lbrace') |
| 60 TOKEN_LBRACKET = intern('lbracket') |
| 61 TOKEN_LPAREN = intern('lparen') |
| 62 TOKEN_LT = intern('lt') |
| 63 TOKEN_LTEQ = intern('lteq') |
| 64 TOKEN_MOD = intern('mod') |
| 65 TOKEN_MUL = intern('mul') |
| 66 TOKEN_NE = intern('ne') |
| 67 TOKEN_PIPE = intern('pipe') |
| 68 TOKEN_POW = intern('pow') |
| 69 TOKEN_RBRACE = intern('rbrace') |
| 70 TOKEN_RBRACKET = intern('rbracket') |
| 71 TOKEN_RPAREN = intern('rparen') |
| 72 TOKEN_SEMICOLON = intern('semicolon') |
| 73 TOKEN_SUB = intern('sub') |
| 74 TOKEN_TILDE = intern('tilde') |
| 75 TOKEN_WHITESPACE = intern('whitespace') |
| 76 TOKEN_FLOAT = intern('float') |
| 77 TOKEN_INTEGER = intern('integer') |
| 78 TOKEN_NAME = intern('name') |
| 79 TOKEN_STRING = intern('string') |
| 80 TOKEN_OPERATOR = intern('operator') |
| 81 TOKEN_BLOCK_BEGIN = intern('block_begin') |
| 82 TOKEN_BLOCK_END = intern('block_end') |
| 83 TOKEN_VARIABLE_BEGIN = intern('variable_begin') |
| 84 TOKEN_VARIABLE_END = intern('variable_end') |
| 85 TOKEN_RAW_BEGIN = intern('raw_begin') |
| 86 TOKEN_RAW_END = intern('raw_end') |
| 87 TOKEN_COMMENT_BEGIN = intern('comment_begin') |
| 88 TOKEN_COMMENT_END = intern('comment_end') |
| 89 TOKEN_COMMENT = intern('comment') |
| 90 TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') |
| 91 TOKEN_LINESTATEMENT_END = intern('linestatement_end') |
| 92 TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') |
| 93 TOKEN_LINECOMMENT_END = intern('linecomment_end') |
| 94 TOKEN_LINECOMMENT = intern('linecomment') |
| 95 TOKEN_DATA = intern('data') |
| 96 TOKEN_INITIAL = intern('initial') |
| 97 TOKEN_EOF = intern('eof') |
| 98 |
| 99 # bind operators to token types |
| 100 operators = { |
| 101 '+': TOKEN_ADD, |
| 102 '-': TOKEN_SUB, |
| 103 '/': TOKEN_DIV, |
| 104 '//': TOKEN_FLOORDIV, |
| 105 '*': TOKEN_MUL, |
| 106 '%': TOKEN_MOD, |
| 107 '**': TOKEN_POW, |
| 108 '~': TOKEN_TILDE, |
| 109 '[': TOKEN_LBRACKET, |
| 110 ']': TOKEN_RBRACKET, |
| 111 '(': TOKEN_LPAREN, |
| 112 ')': TOKEN_RPAREN, |
| 113 '{': TOKEN_LBRACE, |
| 114 '}': TOKEN_RBRACE, |
| 115 '==': TOKEN_EQ, |
| 116 '!=': TOKEN_NE, |
| 117 '>': TOKEN_GT, |
| 118 '>=': TOKEN_GTEQ, |
| 119 '<': TOKEN_LT, |
| 120 '<=': TOKEN_LTEQ, |
| 121 '=': TOKEN_ASSIGN, |
| 122 '.': TOKEN_DOT, |
| 123 ':': TOKEN_COLON, |
| 124 '|': TOKEN_PIPE, |
| 125 ',': TOKEN_COMMA, |
| 126 ';': TOKEN_SEMICOLON |
| 127 } |
| 128 |
| 129 reverse_operators = dict([(v, k) for k, v in operators.iteritems()]) |
| 130 assert len(operators) == len(reverse_operators), 'operators dropped' |
| 131 operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in |
| 132 sorted(operators, key=lambda x: -len(x)))) |
| 133 |
| 134 ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, |
| 135 TOKEN_COMMENT_END, TOKEN_WHITESPACE, |
| 136 TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN, |
| 137 TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT]) |
| 138 ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, |
| 139 TOKEN_COMMENT, TOKEN_LINECOMMENT]) |
| 140 |
| 141 |
| 142 def _describe_token_type(token_type): |
| 143 if token_type in reverse_operators: |
| 144 return reverse_operators[token_type] |
| 145 return { |
| 146 TOKEN_COMMENT_BEGIN: 'begin of comment', |
| 147 TOKEN_COMMENT_END: 'end of comment', |
| 148 TOKEN_COMMENT: 'comment', |
| 149 TOKEN_LINECOMMENT: 'comment', |
| 150 TOKEN_BLOCK_BEGIN: 'begin of statement block', |
| 151 TOKEN_BLOCK_END: 'end of statement block', |
| 152 TOKEN_VARIABLE_BEGIN: 'begin of print statement', |
| 153 TOKEN_VARIABLE_END: 'end of print statement', |
| 154 TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', |
| 155 TOKEN_LINESTATEMENT_END: 'end of line statement', |
| 156 TOKEN_DATA: 'template data / text', |
| 157 TOKEN_EOF: 'end of template' |
| 158 }.get(token_type, token_type) |
| 159 |
| 160 |
| 161 def describe_token(token): |
| 162 """Returns a description of the token.""" |
| 163 if token.type == 'name': |
| 164 return token.value |
| 165 return _describe_token_type(token.type) |
| 166 |
| 167 |
| 168 def describe_token_expr(expr): |
| 169 """Like `describe_token` but for token expressions.""" |
| 170 if ':' in expr: |
| 171 type, value = expr.split(':', 1) |
| 172 if type == 'name': |
| 173 return value |
| 174 else: |
| 175 type = expr |
| 176 return _describe_token_type(type) |
| 177 |
| 178 |
| 179 def count_newlines(value): |
| 180 """Count the number of newline characters in the string. This is |
| 181 useful for extensions that filter a stream. |
| 182 """ |
| 183 return len(newline_re.findall(value)) |
| 184 |
| 185 |
| 186 def compile_rules(environment): |
| 187 """Compiles all the rules from the environment into a list of rules.""" |
| 188 e = re.escape |
| 189 rules = [ |
| 190 (len(environment.comment_start_string), 'comment', |
| 191 e(environment.comment_start_string)), |
| 192 (len(environment.block_start_string), 'block', |
| 193 e(environment.block_start_string)), |
| 194 (len(environment.variable_start_string), 'variable', |
| 195 e(environment.variable_start_string)) |
| 196 ] |
| 197 |
| 198 if environment.line_statement_prefix is not None: |
| 199 rules.append((len(environment.line_statement_prefix), 'linestatement', |
| 200 r'^\s*' + e(environment.line_statement_prefix))) |
| 201 if environment.line_comment_prefix is not None: |
| 202 rules.append((len(environment.line_comment_prefix), 'linecomment', |
| 203 r'(?:^|(?<=\S))[^\S\r\n]*' + |
| 204 e(environment.line_comment_prefix))) |
| 205 |
| 206 return [x[1:] for x in sorted(rules, reverse=True)] |
| 207 |
| 208 |
| 209 class Failure(object): |
| 210 """Class that raises a `TemplateSyntaxError` if called. |
| 211 Used by the `Lexer` to specify known errors. |
| 212 """ |
| 213 |
| 214 def __init__(self, message, cls=TemplateSyntaxError): |
| 215 self.message = message |
| 216 self.error_class = cls |
| 217 |
| 218 def __call__(self, lineno, filename): |
| 219 raise self.error_class(self.message, lineno, filename) |
| 220 |
| 221 |
| 222 class Token(tuple): |
| 223 """Token class.""" |
| 224 __slots__ = () |
| 225 lineno, type, value = (property(itemgetter(x)) for x in range(3)) |
| 226 |
| 227 def __new__(cls, lineno, type, value): |
| 228 return tuple.__new__(cls, (lineno, intern(str(type)), value)) |
| 229 |
| 230 def __str__(self): |
| 231 if self.type in reverse_operators: |
| 232 return reverse_operators[self.type] |
| 233 elif self.type == 'name': |
| 234 return self.value |
| 235 return self.type |
| 236 |
| 237 def test(self, expr): |
| 238 """Test a token against a token expression. This can either be a |
| 239 token type or ``'token_type:token_value'``. This can only test |
| 240 against string values and types. |
| 241 """ |
| 242 # here we do a regular string equality check as test_any is usually |
| 243 # passed an iterable of not interned strings. |
| 244 if self.type == expr: |
| 245 return True |
| 246 elif ':' in expr: |
| 247 return expr.split(':', 1) == [self.type, self.value] |
| 248 return False |
| 249 |
| 250 def test_any(self, *iterable): |
| 251 """Test against multiple token expressions.""" |
| 252 for expr in iterable: |
| 253 if self.test(expr): |
| 254 return True |
| 255 return False |
| 256 |
| 257 def __repr__(self): |
| 258 return 'Token(%r, %r, %r)' % ( |
| 259 self.lineno, |
| 260 self.type, |
| 261 self.value |
| 262 ) |
| 263 |
| 264 |
| 265 class TokenStreamIterator(object): |
| 266 """The iterator for tokenstreams. Iterate over the stream |
| 267 until the eof token is reached. |
| 268 """ |
| 269 |
| 270 def __init__(self, stream): |
| 271 self.stream = stream |
| 272 |
| 273 def __iter__(self): |
| 274 return self |
| 275 |
| 276 def next(self): |
| 277 token = self.stream.current |
| 278 if token.type is TOKEN_EOF: |
| 279 self.stream.close() |
| 280 raise StopIteration() |
| 281 next(self.stream) |
| 282 return token |
| 283 |
| 284 |
| 285 class TokenStream(object): |
| 286 """A token stream is an iterable that yields :class:`Token`\s. The |
| 287 parser however does not iterate over it but calls :meth:`next` to go |
| 288 one token ahead. The current active token is stored as :attr:`current`. |
| 289 """ |
| 290 |
| 291 def __init__(self, generator, name, filename): |
| 292 self._next = iter(generator).next |
| 293 self._pushed = deque() |
| 294 self.name = name |
| 295 self.filename = filename |
| 296 self.closed = False |
| 297 self.current = Token(1, TOKEN_INITIAL, '') |
| 298 next(self) |
| 299 |
| 300 def __iter__(self): |
| 301 return TokenStreamIterator(self) |
| 302 |
| 303 def __nonzero__(self): |
| 304 return bool(self._pushed) or self.current.type is not TOKEN_EOF |
| 305 |
| 306 eos = property(lambda x: not x, doc="Are we at the end of the stream?") |
| 307 |
| 308 def push(self, token): |
| 309 """Push a token back to the stream.""" |
| 310 self._pushed.append(token) |
| 311 |
| 312 def look(self): |
| 313 """Look at the next token.""" |
| 314 old_token = next(self) |
| 315 result = self.current |
| 316 self.push(result) |
| 317 self.current = old_token |
| 318 return result |
| 319 |
| 320 def skip(self, n=1): |
| 321 """Got n tokens ahead.""" |
| 322 for x in xrange(n): |
| 323 next(self) |
| 324 |
| 325 def next_if(self, expr): |
| 326 """Perform the token test and return the token if it matched. |
| 327 Otherwise the return value is `None`. |
| 328 """ |
| 329 if self.current.test(expr): |
| 330 return next(self) |
| 331 |
| 332 def skip_if(self, expr): |
| 333 """Like :meth:`next_if` but only returns `True` or `False`.""" |
| 334 return self.next_if(expr) is not None |
| 335 |
| 336 def next(self): |
| 337 """Go one token ahead and return the old one""" |
| 338 rv = self.current |
| 339 if self._pushed: |
| 340 self.current = self._pushed.popleft() |
| 341 elif self.current.type is not TOKEN_EOF: |
| 342 try: |
| 343 self.current = self._next() |
| 344 except StopIteration: |
| 345 self.close() |
| 346 return rv |
| 347 |
| 348 def close(self): |
| 349 """Close the stream.""" |
| 350 self.current = Token(self.current.lineno, TOKEN_EOF, '') |
| 351 self._next = None |
| 352 self.closed = True |
| 353 |
| 354 def expect(self, expr): |
| 355 """Expect a given token type and return it. This accepts the same |
| 356 argument as :meth:`jinja2.lexer.Token.test`. |
| 357 """ |
| 358 if not self.current.test(expr): |
| 359 expr = describe_token_expr(expr) |
| 360 if self.current.type is TOKEN_EOF: |
| 361 raise TemplateSyntaxError('unexpected end of template, ' |
| 362 'expected %r.' % expr, |
| 363 self.current.lineno, |
| 364 self.name, self.filename) |
| 365 raise TemplateSyntaxError("expected token %r, got %r" % |
| 366 (expr, describe_token(self.current)), |
| 367 self.current.lineno, |
| 368 self.name, self.filename) |
| 369 try: |
| 370 return self.current |
| 371 finally: |
| 372 next(self) |
| 373 |
| 374 |
| 375 def get_lexer(environment): |
| 376 """Return a lexer which is probably cached.""" |
| 377 key = (environment.block_start_string, |
| 378 environment.block_end_string, |
| 379 environment.variable_start_string, |
| 380 environment.variable_end_string, |
| 381 environment.comment_start_string, |
| 382 environment.comment_end_string, |
| 383 environment.line_statement_prefix, |
| 384 environment.line_comment_prefix, |
| 385 environment.trim_blocks, |
| 386 environment.newline_sequence) |
| 387 lexer = _lexer_cache.get(key) |
| 388 if lexer is None: |
| 389 lexer = Lexer(environment) |
| 390 _lexer_cache[key] = lexer |
| 391 return lexer |
| 392 |
| 393 |
| 394 class Lexer(object): |
| 395 """Class that implements a lexer for a given environment. Automatically |
| 396 created by the environment class, usually you don't have to do that. |
| 397 |
| 398 Note that the lexer is not automatically bound to an environment. |
| 399 Multiple environments can share the same lexer. |
| 400 """ |
| 401 |
| 402 def __init__(self, environment): |
| 403 # shortcuts |
| 404 c = lambda x: re.compile(x, re.M | re.S) |
| 405 e = re.escape |
| 406 |
| 407 # lexing rules for tags |
| 408 tag_rules = [ |
| 409 (whitespace_re, TOKEN_WHITESPACE, None), |
| 410 (float_re, TOKEN_FLOAT, None), |
| 411 (integer_re, TOKEN_INTEGER, None), |
| 412 (name_re, TOKEN_NAME, None), |
| 413 (string_re, TOKEN_STRING, None), |
| 414 (operator_re, TOKEN_OPERATOR, None) |
| 415 ] |
| 416 |
| 417 # assamble the root lexing rule. because "|" is ungreedy |
| 418 # we have to sort by length so that the lexer continues working |
| 419 # as expected when we have parsing rules like <% for block and |
| 420 # <%= for variables. (if someone wants asp like syntax) |
| 421 # variables are just part of the rules if variable processing |
| 422 # is required. |
| 423 root_tag_rules = compile_rules(environment) |
| 424 |
| 425 # block suffix if trimming is enabled |
| 426 block_suffix_re = environment.trim_blocks and '\\n?' or '' |
| 427 |
| 428 self.newline_sequence = environment.newline_sequence |
| 429 |
| 430 # global lexing rules |
| 431 self.rules = { |
| 432 'root': [ |
| 433 # directives |
| 434 (c('(.*?)(?:%s)' % '|'.join( |
| 435 [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( |
| 436 e(environment.block_start_string), |
| 437 e(environment.block_start_string), |
| 438 e(environment.block_end_string), |
| 439 e(environment.block_end_string) |
| 440 )] + [ |
| 441 r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r) |
| 442 for n, r in root_tag_rules |
| 443 ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), |
| 444 # data |
| 445 (c('.+'), TOKEN_DATA, None) |
| 446 ], |
| 447 # comments |
| 448 TOKEN_COMMENT_BEGIN: [ |
| 449 (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( |
| 450 e(environment.comment_end_string), |
| 451 e(environment.comment_end_string), |
| 452 block_suffix_re |
| 453 )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), |
| 454 (c('(.)'), (Failure('Missing end of comment tag'),), None) |
| 455 ], |
| 456 # blocks |
| 457 TOKEN_BLOCK_BEGIN: [ |
| 458 (c('(?:\-%s\s*|%s)%s' % ( |
| 459 e(environment.block_end_string), |
| 460 e(environment.block_end_string), |
| 461 block_suffix_re |
| 462 )), TOKEN_BLOCK_END, '#pop'), |
| 463 ] + tag_rules, |
| 464 # variables |
| 465 TOKEN_VARIABLE_BEGIN: [ |
| 466 (c('\-%s\s*|%s' % ( |
| 467 e(environment.variable_end_string), |
| 468 e(environment.variable_end_string) |
| 469 )), TOKEN_VARIABLE_END, '#pop') |
| 470 ] + tag_rules, |
| 471 # raw block |
| 472 TOKEN_RAW_BEGIN: [ |
| 473 (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( |
| 474 e(environment.block_start_string), |
| 475 e(environment.block_start_string), |
| 476 e(environment.block_end_string), |
| 477 e(environment.block_end_string), |
| 478 block_suffix_re |
| 479 )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), |
| 480 (c('(.)'), (Failure('Missing end of raw directive'),), None) |
| 481 ], |
| 482 # line statements |
| 483 TOKEN_LINESTATEMENT_BEGIN: [ |
| 484 (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') |
| 485 ] + tag_rules, |
| 486 # line comments |
| 487 TOKEN_LINECOMMENT_BEGIN: [ |
| 488 (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, |
| 489 TOKEN_LINECOMMENT_END), '#pop') |
| 490 ] |
| 491 } |
| 492 |
| 493 def _normalize_newlines(self, value): |
| 494 """Called for strings and template data to normlize it to unicode.""" |
| 495 return newline_re.sub(self.newline_sequence, value) |
| 496 |
| 497 def tokenize(self, source, name=None, filename=None, state=None): |
| 498 """Calls tokeniter + tokenize and wraps it in a token stream. |
| 499 """ |
| 500 stream = self.tokeniter(source, name, filename, state) |
| 501 return TokenStream(self.wrap(stream, name, filename), name, filename) |
| 502 |
| 503 def wrap(self, stream, name=None, filename=None): |
| 504 """This is called with the stream as returned by `tokenize` and wraps |
| 505 every token in a :class:`Token` and converts the value. |
| 506 """ |
| 507 for lineno, token, value in stream: |
| 508 if token in ignored_tokens: |
| 509 continue |
| 510 elif token == 'linestatement_begin': |
| 511 token = 'block_begin' |
| 512 elif token == 'linestatement_end': |
| 513 token = 'block_end' |
| 514 # we are not interested in those tokens in the parser |
| 515 elif token in ('raw_begin', 'raw_end'): |
| 516 continue |
| 517 elif token == 'data': |
| 518 value = self._normalize_newlines(value) |
| 519 elif token == 'keyword': |
| 520 token = value |
| 521 elif token == 'name': |
| 522 value = str(value) |
| 523 elif token == 'string': |
| 524 # try to unescape string |
| 525 try: |
| 526 value = self._normalize_newlines(value[1:-1]) \ |
| 527 .encode('ascii', 'backslashreplace') \ |
| 528 .decode('unicode-escape') |
| 529 except Exception, e: |
| 530 msg = str(e).split(':')[-1].strip() |
| 531 raise TemplateSyntaxError(msg, lineno, name, filename) |
| 532 # if we can express it as bytestring (ascii only) |
| 533 # we do that for support of semi broken APIs |
| 534 # as datetime.datetime.strftime. On python 3 this |
| 535 # call becomes a noop thanks to 2to3 |
| 536 try: |
| 537 value = str(value) |
| 538 except UnicodeError: |
| 539 pass |
| 540 elif token == 'integer': |
| 541 value = int(value) |
| 542 elif token == 'float': |
| 543 value = float(value) |
| 544 elif token == 'operator': |
| 545 token = operators[value] |
| 546 yield Token(lineno, token, value) |
| 547 |
| 548 def tokeniter(self, source, name, filename=None, state=None): |
| 549 """This method tokenizes the text and returns the tokens in a |
| 550 generator. Use this method if you just want to tokenize a template. |
| 551 """ |
| 552 source = '\n'.join(unicode(source).splitlines()) |
| 553 pos = 0 |
| 554 lineno = 1 |
| 555 stack = ['root'] |
| 556 if state is not None and state != 'root': |
| 557 assert state in ('variable', 'block'), 'invalid state' |
| 558 stack.append(state + '_begin') |
| 559 else: |
| 560 state = 'root' |
| 561 statetokens = self.rules[stack[-1]] |
| 562 source_length = len(source) |
| 563 |
| 564 balancing_stack = [] |
| 565 |
| 566 while 1: |
| 567 # tokenizer loop |
| 568 for regex, tokens, new_state in statetokens: |
| 569 m = regex.match(source, pos) |
| 570 # if no match we try again with the next rule |
| 571 if m is None: |
| 572 continue |
| 573 |
| 574 # we only match blocks and variables if brances / parentheses |
| 575 # are balanced. continue parsing with the lower rule which |
| 576 # is the operator rule. do this only if the end tags look |
| 577 # like operators |
| 578 if balancing_stack and \ |
| 579 tokens in ('variable_end', 'block_end', |
| 580 'linestatement_end'): |
| 581 continue |
| 582 |
| 583 # tuples support more options |
| 584 if isinstance(tokens, tuple): |
| 585 for idx, token in enumerate(tokens): |
| 586 # failure group |
| 587 if token.__class__ is Failure: |
| 588 raise token(lineno, filename) |
| 589 # bygroup is a bit more complex, in that case we |
| 590 # yield for the current token the first named |
| 591 # group that matched |
| 592 elif token == '#bygroup': |
| 593 for key, value in m.groupdict().iteritems(): |
| 594 if value is not None: |
| 595 yield lineno, key, value |
| 596 lineno += value.count('\n') |
| 597 break |
| 598 else: |
| 599 raise RuntimeError('%r wanted to resolve ' |
| 600 'the token dynamically' |
| 601 ' but no group matched' |
| 602 % regex) |
| 603 # normal group |
| 604 else: |
| 605 data = m.group(idx + 1) |
| 606 if data or token not in ignore_if_empty: |
| 607 yield lineno, token, data |
| 608 lineno += data.count('\n') |
| 609 |
| 610 # strings as token just are yielded as it. |
| 611 else: |
| 612 data = m.group() |
| 613 # update brace/parentheses balance |
| 614 if tokens == 'operator': |
| 615 if data == '{': |
| 616 balancing_stack.append('}') |
| 617 elif data == '(': |
| 618 balancing_stack.append(')') |
| 619 elif data == '[': |
| 620 balancing_stack.append(']') |
| 621 elif data in ('}', ')', ']'): |
| 622 if not balancing_stack: |
| 623 raise TemplateSyntaxError('unexpected \'%s\'' % |
| 624 data, lineno, name, |
| 625 filename) |
| 626 expected_op = balancing_stack.pop() |
| 627 if expected_op != data: |
| 628 raise TemplateSyntaxError('unexpected \'%s\', ' |
| 629 'expected \'%s\'' % |
| 630 (data, expected_op), |
| 631 lineno, name, |
| 632 filename) |
| 633 # yield items |
| 634 if data or tokens not in ignore_if_empty: |
| 635 yield lineno, tokens, data |
| 636 lineno += data.count('\n') |
| 637 |
| 638 # fetch new position into new variable so that we can check |
| 639 # if there is a internal parsing error which would result |
| 640 # in an infinite loop |
| 641 pos2 = m.end() |
| 642 |
| 643 # handle state changes |
| 644 if new_state is not None: |
| 645 # remove the uppermost state |
| 646 if new_state == '#pop': |
| 647 stack.pop() |
| 648 # resolve the new state by group checking |
| 649 elif new_state == '#bygroup': |
| 650 for key, value in m.groupdict().iteritems(): |
| 651 if value is not None: |
| 652 stack.append(key) |
| 653 break |
| 654 else: |
| 655 raise RuntimeError('%r wanted to resolve the ' |
| 656 'new state dynamically but' |
| 657 ' no group matched' % |
| 658 regex) |
| 659 # direct state name given |
| 660 else: |
| 661 stack.append(new_state) |
| 662 statetokens = self.rules[stack[-1]] |
| 663 # we are still at the same position and no stack change. |
| 664 # this means a loop without break condition, avoid that and |
| 665 # raise error |
| 666 elif pos2 == pos: |
| 667 raise RuntimeError('%r yielded empty string without ' |
| 668 'stack change' % regex) |
| 669 # publish new function and start again |
| 670 pos = pos2 |
| 671 break |
| 672 # if loop terminated without break we havn't found a single match |
| 673 # either we are at the end of the file or we have a problem |
| 674 else: |
| 675 # end of text |
| 676 if pos >= source_length: |
| 677 return |
| 678 # something went wrong |
| 679 raise TemplateSyntaxError('unexpected char %r at %d' % |
| 680 (source[pos], pos), lineno, |
| 681 name, filename) |
| OLD | NEW |
|---|
Chromium Code Reviews
Issue 14761007:
Add Python templating engine Jinja2 to third_party (Closed)
Base URL: svn://svn.chromium.org/chrome/trunk/src