Cython has moved to github.
cython-devel
view Cython/Compiler/Parsing.py @ 1372:3ba99a87b30d
support parentheses around import-from names
| author | Stefan Behnel <scoder@users.berlios.de> |
|---|---|
| date | Mon Nov 24 12:47:22 2008 +0100 (3 years ago) |
| parents | c50ea4f4ba4e |
| children | 85a5fdd79388 |
line source
1 # cython: auto_cpdef=True
2 #
3 # Pyrex Parser
4 #
6 # This should be done automatically
7 import cython
8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)
10 import os
11 import re
12 import sys
13 from types import ListType, TupleType
14 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
15 import Nodes
16 import ExprNodes
17 import StringEncoding
18 from StringEncoding import EncodedString, BytesLiteral
19 from ModuleNode import ModuleNode
20 from Errors import error, warning, InternalError
21 from Cython import Utils
22 import Future
23 import Options
25 class Ctx(object):
26 # Parsing context
27 level = 'other'
28 visibility = 'private'
29 cdef_flag = 0
30 typedef_flag = 0
31 api = 0
32 overridable = 0
33 nogil = 0
35 def __init__(self, **kwds):
36 self.__dict__.update(kwds)
38 def __call__(self, **kwds):
39 ctx = Ctx()
40 d = ctx.__dict__
41 d.update(self.__dict__)
42 d.update(kwds)
43 return ctx
45 def eat_newlines(s):
46 while s.sy == 'NEWLINE':
47 s.next()
49 def p_ident(s, message = "Expected an identifier"):
50 if s.sy == 'IDENT':
51 name = s.systring
52 s.next()
53 return name
54 else:
55 s.error(message)
57 def p_ident_list(s):
58 names = []
59 while s.sy == 'IDENT':
60 names.append(s.systring)
61 s.next()
62 if s.sy != ',':
63 break
64 s.next()
65 return names
67 #------------------------------------------
68 #
69 # Expressions
70 #
71 #------------------------------------------
73 def p_binop_expr(s, ops, p_sub_expr):
74 n1 = p_sub_expr(s)
75 while s.sy in ops:
76 op = s.sy
77 pos = s.position()
78 s.next()
79 n2 = p_sub_expr(s)
80 n1 = ExprNodes.binop_node(pos, op, n1, n2)
81 return n1
83 #expression: or_test [if or_test else test] | lambda_form
85 def p_simple_expr(s):
86 pos = s.position()
87 expr = p_or_test(s)
88 if s.sy == 'if':
89 s.next()
90 test = p_or_test(s)
91 if s.sy == 'else':
92 s.next()
93 other = p_test(s)
94 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
95 else:
96 s.error("Expected 'else'")
97 else:
98 return expr
100 #test: or_test | lambda_form
102 def p_test(s):
103 return p_or_test(s)
105 #or_test: and_test ('or' and_test)*
107 def p_or_test(s):
108 return p_rassoc_binop_expr(s, ('or',), p_and_test)
110 def p_rassoc_binop_expr(s, ops, p_subexpr):
111 n1 = p_subexpr(s)
112 if s.sy in ops:
113 pos = s.position()
114 op = s.sy
115 s.next()
116 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
117 n1 = ExprNodes.binop_node(pos, op, n1, n2)
118 return n1
120 #and_test: not_test ('and' not_test)*
122 def p_and_test(s):
123 #return p_binop_expr(s, ('and',), p_not_test)
124 return p_rassoc_binop_expr(s, ('and',), p_not_test)
126 #not_test: 'not' not_test | comparison
128 def p_not_test(s):
129 if s.sy == 'not':
130 pos = s.position()
131 s.next()
132 return ExprNodes.NotNode(pos, operand = p_not_test(s))
133 else:
134 return p_comparison(s)
136 #comparison: expr (comp_op expr)*
137 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
139 def p_comparison(s):
140 n1 = p_bit_expr(s)
141 if s.sy in comparison_ops:
142 pos = s.position()
143 op = p_cmp_op(s)
144 n2 = p_bit_expr(s)
145 n1 = ExprNodes.PrimaryCmpNode(pos,
146 operator = op, operand1 = n1, operand2 = n2)
147 if s.sy in comparison_ops:
148 n1.cascade = p_cascaded_cmp(s)
149 return n1
151 def p_cascaded_cmp(s):
152 pos = s.position()
153 op = p_cmp_op(s)
154 n2 = p_bit_expr(s)
155 result = ExprNodes.CascadedCmpNode(pos,
156 operator = op, operand2 = n2)
157 if s.sy in comparison_ops:
158 result.cascade = p_cascaded_cmp(s)
159 return result
161 def p_cmp_op(s):
162 if s.sy == 'not':
163 s.next()
164 s.expect('in')
165 op = 'not_in'
166 elif s.sy == 'is':
167 s.next()
168 if s.sy == 'not':
169 s.next()
170 op = 'is_not'
171 else:
172 op = 'is'
173 else:
174 op = s.sy
175 s.next()
176 if op == '<>':
177 op = '!='
178 return op
180 comparison_ops = (
181 '<', '>', '==', '>=', '<=', '<>', '!=',
182 'in', 'is', 'not'
183 )
185 #expr: xor_expr ('|' xor_expr)*
187 def p_bit_expr(s):
188 return p_binop_expr(s, ('|',), p_xor_expr)
190 #xor_expr: and_expr ('^' and_expr)*
192 def p_xor_expr(s):
193 return p_binop_expr(s, ('^',), p_and_expr)
195 #and_expr: shift_expr ('&' shift_expr)*
197 def p_and_expr(s):
198 return p_binop_expr(s, ('&',), p_shift_expr)
200 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*
202 def p_shift_expr(s):
203 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)
205 #arith_expr: term (('+'|'-') term)*
207 def p_arith_expr(s):
208 return p_binop_expr(s, ('+', '-'), p_term)
210 #term: factor (('*'|'/'|'%') factor)*
212 def p_term(s):
213 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)
215 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
217 def p_factor(s):
218 sy = s.sy
219 if sy in ('+', '-', '~'):
220 op = s.sy
221 pos = s.position()
222 s.next()
223 return ExprNodes.unop_node(pos, op, p_factor(s))
224 elif sy == '&':
225 pos = s.position()
226 s.next()
227 arg = p_factor(s)
228 return ExprNodes.AmpersandNode(pos, operand = arg)
229 elif sy == "<":
230 return p_typecast(s)
231 elif sy == 'IDENT' and s.systring == "sizeof":
232 return p_sizeof(s)
233 else:
234 return p_power(s)
236 def p_typecast(s):
237 # s.sy == "<"
238 pos = s.position()
239 s.next()
240 base_type = p_c_base_type(s)
241 if base_type.name is None:
242 s.error("Unknown type")
243 declarator = p_c_declarator(s, empty = 1)
244 if s.sy == '?':
245 s.next()
246 typecheck = 1
247 else:
248 typecheck = 0
249 s.expect(">")
250 operand = p_factor(s)
251 return ExprNodes.TypecastNode(pos,
252 base_type = base_type,
253 declarator = declarator,
254 operand = operand,
255 typecheck = typecheck)
257 def p_sizeof(s):
258 # s.sy == ident "sizeof"
259 pos = s.position()
260 s.next()
261 s.expect('(')
262 # Here we decide if we are looking at an expression or type
263 # If it is actually a type, but parsable as an expression,
264 # we treat it as an expression here.
265 if looking_at_expr(s):
266 operand = p_simple_expr(s)
267 node = ExprNodes.SizeofVarNode(pos, operand = operand)
268 else:
269 base_type = p_c_base_type(s)
270 declarator = p_c_declarator(s, empty = 1)
271 node = ExprNodes.SizeofTypeNode(pos,
272 base_type = base_type, declarator = declarator)
273 s.expect(')')
274 return node
276 #power: atom trailer* ('**' factor)*
278 def p_power(s):
279 n1 = p_atom(s)
280 while s.sy in ('(', '[', '.'):
281 n1 = p_trailer(s, n1)
282 if s.sy == '**':
283 pos = s.position()
284 s.next()
285 n2 = p_factor(s)
286 n1 = ExprNodes.binop_node(pos, '**', n1, n2)
287 return n1
289 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
291 def p_trailer(s, node1):
292 pos = s.position()
293 if s.sy == '(':
294 return p_call(s, node1)
295 elif s.sy == '[':
296 return p_index(s, node1)
297 else: # s.sy == '.'
298 s.next()
299 name = EncodedString( p_ident(s) )
300 return ExprNodes.AttributeNode(pos,
301 obj = node1, attribute = name)
303 # arglist: argument (',' argument)* [',']
304 # argument: [test '='] test # Really [keyword '='] test
306 def p_call(s, function):
307 # s.sy == '('
308 pos = s.position()
309 s.next()
310 positional_args = []
311 keyword_args = []
312 star_arg = None
313 starstar_arg = None
314 while s.sy not in ('**', ')'):
315 if s.sy == '*':
316 if star_arg:
317 s.error("only one star-arg parameter allowed",
318 pos = s.position())
319 s.next()
320 star_arg = p_simple_expr(s)
321 else:
322 arg = p_simple_expr(s)
323 if s.sy == '=':
324 s.next()
325 if not arg.is_name:
326 s.error("Expected an identifier before '='",
327 pos = arg.pos)
328 encoded_name = EncodedString(arg.name)
329 keyword = ExprNodes.IdentifierStringNode(arg.pos,
330 value = encoded_name)
331 arg = p_simple_expr(s)
332 keyword_args.append((keyword, arg))
333 else:
334 if keyword_args:
335 s.error("Non-keyword arg following keyword arg",
336 pos = arg.pos)
337 if star_arg:
338 s.error("Non-keyword arg following star-arg",
339 pos = arg.pos)
340 positional_args.append(arg)
341 if s.sy != ',':
342 break
343 s.next()
345 if s.sy == '**':
346 s.next()
347 starstar_arg = p_simple_expr(s)
348 if s.sy == ',':
349 s.next()
350 s.expect(')')
351 if not (keyword_args or star_arg or starstar_arg):
352 return ExprNodes.SimpleCallNode(pos,
353 function = function,
354 args = positional_args)
355 else:
356 arg_tuple = None
357 keyword_dict = None
358 if positional_args or not star_arg:
359 arg_tuple = ExprNodes.TupleNode(pos,
360 args = positional_args)
361 if star_arg:
362 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
363 if arg_tuple:
364 arg_tuple = ExprNodes.binop_node(pos,
365 operator = '+', operand1 = arg_tuple,
366 operand2 = star_arg_tuple)
367 else:
368 arg_tuple = star_arg_tuple
369 if keyword_args:
370 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
371 for key, value in keyword_args]
372 keyword_dict = ExprNodes.DictNode(pos,
373 key_value_pairs = keyword_args)
374 return ExprNodes.GeneralCallNode(pos,
375 function = function,
376 positional_args = arg_tuple,
377 keyword_args = keyword_dict,
378 starstar_arg = starstar_arg)
380 #lambdef: 'lambda' [varargslist] ':' test
382 #subscriptlist: subscript (',' subscript)* [',']
384 def p_index(s, base):
385 # s.sy == '['
386 pos = s.position()
387 s.next()
388 subscripts = p_subscript_list(s)
389 if len(subscripts) == 1 and len(subscripts[0]) == 2:
390 start, stop = subscripts[0]
391 result = ExprNodes.SliceIndexNode(pos,
392 base = base, start = start, stop = stop)
393 else:
394 indexes = make_slice_nodes(pos, subscripts)
395 if len(indexes) == 1:
396 index = indexes[0]
397 else:
398 index = ExprNodes.TupleNode(pos, args = indexes)
399 result = ExprNodes.IndexNode(pos,
400 base = base, index = index)
401 s.expect(']')
402 return result
404 def p_subscript_list(s):
405 items = [p_subscript(s)]
406 while s.sy == ',':
407 s.next()
408 if s.sy == ']':
409 break
410 items.append(p_subscript(s))
411 return items
413 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]
415 def p_subscript(s):
416 # Parse a subscript and return a list of
417 # 1, 2 or 3 ExprNodes, depending on how
418 # many slice elements were encountered.
419 pos = s.position()
420 if s.sy == '.':
421 expect_ellipsis(s)
422 return [ExprNodes.EllipsisNode(pos)]
423 else:
424 start = p_slice_element(s, (':',))
425 if s.sy != ':':
426 return [start]
427 s.next()
428 stop = p_slice_element(s, (':', ',', ']'))
429 if s.sy != ':':
430 return [start, stop]
431 s.next()
432 step = p_slice_element(s, (':', ',', ']'))
433 return [start, stop, step]
435 def p_slice_element(s, follow_set):
436 # Simple expression which may be missing iff
437 # it is followed by something in follow_set.
438 if s.sy not in follow_set:
439 return p_simple_expr(s)
440 else:
441 return None
443 def expect_ellipsis(s):
444 s.expect('.')
445 s.expect('.')
446 s.expect('.')
448 def make_slice_nodes(pos, subscripts):
449 # Convert a list of subscripts as returned
450 # by p_subscript_list into a list of ExprNodes,
451 # creating SliceNodes for elements with 2 or
452 # more components.
453 result = []
454 for subscript in subscripts:
455 if len(subscript) == 1:
456 result.append(subscript[0])
457 else:
458 result.append(make_slice_node(pos, *subscript))
459 return result
461 def make_slice_node(pos, start, stop = None, step = None):
462 if not start:
463 start = ExprNodes.NoneNode(pos)
464 if not stop:
465 stop = ExprNodes.NoneNode(pos)
466 if not step:
467 step = ExprNodes.NoneNode(pos)
468 return ExprNodes.SliceNode(pos,
469 start = start, stop = stop, step = step)
471 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
473 def p_atom(s):
474 pos = s.position()
475 sy = s.sy
476 if sy == '(':
477 s.next()
478 if s.sy == ')':
479 result = ExprNodes.TupleNode(pos, args = [])
480 else:
481 result = p_expr(s)
482 s.expect(')')
483 return result
484 elif sy == '[':
485 return p_list_maker(s)
486 elif sy == '{':
487 return p_dict_maker(s)
488 elif sy == '`':
489 return p_backquote_expr(s)
490 elif sy == 'INT':
491 value = s.systring
492 s.next()
493 unsigned = ""
494 longness = ""
495 while value[-1] in "UuLl":
496 if value[-1] in "Ll":
497 longness += "L"
498 else:
499 unsigned += "U"
500 value = value[:-1]
501 return ExprNodes.IntNode(pos,
502 value = value,
503 unsigned = unsigned,
504 longness = longness)
505 elif sy == 'FLOAT':
506 value = s.systring
507 s.next()
508 return ExprNodes.FloatNode(pos, value = value)
509 elif sy == 'IMAG':
510 value = s.systring[:-1]
511 s.next()
512 return ExprNodes.ImagNode(pos, value = value)
513 elif sy == 'BEGIN_STRING':
514 kind, value = p_cat_string_literal(s)
515 if kind == 'c':
516 return ExprNodes.CharNode(pos, value = value)
517 elif kind == 'u':
518 return ExprNodes.UnicodeNode(pos, value = value)
519 else:
520 return ExprNodes.StringNode(pos, value = value)
521 elif sy == 'IDENT':
522 name = EncodedString( s.systring )
523 s.next()
524 if name == "None":
525 return ExprNodes.NoneNode(pos)
526 elif name == "True":
527 return ExprNodes.BoolNode(pos, value=True)
528 elif name == "False":
529 return ExprNodes.BoolNode(pos, value=False)
530 elif name == "NULL":
531 return ExprNodes.NullNode(pos)
532 else:
533 return p_name(s, name)
534 else:
535 s.error("Expected an identifier or literal")
537 def p_name(s, name):
538 pos = s.position()
539 if not s.compile_time_expr and name in s.compile_time_env:
540 value = s.compile_time_env.lookup_here(name)
541 rep = repr(value)
542 if isinstance(value, bool):
543 return ExprNodes.BoolNode(pos, value = value)
544 elif isinstance(value, int):
545 return ExprNodes.IntNode(pos, value = rep)
546 elif isinstance(value, long):
547 return ExprNodes.IntNode(pos, value = rep, longness = "L")
548 elif isinstance(value, float):
549 return ExprNodes.FloatNode(pos, value = rep)
550 elif isinstance(value, (str, unicode)):
551 return ExprNodes.StringNode(pos, value = value)
552 else:
553 error(pos, "Invalid type for compile-time constant: %s"
554 % value.__class__.__name__)
555 return ExprNodes.NameNode(pos, name = name)
557 def p_cat_string_literal(s):
558 # A sequence of one or more adjacent string literals.
559 # Returns (kind, value) where kind in ('b', 'c', 'u')
560 kind, value = p_string_literal(s)
561 if kind != 'c':
562 strings = [value]
563 while s.sy == 'BEGIN_STRING':
564 next_kind, next_value = p_string_literal(s)
565 if next_kind == 'c':
566 error(s.position(),
567 "Cannot concatenate char literal with another string or char literal")
568 elif next_kind != kind:
569 # we have to switch to unicode now
570 if kind == 'b':
571 # concatenating a unicode string to byte strings
572 strings = [u''.join([s.decode(s.encoding) for s in strings])]
573 elif kind == 'u':
574 # concatenating a byte string to unicode strings
575 strings.append(next_value.decode(next_value.encoding))
576 kind = 'u'
577 else:
578 strings.append(next_value)
579 if kind == 'u':
580 value = EncodedString( u''.join(strings) )
581 else:
582 value = BytesLiteral( ''.join(strings) )
583 value.encoding = s.source_encoding
584 return kind, value
586 def p_opt_string_literal(s):
587 if s.sy == 'BEGIN_STRING':
588 return p_string_literal(s)
589 else:
590 return None
592 def p_string_literal(s):
593 # A single string or char literal.
594 # Returns (kind, value) where kind in ('b', 'c', 'u')
595 # s.sy == 'BEGIN_STRING'
596 pos = s.position()
597 is_raw = 0
598 kind = s.systring[:1].lower()
599 if kind == 'r':
600 kind = ''
601 is_raw = 1
602 elif kind in 'ub':
603 is_raw = s.systring[1:2].lower() == 'r'
604 elif kind != 'c':
605 kind = ''
606 if Future.unicode_literals in s.context.future_directives:
607 if kind == '':
608 kind = 'u'
609 elif kind == '':
610 kind = 'b'
611 if kind == 'u':
612 chars = StringEncoding.UnicodeLiteralBuilder()
613 else:
614 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)
615 while 1:
616 s.next()
617 sy = s.sy
618 #print "p_string_literal: sy =", sy, repr(s.systring) ###
619 if sy == 'CHARS':
620 chars.append(s.systring)
621 elif sy == 'ESCAPE':
622 has_escape = True
623 systr = s.systring
624 if is_raw:
625 if systr == u'\\\n':
626 chars.append(u'\\\n')
627 elif systr == u'\\\"':
628 chars.append(u'"')
629 elif systr == u'\\\'':
630 chars.append(u"'")
631 else:
632 chars.append(systr)
633 else:
634 c = systr[1]
635 if c in u"01234567":
636 chars.append_charval( int(systr[1:], 8) )
637 elif c in u"'\"\\":
638 chars.append(c)
639 elif c in u"abfnrtv":
640 chars.append(
641 StringEncoding.char_from_escape_sequence(systr))
642 elif c == u'\n':
643 pass
644 elif c in u'Uux':
645 if kind == 'u' or c == 'x':
646 chrval = int(systr[2:], 16)
647 if chrval > 1114111: # sys.maxunicode:
648 s.error("Invalid unicode escape '%s'" % systr,
649 pos = pos)
650 elif chrval > 65535:
651 warning(s.position(),
652 "Unicode characters above 65535 are not "
653 "necessarily portable across Python installations", 1)
654 chars.append_charval(chrval)
655 else:
656 # unicode escapes in plain byte strings are not unescaped
657 chars.append(systr)
658 else:
659 chars.append(u'\\' + systr[1:])
660 elif sy == 'NEWLINE':
661 chars.append(u'\n')
662 elif sy == 'END_STRING':
663 break
664 elif sy == 'EOF':
665 s.error("Unclosed string literal", pos = pos)
666 else:
667 s.error(
668 "Unexpected token %r:%r in string literal" %
669 (sy, s.systring))
670 if kind == 'c':
671 value = chars.getchar()
672 if len(value) != 1:
673 error(pos, u"invalid character literal: %r" % value)
674 else:
675 value = chars.getstring()
676 s.next()
677 #print "p_string_literal: value =", repr(value) ###
678 return kind, value
680 # list_display ::= "[" [listmaker] "]"
681 # listmaker ::= expression ( list_for | ( "," expression )* [","] )
682 # list_iter ::= list_for | list_if
683 # list_for ::= "for" expression_list "in" testlist [list_iter]
684 # list_if ::= "if" test [list_iter]
686 def p_list_maker(s):
687 # s.sy == '['
688 pos = s.position()
689 s.next()
690 if s.sy == ']':
691 s.expect(']')
692 return ExprNodes.ListNode(pos, args = [])
693 expr = p_simple_expr(s)
694 if s.sy == 'for':
695 loop = p_list_for(s)
696 s.expect(']')
697 inner_loop = loop
698 while not isinstance(inner_loop.body, Nodes.PassStatNode):
699 inner_loop = inner_loop.body
700 if isinstance(inner_loop, Nodes.IfStatNode):
701 inner_loop = inner_loop.if_clauses[0]
702 append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )
703 inner_loop.body = Nodes.ExprStatNode(pos, expr = append)
704 return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)
705 else:
706 exprs = [expr]
707 if s.sy == ',':
708 s.next()
709 exprs += p_simple_expr_list(s)
710 s.expect(']')
711 return ExprNodes.ListNode(pos, args = exprs)
713 def p_list_iter(s):
714 if s.sy == 'for':
715 return p_list_for(s)
716 elif s.sy == 'if':
717 return p_list_if(s)
718 else:
719 return Nodes.PassStatNode(s.position())
721 def p_list_for(s):
722 # s.sy == 'for'
723 pos = s.position()
724 s.next()
725 kw = p_for_bounds(s)
726 kw['else_clause'] = None
727 kw['body'] = p_list_iter(s)
728 return Nodes.ForStatNode(pos, **kw)
730 def p_list_if(s):
731 # s.sy == 'if'
732 pos = s.position()
733 s.next()
734 test = p_test(s)
735 return Nodes.IfStatNode(pos,
736 if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],
737 else_clause = None )
739 #dictmaker: test ':' test (',' test ':' test)* [',']
741 def p_dict_maker(s):
742 # s.sy == '{'
743 pos = s.position()
744 s.next()
745 items = []
746 while s.sy != '}':
747 items.append(p_dict_item(s))
748 if s.sy != ',':
749 break
750 s.next()
751 s.expect('}')
752 return ExprNodes.DictNode(pos, key_value_pairs = items)
754 def p_dict_item(s):
755 key = p_simple_expr(s)
756 s.expect(':')
757 value = p_simple_expr(s)
758 return ExprNodes.DictItemNode(key.pos, key=key, value=value)
760 def p_backquote_expr(s):
761 # s.sy == '`'
762 pos = s.position()
763 s.next()
764 arg = p_expr(s)
765 s.expect('`')
766 return ExprNodes.BackquoteNode(pos, arg = arg)
768 def p_simple_expr_list(s):
769 exprs = []
770 while s.sy not in expr_terminators:
771 exprs.append(p_simple_expr(s))
772 if s.sy != ',':
773 break
774 s.next()
775 return exprs
777 def p_expr(s):
778 pos = s.position()
779 expr = p_simple_expr(s)
780 if s.sy == ',':
781 s.next()
782 exprs = [expr] + p_simple_expr_list(s)
783 return ExprNodes.TupleNode(pos, args = exprs)
784 else:
785 return expr
788 #testlist: test (',' test)* [',']
789 # differs from p_expr only in the fact that it cannot contain conditional expressions
791 def p_testlist(s):
792 pos = s.position()
793 expr = p_test(s)
794 if s.sy == ',':
795 exprs = [expr]
796 while s.sy == ',':
797 s.next()
798 exprs.append(p_test(s))
799 return ExprNodes.TupleNode(pos, args = exprs)
800 else:
801 return expr
803 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')
805 #-------------------------------------------------------
806 #
807 # Statements
808 #
809 #-------------------------------------------------------
811 def p_global_statement(s):
812 # assume s.sy == 'global'
813 pos = s.position()
814 s.next()
815 names = p_ident_list(s)
816 return Nodes.GlobalNode(pos, names = names)
818 def p_expression_or_assignment(s):
819 expr_list = [p_expr(s)]
820 while s.sy == '=':
821 s.next()
822 expr_list.append(p_expr(s))
823 if len(expr_list) == 1:
824 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
825 lhs = expr_list[0]
826 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
827 error(lhs.pos, "Illegal operand for inplace operation.")
828 operator = s.sy[:-1]
829 s.next()
830 rhs = p_expr(s)
831 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
832 expr = expr_list[0]
833 if isinstance(expr, ExprNodes.StringNode):
834 return Nodes.PassStatNode(expr.pos)
835 else:
836 return Nodes.ExprStatNode(expr.pos, expr = expr)
837 else:
838 expr_list_list = []
839 flatten_parallel_assignments(expr_list, expr_list_list)
840 nodes = []
841 for expr_list in expr_list_list:
842 lhs_list = expr_list[:-1]
843 rhs = expr_list[-1]
844 if len(lhs_list) == 1:
845 node = Nodes.SingleAssignmentNode(rhs.pos,
846 lhs = lhs_list[0], rhs = rhs)
847 else:
848 node = Nodes.CascadedAssignmentNode(rhs.pos,
849 lhs_list = lhs_list, rhs = rhs)
850 nodes.append(node)
851 if len(nodes) == 1:
852 return nodes[0]
853 else:
854 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
856 def flatten_parallel_assignments(input, output):
857 # The input is a list of expression nodes, representing
858 # the LHSs and RHS of one (possibly cascaded) assignment
859 # statement. If they are all sequence constructors with
860 # the same number of arguments, rearranges them into a
861 # list of equivalent assignments between the individual
862 # elements. This transformation is applied recursively.
863 size = find_parallel_assignment_size(input)
864 if size >= 0:
865 for i in range(size):
866 new_exprs = [expr.args[i] for expr in input]
867 flatten_parallel_assignments(new_exprs, output)
868 else:
869 output.append(input)
871 def find_parallel_assignment_size(input):
872 # The input is a list of expression nodes. If
873 # they are all sequence constructors with the same number
874 # of arguments, return that number, else return -1.
875 # Produces an error message if they are all sequence
876 # constructors but not all the same size.
877 for expr in input:
878 if not expr.is_sequence_constructor:
879 return -1
880 rhs = input[-1]
881 rhs_size = len(rhs.args)
882 for lhs in input[:-1]:
883 lhs_size = len(lhs.args)
884 if lhs_size != rhs_size:
885 error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"
886 % (lhs_size, rhs_size))
887 return -1
888 return rhs_size
890 def p_print_statement(s):
891 # s.sy == 'print'
892 pos = s.position()
893 s.next()
894 if s.sy == '>>':
895 s.error("'print >>' not yet implemented")
896 args = []
897 ends_with_comma = 0
898 if s.sy not in ('NEWLINE', 'EOF'):
899 args.append(p_simple_expr(s))
900 while s.sy == ',':
901 s.next()
902 if s.sy in ('NEWLINE', 'EOF'):
903 ends_with_comma = 1
904 break
905 args.append(p_simple_expr(s))
906 arg_tuple = ExprNodes.TupleNode(pos, args = args)
907 return Nodes.PrintStatNode(pos,
908 arg_tuple = arg_tuple, append_newline = not ends_with_comma)
910 def p_exec_statement(s):
911 # s.sy == 'exec'
912 pos = s.position()
913 s.next()
914 args = [ p_bit_expr(s) ]
915 if s.sy == 'in':
916 s.next()
917 args.append(p_simple_expr(s))
918 if s.sy == ',':
919 s.next()
920 args.append(p_simple_expr(s))
921 else:
922 error(pos, "'exec' currently requires a target mapping (globals/locals)")
923 return Nodes.ExecStatNode(pos, args = args)
925 def p_del_statement(s):
926 # s.sy == 'del'
927 pos = s.position()
928 s.next()
929 args = p_simple_expr_list(s)
930 return Nodes.DelStatNode(pos, args = args)
932 def p_pass_statement(s, with_newline = 0):
933 pos = s.position()
934 s.expect('pass')
935 if with_newline:
936 s.expect_newline("Expected a newline")
937 return Nodes.PassStatNode(pos)
939 def p_break_statement(s):
940 # s.sy == 'break'
941 pos = s.position()
942 s.next()
943 return Nodes.BreakStatNode(pos)
945 def p_continue_statement(s):
946 # s.sy == 'continue'
947 pos = s.position()
948 s.next()
949 return Nodes.ContinueStatNode(pos)
951 def p_return_statement(s):
952 # s.sy == 'return'
953 pos = s.position()
954 s.next()
955 if s.sy not in statement_terminators:
956 value = p_expr(s)
957 else:
958 value = None
959 return Nodes.ReturnStatNode(pos, value = value)
961 def p_raise_statement(s):
962 # s.sy == 'raise'
963 pos = s.position()
964 s.next()
965 exc_type = None
966 exc_value = None
967 exc_tb = None
968 if s.sy not in statement_terminators:
969 exc_type = p_simple_expr(s)
970 if s.sy == ',':
971 s.next()
972 exc_value = p_simple_expr(s)
973 if s.sy == ',':
974 s.next()
975 exc_tb = p_simple_expr(s)
976 if exc_type or exc_value or exc_tb:
977 return Nodes.RaiseStatNode(pos,
978 exc_type = exc_type,
979 exc_value = exc_value,
980 exc_tb = exc_tb)
981 else:
982 return Nodes.ReraiseStatNode(pos)
984 def p_import_statement(s):
985 # s.sy in ('import', 'cimport')
986 pos = s.position()
987 kind = s.sy
988 s.next()
989 items = [p_dotted_name(s, as_allowed = 1)]
990 while s.sy == ',':
991 s.next()
992 items.append(p_dotted_name(s, as_allowed = 1))
993 stats = []
994 for pos, target_name, dotted_name, as_name in items:
995 dotted_name = EncodedString(dotted_name)
996 if kind == 'cimport':
997 stat = Nodes.CImportStatNode(pos,
998 module_name = dotted_name,
999 as_name = as_name)
1000 else:
1001 if as_name and "." in dotted_name:
1002 name_list = ExprNodes.ListNode(pos, args = [
1003 ExprNodes.IdentifierStringNode(
1004 pos, value = EncodedString("*"))])
1005 else:
1006 name_list = None
1007 stat = Nodes.SingleAssignmentNode(pos,
1008 lhs = ExprNodes.NameNode(pos,
1009 name = as_name or target_name),
1010 rhs = ExprNodes.ImportNode(pos,
1011 module_name = ExprNodes.IdentifierStringNode(
1012 pos, value = dotted_name),
1013 name_list = name_list))
1014 stats.append(stat)
1015 return Nodes.StatListNode(pos, stats = stats)
1017 def p_from_import_statement(s, first_statement = 0):
1018 # s.sy == 'from'
1019 pos = s.position()
1020 s.next()
1021 (dotted_name_pos, _, dotted_name, _) = \
1022 p_dotted_name(s, as_allowed = 0)
1023 if s.sy in ('import', 'cimport'):
1024 kind = s.sy
1025 s.next()
1026 else:
1027 s.error("Expected 'import' or 'cimport'")
1028 is_cimport = kind == 'cimport'
1029 is_parenthesized = False
1030 if s.sy == '*':
1031 imported_names = [(s.position(), "*", None, None)]
1032 s.next()
1033 else:
1034 if s.sy == '(':
1035 is_parenthesized = True
1036 s.next()
1037 eat_newlines(s)
1038 imported_names = [p_imported_name(s, is_cimport)]
1039 if is_parenthesized:
1040 eat_newlines(s)
1041 while s.sy == ',':
1042 s.next()
1043 if is_parenthesized:
1044 eat_newlines(s)
1045 imported_names.append(p_imported_name(s, is_cimport))
1046 if is_parenthesized:
1047 eat_newlines(s)
1048 if is_parenthesized:
1049 s.expect(')')
1050 dotted_name = EncodedString(dotted_name)
1051 if dotted_name == '__future__':
1052 if not first_statement:
1053 s.error("from __future__ imports must occur at the beginning of the file")
1054 else:
1055 for (name_pos, name, as_name, kind) in imported_names:
1056 if name == "braces":
1057 s.error("not a chance", name_pos)
1058 break
1059 try:
1060 directive = getattr(Future, name)
1061 except AttributeError:
1062 s.error("future feature %s is not defined" % name, name_pos)
1063 break
1064 s.context.future_directives.add(directive)
1065 return Nodes.PassStatNode(pos)
1066 elif kind == 'cimport':
1067 return Nodes.FromCImportStatNode(pos,
1068 module_name = dotted_name,
1069 imported_names = imported_names)
1070 else:
1071 imported_name_strings = []
1072 items = []
1073 for (name_pos, name, as_name, kind) in imported_names:
1074 encoded_name = EncodedString(name)
1075 imported_name_strings.append(
1076 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
1077 items.append(
1078 (name,
1079 ExprNodes.NameNode(name_pos,
1080 name = as_name or name)))
1081 import_list = ExprNodes.ListNode(
1082 imported_names[0][0], args = imported_name_strings)
1083 dotted_name = EncodedString(dotted_name)
1084 return Nodes.FromImportStatNode(pos,
1085 module = ExprNodes.ImportNode(dotted_name_pos,
1086 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
1087 name_list = import_list),
1088 items = items)
1090 imported_name_kinds = ('class', 'struct', 'union')
1092 def p_imported_name(s, is_cimport):
1093 pos = s.position()
1094 kind = None
1095 if is_cimport and s.systring in imported_name_kinds:
1096 kind = s.systring
1097 s.next()
1098 name = p_ident(s)
1099 as_name = p_as_name(s)
1100 return (pos, name, as_name, kind)
1102 def p_dotted_name(s, as_allowed):
1103 pos = s.position()
1104 target_name = p_ident(s)
1105 as_name = None
1106 names = [target_name]
1107 while s.sy == '.':
1108 s.next()
1109 names.append(p_ident(s))
1110 if as_allowed:
1111 as_name = p_as_name(s)
1112 return (pos, target_name, u'.'.join(names), as_name)
1114 def p_as_name(s):
1115 if s.sy == 'IDENT' and s.systring == 'as':
1116 s.next()
1117 return p_ident(s)
1118 else:
1119 return None
1121 def p_assert_statement(s):
1122 # s.sy == 'assert'
1123 pos = s.position()
1124 s.next()
1125 cond = p_simple_expr(s)
1126 if s.sy == ',':
1127 s.next()
1128 value = p_simple_expr(s)
1129 else:
1130 value = None
1131 return Nodes.AssertStatNode(pos, cond = cond, value = value)
1133 statement_terminators = (';', 'NEWLINE', 'EOF')
1135 def p_if_statement(s):
1136 # s.sy == 'if'
1137 pos = s.position()
1138 s.next()
1139 if_clauses = [p_if_clause(s)]
1140 while s.sy == 'elif':
1141 s.next()
1142 if_clauses.append(p_if_clause(s))
1143 else_clause = p_else_clause(s)
1144 return Nodes.IfStatNode(pos,
1145 if_clauses = if_clauses, else_clause = else_clause)
1147 def p_if_clause(s):
1148 pos = s.position()
1149 test = p_simple_expr(s)
1150 body = p_suite(s)
1151 return Nodes.IfClauseNode(pos,
1152 condition = test, body = body)
1154 def p_else_clause(s):
1155 if s.sy == 'else':
1156 s.next()
1157 return p_suite(s)
1158 else:
1159 return None
1161 def p_while_statement(s):
1162 # s.sy == 'while'
1163 pos = s.position()
1164 s.next()
1165 test = p_simple_expr(s)
1166 body = p_suite(s)
1167 else_clause = p_else_clause(s)
1168 return Nodes.WhileStatNode(pos,
1169 condition = test, body = body,
1170 else_clause = else_clause)
1172 def p_for_statement(s):
1173 # s.sy == 'for'
1174 pos = s.position()
1175 s.next()
1176 kw = p_for_bounds(s)
1177 kw['body'] = p_suite(s)
1178 kw['else_clause'] = p_else_clause(s)
1179 return Nodes.ForStatNode(pos, **kw)
1181 def p_for_bounds(s):
1182 target = p_for_target(s)
1183 if s.sy == 'in':
1184 s.next()
1185 iterator = p_for_iterator(s)
1186 return { 'target': target, 'iterator': iterator }
1187 else:
1188 if s.sy == 'from':
1189 s.next()
1190 bound1 = p_bit_expr(s)
1191 else:
1192 # Support shorter "for a <= x < b" syntax
1193 bound1, target = target, None
1194 rel1 = p_for_from_relation(s)
1195 name2_pos = s.position()
1196 name2 = p_ident(s)
1197 rel2_pos = s.position()
1198 rel2 = p_for_from_relation(s)
1199 bound2 = p_bit_expr(s)
1200 step = p_for_from_step(s)
1201 if target is None:
1202 target = ExprNodes.NameNode(name2_pos, name = name2)
1203 else:
1204 if not target.is_name:
1205 error(target.pos,
1206 "Target of for-from statement must be a variable name")
1207 elif name2 != target.name:
1208 error(name2_pos,
1209 "Variable name in for-from range does not match target")
1210 if rel1[0] != rel2[0]:
1211 error(rel2_pos,
1212 "Relation directions in for-from do not match")
1213 return {'target': target,
1214 'bound1': bound1,
1215 'relation1': rel1,
1216 'relation2': rel2,
1217 'bound2': bound2,
1218 'step': step }
1220 def p_for_from_relation(s):
1221 if s.sy in inequality_relations:
1222 op = s.sy
1223 s.next()
1224 return op
1225 else:
1226 s.error("Expected one of '<', '<=', '>' '>='")
1228 def p_for_from_step(s):
1229 if s.sy == 'by':
1230 s.next()
1231 step = p_bit_expr(s)
1232 return step
1233 else:
1234 return None
1236 inequality_relations = ('<', '<=', '>', '>=')
1238 def p_target(s, terminator):
1239 pos = s.position()
1240 expr = p_bit_expr(s)
1241 if s.sy == ',':
1242 s.next()
1243 exprs = [expr]
1244 while s.sy != terminator:
1245 exprs.append(p_bit_expr(s))
1246 if s.sy != ',':
1247 break
1248 s.next()
1249 return ExprNodes.TupleNode(pos, args = exprs)
1250 else:
1251 return expr
1253 def p_for_target(s):
1254 return p_target(s, 'in')
1256 def p_for_iterator(s):
1257 pos = s.position()
1258 expr = p_testlist(s)
1259 return ExprNodes.IteratorNode(pos, sequence = expr)
1261 def p_try_statement(s):
1262 # s.sy == 'try'
1263 pos = s.position()
1264 s.next()
1265 body = p_suite(s)
1266 except_clauses = []
1267 else_clause = None
1268 if s.sy in ('except', 'else'):
1269 while s.sy == 'except':
1270 except_clauses.append(p_except_clause(s))
1271 if s.sy == 'else':
1272 s.next()
1273 else_clause = p_suite(s)
1274 body = Nodes.TryExceptStatNode(pos,
1275 body = body, except_clauses = except_clauses,
1276 else_clause = else_clause)
1277 if s.sy != 'finally':
1278 return body
1279 # try-except-finally is equivalent to nested try-except/try-finally
1280 if s.sy == 'finally':
1281 s.next()
1282 finally_clause = p_suite(s)
1283 return Nodes.TryFinallyStatNode(pos,
1284 body = body, finally_clause = finally_clause)
1285 else:
1286 s.error("Expected 'except' or 'finally'")
1288 def p_except_clause(s):
1289 # s.sy == 'except'
1290 pos = s.position()
1291 s.next()
1292 exc_type = None
1293 exc_value = None
1294 if s.sy != ':':
1295 exc_type = p_simple_expr(s)
1296 if s.sy == ',':
1297 s.next()
1298 exc_value = p_simple_expr(s)
1299 body = p_suite(s)
1300 return Nodes.ExceptClauseNode(pos,
1301 pattern = exc_type, target = exc_value, body = body)
1303 def p_include_statement(s, ctx):
1304 pos = s.position()
1305 s.next() # 'include'
1306 _, include_file_name = p_string_literal(s)
1307 s.expect_newline("Syntax error in include statement")
1308 if s.compile_time_eval:
1309 include_file_path = s.context.find_include_file(include_file_name, pos)
1310 if include_file_path:
1311 s.included_files.append(include_file_name)
1312 f = Utils.open_source_file(include_file_path, mode="rU")
1313 source_desc = FileSourceDescriptor(include_file_path)
1314 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
1315 try:
1316 tree = p_statement_list(s2, ctx)
1317 finally:
1318 f.close()
1319 return tree
1320 else:
1321 return None
1322 else:
1323 return Nodes.PassStatNode(pos)
1325 def p_with_statement(s):
1326 pos = s.position()
1327 s.next() # 'with'
1328 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):
1329 if s.sy == 'IDENT' and s.systring == 'nogil':
1330 state = s.systring
1331 s.next()
1332 body = p_suite(s)
1333 return Nodes.GILStatNode(pos, state = state, body = body)
1334 else:
1335 manager = p_expr(s)
1336 target = None
1337 if s.sy == 'IDENT' and s.systring == 'as':
1338 s.next()
1339 allow_multi = (s.sy == '(')
1340 target = p_target(s, ':')
1341 if not allow_multi and isinstance(target, ExprNodes.TupleNode):
1342 s.error("Multiple with statement target values not allowed without paranthesis")
1343 body = p_suite(s)
1344 return Nodes.WithStatNode(pos, manager = manager,
1345 target = target, body = body)
1347 def p_simple_statement(s, first_statement = 0):
1348 #print "p_simple_statement:", s.sy, s.systring ###
1349 if s.sy == 'global':
1350 node = p_global_statement(s)
1351 elif s.sy == 'print':
1352 node = p_print_statement(s)
1353 elif s.sy == 'exec':
1354 node = p_exec_statement(s)
1355 elif s.sy == 'del':
1356 node = p_del_statement(s)
1357 elif s.sy == 'break':
1358 node = p_break_statement(s)
1359 elif s.sy == 'continue':
1360 node = p_continue_statement(s)
1361 elif s.sy == 'return':
1362 node = p_return_statement(s)
1363 elif s.sy == 'raise':
1364 node = p_raise_statement(s)
1365 elif s.sy in ('import', 'cimport'):
1366 node = p_import_statement(s)
1367 elif s.sy == 'from':
1368 node = p_from_import_statement(s, first_statement = first_statement)
1369 elif s.sy == 'assert':
1370 node = p_assert_statement(s)
1371 elif s.sy == 'pass':
1372 node = p_pass_statement(s)
1373 else:
1374 node = p_expression_or_assignment(s)
1375 return node
1377 def p_simple_statement_list(s, ctx, first_statement = 0):
1378 # Parse a series of simple statements on one line
1379 # separated by semicolons.
1380 stat = p_simple_statement(s, first_statement = first_statement)
1381 if s.sy == ';':
1382 stats = [stat]
1383 while s.sy == ';':
1384 #print "p_simple_statement_list: maybe more to follow" ###
1385 s.next()
1386 if s.sy in ('NEWLINE', 'EOF'):
1387 break
1388 stats.append(p_simple_statement(s))
1389 stat = Nodes.StatListNode(stats[0].pos, stats = stats)
1390 s.expect_newline("Syntax error in simple statement list")
1391 return stat
1393 def p_compile_time_expr(s):
1394 old = s.compile_time_expr
1395 s.compile_time_expr = 1
1396 expr = p_expr(s)
1397 s.compile_time_expr = old
1398 return expr
1400 def p_DEF_statement(s):
1401 pos = s.position()
1402 denv = s.compile_time_env
1403 s.next() # 'DEF'
1404 name = p_ident(s)
1405 s.expect('=')
1406 expr = p_compile_time_expr(s)
1407 value = expr.compile_time_value(denv)
1408 #print "p_DEF_statement: %s = %r" % (name, value) ###
1409 denv.declare(name, value)
1410 s.expect_newline()
1411 return Nodes.PassStatNode(pos)
1413 def p_IF_statement(s, ctx):
1414 pos = s.position()
1415 saved_eval = s.compile_time_eval
1416 current_eval = saved_eval
1417 denv = s.compile_time_env
1418 result = None
1419 while 1:
1420 s.next() # 'IF' or 'ELIF'
1421 expr = p_compile_time_expr(s)
1422 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1423 body = p_suite(s, ctx)
1424 if s.compile_time_eval:
1425 result = body
1426 current_eval = 0
1427 if s.sy != 'ELIF':
1428 break
1429 if s.sy == 'ELSE':
1430 s.next()
1431 s.compile_time_eval = current_eval
1432 body = p_suite(s, ctx)
1433 if current_eval:
1434 result = body
1435 if not result:
1436 result = Nodes.PassStatNode(pos)
1437 s.compile_time_eval = saved_eval
1438 return result
1440 def p_statement(s, ctx, first_statement = 0):
1441 cdef_flag = ctx.cdef_flag
1442 if s.sy == 'ctypedef':
1443 if ctx.level not in ('module', 'module_pxd'):
1444 s.error("ctypedef statement not allowed here")
1445 #if ctx.api:
1446 # error(s.position(), "'api' not allowed with 'ctypedef'")
1447 return p_ctypedef_statement(s, ctx)
1448 elif s.sy == 'DEF':
1449 return p_DEF_statement(s)
1450 elif s.sy == 'IF':
1451 return p_IF_statement(s, ctx)
1452 elif s.sy == 'DECORATOR':
1453 if ctx.level not in ('module', 'class', 'c_class', 'property'):
1454 s.error('decorator not allowed here')
1455 s.level = ctx.level
1456 decorators = p_decorators(s)
1457 if s.sy != 'def':
1458 s.error("Decorators can only be followed by functions ")
1459 return p_def_statement(s, decorators)
1460 else:
1461 overridable = 0
1462 if s.sy == 'cdef':
1463 cdef_flag = 1
1464 s.next()
1465 if s.sy == 'cpdef':
1466 cdef_flag = 1
1467 overridable = 1
1468 s.next()
1469 if cdef_flag:
1470 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
1471 s.error('cdef statement not allowed here')
1472 s.level = ctx.level
1473 return p_cdef_statement(s, ctx(overridable = overridable))
1474 else:
1475 if ctx.api:
1476 error(s.pos, "'api' not allowed with this statement")
1477 elif s.sy == 'def':
1478 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):
1479 s.error('def statement not allowed here')
1480 s.level = ctx.level
1481 return p_def_statement(s)
1482 elif s.sy == 'class':
1483 if ctx.level != 'module':
1484 s.error("class definition not allowed here")
1485 return p_class_statement(s)
1486 elif s.sy == 'include':
1487 if ctx.level not in ('module', 'module_pxd'):
1488 s.error("include statement not allowed here")
1489 return p_include_statement(s, ctx)
1490 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
1491 return p_property_decl(s)
1492 elif s.sy == 'pass' and ctx.level != 'property':
1493 return p_pass_statement(s, with_newline = 1)
1494 else:
1495 if ctx.level in ('c_class_pxd', 'property'):
1496 s.error("Executable statement not allowed here")
1497 if s.sy == 'if':
1498 return p_if_statement(s)
1499 elif s.sy == 'while':
1500 return p_while_statement(s)
1501 elif s.sy == 'for':
1502 return p_for_statement(s)
1503 elif s.sy == 'try':
1504 return p_try_statement(s)
1505 elif s.sy == 'with':
1506 return p_with_statement(s)
1507 else:
1508 return p_simple_statement_list(
1509 s, ctx, first_statement = first_statement)
1511 def p_statement_list(s, ctx, first_statement = 0):
1512 # Parse a series of statements separated by newlines.
1513 pos = s.position()
1514 stats = []
1515 while s.sy not in ('DEDENT', 'EOF'):
1516 stats.append(p_statement(s, ctx, first_statement = first_statement))
1517 first_statement = 0
1518 if len(stats) == 1:
1519 return stats[0]
1520 else:
1521 return Nodes.StatListNode(pos, stats = stats)
1523 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
1524 pos = s.position()
1525 s.expect(':')
1526 doc = None
1527 stmts = []
1528 if s.sy == 'NEWLINE':
1529 s.next()
1530 s.expect_indent()
1531 if with_doc or with_pseudo_doc:
1532 doc = p_doc_string(s)
1533 body = p_statement_list(s, ctx)
1534 s.expect_dedent()
1535 else:
1536 if ctx.api:
1537 error(s.pos, "'api' not allowed with this statement")
1538 if ctx.level in ('module', 'class', 'function', 'other'):
1539 body = p_simple_statement_list(s, ctx)
1540 else:
1541 body = p_pass_statement(s)
1542 s.expect_newline("Syntax error in declarations")
1543 if with_doc:
1544 return doc, body
1545 else:
1546 return body
1548 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
1549 """
1550 Parses positional and keyword arguments. end_sy_set
1551 should contain any s.sy that terminate the argument list.
1552 Argument expansion (* and **) are not allowed.
1554 type_positions and type_keywords specifies which argument
1555 positions and/or names which should be interpreted as
1556 types. Other arguments will be treated as expressions.
1558 Returns: (positional_args, keyword_args)
1559 """
1560 positional_args = []
1561 keyword_args = []
1562 pos_idx = 0
1564 while s.sy not in end_sy_set:
1565 if s.sy == '*' or s.sy == '**':
1566 s.error('Argument expansion not allowed here.')
1568 was_keyword = False
1569 parsed_type = False
1570 if s.sy == 'IDENT':
1571 # Since we can have either types or expressions as positional args,
1572 # we use a strategy of looking an extra step forward for a '=' and
1573 # if it is a positional arg we backtrack.
1574 ident = s.systring
1575 s.next()
1576 if s.sy == '=':
1577 s.next()
1578 # Is keyword arg
1579 if ident in type_keywords:
1580 arg = p_c_base_type(s)
1581 parsed_type = True
1582 else:
1583 arg = p_simple_expr(s)
1584 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
1585 value = EncodedString(ident))
1586 keyword_args.append((keyword_node, arg))
1587 was_keyword = True
1588 else:
1589 s.put_back('IDENT', ident)
1591 if not was_keyword:
1592 if pos_idx in type_positions:
1593 arg = p_c_base_type(s)
1594 parsed_type = True
1595 else:
1596 arg = p_simple_expr(s)
1597 positional_args.append(arg)
1598 pos_idx += 1
1599 if len(keyword_args) > 0:
1600 s.error("Non-keyword arg following keyword arg",
1601 pos = arg.pos)
1603 if s.sy != ',':
1604 if s.sy not in end_sy_set:
1605 if parsed_type:
1606 s.error("Expected: type")
1607 else:
1608 s.error("Expected: expression")
1609 break
1610 s.next()
1611 return positional_args, keyword_args
1613 def p_c_base_type(s, self_flag = 0, nonempty = 0):
1614 # If self_flag is true, this is the base type for the
1615 # self argument of a C method of an extension type.
1616 if s.sy == '(':
1617 return p_c_complex_base_type(s)
1618 else:
1619 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)
1621 def p_calling_convention(s):
1622 if s.sy == 'IDENT' and s.systring in calling_convention_words:
1623 result = s.systring
1624 s.next()
1625 return result
1626 else:
1627 return ""
1629 calling_convention_words = ("__stdcall", "__cdecl")
1631 def p_c_complex_base_type(s):
1632 # s.sy == '('
1633 pos = s.position()
1634 s.next()
1635 base_type = p_c_base_type(s)
1636 declarator = p_c_declarator(s, empty = 1)
1637 s.expect(')')
1638 return Nodes.CComplexBaseTypeNode(pos,
1639 base_type = base_type, declarator = declarator)
1641 def p_c_simple_base_type(s, self_flag, nonempty):
1642 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty
1643 is_basic = 0
1644 signed = 1
1645 longness = 0
1646 module_path = []
1647 pos = s.position()
1648 if looking_at_base_type(s):
1649 #print "p_c_simple_base_type: looking_at_base_type at", s.position()
1650 is_basic = 1
1651 signed, longness = p_sign_and_longness(s)
1652 if s.sy == 'IDENT' and s.systring in basic_c_type_names:
1653 name = s.systring
1654 s.next()
1655 else:
1656 name = 'int'
1657 elif looking_at_dotted_name(s):
1658 #print "p_c_simple_base_type: looking_at_type_name at", s.position()
1659 name = s.systring
1660 s.next()
1661 while s.sy == '.':
1662 module_path.append(name)
1663 s.next()
1664 name = p_ident(s)
1665 else:
1666 name = s.systring
1667 s.next()
1668 if nonempty and s.sy != 'IDENT':
1669 # Make sure this is not a declaration of a variable or function.
1670 if s.sy == '(':
1671 s.next()
1672 if s.sy == '*' or s.sy == '**':
1673 s.put_back('(', '(')
1674 else:
1675 s.put_back('(', '(')
1676 s.put_back('IDENT', name)
1677 name = None
1678 elif s.sy not in ('*', '**', '['):
1679 s.put_back('IDENT', name)
1680 name = None
1682 type_node = Nodes.CSimpleBaseTypeNode(pos,
1683 name = name, module_path = module_path,
1684 is_basic_c_type = is_basic, signed = signed,
1685 longness = longness, is_self_arg = self_flag)
1688 # Treat trailing [] on type as buffer access if it appears in a context
1689 # where declarator names are required (so that it cannot mean int[] or
1690 # sizeof(int[SIZE]))...
1691 #
1692 # (This means that buffers cannot occur where there can be empty declarators,
1693 # which is an ok restriction to make.)
1694 if nonempty and s.sy == '[':
1695 return p_buffer_access(s, type_node)
1696 else:
1697 return type_node
1699 def p_buffer_access(s, base_type_node):
1700 # s.sy == '['
1701 pos = s.position()
1702 s.next()
1703 positional_args, keyword_args = (
1704 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
1705 )
1706 s.expect(']')
1708 keyword_dict = ExprNodes.DictNode(pos,
1709 key_value_pairs = [
1710 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
1711 for key, value in keyword_args
1712 ])
1714 result = Nodes.CBufferAccessTypeNode(pos,
1715 positional_args = positional_args,
1716 keyword_args = keyword_dict,
1717 base_type_node = base_type_node)
1718 return result
1721 def looking_at_name(s):
1722 return s.sy == 'IDENT' and not s.systring in calling_convention_words
1724 def looking_at_expr(s):
1725 if s.systring in base_type_start_words:
1726 return False
1727 elif s.sy == 'IDENT':
1728 is_type = False
1729 name = s.systring
1730 dotted_path = []
1731 s.next()
1732 while s.sy == '.':
1733 s.next()
1734 dotted_path.append(s.systring)
1735 s.expect('IDENT')
1736 saved = s.sy, s.systring
1737 if s.sy == 'IDENT':
1738 is_type = True
1739 elif s.sy == '*' or s.sy == '**':
1740 s.next()
1741 is_type = s.sy == ')'
1742 s.put_back(*saved)
1743 elif s.sy == '(':
1744 s.next()
1745 is_type = s.sy == '*'
1746 s.put_back(*saved)
1747 elif s.sy == '[':
1748 s.next()
1749 is_type = s.sy == ']'
1750 s.put_back(*saved)
1751 dotted_path.reverse()
1752 for p in dotted_path:
1753 s.put_back('IDENT', p)
1754 s.put_back('.', '.')
1755 s.put_back('IDENT', name)
1756 return not is_type
1757 else:
1758 return True
1760 def looking_at_base_type(s):
1761 #print "looking_at_base_type?", s.sy, s.systring, s.position()
1762 return s.sy == 'IDENT' and s.systring in base_type_start_words
1764 def looking_at_dotted_name(s):
1765 if s.sy == 'IDENT':
1766 name = s.systring
1767 s.next()
1768 result = s.sy == '.'
1769 s.put_back('IDENT', name)
1770 return result
1771 else:
1772 return 0
1774 basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")
1776 sign_and_longness_words = ("short", "long", "signed", "unsigned")
1778 base_type_start_words = \
1779 basic_c_type_names + sign_and_longness_words
1781 def p_sign_and_longness(s):
1782 signed = 1
1783 longness = 0
1784 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
1785 if s.systring == 'unsigned':
1786 signed = 0
1787 elif s.systring == 'signed':
1788 signed = 2
1789 elif s.systring == 'short':
1790 longness = -1
1791 elif s.systring == 'long':
1792 longness += 1
1793 s.next()
1794 return signed, longness
1796 def p_opt_cname(s):
1797 literal = p_opt_string_literal(s)
1798 if literal:
1799 _, cname = literal
1800 else:
1801 cname = None
1802 return cname
1804 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
1805 assignable = 0, nonempty = 0,
1806 calling_convention_allowed = 0):
1807 # If empty is true, the declarator must be empty. If nonempty is true,
1808 # the declarator must be nonempty. Otherwise we don't care.
1809 # If cmethod_flag is true, then if this declarator declares
1810 # a function, it's a C method of an extension type.
1811 pos = s.position()
1812 if s.sy == '(':
1813 s.next()
1814 if s.sy == ')' or looking_at_name(s):
1815 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)
1816 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
1817 else:
1818 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1819 cmethod_flag = cmethod_flag,
1820 nonempty = nonempty,
1821 calling_convention_allowed = 1)
1822 s.expect(')')
1823 else:
1824 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
1825 assignable, nonempty)
1826 if not calling_convention_allowed and result.calling_convention and s.sy != '(':
1827 error(s.position(), "%s on something that is not a function"
1828 % result.calling_convention)
1829 while s.sy in ('[', '('):
1830 pos = s.position()
1831 if s.sy == '[':
1832 result = p_c_array_declarator(s, result)
1833 else: # sy == '('
1834 s.next()
1835 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
1836 cmethod_flag = 0
1837 return result
1839 def p_c_array_declarator(s, base):
1840 pos = s.position()
1841 s.next() # '['
1842 if s.sy != ']':
1843 dim = p_expr(s)
1844 else:
1845 dim = None
1846 s.expect(']')
1847 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)
1849 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
1850 # Opening paren has already been skipped
1851 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
1852 nonempty_declarators = 0)
1853 ellipsis = p_optional_ellipsis(s)
1854 s.expect(')')
1855 nogil = p_nogil(s)
1856 exc_val, exc_check = p_exception_value_clause(s)
1857 with_gil = p_with_gil(s)
1858 return Nodes.CFuncDeclaratorNode(pos,
1859 base = base, args = args, has_varargs = ellipsis,
1860 exception_value = exc_val, exception_check = exc_check,
1861 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
1863 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
1864 assignable, nonempty):
1865 pos = s.position()
1866 calling_convention = p_calling_convention(s)
1867 if s.sy == '*':
1868 s.next()
1869 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1870 cmethod_flag = cmethod_flag,
1871 assignable = assignable, nonempty = nonempty)
1872 result = Nodes.CPtrDeclaratorNode(pos,
1873 base = base)
1874 elif s.sy == '**': # scanner returns this as a single token
1875 s.next()
1876 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1877 cmethod_flag = cmethod_flag,
1878 assignable = assignable, nonempty = nonempty)
1879 result = Nodes.CPtrDeclaratorNode(pos,
1880 base = Nodes.CPtrDeclaratorNode(pos,
1881 base = base))
1882 else:
1883 rhs = None
1884 if s.sy == 'IDENT':
1885 name = EncodedString(s.systring)
1886 if empty:
1887 error(s.position(), "Declarator should be empty")
1888 s.next()
1889 cname = p_opt_cname(s)
1890 if s.sy == '=' and assignable:
1891 s.next()
1892 rhs = p_simple_expr(s)
1893 else:
1894 if nonempty:
1895 error(s.position(), "Empty declarator")
1896 name = ""
1897 cname = None
1898 result = Nodes.CNameDeclaratorNode(pos,
1899 name = name, cname = cname, default = rhs)
1900 result.calling_convention = calling_convention
1901 return result
1903 def p_nogil(s):
1904 if s.sy == 'IDENT' and s.systring == 'nogil':
1905 s.next()
1906 return 1
1907 else:
1908 return 0
1910 def p_with_gil(s):
1911 if s.sy == 'with':
1912 s.next()
1913 s.expect_keyword('gil')
1914 return 1
1915 else:
1916 return 0
1918 def p_exception_value_clause(s):
1919 exc_val = None
1920 exc_check = 0
1921 if s.sy == 'except':
1922 s.next()
1923 if s.sy == '*':
1924 exc_check = 1
1925 s.next()
1926 elif s.sy == '+':
1927 exc_check = '+'
1928 s.next()
1929 if s.sy == 'IDENT':
1930 name = s.systring
1931 s.next()
1932 exc_val = p_name(s, name)
1933 else:
1934 if s.sy == '?':
1935 exc_check = 1
1936 s.next()
1937 exc_val = p_simple_expr(s)
1938 return exc_val, exc_check
1940 c_arg_list_terminators = ('*', '**', '.', ')')
1942 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
1943 nonempty_declarators = 0, kw_only = 0):
1944 # Comma-separated list of C argument declarations, possibly empty.
1945 # May have a trailing comma.
1946 args = []
1947 is_self_arg = cmethod_flag
1948 while s.sy not in c_arg_list_terminators:
1949 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
1950 nonempty = nonempty_declarators, kw_only = kw_only))
1951 if s.sy != ',':
1952 break
1953 s.next()
1954 is_self_arg = 0
1955 return args
1957 def p_optional_ellipsis(s):
1958 if s.sy == '.':
1959 expect_ellipsis(s)
1960 return 1
1961 else:
1962 return 0
1964 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
1965 pos = s.position()
1966 not_none = 0
1967 default = None
1968 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
1969 declarator = p_c_declarator(s, ctx, nonempty = nonempty)
1970 if s.sy == 'not':
1971 s.next()
1972 if s.sy == 'IDENT' and s.systring == 'None':
1973 s.next()
1974 else:
1975 s.error("Expected 'None'")
1976 if not in_pyfunc:
1977 error(pos, "'not None' only allowed in Python functions")
1978 not_none = 1
1979 if s.sy == '=':
1980 s.next()
1981 if 'pxd' in s.level:
1982 if s.sy not in ['*', '?']:
1983 error(pos, "default values cannot be specified in pxd files, use ? or *")
1984 default = ExprNodes.BoolNode(1)
1985 s.next()
1986 else:
1987 default = p_simple_expr(s)
1988 return Nodes.CArgDeclNode(pos,
1989 base_type = base_type,
1990 declarator = declarator,
1991 not_none = not_none,
1992 default = default,
1993 kw_only = kw_only)
1995 def p_api(s):
1996 if s.sy == 'IDENT' and s.systring == 'api':
1997 s.next()
1998 return 1
1999 else:
2000 return 0
2002 def p_cdef_statement(s, ctx):
2003 pos = s.position()
2004 ctx.visibility = p_visibility(s, ctx.visibility)
2005 ctx.api = ctx.api or p_api(s)
2006 if ctx.api:
2007 if ctx.visibility not in ('private', 'public'):
2008 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
2009 if (ctx.visibility == 'extern') and s.sy == 'from':
2010 return p_cdef_extern_block(s, pos, ctx)
2011 elif s.sy == 'import':
2012 s.next()
2013 return p_cdef_extern_block(s, pos, ctx)
2014 if p_nogil(s):
2015 ctx.nogil = 1
2016 if s.sy == ':':
2017 return p_cdef_block(s, ctx)
2018 elif s.sy == 'class':
2019 if ctx.level not in ('module', 'module_pxd'):
2020 error(pos, "Extension type definition not allowed here")
2021 #if ctx.api:
2022 # error(pos, "'api' not allowed with extension class")
2023 return p_c_class_definition(s, pos, ctx)
2024 elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:
2025 if ctx.level not in ('module', 'module_pxd'):
2026 error(pos, "C struct/union/enum definition not allowed here")
2027 #if ctx.visibility == 'public':
2028 # error(pos, "Public struct/union/enum definition not implemented")
2029 #if ctx.api:
2030 # error(pos, "'api' not allowed with '%s'" % s.systring)
2031 if s.systring == "enum":
2032 return p_c_enum_definition(s, pos, ctx)
2033 else:
2034 return p_c_struct_or_union_definition(s, pos, ctx)
2035 elif s.sy == 'pass':
2036 node = p_pass_statement(s)
2037 s.expect_newline('Expected a newline')
2038 return node
2039 else:
2040 return p_c_func_or_var_declaration(s, pos, ctx)
2042 def p_cdef_block(s, ctx):
2043 return p_suite(s, ctx(cdef_flag = 1))
2045 def p_cdef_extern_block(s, pos, ctx):
2046 include_file = None
2047 s.expect('from')
2048 if s.sy == '*':
2049 s.next()
2050 else:
2051 _, include_file = p_string_literal(s)
2052 ctx = ctx(cdef_flag = 1, visibility = 'extern')
2053 if p_nogil(s):
2054 ctx.nogil = 1
2055 body = p_suite(s, ctx)
2056 return Nodes.CDefExternNode(pos,
2057 include_file = include_file,
2058 body = body)
2060 struct_union_or_enum = (
2061 "struct", "union", "enum"
2062 )
2064 def p_c_enum_definition(s, pos, ctx):
2065 # s.sy == ident 'enum'
2066 s.next()
2067 if s.sy == 'IDENT':
2068 name = s.systring
2069 s.next()
2070 cname = p_opt_cname(s)
2071 else:
2072 name = None
2073 cname = None
2074 items = None
2075 s.expect(':')
2076 items = []
2077 if s.sy != 'NEWLINE':
2078 p_c_enum_line(s, items)
2079 else:
2080 s.next() # 'NEWLINE'
2081 s.expect_indent()
2082 while s.sy not in ('DEDENT', 'EOF'):
2083 p_c_enum_line(s, items)
2084 s.expect_dedent()
2085 return Nodes.CEnumDefNode(
2086 pos, name = name, cname = cname, items = items,
2087 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2088 in_pxd = ctx.level == 'module_pxd')
2090 def p_c_enum_line(s, items):
2091 if s.sy != 'pass':
2092 p_c_enum_item(s, items)
2093 while s.sy == ',':
2094 s.next()
2095 if s.sy in ('NEWLINE', 'EOF'):
2096 break
2097 p_c_enum_item(s, items)
2098 else:
2099 s.next()
2100 s.expect_newline("Syntax error in enum item list")
2102 def p_c_enum_item(s, items):
2103 pos = s.position()
2104 name = p_ident(s)
2105 cname = p_opt_cname(s)
2106 value = None
2107 if s.sy == '=':
2108 s.next()
2109 value = p_simple_expr(s)
2110 items.append(Nodes.CEnumDefItemNode(pos,
2111 name = name, cname = cname, value = value))
2113 def p_c_struct_or_union_definition(s, pos, ctx):
2114 # s.sy == ident 'struct' or 'union'
2115 kind = s.systring
2116 s.next()
2117 name = p_ident(s)
2118 cname = p_opt_cname(s)
2119 attributes = None
2120 if s.sy == ':':
2121 s.next()
2122 s.expect('NEWLINE')
2123 s.expect_indent()
2124 attributes = []
2125 body_ctx = Ctx()
2126 while s.sy != 'DEDENT':
2127 if s.sy != 'pass':
2128 attributes.append(
2129 p_c_func_or_var_declaration(s, s.position(), body_ctx))
2130 else:
2131 s.next()
2132 s.expect_newline("Expected a newline")
2133 s.expect_dedent()
2134 else:
2135 s.expect_newline("Syntax error in struct or union definition")
2136 return Nodes.CStructOrUnionDefNode(pos,
2137 name = name, cname = cname, kind = kind, attributes = attributes,
2138 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2139 in_pxd = ctx.level == 'module_pxd')
2141 def p_visibility(s, prev_visibility):
2142 pos = s.position()
2143 visibility = prev_visibility
2144 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
2145 visibility = s.systring
2146 if prev_visibility != 'private' and visibility != prev_visibility:
2147 s.error("Conflicting visibility options '%s' and '%s'"
2148 % (prev_visibility, visibility))
2149 s.next()
2150 return visibility
2152 def p_c_modifiers(s):
2153 if s.sy == 'IDENT' and s.systring in ('inline',):
2154 modifier = s.systring
2155 s.next()
2156 return [modifier] + p_c_modifiers(s)
2157 return []
2159 def p_c_func_or_var_declaration(s, pos, ctx):
2160 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
2161 modifiers = p_c_modifiers(s)
2162 base_type = p_c_base_type(s, nonempty = 1)
2163 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2164 assignable = 1, nonempty = 1)
2165 declarator.overridable = ctx.overridable
2166 if s.sy == ':':
2167 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):
2168 s.error("C function definition not allowed here")
2169 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
2170 result = Nodes.CFuncDefNode(pos,
2171 visibility = ctx.visibility,
2172 base_type = base_type,
2173 declarator = declarator,
2174 body = suite,
2175 doc = doc,
2176 modifiers = modifiers,
2177 api = ctx.api,
2178 overridable = ctx.overridable)
2179 else:
2180 #if api:
2181 # error(s.pos, "'api' not allowed with variable declaration")
2182 declarators = [declarator]
2183 while s.sy == ',':
2184 s.next()
2185 if s.sy == 'NEWLINE':
2186 break
2187 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2188 assignable = 1, nonempty = 1)
2189 declarators.append(declarator)
2190 s.expect_newline("Syntax error in C variable declaration")
2191 result = Nodes.CVarDefNode(pos,
2192 visibility = ctx.visibility,
2193 base_type = base_type,
2194 declarators = declarators,
2195 in_pxd = ctx.level == 'module_pxd',
2196 api = ctx.api,
2197 overridable = ctx.overridable)
2198 return result
2200 def p_ctypedef_statement(s, ctx):
2201 # s.sy == 'ctypedef'
2202 pos = s.position()
2203 s.next()
2204 visibility = p_visibility(s, ctx.visibility)
2205 api = p_api(s)
2206 ctx = ctx(typedef_flag = 1, visibility = visibility)
2207 if api:
2208 ctx.api = 1
2209 if s.sy == 'class':
2210 return p_c_class_definition(s, pos, ctx)
2211 elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):
2212 if s.systring == 'enum':
2213 return p_c_enum_definition(s, pos, ctx)
2214 else:
2215 return p_c_struct_or_union_definition(s, pos, ctx)
2216 else:
2217 base_type = p_c_base_type(s, nonempty = 1)
2218 if base_type.name is None:
2219 s.error("Syntax error in ctypedef statement")
2220 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
2221 s.expect_newline("Syntax error in ctypedef statement")
2222 return Nodes.CTypeDefNode(
2223 pos, base_type = base_type,
2224 declarator = declarator, visibility = visibility,
2225 in_pxd = ctx.level == 'module_pxd')
2227 def p_decorators(s):
2228 decorators = []
2229 while s.sy == 'DECORATOR':
2230 pos = s.position()
2231 s.next()
2232 decstring = p_dotted_name(s, as_allowed=0)[2]
2233 names = decstring.split('.')
2234 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))
2235 for name in names[1:]:
2236 decorator = ExprNodes.AttributeNode(pos,
2237 attribute=EncodedString(name),
2238 obj=decorator)
2239 if s.sy == '(':
2240 decorator = p_call(s, decorator)
2241 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
2242 s.expect_newline("Expected a newline after decorator")
2243 return decorators
2245 def p_def_statement(s, decorators=None):
2246 # s.sy == 'def'
2247 pos = s.position()
2248 s.next()
2249 name = EncodedString( p_ident(s) )
2250 #args = []
2251 s.expect('(');
2252 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)
2253 star_arg = None
2254 starstar_arg = None
2255 if s.sy == '*':
2256 s.next()
2257 if s.sy == 'IDENT':
2258 star_arg = p_py_arg_decl(s)
2259 if s.sy == ',':
2260 s.next()
2261 args.extend(p_c_arg_list(s, in_pyfunc = 1,
2262 nonempty_declarators = 1, kw_only = 1))
2263 elif s.sy != ')':
2264 s.error("Syntax error in Python function argument list")
2265 if s.sy == '**':
2266 s.next()
2267 starstar_arg = p_py_arg_decl(s)
2268 s.expect(')')
2269 if p_nogil(s):
2270 error(s.pos, "Python function cannot be declared nogil")
2271 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
2272 return Nodes.DefNode(pos, name = name, args = args,
2273 star_arg = star_arg, starstar_arg = starstar_arg,
2274 doc = doc, body = body, decorators = decorators)
2276 def p_py_arg_decl(s):
2277 pos = s.position()
2278 name = p_ident(s)
2279 return Nodes.PyArgDeclNode(pos, name = name)
2281 def p_class_statement(s):
2282 # s.sy == 'class'
2283 pos = s.position()
2284 s.next()
2285 class_name = EncodedString( p_ident(s) )
2286 class_name.encoding = s.source_encoding
2287 if s.sy == '(':
2288 s.next()
2289 base_list = p_simple_expr_list(s)
2290 s.expect(')')
2291 else:
2292 base_list = []
2293 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
2294 return Nodes.PyClassDefNode(pos,
2295 name = class_name,
2296 bases = ExprNodes.TupleNode(pos, args = base_list),
2297 doc = doc, body = body)
2299 def p_c_class_definition(s, pos, ctx):
2300 # s.sy == 'class'
2301 s.next()
2302 module_path = []
2303 class_name = p_ident(s)
2304 while s.sy == '.':
2305 s.next()
2306 module_path.append(class_name)
2307 class_name = p_ident(s)
2308 if module_path and ctx.visibility != 'extern':
2309 error(pos, "Qualified class name only allowed for 'extern' C class")
2310 if module_path and s.sy == 'IDENT' and s.systring == 'as':
2311 s.next()
2312 as_name = p_ident(s)
2313 else:
2314 as_name = class_name
2315 objstruct_name = None
2316 typeobj_name = None
2317 base_class_module = None
2318 base_class_name = None
2319 if s.sy == '(':
2320 s.next()
2321 base_class_path = [p_ident(s)]
2322 while s.sy == '.':
2323 s.next()
2324 base_class_path.append(p_ident(s))
2325 if s.sy == ',':
2326 s.error("C class may only have one base class")
2327 s.expect(')')
2328 base_class_module = ".".join(base_class_path[:-1])
2329 base_class_name = base_class_path[-1]
2330 if s.sy == '[':
2331 if ctx.visibility not in ('public', 'extern'):
2332 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")
2333 objstruct_name, typeobj_name = p_c_class_options(s)
2334 if s.sy == ':':
2335 if ctx.level == 'module_pxd':
2336 body_level = 'c_class_pxd'
2337 else:
2338 body_level = 'c_class'
2339 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
2340 else:
2341 s.expect_newline("Syntax error in C class definition")
2342 doc = None
2343 body = None
2344 if ctx.visibility == 'extern':
2345 if not module_path:
2346 error(pos, "Module name required for 'extern' C class")
2347 if typeobj_name:
2348 error(pos, "Type object name specification not allowed for 'extern' C class")
2349 elif ctx.visibility == 'public':
2350 if not objstruct_name:
2351 error(pos, "Object struct name specification required for 'public' C class")
2352 if not typeobj_name:
2353 error(pos, "Type object name specification required for 'public' C class")
2354 elif ctx.visibility == 'private':
2355 if ctx.api:
2356 error(pos, "Only 'public' C class can be declared 'api'")
2357 else:
2358 error(pos, "Invalid class visibility '%s'" % ctx.visibility)
2359 return Nodes.CClassDefNode(pos,
2360 visibility = ctx.visibility,
2361 typedef_flag = ctx.typedef_flag,
2362 api = ctx.api,
2363 module_name = ".".join(module_path),
2364 class_name = class_name,
2365 as_name = as_name,
2366 base_class_module = base_class_module,
2367 base_class_name = base_class_name,
2368 objstruct_name = objstruct_name,
2369 typeobj_name = typeobj_name,
2370 in_pxd = ctx.level == 'module_pxd',
2371 doc = doc,
2372 body = body)
2374 def p_c_class_options(s):
2375 objstruct_name = None
2376 typeobj_name = None
2377 s.expect('[')
2378 while 1:
2379 if s.sy != 'IDENT':
2380 break
2381 if s.systring == 'object':
2382 s.next()
2383 objstruct_name = p_ident(s)
2384 elif s.systring == 'type':
2385 s.next()
2386 typeobj_name = p_ident(s)
2387 if s.sy != ',':
2388 break
2389 s.next()
2390 s.expect(']', "Expected 'object' or 'type'")
2391 return objstruct_name, typeobj_name
2393 def p_property_decl(s):
2394 pos = s.position()
2395 s.next() # 'property'
2396 name = p_ident(s)
2397 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
2398 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)
2400 def p_doc_string(s):
2401 if s.sy == 'BEGIN_STRING':
2402 pos = s.position()
2403 kind, result = p_cat_string_literal(s)
2404 if s.sy != 'EOF':
2405 s.expect_newline("Syntax error in doc string")
2406 if kind != 'u':
2407 # warning(pos, "Python 3 requires docstrings to be unicode strings")
2408 if kind == 'b':
2409 result.encoding = None # force a unicode string
2410 return result
2411 else:
2412 return None
2414 def p_code(s, level=None):
2415 body = p_statement_list(s, Ctx(level = level), first_statement = 1)
2416 if s.sy != 'EOF':
2417 s.error("Syntax error in statement [%s,%s]" % (
2418 repr(s.sy), repr(s.systring)))
2419 return body
2421 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z_]+)\s*=(.*)$")
2423 def p_compiler_directive_comments(s):
2424 result = {}
2425 while s.sy == 'commentline':
2426 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)
2427 if m:
2428 name = m.group(1)
2429 try:
2430 value = Options.parse_option_value(str(name), str(m.group(2).strip()))
2431 except ValueError, e:
2432 s.error(e.args[0], fatal=False)
2433 if value is not None: # can be False!
2434 result[name] = value
2435 s.next()
2436 return result
2438 def p_module(s, pxd, full_module_name):
2439 pos = s.position()
2441 option_comments = p_compiler_directive_comments(s)
2442 s.parse_comments = False
2444 doc = p_doc_string(s)
2445 if pxd:
2446 level = 'module_pxd'
2447 else:
2448 level = 'module'
2450 body = p_statement_list(s, Ctx(level = level), first_statement = 1)
2451 if s.sy != 'EOF':
2452 s.error("Syntax error in statement [%s,%s]" % (
2453 repr(s.sy), repr(s.systring)))
2454 return ModuleNode(pos, doc = doc, body = body,
2455 full_module_name = full_module_name,
2456 option_comments = option_comments)
2458 #----------------------------------------------
2459 #
2460 # Debugging
2461 #
2462 #----------------------------------------------
2464 def print_parse_tree(f, node, level, key = None):
2465 from Nodes import Node
2466 ind = " " * level
2467 if node:
2468 f.write(ind)
2469 if key:
2470 f.write("%s: " % key)
2471 t = type(node)
2472 if t == TupleType:
2473 f.write("(%s @ %s\n" % (node[0], node[1]))
2474 for i in xrange(2, len(node)):
2475 print_parse_tree(f, node[i], level+1)
2476 f.write("%s)\n" % ind)
2477 return
2478 elif isinstance(node, Node):
2479 try:
2480 tag = node.tag
2481 except AttributeError:
2482 tag = node.__class__.__name__
2483 f.write("%s @ %s\n" % (tag, node.pos))
2484 for name, value in node.__dict__.items():
2485 if name != 'tag' and name != 'pos':
2486 print_parse_tree(f, value, level+1, name)
2487 return
2488 elif t == ListType:
2489 f.write("[\n")
2490 for i in xrange(len(node)):
2491 print_parse_tree(f, node[i], level+1)
2492 f.write("%s]\n" % ind)
2493 return
2494 f.write("%s%s\n" % (ind, node))