Cython has moved to github.
cython-devel
view Cython/Compiler/Parsing.py @ 834:4695bbb3a785
Better integer literal parsing.
Now accepts U and LL suffixes, and large integer literals are longs rather than being truncated as Python objects.
Now accepts U and LL suffixes, and large integer literals are longs rather than being truncated as Python objects.
| author | Robert Bradshaw <robertwb@math.washington.edu> |
|---|---|
| date | Thu Jul 31 00:55:14 2008 -0700 (3 years ago) |
| parents | d5e2a1ea66db |
| children | 7d24ad4f8a9d |
line source
1 #
2 # Pyrex Parser
3 #
5 import os, re
6 from types import ListType, TupleType
7 from Scanning import PyrexScanner, FileSourceDescriptor
8 import Nodes
9 import ExprNodes
10 from ModuleNode import ModuleNode
11 from Errors import error, warning, InternalError
12 from Cython import Utils
13 import Future
15 class Ctx(object):
16 # Parsing context
17 level = 'other'
18 visibility = 'private'
19 cdef_flag = 0
20 typedef_flag = 0
21 api = 0
22 overridable = 0
23 nogil = 0
25 def __init__(self, **kwds):
26 self.__dict__.update(kwds)
28 def __call__(self, **kwds):
29 ctx = Ctx()
30 d = ctx.__dict__
31 d.update(self.__dict__)
32 d.update(kwds)
33 return ctx
35 def p_ident(s, message = "Expected an identifier"):
36 if s.sy == 'IDENT':
37 name = s.systring
38 s.next()
39 return name
40 else:
41 s.error(message)
43 def p_ident_list(s):
44 names = []
45 while s.sy == 'IDENT':
46 names.append(s.systring)
47 s.next()
48 if s.sy != ',':
49 break
50 s.next()
51 return names
53 #------------------------------------------
54 #
55 # Expressions
56 #
57 #------------------------------------------
59 def p_binop_expr(s, ops, p_sub_expr):
60 n1 = p_sub_expr(s)
61 while s.sy in ops:
62 op = s.sy
63 pos = s.position()
64 s.next()
65 n2 = p_sub_expr(s)
66 n1 = ExprNodes.binop_node(pos, op, n1, n2)
67 return n1
69 #expression: or_test [if or_test else test] | lambda_form
71 def p_simple_expr(s):
72 pos = s.position()
73 expr = p_or_test(s)
74 if s.sy == 'if':
75 s.next()
76 test = p_or_test(s)
77 if s.sy == 'else':
78 s.next()
79 other = p_test(s)
80 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
81 else:
82 s.error("Expected 'else'")
83 else:
84 return expr
86 #test: or_test | lambda_form
88 def p_test(s):
89 return p_or_test(s)
91 #or_test: and_test ('or' and_test)*
93 def p_or_test(s):
94 return p_rassoc_binop_expr(s, ('or',), p_and_test)
96 def p_rassoc_binop_expr(s, ops, p_subexpr):
97 n1 = p_subexpr(s)
98 if s.sy in ops:
99 pos = s.position()
100 op = s.sy
101 s.next()
102 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
103 n1 = ExprNodes.binop_node(pos, op, n1, n2)
104 return n1
106 #and_test: not_test ('and' not_test)*
108 def p_and_test(s):
109 #return p_binop_expr(s, ('and',), p_not_test)
110 return p_rassoc_binop_expr(s, ('and',), p_not_test)
112 #not_test: 'not' not_test | comparison
114 def p_not_test(s):
115 if s.sy == 'not':
116 pos = s.position()
117 s.next()
118 return ExprNodes.NotNode(pos, operand = p_not_test(s))
119 else:
120 return p_comparison(s)
122 #comparison: expr (comp_op expr)*
123 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
125 def p_comparison(s):
126 n1 = p_bit_expr(s)
127 if s.sy in comparison_ops:
128 pos = s.position()
129 op = p_cmp_op(s)
130 n2 = p_bit_expr(s)
131 n1 = ExprNodes.PrimaryCmpNode(pos,
132 operator = op, operand1 = n1, operand2 = n2)
133 if s.sy in comparison_ops:
134 n1.cascade = p_cascaded_cmp(s)
135 return n1
137 def p_cascaded_cmp(s):
138 pos = s.position()
139 op = p_cmp_op(s)
140 n2 = p_bit_expr(s)
141 result = ExprNodes.CascadedCmpNode(pos,
142 operator = op, operand2 = n2)
143 if s.sy in comparison_ops:
144 result.cascade = p_cascaded_cmp(s)
145 return result
147 def p_cmp_op(s):
148 if s.sy == 'not':
149 s.next()
150 s.expect('in')
151 op = 'not_in'
152 elif s.sy == 'is':
153 s.next()
154 if s.sy == 'not':
155 s.next()
156 op = 'is_not'
157 else:
158 op = 'is'
159 else:
160 op = s.sy
161 s.next()
162 if op == '<>':
163 op = '!='
164 return op
166 comparison_ops = (
167 '<', '>', '==', '>=', '<=', '<>', '!=',
168 'in', 'is', 'not'
169 )
171 #expr: xor_expr ('|' xor_expr)*
173 def p_bit_expr(s):
174 return p_binop_expr(s, ('|',), p_xor_expr)
176 #xor_expr: and_expr ('^' and_expr)*
178 def p_xor_expr(s):
179 return p_binop_expr(s, ('^',), p_and_expr)
181 #and_expr: shift_expr ('&' shift_expr)*
183 def p_and_expr(s):
184 return p_binop_expr(s, ('&',), p_shift_expr)
186 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*
188 def p_shift_expr(s):
189 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)
191 #arith_expr: term (('+'|'-') term)*
193 def p_arith_expr(s):
194 return p_binop_expr(s, ('+', '-'), p_term)
196 #term: factor (('*'|'/'|'%') factor)*
198 def p_term(s):
199 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)
201 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
203 def p_factor(s):
204 sy = s.sy
205 if sy in ('+', '-', '~'):
206 op = s.sy
207 pos = s.position()
208 s.next()
209 return ExprNodes.unop_node(pos, op, p_factor(s))
210 elif sy == '&':
211 pos = s.position()
212 s.next()
213 arg = p_factor(s)
214 return ExprNodes.AmpersandNode(pos, operand = arg)
215 elif sy == "<":
216 return p_typecast(s)
217 elif sy == 'IDENT' and s.systring == "sizeof":
218 return p_sizeof(s)
219 else:
220 return p_power(s)
222 def p_typecast(s):
223 # s.sy == "<"
224 pos = s.position()
225 s.next()
226 base_type = p_c_base_type(s)
227 declarator = p_c_declarator(s, empty = 1)
228 if s.sy == '?':
229 s.next()
230 typecheck = 1
231 else:
232 typecheck = 0
233 s.expect(">")
234 operand = p_factor(s)
235 return ExprNodes.TypecastNode(pos,
236 base_type = base_type,
237 declarator = declarator,
238 operand = operand,
239 typecheck = typecheck)
241 def p_sizeof(s):
242 # s.sy == ident "sizeof"
243 pos = s.position()
244 s.next()
245 s.expect('(')
246 if looking_at_type(s) or looking_at_dotted_name(s):
247 base_type = p_c_base_type(s)
248 declarator = p_c_declarator(s, empty = 1)
249 node = ExprNodes.SizeofTypeNode(pos,
250 base_type = base_type, declarator = declarator)
251 else:
252 operand = p_simple_expr(s)
253 node = ExprNodes.SizeofVarNode(pos, operand = operand)
254 s.expect(')')
255 return node
257 #power: atom trailer* ('**' factor)*
259 def p_power(s):
260 n1 = p_atom(s)
261 while s.sy in ('(', '[', '.'):
262 n1 = p_trailer(s, n1)
263 if s.sy == '**':
264 pos = s.position()
265 s.next()
266 n2 = p_factor(s)
267 n1 = ExprNodes.binop_node(pos, '**', n1, n2)
268 return n1
270 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
272 def p_trailer(s, node1):
273 pos = s.position()
274 if s.sy == '(':
275 return p_call(s, node1)
276 elif s.sy == '[':
277 return p_index(s, node1)
278 else: # s.sy == '.'
279 s.next()
280 name = Utils.EncodedString( p_ident(s) )
281 return ExprNodes.AttributeNode(pos,
282 obj = node1, attribute = name)
284 # arglist: argument (',' argument)* [',']
285 # argument: [test '='] test # Really [keyword '='] test
287 def p_call(s, function):
288 # s.sy == '('
289 pos = s.position()
290 s.next()
291 positional_args = []
292 keyword_args = []
293 star_arg = None
294 starstar_arg = None
295 while s.sy not in ('*', '**', ')'):
296 arg = p_simple_expr(s)
297 if s.sy == '=':
298 s.next()
299 if not arg.is_name:
300 s.error("Expected an identifier before '='",
301 pos = arg.pos)
302 encoded_name = Utils.EncodedString(arg.name)
303 keyword = ExprNodes.IdentifierStringNode(arg.pos,
304 value = encoded_name)
305 arg = p_simple_expr(s)
306 keyword_args.append((keyword, arg))
307 else:
308 if keyword_args:
309 s.error("Non-keyword arg following keyword arg",
310 pos = arg.pos)
311 positional_args.append(arg)
312 if s.sy != ',':
313 break
314 s.next()
316 if s.sy == '*':
317 s.next()
318 star_arg = p_simple_expr(s)
319 if s.sy == ',':
320 s.next()
321 if s.sy == '**':
322 s.next()
323 starstar_arg = p_simple_expr(s)
324 if s.sy == ',':
325 s.next()
326 s.expect(')')
327 if not (keyword_args or star_arg or starstar_arg):
328 return ExprNodes.SimpleCallNode(pos,
329 function = function,
330 args = positional_args)
331 else:
332 arg_tuple = None
333 keyword_dict = None
334 if positional_args or not star_arg:
335 arg_tuple = ExprNodes.TupleNode(pos,
336 args = positional_args)
337 if star_arg:
338 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
339 if arg_tuple:
340 arg_tuple = ExprNodes.binop_node(pos,
341 operator = '+', operand1 = arg_tuple,
342 operand2 = star_arg_tuple)
343 else:
344 arg_tuple = star_arg_tuple
345 if keyword_args:
346 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
347 for key, value in keyword_args]
348 keyword_dict = ExprNodes.DictNode(pos,
349 key_value_pairs = keyword_args)
350 return ExprNodes.GeneralCallNode(pos,
351 function = function,
352 positional_args = arg_tuple,
353 keyword_args = keyword_dict,
354 starstar_arg = starstar_arg)
356 #lambdef: 'lambda' [varargslist] ':' test
358 #subscriptlist: subscript (',' subscript)* [',']
360 def p_index(s, base):
361 # s.sy == '['
362 pos = s.position()
363 s.next()
364 subscripts = p_subscript_list(s)
365 if len(subscripts) == 1 and len(subscripts[0]) == 2:
366 start, stop = subscripts[0]
367 result = ExprNodes.SliceIndexNode(pos,
368 base = base, start = start, stop = stop)
369 else:
370 indexes = make_slice_nodes(pos, subscripts)
371 if len(indexes) == 1:
372 index = indexes[0]
373 else:
374 index = ExprNodes.TupleNode(pos, args = indexes)
375 result = ExprNodes.IndexNode(pos,
376 base = base, index = index)
377 s.expect(']')
378 return result
380 def p_subscript_list(s):
381 items = [p_subscript(s)]
382 while s.sy == ',':
383 s.next()
384 if s.sy == ']':
385 break
386 items.append(p_subscript(s))
387 return items
389 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]
391 def p_subscript(s):
392 # Parse a subscript and return a list of
393 # 1, 2 or 3 ExprNodes, depending on how
394 # many slice elements were encountered.
395 pos = s.position()
396 if s.sy == '.':
397 expect_ellipsis(s)
398 return [ExprNodes.EllipsisNode(pos)]
399 else:
400 start = p_slice_element(s, (':',))
401 if s.sy != ':':
402 return [start]
403 s.next()
404 stop = p_slice_element(s, (':', ',', ']'))
405 if s.sy != ':':
406 return [start, stop]
407 s.next()
408 step = p_slice_element(s, (':', ',', ']'))
409 return [start, stop, step]
411 def p_slice_element(s, follow_set):
412 # Simple expression which may be missing iff
413 # it is followed by something in follow_set.
414 if s.sy not in follow_set:
415 return p_simple_expr(s)
416 else:
417 return None
419 def expect_ellipsis(s):
420 s.expect('.')
421 s.expect('.')
422 s.expect('.')
424 def make_slice_nodes(pos, subscripts):
425 # Convert a list of subscripts as returned
426 # by p_subscript_list into a list of ExprNodes,
427 # creating SliceNodes for elements with 2 or
428 # more components.
429 result = []
430 for subscript in subscripts:
431 if len(subscript) == 1:
432 result.append(subscript[0])
433 else:
434 result.append(make_slice_node(pos, *subscript))
435 return result
437 def make_slice_node(pos, start, stop = None, step = None):
438 if not start:
439 start = ExprNodes.NoneNode(pos)
440 if not stop:
441 stop = ExprNodes.NoneNode(pos)
442 if not step:
443 step = ExprNodes.NoneNode(pos)
444 return ExprNodes.SliceNode(pos,
445 start = start, stop = stop, step = step)
447 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
449 def p_atom(s):
450 pos = s.position()
451 sy = s.sy
452 if sy == '(':
453 s.next()
454 if s.sy == ')':
455 result = ExprNodes.TupleNode(pos, args = [])
456 else:
457 result = p_expr(s)
458 s.expect(')')
459 return result
460 elif sy == '[':
461 return p_list_maker(s)
462 elif sy == '{':
463 return p_dict_maker(s)
464 elif sy == '`':
465 return p_backquote_expr(s)
466 elif sy == 'INT':
467 value = s.systring
468 s.next()
469 unsigned = ""
470 longness = ""
471 while value[-1] in "UuLl":
472 if value[-1] in "Ll":
473 longness += "L"
474 else:
475 unsigned += "U"
476 value = value[:-1]
477 return ExprNodes.IntNode(pos,
478 value = value,
479 unsigned = unsigned,
480 longness = longness)
481 elif sy == 'FLOAT':
482 value = s.systring
483 s.next()
484 return ExprNodes.FloatNode(pos, value = value)
485 elif sy == 'IMAG':
486 value = s.systring[:-1]
487 s.next()
488 return ExprNodes.ImagNode(pos, value = value)
489 elif sy == 'BEGIN_STRING':
490 kind, value = p_cat_string_literal(s)
491 if kind == 'c':
492 return ExprNodes.CharNode(pos, value = value)
493 else:
494 return ExprNodes.StringNode(pos, value = value)
495 elif sy == 'IDENT':
496 name = Utils.EncodedString( s.systring )
497 s.next()
498 if name == "None":
499 return ExprNodes.NoneNode(pos)
500 elif name == "True":
501 return ExprNodes.BoolNode(pos, value=True)
502 elif name == "False":
503 return ExprNodes.BoolNode(pos, value=False)
504 else:
505 return p_name(s, name)
506 elif sy == 'NULL':
507 s.next()
508 return ExprNodes.NullNode(pos)
509 else:
510 s.error("Expected an identifier or literal")
512 def p_name(s, name):
513 pos = s.position()
514 if not s.compile_time_expr:
515 try:
516 value = s.compile_time_env.lookup_here(name)
517 except KeyError:
518 pass
519 else:
520 rep = repr(value)
521 if isinstance(value, bool):
522 return ExprNodes.BoolNode(pos, value = value)
523 elif isinstance(value, int):
524 return ExprNodes.IntNode(pos, value = rep)
525 elif isinstance(value, long):
526 return ExprNodes.IntNode(pos, value = rep, longness = "L")
527 elif isinstance(value, float):
528 return ExprNodes.FloatNode(pos, value = rep)
529 elif isinstance(value, unicode):
530 return ExprNodes.StringNode(pos, value = value)
531 else:
532 error(pos, "Invalid type for compile-time constant: %s"
533 % value.__class__.__name__)
534 return ExprNodes.NameNode(pos, name = name)
536 def p_cat_string_literal(s):
537 # A sequence of one or more adjacent string literals.
538 # Returns (kind, value) where kind in ('b', 'c', 'u')
539 kind, value = p_string_literal(s)
540 if kind != 'c':
541 strings = [value]
542 while s.sy == 'BEGIN_STRING':
543 next_kind, next_value = p_string_literal(s)
544 if next_kind == 'c':
545 error(s.position(),
546 "Cannot concatenate char literal with another string or char literal")
547 elif next_kind == 'u':
548 kind = 'u'
549 strings.append(next_value)
550 value = Utils.EncodedString( u''.join(strings) )
551 if kind != 'u':
552 value.encoding = s.source_encoding
553 return kind, value
555 def p_opt_string_literal(s):
556 if s.sy == 'BEGIN_STRING':
557 return p_string_literal(s)
558 else:
559 return None
561 def p_string_literal(s):
562 # A single string or char literal.
563 # Returns (kind, value) where kind in ('b', 'c', 'u')
564 # s.sy == 'BEGIN_STRING'
565 pos = s.position()
566 is_raw = 0
567 kind = s.systring[:1].lower()
568 if kind == 'r':
569 kind = ''
570 is_raw = 1
571 elif kind in 'ub':
572 is_raw = s.systring[1:2].lower() == 'r'
573 elif kind != 'c':
574 kind = ''
575 if Future.unicode_literals in s.context.future_directives:
576 if kind == '':
577 kind = 'u'
578 elif kind == '':
579 kind = 'b'
580 chars = []
581 while 1:
582 s.next()
583 sy = s.sy
584 #print "p_string_literal: sy =", sy, repr(s.systring) ###
585 if sy == 'CHARS':
586 systr = s.systring
587 if len(systr) == 1 and systr in "'\"\n":
588 chars.append('\\')
589 chars.append(systr)
590 elif sy == 'ESCAPE':
591 systr = s.systring
592 if is_raw:
593 if systr == '\\\n':
594 chars.append(r'\\\n')
595 elif systr == r'\"':
596 chars.append(r'\\\"')
597 elif systr == r'\\':
598 chars.append(r'\\\\')
599 else:
600 chars.append('\\' + systr)
601 else:
602 c = systr[1]
603 if c in "'\"\\abfnrtv01234567":
604 chars.append(systr)
605 elif c == '\n':
606 pass
607 elif c in 'ux':
608 if kind == 'u':
609 try:
610 chars.append(
611 systr.encode("ASCII").decode('unicode_escape'))
612 except UnicodeDecodeError:
613 s.error("Invalid unicode escape '%s'" % systr,
614 pos = pos)
615 elif c == 'x':
616 chars.append('\\x0' + systr[2:])
617 else:
618 chars.append(systr)
619 else:
620 chars.append(r'\\' + systr[1:])
621 elif sy == 'NEWLINE':
622 chars.append(r'\n')
623 elif sy == 'END_STRING':
624 break
625 elif sy == 'EOF':
626 s.error("Unclosed string literal", pos = pos)
627 else:
628 s.error(
629 "Unexpected token %r:%r in string literal" %
630 (sy, s.systring))
631 s.next()
632 value = Utils.EncodedString( u''.join(chars) )
633 if kind != 'u':
634 value.encoding = s.source_encoding
635 #print "p_string_literal: value =", repr(value) ###
636 return kind, value
638 # list_display ::= "[" [listmaker] "]"
639 # listmaker ::= expression ( list_for | ( "," expression )* [","] )
640 # list_iter ::= list_for | list_if
641 # list_for ::= "for" expression_list "in" testlist [list_iter]
642 # list_if ::= "if" test [list_iter]
644 def p_list_maker(s):
645 # s.sy == '['
646 pos = s.position()
647 s.next()
648 if s.sy == ']':
649 s.expect(']')
650 return ExprNodes.ListNode(pos, args = [])
651 expr = p_simple_expr(s)
652 if s.sy == 'for':
653 loop = p_list_for(s)
654 s.expect(']')
655 inner_loop = loop
656 while not isinstance(inner_loop.body, Nodes.PassStatNode):
657 inner_loop = inner_loop.body
658 if isinstance(inner_loop, Nodes.IfStatNode):
659 inner_loop = inner_loop.if_clauses[0]
660 append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )
661 inner_loop.body = Nodes.ExprStatNode(pos, expr = append)
662 return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)
663 else:
664 exprs = [expr]
665 if s.sy == ',':
666 s.next()
667 exprs += p_simple_expr_list(s)
668 s.expect(']')
669 return ExprNodes.ListNode(pos, args = exprs)
671 def p_list_iter(s):
672 if s.sy == 'for':
673 return p_list_for(s)
674 elif s.sy == 'if':
675 return p_list_if(s)
676 else:
677 return Nodes.PassStatNode(s.position())
679 def p_list_for(s):
680 # s.sy == 'for'
681 pos = s.position()
682 s.next()
683 kw = p_for_bounds(s)
684 kw['else_clause'] = None
685 kw['body'] = p_list_iter(s)
686 return Nodes.ForStatNode(pos, **kw)
688 def p_list_if(s):
689 # s.sy == 'if'
690 pos = s.position()
691 s.next()
692 test = p_test(s)
693 return Nodes.IfStatNode(pos,
694 if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],
695 else_clause = None )
697 #dictmaker: test ':' test (',' test ':' test)* [',']
699 def p_dict_maker(s):
700 # s.sy == '{'
701 pos = s.position()
702 s.next()
703 items = []
704 while s.sy != '}':
705 items.append(p_dict_item(s))
706 if s.sy != ',':
707 break
708 s.next()
709 s.expect('}')
710 return ExprNodes.DictNode(pos, key_value_pairs = items)
712 def p_dict_item(s):
713 key = p_simple_expr(s)
714 s.expect(':')
715 value = p_simple_expr(s)
716 return ExprNodes.DictItemNode(key.pos, key=key, value=value)
718 def p_backquote_expr(s):
719 # s.sy == '`'
720 pos = s.position()
721 s.next()
722 arg = p_expr(s)
723 s.expect('`')
724 return ExprNodes.BackquoteNode(pos, arg = arg)
726 def p_simple_expr_list(s):
727 exprs = []
728 while s.sy not in expr_terminators:
729 exprs.append(p_simple_expr(s))
730 if s.sy != ',':
731 break
732 s.next()
733 return exprs
735 def p_expr(s):
736 pos = s.position()
737 expr = p_simple_expr(s)
738 if s.sy == ',':
739 s.next()
740 exprs = [expr] + p_simple_expr_list(s)
741 return ExprNodes.TupleNode(pos, args = exprs)
742 else:
743 return expr
746 #testlist: test (',' test)* [',']
747 # differs from p_expr only in the fact that it cannot contain conditional expressions
749 def p_testlist(s):
750 pos = s.position()
751 expr = p_test(s)
752 if s.sy == ',':
753 exprs = [expr]
754 while s.sy == ',':
755 s.next()
756 exprs.append(p_test(s))
757 return ExprNodes.TupleNode(pos, args = exprs)
758 else:
759 return expr
761 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')
763 #-------------------------------------------------------
764 #
765 # Statements
766 #
767 #-------------------------------------------------------
769 def p_global_statement(s):
770 # assume s.sy == 'global'
771 pos = s.position()
772 s.next()
773 names = p_ident_list(s)
774 return Nodes.GlobalNode(pos, names = names)
776 def p_expression_or_assignment(s):
777 expr_list = [p_expr(s)]
778 while s.sy == '=':
779 s.next()
780 expr_list.append(p_expr(s))
781 if len(expr_list) == 1:
782 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
783 lhs = expr_list[0]
784 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
785 error(lhs.pos, "Illegal operand for inplace operation.")
786 operator = s.sy[:-1]
787 s.next()
788 rhs = p_expr(s)
789 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
790 expr = expr_list[0]
791 if isinstance(expr, ExprNodes.StringNode):
792 return Nodes.PassStatNode(expr.pos)
793 else:
794 return Nodes.ExprStatNode(expr.pos, expr = expr)
795 else:
796 expr_list_list = []
797 flatten_parallel_assignments(expr_list, expr_list_list)
798 nodes = []
799 for expr_list in expr_list_list:
800 lhs_list = expr_list[:-1]
801 rhs = expr_list[-1]
802 if len(lhs_list) == 1:
803 node = Nodes.SingleAssignmentNode(rhs.pos,
804 lhs = lhs_list[0], rhs = rhs)
805 else:
806 node = Nodes.CascadedAssignmentNode(rhs.pos,
807 lhs_list = lhs_list, rhs = rhs)
808 nodes.append(node)
809 if len(nodes) == 1:
810 return nodes[0]
811 else:
812 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)
814 def flatten_parallel_assignments(input, output):
815 # The input is a list of expression nodes, representing
816 # the LHSs and RHS of one (possibly cascaded) assignment
817 # statement. If they are all sequence constructors with
818 # the same number of arguments, rearranges them into a
819 # list of equivalent assignments between the individual
820 # elements. This transformation is applied recursively.
821 size = find_parallel_assignment_size(input)
822 if size >= 0:
823 for i in range(size):
824 new_exprs = [expr.args[i] for expr in input]
825 flatten_parallel_assignments(new_exprs, output)
826 else:
827 output.append(input)
829 def find_parallel_assignment_size(input):
830 # The input is a list of expression nodes. If
831 # they are all sequence constructors with the same number
832 # of arguments, return that number, else return -1.
833 # Produces an error message if they are all sequence
834 # constructors but not all the same size.
835 for expr in input:
836 if not expr.is_sequence_constructor:
837 return -1
838 rhs = input[-1]
839 rhs_size = len(rhs.args)
840 for lhs in input[:-1]:
841 lhs_size = len(lhs.args)
842 if lhs_size != rhs_size:
843 error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"
844 % (lhs_size, rhs_size))
845 return -1
846 return rhs_size
848 def p_print_statement(s):
849 # s.sy == 'print'
850 pos = s.position()
851 s.next()
852 if s.sy == '>>':
853 s.error("'print >>' not yet implemented")
854 args = []
855 ends_with_comma = 0
856 if s.sy not in ('NEWLINE', 'EOF'):
857 args.append(p_simple_expr(s))
858 while s.sy == ',':
859 s.next()
860 if s.sy in ('NEWLINE', 'EOF'):
861 ends_with_comma = 1
862 break
863 args.append(p_simple_expr(s))
864 arg_tuple = ExprNodes.TupleNode(pos, args = args)
865 return Nodes.PrintStatNode(pos,
866 arg_tuple = arg_tuple, append_newline = not ends_with_comma)
868 def p_del_statement(s):
869 # s.sy == 'del'
870 pos = s.position()
871 s.next()
872 args = p_simple_expr_list(s)
873 return Nodes.DelStatNode(pos, args = args)
875 def p_pass_statement(s, with_newline = 0):
876 pos = s.position()
877 s.expect('pass')
878 if with_newline:
879 s.expect_newline("Expected a newline")
880 return Nodes.PassStatNode(pos)
882 def p_break_statement(s):
883 # s.sy == 'break'
884 pos = s.position()
885 s.next()
886 return Nodes.BreakStatNode(pos)
888 def p_continue_statement(s):
889 # s.sy == 'continue'
890 pos = s.position()
891 s.next()
892 return Nodes.ContinueStatNode(pos)
894 def p_return_statement(s):
895 # s.sy == 'return'
896 pos = s.position()
897 s.next()
898 if s.sy not in statement_terminators:
899 value = p_expr(s)
900 else:
901 value = None
902 return Nodes.ReturnStatNode(pos, value = value)
904 def p_raise_statement(s):
905 # s.sy == 'raise'
906 pos = s.position()
907 s.next()
908 exc_type = None
909 exc_value = None
910 exc_tb = None
911 if s.sy not in statement_terminators:
912 exc_type = p_simple_expr(s)
913 if s.sy == ',':
914 s.next()
915 exc_value = p_simple_expr(s)
916 if s.sy == ',':
917 s.next()
918 exc_tb = p_simple_expr(s)
919 if exc_type or exc_value or exc_tb:
920 return Nodes.RaiseStatNode(pos,
921 exc_type = exc_type,
922 exc_value = exc_value,
923 exc_tb = exc_tb)
924 else:
925 return Nodes.ReraiseStatNode(pos)
927 def p_import_statement(s):
928 # s.sy in ('import', 'cimport')
929 pos = s.position()
930 kind = s.sy
931 s.next()
932 items = [p_dotted_name(s, as_allowed = 1)]
933 while s.sy == ',':
934 s.next()
935 items.append(p_dotted_name(s, as_allowed = 1))
936 stats = []
937 for pos, target_name, dotted_name, as_name in items:
938 dotted_name = Utils.EncodedString(dotted_name)
939 if kind == 'cimport':
940 stat = Nodes.CImportStatNode(pos,
941 module_name = dotted_name,
942 as_name = as_name)
943 else:
944 if as_name and "." in dotted_name:
945 name_list = ExprNodes.ListNode(pos, args = [
946 ExprNodes.StringNode(pos, value = Utils.EncodedString("*"))])
947 else:
948 name_list = None
949 stat = Nodes.SingleAssignmentNode(pos,
950 lhs = ExprNodes.NameNode(pos,
951 name = as_name or target_name),
952 rhs = ExprNodes.ImportNode(pos,
953 module_name = ExprNodes.IdentifierStringNode(
954 pos, value = dotted_name),
955 name_list = name_list))
956 stats.append(stat)
957 return Nodes.StatListNode(pos, stats = stats)
959 def p_from_import_statement(s, first_statement = 0):
960 # s.sy == 'from'
961 pos = s.position()
962 s.next()
963 (dotted_name_pos, _, dotted_name, _) = \
964 p_dotted_name(s, as_allowed = 0)
965 if s.sy in ('import', 'cimport'):
966 kind = s.sy
967 s.next()
968 else:
969 s.error("Expected 'import' or 'cimport'")
970 is_cimport = kind == 'cimport'
971 if s.sy == '*':
972 imported_names = [(s.position(), "*", None, None)]
973 s.next()
974 else:
975 imported_names = [p_imported_name(s, is_cimport)]
976 while s.sy == ',':
977 s.next()
978 imported_names.append(p_imported_name(s, is_cimport))
979 dotted_name = Utils.EncodedString(dotted_name)
980 if dotted_name == '__future__':
981 if not first_statement:
982 s.error("from __future__ imports must occur at the beginning of the file")
983 else:
984 for (name_pos, name, as_name, kind) in imported_names:
985 if name == "braces":
986 s.error("not a chance", name_pos)
987 break
988 try:
989 directive = getattr(Future, name)
990 except AttributeError:
991 s.error("future feature %s is not defined" % name, name_pos)
992 break
993 s.context.future_directives.add(directive)
994 return Nodes.PassStatNode(pos)
995 elif kind == 'cimport':
996 for (name_pos, name, as_name, kind) in imported_names:
997 local_name = as_name or name
998 s.add_type_name(local_name)
999 return Nodes.FromCImportStatNode(pos,
1000 module_name = dotted_name,
1001 imported_names = imported_names)
1002 else:
1003 imported_name_strings = []
1004 items = []
1005 for (name_pos, name, as_name, kind) in imported_names:
1006 encoded_name = Utils.EncodedString(name)
1007 imported_name_strings.append(
1008 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
1009 items.append(
1010 (name,
1011 ExprNodes.NameNode(name_pos,
1012 name = as_name or name)))
1013 import_list = ExprNodes.ListNode(
1014 imported_names[0][0], args = imported_name_strings)
1015 dotted_name = Utils.EncodedString(dotted_name)
1016 return Nodes.FromImportStatNode(pos,
1017 module = ExprNodes.ImportNode(dotted_name_pos,
1018 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
1019 name_list = import_list),
1020 items = items)
1022 imported_name_kinds = ('class', 'struct', 'union')
1024 def p_imported_name(s, is_cimport):
1025 pos = s.position()
1026 kind = None
1027 if is_cimport and s.systring in imported_name_kinds:
1028 kind = s.systring
1029 s.next()
1030 name = p_ident(s)
1031 as_name = p_as_name(s)
1032 return (pos, name, as_name, kind)
1034 def p_dotted_name(s, as_allowed):
1035 pos = s.position()
1036 target_name = p_ident(s)
1037 as_name = None
1038 names = [target_name]
1039 while s.sy == '.':
1040 s.next()
1041 names.append(p_ident(s))
1042 if as_allowed:
1043 as_name = p_as_name(s)
1044 return (pos, target_name, u'.'.join(names), as_name)
1046 def p_as_name(s):
1047 if s.sy == 'IDENT' and s.systring == 'as':
1048 s.next()
1049 return p_ident(s)
1050 else:
1051 return None
1053 def p_assert_statement(s):
1054 # s.sy == 'assert'
1055 pos = s.position()
1056 s.next()
1057 cond = p_simple_expr(s)
1058 if s.sy == ',':
1059 s.next()
1060 value = p_simple_expr(s)
1061 else:
1062 value = None
1063 return Nodes.AssertStatNode(pos, cond = cond, value = value)
1065 statement_terminators = (';', 'NEWLINE', 'EOF')
1067 def p_if_statement(s):
1068 # s.sy == 'if'
1069 pos = s.position()
1070 s.next()
1071 if_clauses = [p_if_clause(s)]
1072 while s.sy == 'elif':
1073 s.next()
1074 if_clauses.append(p_if_clause(s))
1075 else_clause = p_else_clause(s)
1076 return Nodes.IfStatNode(pos,
1077 if_clauses = if_clauses, else_clause = else_clause)
1079 def p_if_clause(s):
1080 pos = s.position()
1081 test = p_simple_expr(s)
1082 body = p_suite(s)
1083 return Nodes.IfClauseNode(pos,
1084 condition = test, body = body)
1086 def p_else_clause(s):
1087 if s.sy == 'else':
1088 s.next()
1089 return p_suite(s)
1090 else:
1091 return None
1093 def p_while_statement(s):
1094 # s.sy == 'while'
1095 pos = s.position()
1096 s.next()
1097 test = p_simple_expr(s)
1098 body = p_suite(s)
1099 else_clause = p_else_clause(s)
1100 return Nodes.WhileStatNode(pos,
1101 condition = test, body = body,
1102 else_clause = else_clause)
1104 def p_for_statement(s):
1105 # s.sy == 'for'
1106 pos = s.position()
1107 s.next()
1108 kw = p_for_bounds(s)
1109 kw['body'] = p_suite(s)
1110 kw['else_clause'] = p_else_clause(s)
1111 return Nodes.ForStatNode(pos, **kw)
1113 def p_for_bounds(s):
1114 target = p_for_target(s)
1115 if s.sy == 'in':
1116 s.next()
1117 iterator = p_for_iterator(s)
1118 return { 'target': target, 'iterator': iterator }
1119 else:
1120 if s.sy == 'from':
1121 s.next()
1122 bound1 = p_bit_expr(s)
1123 else:
1124 # Support shorter "for a <= x < b" syntax
1125 bound1, target = target, None
1126 rel1 = p_for_from_relation(s)
1127 name2_pos = s.position()
1128 name2 = p_ident(s)
1129 rel2_pos = s.position()
1130 rel2 = p_for_from_relation(s)
1131 bound2 = p_bit_expr(s)
1132 step = p_for_from_step(s)
1133 if target is None:
1134 target = ExprNodes.NameNode(name2_pos, name = name2)
1135 else:
1136 if not target.is_name:
1137 error(target.pos,
1138 "Target of for-from statement must be a variable name")
1139 elif name2 != target.name:
1140 error(name2_pos,
1141 "Variable name in for-from range does not match target")
1142 if rel1[0] != rel2[0]:
1143 error(rel2_pos,
1144 "Relation directions in for-from do not match")
1145 return {'target': target,
1146 'bound1': bound1,
1147 'relation1': rel1,
1148 'relation2': rel2,
1149 'bound2': bound2,
1150 'step': step }
1152 def p_for_from_relation(s):
1153 if s.sy in inequality_relations:
1154 op = s.sy
1155 s.next()
1156 return op
1157 else:
1158 s.error("Expected one of '<', '<=', '>' '>='")
1160 def p_for_from_step(s):
1161 if s.sy == 'by':
1162 s.next()
1163 step = p_bit_expr(s)
1164 return step
1165 else:
1166 return None
1168 inequality_relations = ('<', '<=', '>', '>=')
1170 def p_target(s, terminator):
1171 pos = s.position()
1172 expr = p_bit_expr(s)
1173 if s.sy == ',':
1174 s.next()
1175 exprs = [expr]
1176 while s.sy != terminator:
1177 exprs.append(p_bit_expr(s))
1178 if s.sy != ',':
1179 break
1180 s.next()
1181 return ExprNodes.TupleNode(pos, args = exprs)
1182 else:
1183 return expr
1185 def p_for_target(s):
1186 return p_target(s, 'in')
1188 def p_for_iterator(s):
1189 pos = s.position()
1190 expr = p_testlist(s)
1191 return ExprNodes.IteratorNode(pos, sequence = expr)
1193 def p_try_statement(s):
1194 # s.sy == 'try'
1195 pos = s.position()
1196 s.next()
1197 body = p_suite(s)
1198 except_clauses = []
1199 else_clause = None
1200 if s.sy in ('except', 'else'):
1201 while s.sy == 'except':
1202 except_clauses.append(p_except_clause(s))
1203 if s.sy == 'else':
1204 s.next()
1205 else_clause = p_suite(s)
1206 body = Nodes.TryExceptStatNode(pos,
1207 body = body, except_clauses = except_clauses,
1208 else_clause = else_clause)
1209 if s.sy != 'finally':
1210 return body
1211 # try-except-finally is equivalent to nested try-except/try-finally
1212 if s.sy == 'finally':
1213 s.next()
1214 finally_clause = p_suite(s)
1215 return Nodes.TryFinallyStatNode(pos,
1216 body = body, finally_clause = finally_clause)
1217 else:
1218 s.error("Expected 'except' or 'finally'")
1220 def p_except_clause(s):
1221 # s.sy == 'except'
1222 pos = s.position()
1223 s.next()
1224 exc_type = None
1225 exc_value = None
1226 if s.sy != ':':
1227 exc_type = p_simple_expr(s)
1228 if s.sy == ',':
1229 s.next()
1230 exc_value = p_simple_expr(s)
1231 body = p_suite(s)
1232 return Nodes.ExceptClauseNode(pos,
1233 pattern = exc_type, target = exc_value, body = body)
1235 def p_include_statement(s, ctx):
1236 pos = s.position()
1237 s.next() # 'include'
1238 _, include_file_name = p_string_literal(s)
1239 s.expect_newline("Syntax error in include statement")
1240 if s.compile_time_eval:
1241 include_file_path = s.context.find_include_file(include_file_name, pos)
1242 if include_file_path:
1243 s.included_files.append(include_file_name)
1244 f = Utils.open_source_file(include_file_path, mode="rU")
1245 source_desc = FileSourceDescriptor(include_file_path)
1246 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding)
1247 try:
1248 tree = p_statement_list(s2, ctx)
1249 finally:
1250 f.close()
1251 return tree
1252 else:
1253 return None
1254 else:
1255 return Nodes.PassStatNode(pos)
1257 def p_with_statement(s):
1258 pos = s.position()
1259 s.next() # 'with'
1260 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):
1261 if s.sy == 'IDENT' and s.systring == 'nogil':
1262 state = s.systring
1263 s.next()
1264 body = p_suite(s)
1265 return Nodes.GILStatNode(pos, state = state, body = body)
1266 else:
1267 manager = p_expr(s)
1268 target = None
1269 if s.sy == 'IDENT' and s.systring == 'as':
1270 s.next()
1271 allow_multi = (s.sy == '(')
1272 target = p_target(s, ':')
1273 if not allow_multi and isinstance(target, ExprNodes.TupleNode):
1274 s.error("Multiple with statement target values not allowed without paranthesis")
1275 body = p_suite(s)
1276 return Nodes.WithStatNode(pos, manager = manager,
1277 target = target, body = body)
1279 def p_simple_statement(s, first_statement = 0):
1280 #print "p_simple_statement:", s.sy, s.systring ###
1281 if s.sy == 'global':
1282 node = p_global_statement(s)
1283 elif s.sy == 'print':
1284 node = p_print_statement(s)
1285 elif s.sy == 'del':
1286 node = p_del_statement(s)
1287 elif s.sy == 'break':
1288 node = p_break_statement(s)
1289 elif s.sy == 'continue':
1290 node = p_continue_statement(s)
1291 elif s.sy == 'return':
1292 node = p_return_statement(s)
1293 elif s.sy == 'raise':
1294 node = p_raise_statement(s)
1295 elif s.sy in ('import', 'cimport'):
1296 node = p_import_statement(s)
1297 elif s.sy == 'from':
1298 node = p_from_import_statement(s, first_statement = first_statement)
1299 elif s.sy == 'assert':
1300 node = p_assert_statement(s)
1301 elif s.sy == 'pass':
1302 node = p_pass_statement(s)
1303 else:
1304 node = p_expression_or_assignment(s)
1305 return node
1307 def p_simple_statement_list(s, ctx, first_statement = 0):
1308 # Parse a series of simple statements on one line
1309 # separated by semicolons.
1310 stat = p_simple_statement(s, first_statement = first_statement)
1311 if s.sy == ';':
1312 stats = [stat]
1313 while s.sy == ';':
1314 #print "p_simple_statement_list: maybe more to follow" ###
1315 s.next()
1316 if s.sy in ('NEWLINE', 'EOF'):
1317 break
1318 stats.append(p_simple_statement(s))
1319 stat = Nodes.StatListNode(stats[0].pos, stats = stats)
1320 s.expect_newline("Syntax error in simple statement list")
1321 return stat
1323 def p_compile_time_expr(s):
1324 old = s.compile_time_expr
1325 s.compile_time_expr = 1
1326 expr = p_expr(s)
1327 s.compile_time_expr = old
1328 return expr
1330 def p_DEF_statement(s):
1331 pos = s.position()
1332 denv = s.compile_time_env
1333 s.next() # 'DEF'
1334 name = p_ident(s)
1335 s.expect('=')
1336 expr = p_compile_time_expr(s)
1337 value = expr.compile_time_value(denv)
1338 #print "p_DEF_statement: %s = %r" % (name, value) ###
1339 denv.declare(name, value)
1340 s.expect_newline()
1341 return Nodes.PassStatNode(pos)
1343 def p_IF_statement(s, ctx):
1344 pos = s.position()
1345 saved_eval = s.compile_time_eval
1346 current_eval = saved_eval
1347 denv = s.compile_time_env
1348 result = None
1349 while 1:
1350 s.next() # 'IF' or 'ELIF'
1351 expr = p_compile_time_expr(s)
1352 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1353 body = p_suite(s, ctx)
1354 if s.compile_time_eval:
1355 result = body
1356 current_eval = 0
1357 if s.sy != 'ELIF':
1358 break
1359 if s.sy == 'ELSE':
1360 s.next()
1361 s.compile_time_eval = current_eval
1362 body = p_suite(s, ctx)
1363 if current_eval:
1364 result = body
1365 if not result:
1366 result = Nodes.PassStatNode(pos)
1367 s.compile_time_eval = saved_eval
1368 return result
1370 def p_statement(s, ctx, first_statement = 0):
1371 cdef_flag = ctx.cdef_flag
1372 if s.sy == 'ctypedef':
1373 if ctx.level not in ('module', 'module_pxd'):
1374 s.error("ctypedef statement not allowed here")
1375 if ctx.api:
1376 error(s.position(), "'api' not allowed with 'ctypedef'")
1377 return p_ctypedef_statement(s, ctx)
1378 elif s.sy == 'DEF':
1379 return p_DEF_statement(s)
1380 elif s.sy == 'IF':
1381 return p_IF_statement(s, ctx)
1382 elif s.sy == 'DECORATOR':
1383 if ctx.level not in ('module', 'class', 'c_class', 'property'):
1384 s.error('decorator not allowed here')
1385 s.level = ctx.level
1386 decorators = p_decorators(s)
1387 if s.sy != 'def':
1388 s.error("Decorators can only be followed by functions ")
1389 return p_def_statement(s, decorators)
1390 else:
1391 overridable = 0
1392 if s.sy == 'cdef':
1393 cdef_flag = 1
1394 s.next()
1395 if s.sy == 'cpdef':
1396 cdef_flag = 1
1397 overridable = 1
1398 s.next()
1399 if cdef_flag:
1400 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
1401 s.error('cdef statement not allowed here')
1402 s.level = ctx.level
1403 return p_cdef_statement(s, ctx(overridable = overridable))
1404 else:
1405 if ctx.api:
1406 error(s.pos, "'api' not allowed with this statement")
1407 elif s.sy == 'def':
1408 if ctx.level not in ('module', 'class', 'c_class', 'property'):
1409 s.error('def statement not allowed here')
1410 s.level = ctx.level
1411 return p_def_statement(s)
1412 elif s.sy == 'class':
1413 if ctx.level != 'module':
1414 s.error("class definition not allowed here")
1415 return p_class_statement(s)
1416 elif s.sy == 'include':
1417 if ctx.level not in ('module', 'module_pxd'):
1418 s.error("include statement not allowed here")
1419 return p_include_statement(s, ctx)
1420 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
1421 return p_property_decl(s)
1422 elif s.sy == 'pass' and ctx.level != 'property':
1423 return p_pass_statement(s, with_newline = 1)
1424 else:
1425 if ctx.level in ('c_class_pxd', 'property'):
1426 s.error("Executable statement not allowed here")
1427 if s.sy == 'if':
1428 return p_if_statement(s)
1429 elif s.sy == 'while':
1430 return p_while_statement(s)
1431 elif s.sy == 'for':
1432 return p_for_statement(s)
1433 elif s.sy == 'try':
1434 return p_try_statement(s)
1435 elif s.sy == 'with':
1436 return p_with_statement(s)
1437 else:
1438 return p_simple_statement_list(
1439 s, ctx, first_statement = first_statement)
1441 def p_statement_list(s, ctx, first_statement = 0):
1442 # Parse a series of statements separated by newlines.
1443 pos = s.position()
1444 stats = []
1445 while s.sy not in ('DEDENT', 'EOF'):
1446 stats.append(p_statement(s, ctx, first_statement = first_statement))
1447 first_statement = 0
1448 if len(stats) == 1:
1449 return stats[0]
1450 else:
1451 return Nodes.StatListNode(pos, stats = stats)
1453 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):
1454 pos = s.position()
1455 s.expect(':')
1456 doc = None
1457 stmts = []
1458 if s.sy == 'NEWLINE':
1459 s.next()
1460 s.expect_indent()
1461 if with_doc or with_pseudo_doc:
1462 doc = p_doc_string(s)
1463 body = p_statement_list(s, ctx)
1464 s.expect_dedent()
1465 else:
1466 if ctx.api:
1467 error(s.pos, "'api' not allowed with this statement")
1468 if ctx.level in ('module', 'class', 'function', 'other'):
1469 body = p_simple_statement_list(s, ctx)
1470 else:
1471 body = p_pass_statement(s)
1472 s.expect_newline("Syntax error in declarations")
1473 if with_doc:
1474 return doc, body
1475 else:
1476 return body
1478 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):
1479 """
1480 Parses positional and keyword arguments. end_sy_set
1481 should contain any s.sy that terminate the argument list.
1482 Argument expansion (* and **) are not allowed.
1484 type_positions and type_keywords specifies which argument
1485 positions and/or names which should be interpreted as
1486 types. Other arguments will be treated as expressions.
1488 Returns: (positional_args, keyword_args)
1489 """
1490 positional_args = []
1491 keyword_args = []
1492 pos_idx = 0
1494 while s.sy not in end_sy_set:
1495 if s.sy == '*' or s.sy == '**':
1496 s.error('Argument expansion not allowed here.')
1498 was_keyword = False
1499 parsed_type = False
1500 if s.sy == 'IDENT':
1501 # Since we can have either types or expressions as positional args,
1502 # we use a strategy of looking an extra step forward for a '=' and
1503 # if it is a positional arg we backtrack.
1504 ident = s.systring
1505 s.next()
1506 if s.sy == '=':
1507 s.next()
1508 # Is keyword arg
1509 if ident in type_keywords:
1510 arg = p_c_base_type(s)
1511 parsed_type = True
1512 else:
1513 arg = p_simple_expr(s)
1514 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,
1515 value = Utils.EncodedString(ident))
1516 keyword_args.append((keyword_node, arg))
1517 was_keyword = True
1518 else:
1519 s.put_back('IDENT', ident)
1521 if not was_keyword:
1522 if pos_idx in type_positions:
1523 arg = p_c_base_type(s)
1524 parsed_type = True
1525 else:
1526 arg = p_simple_expr(s)
1527 positional_args.append(arg)
1528 pos_idx += 1
1529 if len(keyword_args) > 0:
1530 s.error("Non-keyword arg following keyword arg",
1531 pos = arg.pos)
1533 if s.sy != ',':
1534 if s.sy not in end_sy_set:
1535 if parsed_type:
1536 s.error("Expected: type")
1537 else:
1538 s.error("Expected: expression")
1539 break
1540 s.next()
1541 return positional_args, keyword_args
1543 def p_c_base_type(s, self_flag = 0, nonempty = 0):
1544 # If self_flag is true, this is the base type for the
1545 # self argument of a C method of an extension type.
1546 if s.sy == '(':
1547 return p_c_complex_base_type(s)
1548 else:
1549 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)
1551 def p_calling_convention(s):
1552 if s.sy == 'IDENT' and s.systring in calling_convention_words:
1553 result = s.systring
1554 s.next()
1555 return result
1556 else:
1557 return ""
1559 calling_convention_words = ("__stdcall", "__cdecl")
1561 def p_c_complex_base_type(s):
1562 # s.sy == '('
1563 pos = s.position()
1564 s.next()
1565 base_type = p_c_base_type(s)
1566 declarator = p_c_declarator(s, empty = 1)
1567 s.expect(')')
1568 return Nodes.CComplexBaseTypeNode(pos,
1569 base_type = base_type, declarator = declarator)
1571 def p_c_simple_base_type(s, self_flag, nonempty):
1572 #print "p_c_simple_base_type: self_flag =", self_flag
1573 is_basic = 0
1574 signed = 1
1575 longness = 0
1576 module_path = []
1577 pos = s.position()
1578 if looking_at_base_type(s):
1579 #print "p_c_simple_base_type: looking_at_base_type at", s.position()
1580 is_basic = 1
1581 signed, longness = p_sign_and_longness(s)
1582 if s.sy == 'IDENT' and s.systring in basic_c_type_names:
1583 name = s.systring
1584 s.next()
1585 else:
1586 name = 'int'
1587 elif s.looking_at_type_name():
1588 name = s.systring
1589 s.next()
1590 if nonempty and s.sy != 'IDENT':
1591 # Make sure this is not a declaration of a variable or
1592 # function with the same name as a type.
1593 if s.sy == '(':
1594 s.next()
1595 if s.sy == '*' or s.sy == '**':
1596 s.put_back('(', '(')
1597 else:
1598 s.put_back('(', '(')
1599 s.put_back('IDENT', name)
1600 name = None
1601 elif s.sy not in ('*', '**', '['):
1602 s.put_back('IDENT', name)
1603 name = None
1604 elif looking_at_dotted_name(s):
1605 #print "p_c_simple_base_type: looking_at_type_name at", s.position()
1606 name = s.systring
1607 s.next()
1608 while s.sy == '.':
1609 module_path.append(name)
1610 s.next()
1611 name = p_ident(s)
1612 else:
1613 #print "p_c_simple_base_type: not looking at type at", s.position()
1614 name = None
1616 type_node = Nodes.CSimpleBaseTypeNode(pos,
1617 name = name, module_path = module_path,
1618 is_basic_c_type = is_basic, signed = signed,
1619 longness = longness, is_self_arg = self_flag)
1622 # Treat trailing [] on type as buffer access
1623 if 0: # s.sy == '[':
1624 if is_basic:
1625 s.error("Basic C types do not support buffer access")
1626 return p_buffer_access(s, type_node)
1627 else:
1628 return type_node
1630 def p_buffer_access(s, type_node):
1631 # s.sy == '['
1632 pos = s.position()
1633 s.next()
1634 positional_args, keyword_args = (
1635 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))
1636 )
1637 s.expect(']')
1639 keyword_dict = ExprNodes.DictNode(pos,
1640 key_value_pairs = [
1641 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
1642 for key, value in keyword_args
1643 ])
1645 result = Nodes.CBufferAccessTypeNode(pos,
1646 positional_args = positional_args,
1647 keyword_args = keyword_dict,
1648 base_type_node = type_node)
1649 return result
1652 def looking_at_type(s):
1653 return looking_at_base_type(s) or s.looking_at_type_name()
1655 def looking_at_base_type(s):
1656 #print "looking_at_base_type?", s.sy, s.systring, s.position()
1657 return s.sy == 'IDENT' and s.systring in base_type_start_words
1659 def looking_at_dotted_name(s):
1660 if s.sy == 'IDENT':
1661 name = s.systring
1662 s.next()
1663 result = s.sy == '.'
1664 s.put_back('IDENT', name)
1665 return result
1666 else:
1667 return 0
1669 basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")
1671 sign_and_longness_words = ("short", "long", "signed", "unsigned")
1673 base_type_start_words = \
1674 basic_c_type_names + sign_and_longness_words
1676 def p_sign_and_longness(s):
1677 signed = 1
1678 longness = 0
1679 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
1680 if s.systring == 'unsigned':
1681 signed = 0
1682 elif s.systring == 'signed':
1683 signed = 2
1684 elif s.systring == 'short':
1685 longness = -1
1686 elif s.systring == 'long':
1687 longness += 1
1688 s.next()
1689 return signed, longness
1691 def p_opt_cname(s):
1692 literal = p_opt_string_literal(s)
1693 if literal:
1694 _, cname = literal
1695 else:
1696 cname = None
1697 return cname
1699 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
1700 assignable = 0, nonempty = 0,
1701 calling_convention_allowed = 0):
1702 # If empty is true, the declarator must be empty. If nonempty is true,
1703 # the declarator must be nonempty. Otherwise we don't care.
1704 # If cmethod_flag is true, then if this declarator declares
1705 # a function, it's a C method of an extension type.
1706 pos = s.position()
1707 if s.sy == '(':
1708 s.next()
1709 if s.sy == ')' or looking_at_type(s):
1710 base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None)
1711 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
1712 else:
1713 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1714 cmethod_flag = cmethod_flag,
1715 nonempty = nonempty,
1716 calling_convention_allowed = 1)
1717 s.expect(')')
1718 else:
1719 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
1720 assignable, nonempty)
1721 if not calling_convention_allowed and result.calling_convention and s.sy != '(':
1722 error(s.position(), "%s on something that is not a function"
1723 % result.calling_convention)
1724 while s.sy in ('[', '('):
1725 pos = s.position()
1726 if s.sy == '[':
1727 result = p_c_array_declarator(s, result)
1728 else: # sy == '('
1729 s.next()
1730 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
1731 cmethod_flag = 0
1732 return result
1734 def p_c_array_declarator(s, base):
1735 pos = s.position()
1736 s.next() # '['
1737 if s.sy != ']':
1738 dim = p_expr(s)
1739 else:
1740 dim = None
1741 s.expect(']')
1742 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)
1744 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
1745 # Opening paren has already been skipped
1746 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
1747 nonempty_declarators = 0)
1748 ellipsis = p_optional_ellipsis(s)
1749 s.expect(')')
1750 nogil = p_nogil(s)
1751 exc_val, exc_check = p_exception_value_clause(s)
1752 with_gil = p_with_gil(s)
1753 return Nodes.CFuncDeclaratorNode(pos,
1754 base = base, args = args, has_varargs = ellipsis,
1755 exception_value = exc_val, exception_check = exc_check,
1756 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
1758 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
1759 assignable, nonempty):
1760 pos = s.position()
1761 calling_convention = p_calling_convention(s)
1762 if s.sy == '*':
1763 s.next()
1764 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1765 cmethod_flag = cmethod_flag,
1766 assignable = assignable, nonempty = nonempty)
1767 result = Nodes.CPtrDeclaratorNode(pos,
1768 base = base)
1769 elif s.sy == '**': # scanner returns this as a single token
1770 s.next()
1771 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
1772 cmethod_flag = cmethod_flag,
1773 assignable = assignable, nonempty = nonempty)
1774 result = Nodes.CPtrDeclaratorNode(pos,
1775 base = Nodes.CPtrDeclaratorNode(pos,
1776 base = base))
1777 else:
1778 rhs = None
1779 if s.sy == 'IDENT':
1780 name = s.systring
1781 if is_type:
1782 s.add_type_name(name)
1783 if empty:
1784 error(s.position(), "Declarator should be empty")
1785 s.next()
1786 cname = p_opt_cname(s)
1787 if s.sy == '=' and assignable:
1788 s.next()
1789 rhs = p_simple_expr(s)
1790 else:
1791 if nonempty:
1792 error(s.position(), "Empty declarator")
1793 name = ""
1794 cname = None
1795 result = Nodes.CNameDeclaratorNode(pos,
1796 name = name, cname = cname, default = rhs)
1797 result.calling_convention = calling_convention
1798 return result
1800 def p_nogil(s):
1801 if s.sy == 'IDENT' and s.systring == 'nogil':
1802 s.next()
1803 return 1
1804 else:
1805 return 0
1807 def p_with_gil(s):
1808 if s.sy == 'with':
1809 s.next()
1810 s.expect_keyword('gil')
1811 return 1
1812 else:
1813 return 0
1815 def p_exception_value_clause(s):
1816 exc_val = None
1817 exc_check = 0
1818 if s.sy == 'except':
1819 s.next()
1820 if s.sy == '*':
1821 exc_check = 1
1822 s.next()
1823 elif s.sy == '+':
1824 exc_check = '+'
1825 s.next()
1826 if s.sy == 'IDENT':
1827 name = s.systring
1828 s.next()
1829 exc_val = p_name(s, name)
1830 else:
1831 if s.sy == '?':
1832 exc_check = 1
1833 s.next()
1834 exc_val = p_simple_expr(s)
1835 return exc_val, exc_check
1837 c_arg_list_terminators = ('*', '**', '.', ')')
1839 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
1840 nonempty_declarators = 0, kw_only = 0):
1841 # Comma-separated list of C argument declarations, possibly empty.
1842 # May have a trailing comma.
1843 args = []
1844 is_self_arg = cmethod_flag
1845 while s.sy not in c_arg_list_terminators:
1846 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
1847 nonempty = nonempty_declarators, kw_only = kw_only))
1848 if s.sy != ',':
1849 break
1850 s.next()
1851 is_self_arg = 0
1852 return args
1854 def p_optional_ellipsis(s):
1855 if s.sy == '.':
1856 expect_ellipsis(s)
1857 return 1
1858 else:
1859 return 0
1861 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):
1862 pos = s.position()
1863 not_none = 0
1864 default = None
1865 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
1866 declarator = p_c_declarator(s, ctx, nonempty = nonempty)
1867 if s.sy == 'not':
1868 s.next()
1869 if s.sy == 'IDENT' and s.systring == 'None':
1870 s.next()
1871 else:
1872 s.error("Expected 'None'")
1873 if not in_pyfunc:
1874 error(pos, "'not None' only allowed in Python functions")
1875 not_none = 1
1876 if s.sy == '=':
1877 s.next()
1878 if 'pxd' in s.level:
1879 if s.sy not in ['*', '?']:
1880 error(pos, "default values cannot be specified in pxd files, use ? or *")
1881 default = 1
1882 s.next()
1883 else:
1884 default = p_simple_expr(s)
1885 return Nodes.CArgDeclNode(pos,
1886 base_type = base_type,
1887 declarator = declarator,
1888 not_none = not_none,
1889 default = default,
1890 kw_only = kw_only)
1892 def p_api(s):
1893 if s.sy == 'IDENT' and s.systring == 'api':
1894 s.next()
1895 return 1
1896 else:
1897 return 0
1899 def p_cdef_statement(s, ctx):
1900 pos = s.position()
1901 ctx.visibility = p_visibility(s, ctx.visibility)
1902 ctx.api = ctx.api or p_api(s)
1903 if ctx.api:
1904 if ctx.visibility not in ('private', 'public'):
1905 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
1906 if (ctx.visibility == 'extern') and s.sy == 'from':
1907 return p_cdef_extern_block(s, pos, ctx)
1908 elif s.sy == 'import':
1909 s.next()
1910 return p_cdef_extern_block(s, pos, ctx)
1911 if p_nogil(s):
1912 ctx.nogil = 1
1913 if s.sy == ':':
1914 return p_cdef_block(s, ctx)
1915 elif s.sy == 'class':
1916 if ctx.level not in ('module', 'module_pxd'):
1917 error(pos, "Extension type definition not allowed here")
1918 #if ctx.api:
1919 # error(pos, "'api' not allowed with extension class")
1920 return p_c_class_definition(s, pos, ctx)
1921 elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:
1922 if ctx.level not in ('module', 'module_pxd'):
1923 error(pos, "C struct/union/enum definition not allowed here")
1924 #if ctx.visibility == 'public':
1925 # error(pos, "Public struct/union/enum definition not implemented")
1926 #if ctx.api:
1927 # error(pos, "'api' not allowed with '%s'" % s.systring)
1928 if s.systring == "enum":
1929 return p_c_enum_definition(s, pos, ctx)
1930 else:
1931 return p_c_struct_or_union_definition(s, pos, ctx)
1932 elif s.sy == 'pass':
1933 node = p_pass_statement(s)
1934 s.expect_newline('Expected a newline')
1935 return node
1936 else:
1937 return p_c_func_or_var_declaration(s, pos, ctx)
1939 def p_cdef_block(s, ctx):
1940 return p_suite(s, ctx(cdef_flag = 1))
1942 def p_cdef_extern_block(s, pos, ctx):
1943 include_file = None
1944 s.expect('from')
1945 if s.sy == '*':
1946 s.next()
1947 else:
1948 _, include_file = p_string_literal(s)
1949 ctx = ctx(cdef_flag = 1, visibility = 'extern')
1950 if p_nogil(s):
1951 ctx.nogil = 1
1952 body = p_suite(s, ctx)
1953 return Nodes.CDefExternNode(pos,
1954 include_file = include_file,
1955 body = body)
1957 struct_union_or_enum = (
1958 "struct", "union", "enum"
1959 )
1961 def p_c_enum_definition(s, pos, ctx):
1962 # s.sy == ident 'enum'
1963 s.next()
1964 if s.sy == 'IDENT':
1965 name = s.systring
1966 s.next()
1967 s.add_type_name(name)
1968 cname = p_opt_cname(s)
1969 else:
1970 name = None
1971 cname = None
1972 items = None
1973 s.expect(':')
1974 items = []
1975 if s.sy != 'NEWLINE':
1976 p_c_enum_line(s, items)
1977 else:
1978 s.next() # 'NEWLINE'
1979 s.expect_indent()
1980 while s.sy not in ('DEDENT', 'EOF'):
1981 p_c_enum_line(s, items)
1982 s.expect_dedent()
1983 return Nodes.CEnumDefNode(
1984 pos, name = name, cname = cname, items = items,
1985 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
1986 in_pxd = ctx.level == 'module_pxd')
1988 def p_c_enum_line(s, items):
1989 if s.sy != 'pass':
1990 p_c_enum_item(s, items)
1991 while s.sy == ',':
1992 s.next()
1993 if s.sy in ('NEWLINE', 'EOF'):
1994 break
1995 p_c_enum_item(s, items)
1996 else:
1997 s.next()
1998 s.expect_newline("Syntax error in enum item list")
2000 def p_c_enum_item(s, items):
2001 pos = s.position()
2002 name = p_ident(s)
2003 cname = p_opt_cname(s)
2004 value = None
2005 if s.sy == '=':
2006 s.next()
2007 value = p_simple_expr(s)
2008 items.append(Nodes.CEnumDefItemNode(pos,
2009 name = name, cname = cname, value = value))
2011 def p_c_struct_or_union_definition(s, pos, ctx):
2012 # s.sy == ident 'struct' or 'union'
2013 kind = s.systring
2014 s.next()
2015 name = p_ident(s)
2016 cname = p_opt_cname(s)
2017 s.add_type_name(name)
2018 attributes = None
2019 if s.sy == ':':
2020 s.next()
2021 s.expect('NEWLINE')
2022 s.expect_indent()
2023 attributes = []
2024 body_ctx = Ctx()
2025 while s.sy != 'DEDENT':
2026 if s.sy != 'pass':
2027 attributes.append(
2028 p_c_func_or_var_declaration(s, s.position(), body_ctx))
2029 else:
2030 s.next()
2031 s.expect_newline("Expected a newline")
2032 s.expect_dedent()
2033 else:
2034 s.expect_newline("Syntax error in struct or union definition")
2035 return Nodes.CStructOrUnionDefNode(pos,
2036 name = name, cname = cname, kind = kind, attributes = attributes,
2037 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2038 in_pxd = ctx.level == 'module_pxd')
2040 def p_visibility(s, prev_visibility):
2041 pos = s.position()
2042 visibility = prev_visibility
2043 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
2044 visibility = s.systring
2045 if prev_visibility != 'private' and visibility != prev_visibility:
2046 s.error("Conflicting visibility options '%s' and '%s'"
2047 % (prev_visibility, visibility))
2048 s.next()
2049 return visibility
2051 def p_c_modifiers(s):
2052 if s.sy == 'IDENT' and s.systring in ('inline',):
2053 modifier = s.systring
2054 s.next()
2055 return [modifier] + p_c_modifiers(s)
2056 return []
2058 def p_c_func_or_var_declaration(s, pos, ctx):
2059 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
2060 modifiers = p_c_modifiers(s)
2061 base_type = p_c_base_type(s, nonempty = 1)
2062 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2063 assignable = 1, nonempty = 1)
2064 declarator.overridable = ctx.overridable
2065 if s.sy == ':':
2066 if ctx.level not in ('module', 'c_class'):
2067 s.error("C function definition not allowed here")
2068 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)
2069 result = Nodes.CFuncDefNode(pos,
2070 visibility = ctx.visibility,
2071 base_type = base_type,
2072 declarator = declarator,
2073 body = suite,
2074 doc = doc,
2075 modifiers = modifiers,
2076 api = ctx.api,
2077 overridable = ctx.overridable)
2078 else:
2079 #if api:
2080 # error(s.pos, "'api' not allowed with variable declaration")
2081 declarators = [declarator]
2082 while s.sy == ',':
2083 s.next()
2084 if s.sy == 'NEWLINE':
2085 break
2086 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2087 assignable = 1, nonempty = 1)
2088 declarators.append(declarator)
2089 s.expect_newline("Syntax error in C variable declaration")
2090 result = Nodes.CVarDefNode(pos,
2091 visibility = ctx.visibility,
2092 base_type = base_type,
2093 declarators = declarators,
2094 in_pxd = ctx.level == 'module_pxd',
2095 api = ctx.api,
2096 overridable = ctx.overridable)
2097 return result
2099 def p_ctypedef_statement(s, ctx):
2100 # s.sy == 'ctypedef'
2101 pos = s.position()
2102 s.next()
2103 visibility = p_visibility(s, ctx.visibility)
2104 ctx = ctx(typedef_flag = 1, visibility = visibility)
2105 if s.sy == 'class':
2106 return p_c_class_definition(s, pos, ctx)
2107 elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):
2108 if s.systring == 'enum':
2109 return p_c_enum_definition(s, pos, ctx)
2110 else:
2111 return p_c_struct_or_union_definition(s, pos, ctx)
2112 else:
2113 base_type = p_c_base_type(s, nonempty = 1)
2114 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
2115 s.expect_newline("Syntax error in ctypedef statement")
2116 return Nodes.CTypeDefNode(
2117 pos, base_type = base_type,
2118 declarator = declarator, visibility = visibility,
2119 in_pxd = ctx.level == 'module_pxd')
2121 def p_decorators(s):
2122 decorators = []
2123 while s.sy == 'DECORATOR':
2124 pos = s.position()
2125 s.next()
2126 decorator = ExprNodes.NameNode(
2127 pos, name = Utils.EncodedString(
2128 p_dotted_name(s, as_allowed=0)[2] ))
2129 if s.sy == '(':
2130 decorator = p_call(s, decorator)
2131 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
2132 s.expect_newline("Expected a newline after decorator")
2133 return decorators
2135 def p_def_statement(s, decorators=None):
2136 # s.sy == 'def'
2137 pos = s.position()
2138 s.next()
2139 name = p_ident(s)
2140 #args = []
2141 s.expect('(');
2142 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)
2143 star_arg = None
2144 starstar_arg = None
2145 if s.sy == '*':
2146 s.next()
2147 if s.sy == 'IDENT':
2148 star_arg = p_py_arg_decl(s)
2149 if s.sy == ',':
2150 s.next()
2151 args.extend(p_c_arg_list(s, in_pyfunc = 1,
2152 nonempty_declarators = 1, kw_only = 1))
2153 elif s.sy != ')':
2154 s.error("Syntax error in Python function argument list")
2155 if s.sy == '**':
2156 s.next()
2157 starstar_arg = p_py_arg_decl(s)
2158 s.expect(')')
2159 if p_nogil(s):
2160 error(s.pos, "Python function cannot be declared nogil")
2161 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)
2162 return Nodes.DefNode(pos, name = name, args = args,
2163 star_arg = star_arg, starstar_arg = starstar_arg,
2164 doc = doc, body = body, decorators = decorators)
2166 def p_py_arg_decl(s):
2167 pos = s.position()
2168 name = p_ident(s)
2169 return Nodes.PyArgDeclNode(pos, name = name)
2171 def p_class_statement(s):
2172 # s.sy == 'class'
2173 pos = s.position()
2174 s.next()
2175 class_name = Utils.EncodedString( p_ident(s) )
2176 class_name.encoding = s.source_encoding
2177 if s.sy == '(':
2178 s.next()
2179 base_list = p_simple_expr_list(s)
2180 s.expect(')')
2181 else:
2182 base_list = []
2183 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)
2184 return Nodes.PyClassDefNode(pos,
2185 name = class_name,
2186 bases = ExprNodes.TupleNode(pos, args = base_list),
2187 doc = doc, body = body)
2189 def p_c_class_definition(s, pos, ctx):
2190 # s.sy == 'class'
2191 s.next()
2192 module_path = []
2193 class_name = p_ident(s)
2194 while s.sy == '.':
2195 s.next()
2196 module_path.append(class_name)
2197 class_name = p_ident(s)
2198 if module_path and ctx.visibility != 'extern':
2199 error(pos, "Qualified class name only allowed for 'extern' C class")
2200 if module_path and s.sy == 'IDENT' and s.systring == 'as':
2201 s.next()
2202 as_name = p_ident(s)
2203 else:
2204 as_name = class_name
2205 s.add_type_name(as_name)
2206 objstruct_name = None
2207 typeobj_name = None
2208 base_class_module = None
2209 base_class_name = None
2210 if s.sy == '(':
2211 s.next()
2212 base_class_path = [p_ident(s)]
2213 while s.sy == '.':
2214 s.next()
2215 base_class_path.append(p_ident(s))
2216 if s.sy == ',':
2217 s.error("C class may only have one base class")
2218 s.expect(')')
2219 base_class_module = ".".join(base_class_path[:-1])
2220 base_class_name = base_class_path[-1]
2221 if s.sy == '[':
2222 if ctx.visibility not in ('public', 'extern'):
2223 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")
2224 objstruct_name, typeobj_name = p_c_class_options(s)
2225 if s.sy == ':':
2226 if ctx.level == 'module_pxd':
2227 body_level = 'c_class_pxd'
2228 else:
2229 body_level = 'c_class'
2230 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)
2231 else:
2232 s.expect_newline("Syntax error in C class definition")
2233 doc = None
2234 body = None
2235 if ctx.visibility == 'extern':
2236 if not module_path:
2237 error(pos, "Module name required for 'extern' C class")
2238 if typeobj_name:
2239 error(pos, "Type object name specification not allowed for 'extern' C class")
2240 elif ctx.visibility == 'public':
2241 if not objstruct_name:
2242 error(pos, "Object struct name specification required for 'public' C class")
2243 if not typeobj_name:
2244 error(pos, "Type object name specification required for 'public' C class")
2245 elif ctx.visibility == 'private':
2246 if ctx.api:
2247 error(pos, "Only 'public' C class can be declared 'api'")
2248 else:
2249 error(pos, "Invalid class visibility '%s'" % ctx.visibility)
2250 return Nodes.CClassDefNode(pos,
2251 visibility = ctx.visibility,
2252 typedef_flag = ctx.typedef_flag,
2253 api = ctx.api,
2254 module_name = ".".join(module_path),
2255 class_name = class_name,
2256 as_name = as_name,
2257 base_class_module = base_class_module,
2258 base_class_name = base_class_name,
2259 objstruct_name = objstruct_name,
2260 typeobj_name = typeobj_name,
2261 in_pxd = ctx.level == 'module_pxd',
2262 doc = doc,
2263 body = body)
2265 def p_c_class_options(s):
2266 objstruct_name = None
2267 typeobj_name = None
2268 s.expect('[')
2269 while 1:
2270 if s.sy != 'IDENT':
2271 break
2272 if s.systring == 'object':
2273 s.next()
2274 objstruct_name = p_ident(s)
2275 elif s.systring == 'type':
2276 s.next()
2277 typeobj_name = p_ident(s)
2278 if s.sy != ',':
2279 break
2280 s.next()
2281 s.expect(']', "Expected 'object' or 'type'")
2282 return objstruct_name, typeobj_name
2284 def p_property_decl(s):
2285 pos = s.position()
2286 s.next() # 'property'
2287 name = p_ident(s)
2288 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)
2289 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)
2291 def p_doc_string(s):
2292 if s.sy == 'BEGIN_STRING':
2293 pos = s.position()
2294 kind, result = p_cat_string_literal(s)
2295 if s.sy != 'EOF':
2296 s.expect_newline("Syntax error in doc string")
2297 if kind != 'u':
2298 # warning(pos, "Python 3 requires docstrings to be unicode strings")
2299 if kind == 'b':
2300 result.encoding = None # force a unicode string
2301 return result
2302 else:
2303 return None
2305 def p_module(s, pxd, full_module_name):
2306 s.add_type_name("object")
2307 s.add_type_name("Py_buffer")
2308 pos = s.position()
2309 doc = p_doc_string(s)
2310 if pxd:
2311 level = 'module_pxd'
2312 else:
2313 level = 'module'
2314 body = p_statement_list(s, Ctx(level = level), first_statement = 1)
2315 if s.sy != 'EOF':
2316 s.error("Syntax error in statement [%s,%s]" % (
2317 repr(s.sy), repr(s.systring)))
2318 return ModuleNode(pos, doc = doc, body = body, full_module_name = full_module_name)
2320 #----------------------------------------------
2321 #
2322 # Debugging
2323 #
2324 #----------------------------------------------
2326 def print_parse_tree(f, node, level, key = None):
2327 from Nodes import Node
2328 ind = " " * level
2329 if node:
2330 f.write(ind)
2331 if key:
2332 f.write("%s: " % key)
2333 t = type(node)
2334 if t == TupleType:
2335 f.write("(%s @ %s\n" % (node[0], node[1]))
2336 for i in xrange(2, len(node)):
2337 print_parse_tree(f, node[i], level+1)
2338 f.write("%s)\n" % ind)
2339 return
2340 elif isinstance(node, Node):
2341 try:
2342 tag = node.tag
2343 except AttributeError:
2344 tag = node.__class__.__name__
2345 f.write("%s @ %s\n" % (tag, node.pos))
2346 for name, value in node.__dict__.items():
2347 if name != 'tag' and name != 'pos':
2348 print_parse_tree(f, value, level+1, name)
2349 return
2350 elif t == ListType:
2351 f.write("[\n")
2352 for i in xrange(len(node)):
2353 print_parse_tree(f, node[i], level+1)
2354 f.write("%s]\n" % ind)
2355 return
2356 f.write("%s%s\n" % (ind, node))