1 # cython: auto_cpdef=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13 from types import ListType, TupleType

14 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

15 import Nodes

16 import ExprNodes

17 import StringEncoding

18 from StringEncoding import EncodedString, BytesLiteral

19 from ModuleNode import ModuleNode

20 from Errors import error, warning, InternalError

21 from Cython import Utils

22 import Future

23 import Options

24

25 class Ctx(object):

26 # Parsing context

27 level = 'other'

28 visibility = 'private'

29 cdef_flag = 0

30 typedef_flag = 0

31 api = 0

32 overridable = 0

33 nogil = 0

34

35 def __init__(self, **kwds):

36 self.__dict__.update(kwds)

37

38 def __call__(self, **kwds):

39 ctx = Ctx()

40 d = ctx.__dict__

41 d.update(self.__dict__)

42 d.update(kwds)

43 return ctx

44

45 def p_ident(s, message = "Expected an identifier"):

46 if s.sy == 'IDENT':

47 name = s.systring

48 s.next()

49 return name

50 else:

51 s.error(message)

52

53 def p_ident_list(s):

54 names = []

55 while s.sy == 'IDENT':

56 names.append(s.systring)

57 s.next()

58 if s.sy != ',':

59 break

60 s.next()

61 return names

62

63 #------------------------------------------

64 #

65 # Expressions

66 #

67 #------------------------------------------

68

69 def p_binop_expr(s, ops, p_sub_expr):

70 n1 = p_sub_expr(s)

71 while s.sy in ops:

72 op = s.sy

73 pos = s.position()

74 s.next()

75 n2 = p_sub_expr(s)

76 n1 = ExprNodes.binop_node(pos, op, n1, n2)

77 return n1

78

79 #expression: or_test [if or_test else test] | lambda_form

80

81 def p_simple_expr(s):

82 pos = s.position()

83 expr = p_or_test(s)

84 if s.sy == 'if':

85 s.next()

86 test = p_or_test(s)

87 if s.sy == 'else':

88 s.next()

89 other = p_test(s)

90 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

91 else:

92 s.error("Expected 'else'")

93 else:

94 return expr

95

96 #test: or_test | lambda_form

97

98 def p_test(s):

99 return p_or_test(s)

100

101 #or_test: and_test ('or' and_test)*

102

103 def p_or_test(s):

104 return p_rassoc_binop_expr(s, ('or',), p_and_test)

105

106 def p_rassoc_binop_expr(s, ops, p_subexpr):

107 n1 = p_subexpr(s)

108 if s.sy in ops:

109 pos = s.position()

110 op = s.sy

111 s.next()

112 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

113 n1 = ExprNodes.binop_node(pos, op, n1, n2)

114 return n1

115

116 #and_test: not_test ('and' not_test)*

117

118 def p_and_test(s):

119 #return p_binop_expr(s, ('and',), p_not_test)

120 return p_rassoc_binop_expr(s, ('and',), p_not_test)

121

122 #not_test: 'not' not_test | comparison

123

124 def p_not_test(s):

125 if s.sy == 'not':

126 pos = s.position()

127 s.next()

128 return ExprNodes.NotNode(pos, operand = p_not_test(s))

129 else:

130 return p_comparison(s)

131

132 #comparison: expr (comp_op expr)*

133 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

134

135 def p_comparison(s):

136 n1 = p_bit_expr(s)

137 if s.sy in comparison_ops:

138 pos = s.position()

139 op = p_cmp_op(s)

140 n2 = p_bit_expr(s)

141 n1 = ExprNodes.PrimaryCmpNode(pos,

142 operator = op, operand1 = n1, operand2 = n2)

143 if s.sy in comparison_ops:

144 n1.cascade = p_cascaded_cmp(s)

145 return n1

146

147 def p_cascaded_cmp(s):

148 pos = s.position()

149 op = p_cmp_op(s)

150 n2 = p_bit_expr(s)

151 result = ExprNodes.CascadedCmpNode(pos,

152 operator = op, operand2 = n2)

153 if s.sy in comparison_ops:

154 result.cascade = p_cascaded_cmp(s)

155 return result

156

157 def p_cmp_op(s):

158 if s.sy == 'not':

159 s.next()

160 s.expect('in')

161 op = 'not_in'

162 elif s.sy == 'is':

163 s.next()

164 if s.sy == 'not':

165 s.next()

166 op = 'is_not'

167 else:

168 op = 'is'

169 else:

170 op = s.sy

171 s.next()

172 if op == '<>':

173 op = '!='

174 return op

175

176 comparison_ops = (

177 '<', '>', '==', '>=', '<=', '<>', '!=',

178 'in', 'is', 'not'

179 )

180

181 #expr: xor_expr ('|' xor_expr)*

182

183 def p_bit_expr(s):

184 return p_binop_expr(s, ('|',), p_xor_expr)

185

186 #xor_expr: and_expr ('^' and_expr)*

187

188 def p_xor_expr(s):

189 return p_binop_expr(s, ('^',), p_and_expr)

190

191 #and_expr: shift_expr ('&' shift_expr)*

192

193 def p_and_expr(s):

194 return p_binop_expr(s, ('&',), p_shift_expr)

195

196 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

197

198 def p_shift_expr(s):

199 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

200

201 #arith_expr: term (('+'|'-') term)*

202

203 def p_arith_expr(s):

204 return p_binop_expr(s, ('+', '-'), p_term)

205

206 #term: factor (('*'|'/'|'%') factor)*

207

208 def p_term(s):

209 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

210

211 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

212

213 def p_factor(s):

214 sy = s.sy

215 if sy in ('+', '-', '~'):

216 op = s.sy

217 pos = s.position()

218 s.next()

219 return ExprNodes.unop_node(pos, op, p_factor(s))

220 elif sy == '&':

221 pos = s.position()

222 s.next()

223 arg = p_factor(s)

224 return ExprNodes.AmpersandNode(pos, operand = arg)

225 elif sy == "<":

226 return p_typecast(s)

227 elif sy == 'IDENT' and s.systring == "sizeof":

228 return p_sizeof(s)

229 else:

230 return p_power(s)

231

232 def p_typecast(s):

233 # s.sy == "<"

234 pos = s.position()

235 s.next()

236 base_type = p_c_base_type(s)

237 if base_type.name is None:

238 s.error("Unknown type")

239 declarator = p_c_declarator(s, empty = 1)

240 if s.sy == '?':

241 s.next()

242 typecheck = 1

243 else:

244 typecheck = 0

245 s.expect(">")

246 operand = p_factor(s)

247 return ExprNodes.TypecastNode(pos,

248 base_type = base_type,

249 declarator = declarator,

250 operand = operand,

251 typecheck = typecheck)

252

253 def p_sizeof(s):

254 # s.sy == ident "sizeof"

255 pos = s.position()

256 s.next()

257 s.expect('(')

258 # Here we decide if we are looking at an expression or type

259 # If it is actually a type, but parsable as an expression,

260 # we treat it as an expression here.

261 if looking_at_expr(s):

262 operand = p_simple_expr(s)

263 node = ExprNodes.SizeofVarNode(pos, operand = operand)

264 else:

265 base_type = p_c_base_type(s)

266 declarator = p_c_declarator(s, empty = 1)

267 node = ExprNodes.SizeofTypeNode(pos,

268 base_type = base_type, declarator = declarator)

269 s.expect(')')

270 return node

271

272 #power: atom trailer* ('**' factor)*

273

274 def p_power(s):

275 n1 = p_atom(s)

276 while s.sy in ('(', '[', '.'):

277 n1 = p_trailer(s, n1)

278 if s.sy == '**':

279 pos = s.position()

280 s.next()

281 n2 = p_factor(s)

282 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

283 return n1

284

285 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

286

287 def p_trailer(s, node1):

288 pos = s.position()

289 if s.sy == '(':

290 return p_call(s, node1)

291 elif s.sy == '[':

292 return p_index(s, node1)

293 else: # s.sy == '.'

294 s.next()

295 name = EncodedString( p_ident(s) )

296 return ExprNodes.AttributeNode(pos,

297 obj = node1, attribute = name)

298

299 # arglist: argument (',' argument)* [',']

300 # argument: [test '='] test # Really [keyword '='] test

301

302 def p_call(s, function):

303 # s.sy == '('

304 pos = s.position()

305 s.next()

306 positional_args = []

307 keyword_args = []

308 star_arg = None

309 starstar_arg = None

310 while s.sy not in ('**', ')'):

311 if s.sy == '*':

312 if star_arg:

313 s.error("only one star-arg parameter allowed",

314 pos = s.position())

315 s.next()

316 star_arg = p_simple_expr(s)

317 else:

318 arg = p_simple_expr(s)

319 if s.sy == '=':

320 s.next()

321 if not arg.is_name:

322 s.error("Expected an identifier before '='",

323 pos = arg.pos)

324 encoded_name = EncodedString(arg.name)

325 keyword = ExprNodes.IdentifierStringNode(arg.pos,

326 value = encoded_name)

327 arg = p_simple_expr(s)

328 keyword_args.append((keyword, arg))

329 else:

330 if keyword_args:

331 s.error("Non-keyword arg following keyword arg",

332 pos = arg.pos)

333 if star_arg:

334 s.error("Non-keyword arg following star-arg",

335 pos = arg.pos)

336 positional_args.append(arg)

337 if s.sy != ',':

338 break

339 s.next()

340

341 if s.sy == '**':

342 s.next()

343 starstar_arg = p_simple_expr(s)

344 if s.sy == ',':

345 s.next()

346 s.expect(')')

347 if not (keyword_args or star_arg or starstar_arg):

348 return ExprNodes.SimpleCallNode(pos,

349 function = function,

350 args = positional_args)

351 else:

352 arg_tuple = None

353 keyword_dict = None

354 if positional_args or not star_arg:

355 arg_tuple = ExprNodes.TupleNode(pos,

356 args = positional_args)

357 if star_arg:

358 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

359 if arg_tuple:

360 arg_tuple = ExprNodes.binop_node(pos,

361 operator = '+', operand1 = arg_tuple,

362 operand2 = star_arg_tuple)

363 else:

364 arg_tuple = star_arg_tuple

365 if keyword_args:

366 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

367 for key, value in keyword_args]

368 keyword_dict = ExprNodes.DictNode(pos,

369 key_value_pairs = keyword_args)

370 return ExprNodes.GeneralCallNode(pos,

371 function = function,

372 positional_args = arg_tuple,

373 keyword_args = keyword_dict,

374 starstar_arg = starstar_arg)

375

376 #lambdef: 'lambda' [varargslist] ':' test

377

378 #subscriptlist: subscript (',' subscript)* [',']

379

380 def p_index(s, base):

381 # s.sy == '['

382 pos = s.position()

383 s.next()

384 subscripts = p_subscript_list(s)

385 if len(subscripts) == 1 and len(subscripts[0]) == 2:

386 start, stop = subscripts[0]

387 result = ExprNodes.SliceIndexNode(pos,

388 base = base, start = start, stop = stop)

389 else:

390 indexes = make_slice_nodes(pos, subscripts)

391 if len(indexes) == 1:

392 index = indexes[0]

393 else:

394 index = ExprNodes.TupleNode(pos, args = indexes)

395 result = ExprNodes.IndexNode(pos,

396 base = base, index = index)

397 s.expect(']')

398 return result

399

400 def p_subscript_list(s):

401 items = [p_subscript(s)]

402 while s.sy == ',':

403 s.next()

404 if s.sy == ']':

405 break

406 items.append(p_subscript(s))

407 return items

408

409 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

410

411 def p_subscript(s):

412 # Parse a subscript and return a list of

413 # 1, 2 or 3 ExprNodes, depending on how

414 # many slice elements were encountered.

415 pos = s.position()

416 if s.sy == '.':

417 expect_ellipsis(s)

418 return [ExprNodes.EllipsisNode(pos)]

419 else:

420 start = p_slice_element(s, (':',))

421 if s.sy != ':':

422 return [start]

423 s.next()

424 stop = p_slice_element(s, (':', ',', ']'))

425 if s.sy != ':':

426 return [start, stop]

427 s.next()

428 step = p_slice_element(s, (':', ',', ']'))

429 return [start, stop, step]

430

431 def p_slice_element(s, follow_set):

432 # Simple expression which may be missing iff

433 # it is followed by something in follow_set.

434 if s.sy not in follow_set:

435 return p_simple_expr(s)

436 else:

437 return None

438

439 def expect_ellipsis(s):

440 s.expect('.')

441 s.expect('.')

442 s.expect('.')

443

444 def make_slice_nodes(pos, subscripts):

445 # Convert a list of subscripts as returned

446 # by p_subscript_list into a list of ExprNodes,

447 # creating SliceNodes for elements with 2 or

448 # more components.

449 result = []

450 for subscript in subscripts:

451 if len(subscript) == 1:

452 result.append(subscript[0])

453 else:

454 result.append(make_slice_node(pos, *subscript))

455 return result

456

457 def make_slice_node(pos, start, stop = None, step = None):

458 if not start:

459 start = ExprNodes.NoneNode(pos)

460 if not stop:

461 stop = ExprNodes.NoneNode(pos)

462 if not step:

463 step = ExprNodes.NoneNode(pos)

464 return ExprNodes.SliceNode(pos,

465 start = start, stop = stop, step = step)

466

467 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

468

469 def p_atom(s):

470 pos = s.position()

471 sy = s.sy

472 if sy == '(':

473 s.next()

474 if s.sy == ')':

475 result = ExprNodes.TupleNode(pos, args = [])

476 else:

477 result = p_expr(s)

478 s.expect(')')

479 return result

480 elif sy == '[':

481 return p_list_maker(s)

482 elif sy == '{':

483 return p_dict_maker(s)

484 elif sy == '`':

485 return p_backquote_expr(s)

486 elif sy == 'INT':

487 value = s.systring

488 s.next()

489 unsigned = ""

490 longness = ""

491 while value[-1] in "UuLl":

492 if value[-1] in "Ll":

493 longness += "L"

494 else:

495 unsigned += "U"

496 value = value[:-1]

497 return ExprNodes.IntNode(pos,

498 value = value,

499 unsigned = unsigned,

500 longness = longness)

501 elif sy == 'FLOAT':

502 value = s.systring

503 s.next()

504 return ExprNodes.FloatNode(pos, value = value)

505 elif sy == 'IMAG':

506 value = s.systring[:-1]

507 s.next()

508 return ExprNodes.ImagNode(pos, value = value)

509 elif sy == 'BEGIN_STRING':

510 kind, value = p_cat_string_literal(s)

511 if kind == 'c':

512 return ExprNodes.CharNode(pos, value = value)

513 elif kind == 'u':

514 return ExprNodes.UnicodeNode(pos, value = value)

515 else:

516 return ExprNodes.StringNode(pos, value = value)

517 elif sy == 'IDENT':

518 name = EncodedString( s.systring )

519 s.next()

520 if name == "None":

521 return ExprNodes.NoneNode(pos)

522 elif name == "True":

523 return ExprNodes.BoolNode(pos, value=True)

524 elif name == "False":

525 return ExprNodes.BoolNode(pos, value=False)

526 elif name == "NULL":

527 return ExprNodes.NullNode(pos)

528 else:

529 return p_name(s, name)

530 else:

531 s.error("Expected an identifier or literal")

532

533 def p_name(s, name):

534 pos = s.position()

535 if not s.compile_time_expr and name in s.compile_time_env:

536 value = s.compile_time_env.lookup_here(name)

537 rep = repr(value)

538 if isinstance(value, bool):

539 return ExprNodes.BoolNode(pos, value = value)

540 elif isinstance(value, int):

541 return ExprNodes.IntNode(pos, value = rep)

542 elif isinstance(value, long):

543 return ExprNodes.IntNode(pos, value = rep, longness = "L")

544 elif isinstance(value, float):

545 return ExprNodes.FloatNode(pos, value = rep)

546 elif isinstance(value, (str, unicode)):

547 return ExprNodes.StringNode(pos, value = value)

548 else:

549 error(pos, "Invalid type for compile-time constant: %s"

550 % value.__class__.__name__)

551 return ExprNodes.NameNode(pos, name = name)

552

553 def p_cat_string_literal(s):

554 # A sequence of one or more adjacent string literals.

555 # Returns (kind, value) where kind in ('b', 'c', 'u')

556 kind, value = p_string_literal(s)

557 if kind != 'c':

558 strings = [value]

559 while s.sy == 'BEGIN_STRING':

560 next_kind, next_value = p_string_literal(s)

561 if next_kind == 'c':

562 error(s.position(),

563 "Cannot concatenate char literal with another string or char literal")

564 elif next_kind != kind:

565 # we have to switch to unicode now

566 if kind == 'b':

567 # concatenating a unicode string to byte strings

568 strings = [u''.join([s.decode(s.encoding) for s in strings])]

569 elif kind == 'u':

570 # concatenating a byte string to unicode strings

571 strings.append(next_value.decode(next_value.encoding))

572 kind = 'u'

573 else:

574 strings.append(next_value)

575 if kind == 'u':

576 value = EncodedString( u''.join(strings) )

577 else:

578 value = BytesLiteral( ''.join(strings) )

579 value.encoding = s.source_encoding

580 return kind, value

581

582 def p_opt_string_literal(s):

583 if s.sy == 'BEGIN_STRING':

584 return p_string_literal(s)

585 else:

586 return None

587

588 def p_string_literal(s):

589 # A single string or char literal.

590 # Returns (kind, value) where kind in ('b', 'c', 'u')

591 # s.sy == 'BEGIN_STRING'

592 pos = s.position()

593 is_raw = 0

594 kind = s.systring[:1].lower()

595 if kind == 'r':

596 kind = ''

597 is_raw = 1

598 elif kind in 'ub':

599 is_raw = s.systring[1:2].lower() == 'r'

600 elif kind != 'c':

601 kind = ''

602 if Future.unicode_literals in s.context.future_directives:

603 if kind == '':

604 kind = 'u'

605 elif kind == '':

606 kind = 'b'

607 if kind == 'u':

608 chars = StringEncoding.UnicodeLiteralBuilder()

609 else:

610 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

611 while 1:

612 s.next()

613 sy = s.sy

614 #print "p_string_literal: sy =", sy, repr(s.systring) ###

615 if sy == 'CHARS':

616 chars.append(s.systring)

617 elif sy == 'ESCAPE':

618 has_escape = True

619 systr = s.systring

620 if is_raw:

621 if systr == u'\\\n':

622 chars.append(u'\\\n')

623 elif systr == u'\\\"':

624 chars.append(u'"')

625 elif systr == u'\\\'':

626 chars.append(u"'")

627 else:

628 chars.append(systr)

629 else:

630 c = systr[1]

631 if c in u"01234567":

632 chars.append_charval( int(systr[1:], 8) )

633 elif c in u"'\"\\":

634 chars.append(c)

635 elif c in u"abfnrtv":

636 chars.append(

637 StringEncoding.char_from_escape_sequence(systr))

638 elif c == u'\n':

639 pass

640 elif c in u'Uux':

641 if kind == 'u' or c == 'x':

642 chrval = int(systr[2:], 16)

643 if chrval > 1114111: # sys.maxunicode:

644 s.error("Invalid unicode escape '%s'" % systr,

645 pos = pos)

646 elif chrval > 65535:

647 warning(s.position(),

648 "Unicode characters above 65535 are not "

649 "necessarily portable across Python installations", 1)

650 chars.append_charval(chrval)

651 else:

652 # unicode escapes in plain byte strings are not unescaped

653 chars.append(systr)

654 else:

655 chars.append(u'\\' + systr[1:])

656 elif sy == 'NEWLINE':

657 chars.append(u'\n')

658 elif sy == 'END_STRING':

659 break

660 elif sy == 'EOF':

661 s.error("Unclosed string literal", pos = pos)

662 else:

663 s.error(

664 "Unexpected token %r:%r in string literal" %

665 (sy, s.systring))

666 if kind == 'c':

667 value = chars.getchar()

668 if len(value) != 1:

669 error(pos, u"invalid character literal: %r" % value)

670 else:

671 value = chars.getstring()

672 s.next()

673 #print "p_string_literal: value =", repr(value) ###

674 return kind, value

675

676 # list_display ::= "[" [listmaker] "]"

677 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

678 # list_iter ::= list_for | list_if

679 # list_for ::= "for" expression_list "in" testlist [list_iter]

680 # list_if ::= "if" test [list_iter]

681

682 def p_list_maker(s):

683 # s.sy == '['

684 pos = s.position()

685 s.next()

686 if s.sy == ']':

687 s.expect(']')

688 return ExprNodes.ListNode(pos, args = [])

689 expr = p_simple_expr(s)

690 if s.sy == 'for':

691 loop = p_list_for(s)

692 s.expect(']')

693 inner_loop = loop

694 while not isinstance(inner_loop.body, Nodes.PassStatNode):

695 inner_loop = inner_loop.body

696 if isinstance(inner_loop, Nodes.IfStatNode):

697 inner_loop = inner_loop.if_clauses[0]

698 append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )

699 inner_loop.body = Nodes.ExprStatNode(pos, expr = append)

700 return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)

701 else:

702 exprs = [expr]

703 if s.sy == ',':

704 s.next()

705 exprs += p_simple_expr_list(s)

706 s.expect(']')

707 return ExprNodes.ListNode(pos, args = exprs)

708

709 def p_list_iter(s):

710 if s.sy == 'for':

711 return p_list_for(s)

712 elif s.sy == 'if':

713 return p_list_if(s)

714 else:

715 return Nodes.PassStatNode(s.position())

716

717 def p_list_for(s):

718 # s.sy == 'for'

719 pos = s.position()

720 s.next()

721 kw = p_for_bounds(s)

722 kw['else_clause'] = None

723 kw['body'] = p_list_iter(s)

724 return Nodes.ForStatNode(pos, **kw)

725

726 def p_list_if(s):

727 # s.sy == 'if'

728 pos = s.position()

729 s.next()

730 test = p_test(s)

731 return Nodes.IfStatNode(pos,

732 if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],

733 else_clause = None )

734

735 #dictmaker: test ':' test (',' test ':' test)* [',']

736

737 def p_dict_maker(s):

738 # s.sy == '{'

739 pos = s.position()

740 s.next()

741 items = []

742 while s.sy != '}':

743 items.append(p_dict_item(s))

744 if s.sy != ',':

745 break

746 s.next()

747 s.expect('}')

748 return ExprNodes.DictNode(pos, key_value_pairs = items)

749

750 def p_dict_item(s):

751 key = p_simple_expr(s)

752 s.expect(':')

753 value = p_simple_expr(s)

754 return ExprNodes.DictItemNode(key.pos, key=key, value=value)

755

756 def p_backquote_expr(s):

757 # s.sy == '`'

758 pos = s.position()

759 s.next()

760 arg = p_expr(s)

761 s.expect('`')

762 return ExprNodes.BackquoteNode(pos, arg = arg)

763

764 def p_simple_expr_list(s):

765 exprs = []

766 while s.sy not in expr_terminators:

767 exprs.append(p_simple_expr(s))

768 if s.sy != ',':

769 break

770 s.next()

771 return exprs

772

773 def p_expr(s):

774 pos = s.position()

775 expr = p_simple_expr(s)

776 if s.sy == ',':

777 s.next()

778 exprs = [expr] + p_simple_expr_list(s)

779 return ExprNodes.TupleNode(pos, args = exprs)

780 else:

781 return expr

782

783

784 #testlist: test (',' test)* [',']

785 # differs from p_expr only in the fact that it cannot contain conditional expressions

786

787 def p_testlist(s):

788 pos = s.position()

789 expr = p_test(s)

790 if s.sy == ',':

791 exprs = [expr]

792 while s.sy == ',':

793 s.next()

794 exprs.append(p_test(s))

795 return ExprNodes.TupleNode(pos, args = exprs)

796 else:

797 return expr

798

799 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

800

801 #-------------------------------------------------------

802 #

803 # Statements

804 #

805 #-------------------------------------------------------

806

807 def p_global_statement(s):

808 # assume s.sy == 'global'

809 pos = s.position()

810 s.next()

811 names = p_ident_list(s)

812 return Nodes.GlobalNode(pos, names = names)

813

814 def p_expression_or_assignment(s):

815 expr_list = [p_expr(s)]

816 while s.sy == '=':

817 s.next()

818 expr_list.append(p_expr(s))

819 if len(expr_list) == 1:

820 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

821 lhs = expr_list[0]

822 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

823 error(lhs.pos, "Illegal operand for inplace operation.")

824 operator = s.sy[:-1]

825 s.next()

826 rhs = p_expr(s)

827 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

828 expr = expr_list[0]

829 if isinstance(expr, ExprNodes.StringNode):

830 return Nodes.PassStatNode(expr.pos)

831 else:

832 return Nodes.ExprStatNode(expr.pos, expr = expr)

833 else:

834 expr_list_list = []

835 flatten_parallel_assignments(expr_list, expr_list_list)

836 nodes = []

837 for expr_list in expr_list_list:

838 lhs_list = expr_list[:-1]

839 rhs = expr_list[-1]

840 if len(lhs_list) == 1:

841 node = Nodes.SingleAssignmentNode(rhs.pos,

842 lhs = lhs_list[0], rhs = rhs)

843 else:

844 node = Nodes.CascadedAssignmentNode(rhs.pos,

845 lhs_list = lhs_list, rhs = rhs)

846 nodes.append(node)

847 if len(nodes) == 1:

848 return nodes[0]

849 else:

850 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

851

852 def flatten_parallel_assignments(input, output):

853 # The input is a list of expression nodes, representing

854 # the LHSs and RHS of one (possibly cascaded) assignment

855 # statement. If they are all sequence constructors with

856 # the same number of arguments, rearranges them into a

857 # list of equivalent assignments between the individual

858 # elements. This transformation is applied recursively.

859 size = find_parallel_assignment_size(input)

860 if size >= 0:

861 for i in range(size):

862 new_exprs = [expr.args[i] for expr in input]

863 flatten_parallel_assignments(new_exprs, output)

864 else:

865 output.append(input)

866

867 def find_parallel_assignment_size(input):

868 # The input is a list of expression nodes. If

869 # they are all sequence constructors with the same number

870 # of arguments, return that number, else return -1.

871 # Produces an error message if they are all sequence

872 # constructors but not all the same size.

873 for expr in input:

874 if not expr.is_sequence_constructor:

875 return -1

876 rhs = input[-1]

877 rhs_size = len(rhs.args)

878 for lhs in input[:-1]:

879 lhs_size = len(lhs.args)

880 if lhs_size != rhs_size:

881 error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"

882 % (lhs_size, rhs_size))

883 return -1

884 return rhs_size

885

886 def p_print_statement(s):

887 # s.sy == 'print'

888 pos = s.position()

889 s.next()

890 if s.sy == '>>':

891 s.error("'print >>' not yet implemented")

892 args = []

893 ends_with_comma = 0

894 if s.sy not in ('NEWLINE', 'EOF'):

895 args.append(p_simple_expr(s))

896 while s.sy == ',':

897 s.next()

898 if s.sy in ('NEWLINE', 'EOF'):

899 ends_with_comma = 1

900 break

901 args.append(p_simple_expr(s))

902 arg_tuple = ExprNodes.TupleNode(pos, args = args)

903 return Nodes.PrintStatNode(pos,

904 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

905

906 def p_exec_statement(s):

907 # s.sy == 'exec'

908 pos = s.position()

909 s.next()

910 args = [ p_bit_expr(s) ]

911 if s.sy == 'in':

912 s.next()

913 args.append(p_simple_expr(s))

914 if s.sy == ',':

915 s.next()

916 args.append(p_simple_expr(s))

917 else:

918 error(pos, "'exec' currently requires a target mapping (globals/locals)")

919 return Nodes.ExecStatNode(pos, args = args)

920

921 def p_del_statement(s):

922 # s.sy == 'del'

923 pos = s.position()

924 s.next()

925 args = p_simple_expr_list(s)

926 return Nodes.DelStatNode(pos, args = args)

927

928 def p_pass_statement(s, with_newline = 0):

929 pos = s.position()

930 s.expect('pass')

931 if with_newline:

932 s.expect_newline("Expected a newline")

933 return Nodes.PassStatNode(pos)

934

935 def p_break_statement(s):

936 # s.sy == 'break'

937 pos = s.position()

938 s.next()

939 return Nodes.BreakStatNode(pos)

940

941 def p_continue_statement(s):

942 # s.sy == 'continue'

943 pos = s.position()

944 s.next()

945 return Nodes.ContinueStatNode(pos)

946

947 def p_return_statement(s):

948 # s.sy == 'return'

949 pos = s.position()

950 s.next()

951 if s.sy not in statement_terminators:

952 value = p_expr(s)

953 else:

954 value = None

955 return Nodes.ReturnStatNode(pos, value = value)

956

957 def p_raise_statement(s):

958 # s.sy == 'raise'

959 pos = s.position()

960 s.next()

961 exc_type = None

962 exc_value = None

963 exc_tb = None

964 if s.sy not in statement_terminators:

965 exc_type = p_simple_expr(s)

966 if s.sy == ',':

967 s.next()

968 exc_value = p_simple_expr(s)

969 if s.sy == ',':

970 s.next()

971 exc_tb = p_simple_expr(s)

972 if exc_type or exc_value or exc_tb:

973 return Nodes.RaiseStatNode(pos,

974 exc_type = exc_type,

975 exc_value = exc_value,

976 exc_tb = exc_tb)

977 else:

978 return Nodes.ReraiseStatNode(pos)

979

980 def p_import_statement(s):

981 # s.sy in ('import', 'cimport')

982 pos = s.position()

983 kind = s.sy

984 s.next()

985 items = [p_dotted_name(s, as_allowed = 1)]

986 while s.sy == ',':

987 s.next()

988 items.append(p_dotted_name(s, as_allowed = 1))

989 stats = []

990 for pos, target_name, dotted_name, as_name in items:

991 dotted_name = EncodedString(dotted_name)

992 if kind == 'cimport':

993 stat = Nodes.CImportStatNode(pos,

994 module_name = dotted_name,

995 as_name = as_name)

996 else:

997 if as_name and "." in dotted_name:

998 name_list = ExprNodes.ListNode(pos, args = [

999 ExprNodes.IdentifierStringNode(

1000 pos, value = EncodedString("*"))])

1001 else:

1002 name_list = None

1003 stat = Nodes.SingleAssignmentNode(pos,

1004 lhs = ExprNodes.NameNode(pos,

1005 name = as_name or target_name),

1006 rhs = ExprNodes.ImportNode(pos,

1007 module_name = ExprNodes.IdentifierStringNode(

1008 pos, value = dotted_name),

1009 name_list = name_list))

1010 stats.append(stat)

1011 return Nodes.StatListNode(pos, stats = stats)

1012

1013 def p_from_import_statement(s, first_statement = 0):

1014 # s.sy == 'from'

1015 pos = s.position()

1016 s.next()

1017 (dotted_name_pos, _, dotted_name, _) = \

1018 p_dotted_name(s, as_allowed = 0)

1019 if s.sy in ('import', 'cimport'):

1020 kind = s.sy

1021 s.next()

1022 else:

1023 s.error("Expected 'import' or 'cimport'")

1024 is_cimport = kind == 'cimport'

1025 if s.sy == '*':

1026 imported_names = [(s.position(), "*", None, None)]

1027 s.next()

1028 else:

1029 imported_names = [p_imported_name(s, is_cimport)]

1030 while s.sy == ',':

1031 s.next()

1032 imported_names.append(p_imported_name(s, is_cimport))

1033 dotted_name = EncodedString(dotted_name)

1034 if dotted_name == '__future__':

1035 if not first_statement:

1036 s.error("from __future__ imports must occur at the beginning of the file")

1037 else:

1038 for (name_pos, name, as_name, kind) in imported_names:

1039 if name == "braces":

1040 s.error("not a chance", name_pos)

1041 break

1042 try:

1043 directive = getattr(Future, name)

1044 except AttributeError:

1045 s.error("future feature %s is not defined" % name, name_pos)

1046 break

1047 s.context.future_directives.add(directive)

1048 return Nodes.PassStatNode(pos)

1049 elif kind == 'cimport':

1050 return Nodes.FromCImportStatNode(pos,

1051 module_name = dotted_name,

1052 imported_names = imported_names)

1053 else:

1054 imported_name_strings = []

1055 items = []

1056 for (name_pos, name, as_name, kind) in imported_names:

1057 encoded_name = EncodedString(name)

1058 imported_name_strings.append(

1059 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1060 items.append(

1061 (name,

1062 ExprNodes.NameNode(name_pos,

1063 name = as_name or name)))

1064 import_list = ExprNodes.ListNode(

1065 imported_names[0][0], args = imported_name_strings)

1066 dotted_name = EncodedString(dotted_name)

1067 return Nodes.FromImportStatNode(pos,

1068 module = ExprNodes.ImportNode(dotted_name_pos,

1069 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1070 name_list = import_list),

1071 items = items)

1072

1073 imported_name_kinds = ('class', 'struct', 'union')

1074

1075 def p_imported_name(s, is_cimport):

1076 pos = s.position()

1077 kind = None

1078 if is_cimport and s.systring in imported_name_kinds:

1079 kind = s.systring

1080 s.next()

1081 name = p_ident(s)

1082 as_name = p_as_name(s)

1083 return (pos, name, as_name, kind)

1084

1085 def p_dotted_name(s, as_allowed):

1086 pos = s.position()

1087 target_name = p_ident(s)

1088 as_name = None

1089 names = [target_name]

1090 while s.sy == '.':

1091 s.next()

1092 names.append(p_ident(s))

1093 if as_allowed:

1094 as_name = p_as_name(s)

1095 return (pos, target_name, u'.'.join(names), as_name)

1096

1097 def p_as_name(s):

1098 if s.sy == 'IDENT' and s.systring == 'as':

1099 s.next()

1100 return p_ident(s)

1101 else:

1102 return None

1103

1104 def p_assert_statement(s):

1105 # s.sy == 'assert'

1106 pos = s.position()

1107 s.next()

1108 cond = p_simple_expr(s)

1109 if s.sy == ',':

1110 s.next()

1111 value = p_simple_expr(s)

1112 else:

1113 value = None

1114 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1115

1116 statement_terminators = (';', 'NEWLINE', 'EOF')

1117

1118 def p_if_statement(s):

1119 # s.sy == 'if'

1120 pos = s.position()

1121 s.next()

1122 if_clauses = [p_if_clause(s)]

1123 while s.sy == 'elif':

1124 s.next()

1125 if_clauses.append(p_if_clause(s))

1126 else_clause = p_else_clause(s)

1127 return Nodes.IfStatNode(pos,

1128 if_clauses = if_clauses, else_clause = else_clause)

1129

1130 def p_if_clause(s):

1131 pos = s.position()

1132 test = p_simple_expr(s)

1133 body = p_suite(s)

1134 return Nodes.IfClauseNode(pos,

1135 condition = test, body = body)

1136

1137 def p_else_clause(s):

1138 if s.sy == 'else':

1139 s.next()

1140 return p_suite(s)

1141 else:

1142 return None

1143

1144 def p_while_statement(s):

1145 # s.sy == 'while'

1146 pos = s.position()

1147 s.next()

1148 test = p_simple_expr(s)

1149 body = p_suite(s)

1150 else_clause = p_else_clause(s)

1151 return Nodes.WhileStatNode(pos,

1152 condition = test, body = body,

1153 else_clause = else_clause)

1154

1155 def p_for_statement(s):

1156 # s.sy == 'for'

1157 pos = s.position()

1158 s.next()

1159 kw = p_for_bounds(s)

1160 kw['body'] = p_suite(s)

1161 kw['else_clause'] = p_else_clause(s)

1162 return Nodes.ForStatNode(pos, **kw)

1163

1164 def p_for_bounds(s):

1165 target = p_for_target(s)

1166 if s.sy == 'in':

1167 s.next()

1168 iterator = p_for_iterator(s)

1169 return { 'target': target, 'iterator': iterator }

1170 else:

1171 if s.sy == 'from':

1172 s.next()

1173 bound1 = p_bit_expr(s)

1174 else:

1175 # Support shorter "for a <= x < b" syntax

1176 bound1, target = target, None

1177 rel1 = p_for_from_relation(s)

1178 name2_pos = s.position()

1179 name2 = p_ident(s)

1180 rel2_pos = s.position()

1181 rel2 = p_for_from_relation(s)

1182 bound2 = p_bit_expr(s)

1183 step = p_for_from_step(s)

1184 if target is None:

1185 target = ExprNodes.NameNode(name2_pos, name = name2)

1186 else:

1187 if not target.is_name:

1188 error(target.pos,

1189 "Target of for-from statement must be a variable name")

1190 elif name2 != target.name:

1191 error(name2_pos,

1192 "Variable name in for-from range does not match target")

1193 if rel1[0] != rel2[0]:

1194 error(rel2_pos,

1195 "Relation directions in for-from do not match")

1196 return {'target': target,

1197 'bound1': bound1,

1198 'relation1': rel1,

1199 'relation2': rel2,

1200 'bound2': bound2,

1201 'step': step }

1202

1203 def p_for_from_relation(s):

1204 if s.sy in inequality_relations:

1205 op = s.sy

1206 s.next()

1207 return op

1208 else:

1209 s.error("Expected one of '<', '<=', '>' '>='")

1210

1211 def p_for_from_step(s):

1212 if s.sy == 'by':

1213 s.next()

1214 step = p_bit_expr(s)

1215 return step

1216 else:

1217 return None

1218

1219 inequality_relations = ('<', '<=', '>', '>=')

1220

1221 def p_target(s, terminator):

1222 pos = s.position()

1223 expr = p_bit_expr(s)

1224 if s.sy == ',':

1225 s.next()

1226 exprs = [expr]

1227 while s.sy != terminator:

1228 exprs.append(p_bit_expr(s))

1229 if s.sy != ',':

1230 break

1231 s.next()

1232 return ExprNodes.TupleNode(pos, args = exprs)

1233 else:

1234 return expr

1235

1236 def p_for_target(s):

1237 return p_target(s, 'in')

1238

1239 def p_for_iterator(s):

1240 pos = s.position()

1241 expr = p_testlist(s)

1242 return ExprNodes.IteratorNode(pos, sequence = expr)

1243

1244 def p_try_statement(s):

1245 # s.sy == 'try'

1246 pos = s.position()

1247 s.next()

1248 body = p_suite(s)

1249 except_clauses = []

1250 else_clause = None

1251 if s.sy in ('except', 'else'):

1252 while s.sy == 'except':

1253 except_clauses.append(p_except_clause(s))

1254 if s.sy == 'else':

1255 s.next()

1256 else_clause = p_suite(s)

1257 body = Nodes.TryExceptStatNode(pos,

1258 body = body, except_clauses = except_clauses,

1259 else_clause = else_clause)

1260 if s.sy != 'finally':

1261 return body

1262 # try-except-finally is equivalent to nested try-except/try-finally

1263 if s.sy == 'finally':

1264 s.next()

1265 finally_clause = p_suite(s)

1266 return Nodes.TryFinallyStatNode(pos,

1267 body = body, finally_clause = finally_clause)

1268 else:

1269 s.error("Expected 'except' or 'finally'")

1270

1271 def p_except_clause(s):

1272 # s.sy == 'except'

1273 pos = s.position()

1274 s.next()

1275 exc_type = None

1276 exc_value = None

1277 if s.sy != ':':

1278 exc_type = p_simple_expr(s)

1279 if s.sy == ',':

1280 s.next()

1281 exc_value = p_simple_expr(s)

1282 body = p_suite(s)

1283 return Nodes.ExceptClauseNode(pos,

1284 pattern = exc_type, target = exc_value, body = body)

1285

1286 def p_include_statement(s, ctx):

1287 pos = s.position()

1288 s.next() # 'include'

1289 _, include_file_name = p_string_literal(s)

1290 s.expect_newline("Syntax error in include statement")

1291 if s.compile_time_eval:

1292 include_file_path = s.context.find_include_file(include_file_name, pos)

1293 if include_file_path:

1294 s.included_files.append(include_file_name)

1295 f = Utils.open_source_file(include_file_path, mode="rU")

1296 source_desc = FileSourceDescriptor(include_file_path)

1297 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1298 try:

1299 tree = p_statement_list(s2, ctx)

1300 finally:

1301 f.close()

1302 return tree

1303 else:

1304 return None

1305 else:

1306 return Nodes.PassStatNode(pos)

1307

1308 def p_with_statement(s):

1309 pos = s.position()

1310 s.next() # 'with'

1311 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1312 if s.sy == 'IDENT' and s.systring == 'nogil':

1313 state = s.systring

1314 s.next()

1315 body = p_suite(s)

1316 return Nodes.GILStatNode(pos, state = state, body = body)

1317 else:

1318 manager = p_expr(s)

1319 target = None

1320 if s.sy == 'IDENT' and s.systring == 'as':

1321 s.next()

1322 allow_multi = (s.sy == '(')

1323 target = p_target(s, ':')

1324 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1325 s.error("Multiple with statement target values not allowed without paranthesis")

1326 body = p_suite(s)

1327 return Nodes.WithStatNode(pos, manager = manager,

1328 target = target, body = body)

1329

1330 def p_simple_statement(s, first_statement = 0):

1331 #print "p_simple_statement:", s.sy, s.systring ###

1332 if s.sy == 'global':

1333 node = p_global_statement(s)

1334 elif s.sy == 'print':

1335 node = p_print_statement(s)

1336 elif s.sy == 'exec':

1337 node = p_exec_statement(s)

1338 elif s.sy == 'del':

1339 node = p_del_statement(s)

1340 elif s.sy == 'break':

1341 node = p_break_statement(s)

1342 elif s.sy == 'continue':

1343 node = p_continue_statement(s)

1344 elif s.sy == 'return':

1345 node = p_return_statement(s)

1346 elif s.sy == 'raise':

1347 node = p_raise_statement(s)

1348 elif s.sy in ('import', 'cimport'):

1349 node = p_import_statement(s)

1350 elif s.sy == 'from':

1351 node = p_from_import_statement(s, first_statement = first_statement)

1352 elif s.sy == 'assert':

1353 node = p_assert_statement(s)

1354 elif s.sy == 'pass':

1355 node = p_pass_statement(s)

1356 else:

1357 node = p_expression_or_assignment(s)

1358 return node

1359

1360 def p_simple_statement_list(s, ctx, first_statement = 0):

1361 # Parse a series of simple statements on one line

1362 # separated by semicolons.

1363 stat = p_simple_statement(s, first_statement = first_statement)

1364 if s.sy == ';':

1365 stats = [stat]

1366 while s.sy == ';':

1367 #print "p_simple_statement_list: maybe more to follow" ###

1368 s.next()

1369 if s.sy in ('NEWLINE', 'EOF'):

1370 break

1371 stats.append(p_simple_statement(s))

1372 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1373 s.expect_newline("Syntax error in simple statement list")

1374 return stat

1375

1376 def p_compile_time_expr(s):

1377 old = s.compile_time_expr

1378 s.compile_time_expr = 1

1379 expr = p_expr(s)

1380 s.compile_time_expr = old

1381 return expr

1382

1383 def p_DEF_statement(s):

1384 pos = s.position()

1385 denv = s.compile_time_env

1386 s.next() # 'DEF'

1387 name = p_ident(s)

1388 s.expect('=')

1389 expr = p_compile_time_expr(s)

1390 value = expr.compile_time_value(denv)

1391 #print "p_DEF_statement: %s = %r" % (name, value) ###

1392 denv.declare(name, value)

1393 s.expect_newline()

1394 return Nodes.PassStatNode(pos)

1395

1396 def p_IF_statement(s, ctx):

1397 pos = s.position()

1398 saved_eval = s.compile_time_eval

1399 current_eval = saved_eval

1400 denv = s.compile_time_env

1401 result = None

1402 while 1:

1403 s.next() # 'IF' or 'ELIF'

1404 expr = p_compile_time_expr(s)

1405 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1406 body = p_suite(s, ctx)

1407 if s.compile_time_eval:

1408 result = body

1409 current_eval = 0

1410 if s.sy != 'ELIF':

1411 break

1412 if s.sy == 'ELSE':

1413 s.next()

1414 s.compile_time_eval = current_eval

1415 body = p_suite(s, ctx)

1416 if current_eval:

1417 result = body

1418 if not result:

1419 result = Nodes.PassStatNode(pos)

1420 s.compile_time_eval = saved_eval

1421 return result

1422

1423 def p_statement(s, ctx, first_statement = 0):

1424 cdef_flag = ctx.cdef_flag

1425 if s.sy == 'ctypedef':

1426 if ctx.level not in ('module', 'module_pxd'):

1427 s.error("ctypedef statement not allowed here")

1428 #if ctx.api:

1429 # error(s.position(), "'api' not allowed with 'ctypedef'")

1430 return p_ctypedef_statement(s, ctx)

1431 elif s.sy == 'DEF':

1432 return p_DEF_statement(s)

1433 elif s.sy == 'IF':

1434 return p_IF_statement(s, ctx)

1435 elif s.sy == 'DECORATOR':

1436 if ctx.level not in ('module', 'class', 'c_class', 'property'):

1437 s.error('decorator not allowed here')

1438 s.level = ctx.level

1439 decorators = p_decorators(s)

1440 if s.sy != 'def':

1441 s.error("Decorators can only be followed by functions ")

1442 return p_def_statement(s, decorators)

1443 else:

1444 overridable = 0

1445 if s.sy == 'cdef':

1446 cdef_flag = 1

1447 s.next()

1448 if s.sy == 'cpdef':

1449 cdef_flag = 1

1450 overridable = 1

1451 s.next()

1452 if cdef_flag:

1453 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1454 s.error('cdef statement not allowed here')

1455 s.level = ctx.level

1456 return p_cdef_statement(s, ctx(overridable = overridable))

1457 else:

1458 if ctx.api:

1459 error(s.pos, "'api' not allowed with this statement")

1460 elif s.sy == 'def':

1461 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1462 s.error('def statement not allowed here')

1463 s.level = ctx.level

1464 return p_def_statement(s)

1465 elif s.sy == 'class':

1466 if ctx.level != 'module':

1467 s.error("class definition not allowed here")

1468 return p_class_statement(s)

1469 elif s.sy == 'include':

1470 if ctx.level not in ('module', 'module_pxd'):

1471 s.error("include statement not allowed here")

1472 return p_include_statement(s, ctx)

1473 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1474 return p_property_decl(s)

1475 elif s.sy == 'pass' and ctx.level != 'property':

1476 return p_pass_statement(s, with_newline = 1)

1477 else:

1478 if ctx.level in ('c_class_pxd', 'property'):

1479 s.error("Executable statement not allowed here")

1480 if s.sy == 'if':

1481 return p_if_statement(s)

1482 elif s.sy == 'while':

1483 return p_while_statement(s)

1484 elif s.sy == 'for':

1485 return p_for_statement(s)

1486 elif s.sy == 'try':

1487 return p_try_statement(s)

1488 elif s.sy == 'with':

1489 return p_with_statement(s)

1490 else:

1491 return p_simple_statement_list(

1492 s, ctx, first_statement = first_statement)

1493

1494 def p_statement_list(s, ctx, first_statement = 0):

1495 # Parse a series of statements separated by newlines.

1496 pos = s.position()

1497 stats = []

1498 while s.sy not in ('DEDENT', 'EOF'):

1499 stats.append(p_statement(s, ctx, first_statement = first_statement))

1500 first_statement = 0

1501 if len(stats) == 1:

1502 return stats[0]

1503 else:

1504 return Nodes.StatListNode(pos, stats = stats)

1505

1506 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1507 pos = s.position()

1508 s.expect(':')

1509 doc = None

1510 stmts = []

1511 if s.sy == 'NEWLINE':

1512 s.next()

1513 s.expect_indent()

1514 if with_doc or with_pseudo_doc:

1515 doc = p_doc_string(s)

1516 body = p_statement_list(s, ctx)

1517 s.expect_dedent()

1518 else:

1519 if ctx.api:

1520 error(s.pos, "'api' not allowed with this statement")

1521 if ctx.level in ('module', 'class', 'function', 'other'):

1522 body = p_simple_statement_list(s, ctx)

1523 else:

1524 body = p_pass_statement(s)

1525 s.expect_newline("Syntax error in declarations")

1526 if with_doc:

1527 return doc, body

1528 else:

1529 return body

1530

1531 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1532 """

1533 Parses positional and keyword arguments. end_sy_set

1534 should contain any s.sy that terminate the argument list.

1535 Argument expansion (* and **) are not allowed.

1536

1537 type_positions and type_keywords specifies which argument

1538 positions and/or names which should be interpreted as

1539 types. Other arguments will be treated as expressions.

1540

1541 Returns: (positional_args, keyword_args)

1542 """

1543 positional_args = []

1544 keyword_args = []

1545 pos_idx = 0

1546

1547 while s.sy not in end_sy_set:

1548 if s.sy == '*' or s.sy == '**':

1549 s.error('Argument expansion not allowed here.')

1550

1551 was_keyword = False

1552 parsed_type = False

1553 if s.sy == 'IDENT':

1554 # Since we can have either types or expressions as positional args,

1555 # we use a strategy of looking an extra step forward for a '=' and

1556 # if it is a positional arg we backtrack.

1557 ident = s.systring

1558 s.next()

1559 if s.sy == '=':

1560 s.next()

1561 # Is keyword arg

1562 if ident in type_keywords:

1563 arg = p_c_base_type(s)

1564 parsed_type = True

1565 else:

1566 arg = p_simple_expr(s)

1567 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,

1568 value = EncodedString(ident))

1569 keyword_args.append((keyword_node, arg))

1570 was_keyword = True

1571 else:

1572 s.put_back('IDENT', ident)

1573

1574 if not was_keyword:

1575 if pos_idx in type_positions:

1576 arg = p_c_base_type(s)

1577 parsed_type = True

1578 else:

1579 arg = p_simple_expr(s)

1580 positional_args.append(arg)

1581 pos_idx += 1

1582 if len(keyword_args) > 0:

1583 s.error("Non-keyword arg following keyword arg",

1584 pos = arg.pos)

1585

1586 if s.sy != ',':

1587 if s.sy not in end_sy_set:

1588 if parsed_type:

1589 s.error("Expected: type")

1590 else:

1591 s.error("Expected: expression")

1592 break

1593 s.next()

1594 return positional_args, keyword_args

1595

1596 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1597 # If self_flag is true, this is the base type for the

1598 # self argument of a C method of an extension type.

1599 if s.sy == '(':

1600 return p_c_complex_base_type(s)

1601 else:

1602 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1603

1604 def p_calling_convention(s):

1605 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1606 result = s.systring

1607 s.next()

1608 return result

1609 else:

1610 return ""

1611

1612 calling_convention_words = ("__stdcall", "__cdecl")

1613

1614 def p_c_complex_base_type(s):

1615 # s.sy == '('

1616 pos = s.position()

1617 s.next()

1618 base_type = p_c_base_type(s)

1619 declarator = p_c_declarator(s, empty = 1)

1620 s.expect(')')

1621 return Nodes.CComplexBaseTypeNode(pos,

1622 base_type = base_type, declarator = declarator)

1623

1624 def p_c_simple_base_type(s, self_flag, nonempty):

1625 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1626 is_basic = 0

1627 signed = 1

1628 longness = 0

1629 module_path = []

1630 pos = s.position()

1631 if looking_at_base_type(s):

1632 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1633 is_basic = 1

1634 signed, longness = p_sign_and_longness(s)

1635 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1636 name = s.systring

1637 s.next()

1638 else:

1639 name = 'int'

1640 elif looking_at_dotted_name(s):

1641 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1642 name = s.systring

1643 s.next()

1644 while s.sy == '.':

1645 module_path.append(name)

1646 s.next()

1647 name = p_ident(s)

1648 else:

1649 name = s.systring

1650 s.next()

1651 if nonempty and s.sy != 'IDENT':

1652 # Make sure this is not a declaration of a variable or function.

1653 if s.sy == '(':

1654 s.next()

1655 if s.sy == '*' or s.sy == '**':

1656 s.put_back('(', '(')

1657 else:

1658 s.put_back('(', '(')

1659 s.put_back('IDENT', name)

1660 name = None

1661 elif s.sy not in ('*', '**', '['):

1662 s.put_back('IDENT', name)

1663 name = None

1664

1665 type_node = Nodes.CSimpleBaseTypeNode(pos,

1666 name = name, module_path = module_path,

1667 is_basic_c_type = is_basic, signed = signed,

1668 longness = longness, is_self_arg = self_flag)

1669

1670

1671 # Treat trailing [] on type as buffer access if it appears in a context

1672 # where declarator names are required (so that it cannot mean int[] or

1673 # sizeof(int[SIZE]))...

1674 #

1675 # (This means that buffers cannot occur where there can be empty declarators,

1676 # which is an ok restriction to make.)

1677 if nonempty and s.sy == '[':

1678 return p_buffer_access(s, type_node)

1679 else:

1680 return type_node

1681

1682 def p_buffer_access(s, base_type_node):

1683 # s.sy == '['

1684 pos = s.position()

1685 s.next()

1686 positional_args, keyword_args = (

1687 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1688 )

1689 s.expect(']')

1690

1691 keyword_dict = ExprNodes.DictNode(pos,

1692 key_value_pairs = [

1693 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1694 for key, value in keyword_args

1695 ])

1696

1697 result = Nodes.CBufferAccessTypeNode(pos,

1698 positional_args = positional_args,

1699 keyword_args = keyword_dict,

1700 base_type_node = base_type_node)

1701 return result

1702

1703

1704 def looking_at_name(s):

1705 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1706

1707 def looking_at_expr(s):

1708 if s.systring in base_type_start_words:

1709 return False

1710 elif s.sy == 'IDENT':

1711 is_type = False

1712 name = s.systring

1713 dotted_path = []

1714 s.next()

1715 while s.sy == '.':

1716 s.next()

1717 dotted_path.append(s.systring)

1718 s.expect('IDENT')

1719 saved = s.sy, s.systring

1720 if s.sy == 'IDENT':

1721 is_type = True

1722 elif s.sy == '*' or s.sy == '**':

1723 s.next()

1724 is_type = s.sy == ')'

1725 s.put_back(*saved)

1726 elif s.sy == '(':

1727 s.next()

1728 is_type = s.sy == '*'

1729 s.put_back(*saved)

1730 elif s.sy == '[':

1731 s.next()

1732 is_type = s.sy == ']'

1733 s.put_back(*saved)

1734 dotted_path.reverse()

1735 for p in dotted_path:

1736 s.put_back('IDENT', p)

1737 s.put_back('.', '.')

1738 s.put_back('IDENT', name)

1739 return not is_type

1740 else:

1741 return True

1742

1743 def looking_at_base_type(s):

1744 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1745 return s.sy == 'IDENT' and s.systring in base_type_start_words

1746

1747 def looking_at_dotted_name(s):

1748 if s.sy == 'IDENT':

1749 name = s.systring

1750 s.next()

1751 result = s.sy == '.'

1752 s.put_back('IDENT', name)

1753 return result

1754 else:

1755 return 0

1756

1757 basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")

1758

1759 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1760

1761 base_type_start_words = \

1762 basic_c_type_names + sign_and_longness_words

1763

1764 def p_sign_and_longness(s):

1765 signed = 1

1766 longness = 0

1767 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1768 if s.systring == 'unsigned':

1769 signed = 0

1770 elif s.systring == 'signed':

1771 signed = 2

1772 elif s.systring == 'short':

1773 longness = -1

1774 elif s.systring == 'long':

1775 longness += 1

1776 s.next()

1777 return signed, longness

1778

1779 def p_opt_cname(s):

1780 literal = p_opt_string_literal(s)

1781 if literal:

1782 _, cname = literal

1783 else:

1784 cname = None

1785 return cname

1786

1787 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1788 assignable = 0, nonempty = 0,

1789 calling_convention_allowed = 0):

1790 # If empty is true, the declarator must be empty. If nonempty is true,

1791 # the declarator must be nonempty. Otherwise we don't care.

1792 # If cmethod_flag is true, then if this declarator declares

1793 # a function, it's a C method of an extension type.

1794 pos = s.position()

1795 if s.sy == '(':

1796 s.next()

1797 if s.sy == ')' or looking_at_name(s):

1798 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1799 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1800 else:

1801 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1802 cmethod_flag = cmethod_flag,

1803 nonempty = nonempty,

1804 calling_convention_allowed = 1)

1805 s.expect(')')

1806 else:

1807 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1808 assignable, nonempty)

1809 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1810 error(s.position(), "%s on something that is not a function"

1811 % result.calling_convention)

1812 while s.sy in ('[', '('):

1813 pos = s.position()

1814 if s.sy == '[':

1815 result = p_c_array_declarator(s, result)

1816 else: # sy == '('

1817 s.next()

1818 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1819 cmethod_flag = 0

1820 return result

1821

1822 def p_c_array_declarator(s, base):

1823 pos = s.position()

1824 s.next() # '['

1825 if s.sy != ']':

1826 dim = p_expr(s)

1827 else:

1828 dim = None

1829 s.expect(']')

1830 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1831

1832 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

1833 # Opening paren has already been skipped

1834 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

1835 nonempty_declarators = 0)

1836 ellipsis = p_optional_ellipsis(s)

1837 s.expect(')')

1838 nogil = p_nogil(s)

1839 exc_val, exc_check = p_exception_value_clause(s)

1840 with_gil = p_with_gil(s)

1841 return Nodes.CFuncDeclaratorNode(pos,

1842 base = base, args = args, has_varargs = ellipsis,

1843 exception_value = exc_val, exception_check = exc_check,

1844 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

1845

1846 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1847 assignable, nonempty):

1848 pos = s.position()

1849 calling_convention = p_calling_convention(s)

1850 if s.sy == '*':

1851 s.next()

1852 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1853 cmethod_flag = cmethod_flag,

1854 assignable = assignable, nonempty = nonempty)

1855 result = Nodes.CPtrDeclaratorNode(pos,

1856 base = base)

1857 elif s.sy == '**': # scanner returns this as a single token

1858 s.next()

1859 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1860 cmethod_flag = cmethod_flag,

1861 assignable = assignable, nonempty = nonempty)

1862 result = Nodes.CPtrDeclaratorNode(pos,

1863 base = Nodes.CPtrDeclaratorNode(pos,

1864 base = base))

1865 else:

1866 rhs = None

1867 if s.sy == 'IDENT':

1868 name = EncodedString(s.systring)

1869 if empty:

1870 error(s.position(), "Declarator should be empty")

1871 s.next()

1872 cname = p_opt_cname(s)

1873 if s.sy == '=' and assignable:

1874 s.next()

1875 rhs = p_simple_expr(s)

1876 else:

1877 if nonempty:

1878 error(s.position(), "Empty declarator")

1879 name = ""

1880 cname = None

1881 result = Nodes.CNameDeclaratorNode(pos,

1882 name = name, cname = cname, default = rhs)

1883 result.calling_convention = calling_convention

1884 return result

1885

1886 def p_nogil(s):

1887 if s.sy == 'IDENT' and s.systring == 'nogil':

1888 s.next()

1889 return 1

1890 else:

1891 return 0

1892

1893 def p_with_gil(s):

1894 if s.sy == 'with':

1895 s.next()

1896 s.expect_keyword('gil')

1897 return 1

1898 else:

1899 return 0

1900

1901 def p_exception_value_clause(s):

1902 exc_val = None

1903 exc_check = 0

1904 if s.sy == 'except':

1905 s.next()

1906 if s.sy == '*':

1907 exc_check = 1

1908 s.next()

1909 elif s.sy == '+':

1910 exc_check = '+'

1911 s.next()

1912 if s.sy == 'IDENT':

1913 name = s.systring

1914 s.next()

1915 exc_val = p_name(s, name)

1916 else:

1917 if s.sy == '?':

1918 exc_check = 1

1919 s.next()

1920 exc_val = p_simple_expr(s)

1921 return exc_val, exc_check

1922

1923 c_arg_list_terminators = ('*', '**', '.', ')')

1924

1925 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

1926 nonempty_declarators = 0, kw_only = 0):

1927 # Comma-separated list of C argument declarations, possibly empty.

1928 # May have a trailing comma.

1929 args = []

1930 is_self_arg = cmethod_flag

1931 while s.sy not in c_arg_list_terminators:

1932 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

1933 nonempty = nonempty_declarators, kw_only = kw_only))

1934 if s.sy != ',':

1935 break

1936 s.next()

1937 is_self_arg = 0

1938 return args

1939

1940 def p_optional_ellipsis(s):

1941 if s.sy == '.':

1942 expect_ellipsis(s)

1943 return 1

1944 else:

1945 return 0

1946

1947 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

1948 pos = s.position()

1949 not_none = 0

1950 default = None

1951 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

1952 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

1953 if s.sy == 'not':

1954 s.next()

1955 if s.sy == 'IDENT' and s.systring == 'None':

1956 s.next()

1957 else:

1958 s.error("Expected 'None'")

1959 if not in_pyfunc:

1960 error(pos, "'not None' only allowed in Python functions")

1961 not_none = 1

1962 if s.sy == '=':

1963 s.next()

1964 if 'pxd' in s.level:

1965 if s.sy not in ['*', '?']:

1966 error(pos, "default values cannot be specified in pxd files, use ? or *")

1967 default = ExprNodes.BoolNode(1)

1968 s.next()

1969 else:

1970 default = p_simple_expr(s)

1971 return Nodes.CArgDeclNode(pos,

1972 base_type = base_type,

1973 declarator = declarator,

1974 not_none = not_none,

1975 default = default,

1976 kw_only = kw_only)

1977

1978 def p_api(s):

1979 if s.sy == 'IDENT' and s.systring == 'api':

1980 s.next()

1981 return 1

1982 else:

1983 return 0

1984

1985 def p_cdef_statement(s, ctx):

1986 pos = s.position()

1987 ctx.visibility = p_visibility(s, ctx.visibility)

1988 ctx.api = ctx.api or p_api(s)

1989 if ctx.api:

1990 if ctx.visibility not in ('private', 'public'):

1991 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

1992 if (ctx.visibility == 'extern') and s.sy == 'from':

1993 return p_cdef_extern_block(s, pos, ctx)

1994 elif s.sy == 'import':

1995 s.next()

1996 return p_cdef_extern_block(s, pos, ctx)

1997 if p_nogil(s):

1998 ctx.nogil = 1

1999 if s.sy == ':':

2000 return p_cdef_block(s, ctx)

2001 elif s.sy == 'class':

2002 if ctx.level not in ('module', 'module_pxd'):

2003 error(pos, "Extension type definition not allowed here")

2004 #if ctx.api:

2005 # error(pos, "'api' not allowed with extension class")

2006 return p_c_class_definition(s, pos, ctx)

2007 elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:

2008 if ctx.level not in ('module', 'module_pxd'):

2009 error(pos, "C struct/union/enum definition not allowed here")

2010 #if ctx.visibility == 'public':

2011 # error(pos, "Public struct/union/enum definition not implemented")

2012 #if ctx.api:

2013 # error(pos, "'api' not allowed with '%s'" % s.systring)

2014 if s.systring == "enum":

2015 return p_c_enum_definition(s, pos, ctx)

2016 else:

2017 return p_c_struct_or_union_definition(s, pos, ctx)

2018 elif s.sy == 'pass':

2019 node = p_pass_statement(s)

2020 s.expect_newline('Expected a newline')

2021 return node

2022 else:

2023 return p_c_func_or_var_declaration(s, pos, ctx)

2024

2025 def p_cdef_block(s, ctx):

2026 return p_suite(s, ctx(cdef_flag = 1))

2027

2028 def p_cdef_extern_block(s, pos, ctx):

2029 include_file = None

2030 s.expect('from')

2031 if s.sy == '*':

2032 s.next()

2033 else:

2034 _, include_file = p_string_literal(s)

2035 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2036 if p_nogil(s):

2037 ctx.nogil = 1

2038 body = p_suite(s, ctx)

2039 return Nodes.CDefExternNode(pos,

2040 include_file = include_file,

2041 body = body)

2042

2043 struct_union_or_enum = (

2044 "struct", "union", "enum"

2045 )

2046

2047 def p_c_enum_definition(s, pos, ctx):

2048 # s.sy == ident 'enum'

2049 s.next()

2050 if s.sy == 'IDENT':

2051 name = s.systring

2052 s.next()

2053 cname = p_opt_cname(s)

2054 else:

2055 name = None

2056 cname = None

2057 items = None

2058 s.expect(':')

2059 items = []

2060 if s.sy != 'NEWLINE':

2061 p_c_enum_line(s, items)

2062 else:

2063 s.next() # 'NEWLINE'

2064 s.expect_indent()

2065 while s.sy not in ('DEDENT', 'EOF'):

2066 p_c_enum_line(s, items)

2067 s.expect_dedent()

2068 return Nodes.CEnumDefNode(

2069 pos, name = name, cname = cname, items = items,

2070 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2071 in_pxd = ctx.level == 'module_pxd')

2072

2073 def p_c_enum_line(s, items):

2074 if s.sy != 'pass':

2075 p_c_enum_item(s, items)

2076 while s.sy == ',':

2077 s.next()

2078 if s.sy in ('NEWLINE', 'EOF'):

2079 break

2080 p_c_enum_item(s, items)

2081 else:

2082 s.next()

2083 s.expect_newline("Syntax error in enum item list")

2084

2085 def p_c_enum_item(s, items):

2086 pos = s.position()

2087 name = p_ident(s)

2088 cname = p_opt_cname(s)

2089 value = None

2090 if s.sy == '=':

2091 s.next()

2092 value = p_simple_expr(s)

2093 items.append(Nodes.CEnumDefItemNode(pos,

2094 name = name, cname = cname, value = value))

2095

2096 def p_c_struct_or_union_definition(s, pos, ctx):

2097 # s.sy == ident 'struct' or 'union'

2098 kind = s.systring

2099 s.next()

2100 name = p_ident(s)

2101 cname = p_opt_cname(s)

2102 attributes = None

2103 if s.sy == ':':

2104 s.next()

2105 s.expect('NEWLINE')

2106 s.expect_indent()

2107 attributes = []

2108 body_ctx = Ctx()

2109 while s.sy != 'DEDENT':

2110 if s.sy != 'pass':

2111 attributes.append(

2112 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2113 else:

2114 s.next()

2115 s.expect_newline("Expected a newline")

2116 s.expect_dedent()

2117 else:

2118 s.expect_newline("Syntax error in struct or union definition")

2119 return Nodes.CStructOrUnionDefNode(pos,

2120 name = name, cname = cname, kind = kind, attributes = attributes,

2121 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2122 in_pxd = ctx.level == 'module_pxd')

2123

2124 def p_visibility(s, prev_visibility):

2125 pos = s.position()

2126 visibility = prev_visibility

2127 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2128 visibility = s.systring

2129 if prev_visibility != 'private' and visibility != prev_visibility:

2130 s.error("Conflicting visibility options '%s' and '%s'"

2131 % (prev_visibility, visibility))

2132 s.next()

2133 return visibility

2134

2135 def p_c_modifiers(s):

2136 if s.sy == 'IDENT' and s.systring in ('inline',):

2137 modifier = s.systring

2138 s.next()

2139 return [modifier] + p_c_modifiers(s)

2140 return []

2141

2142 def p_c_func_or_var_declaration(s, pos, ctx):

2143 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2144 modifiers = p_c_modifiers(s)

2145 base_type = p_c_base_type(s, nonempty = 1)

2146 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2147 assignable = 1, nonempty = 1)

2148 declarator.overridable = ctx.overridable

2149 if s.sy == ':':

2150 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):

2151 s.error("C function definition not allowed here")

2152 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2153 result = Nodes.CFuncDefNode(pos,

2154 visibility = ctx.visibility,

2155 base_type = base_type,

2156 declarator = declarator,

2157 body = suite,

2158 doc = doc,

2159 modifiers = modifiers,

2160 api = ctx.api,

2161 overridable = ctx.overridable)

2162 else:

2163 #if api:

2164 # error(s.pos, "'api' not allowed with variable declaration")

2165 declarators = [declarator]

2166 while s.sy == ',':

2167 s.next()

2168 if s.sy == 'NEWLINE':

2169 break

2170 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2171 assignable = 1, nonempty = 1)

2172 declarators.append(declarator)

2173 s.expect_newline("Syntax error in C variable declaration")

2174 result = Nodes.CVarDefNode(pos,

2175 visibility = ctx.visibility,

2176 base_type = base_type,

2177 declarators = declarators,

2178 in_pxd = ctx.level == 'module_pxd',

2179 api = ctx.api,

2180 overridable = ctx.overridable)

2181 return result

2182

2183 def p_ctypedef_statement(s, ctx):

2184 # s.sy == 'ctypedef'

2185 pos = s.position()

2186 s.next()

2187 visibility = p_visibility(s, ctx.visibility)

2188 api = p_api(s)

2189 ctx = ctx(typedef_flag = 1, visibility = visibility)

2190 if api:

2191 ctx.api = 1

2192 if s.sy == 'class':

2193 return p_c_class_definition(s, pos, ctx)

2194 elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):

2195 if s.systring == 'enum':

2196 return p_c_enum_definition(s, pos, ctx)

2197 else:

2198 return p_c_struct_or_union_definition(s, pos, ctx)

2199 else:

2200 base_type = p_c_base_type(s, nonempty = 1)

2201 if base_type.name is None:

2202 s.error("Syntax error in ctypedef statement")

2203 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2204 s.expect_newline("Syntax error in ctypedef statement")

2205 return Nodes.CTypeDefNode(

2206 pos, base_type = base_type,

2207 declarator = declarator, visibility = visibility,

2208 in_pxd = ctx.level == 'module_pxd')

2209

2210 def p_decorators(s):

2211 decorators = []

2212 while s.sy == 'DECORATOR':

2213 pos = s.position()

2214 s.next()

2215 decstring = p_dotted_name(s, as_allowed=0)[2]

2216 names = decstring.split('.')

2217 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2218 for name in names[1:]:

2219 decorator = ExprNodes.AttributeNode(pos,

2220 attribute=EncodedString(name),

2221 obj=decorator)

2222 if s.sy == '(':

2223 decorator = p_call(s, decorator)

2224 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2225 s.expect_newline("Expected a newline after decorator")

2226 return decorators

2227

2228 def p_def_statement(s, decorators=None):

2229 # s.sy == 'def'

2230 pos = s.position()

2231 s.next()

2232 name = EncodedString( p_ident(s) )

2233 #args = []

2234 s.expect('(');

2235 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2236 star_arg = None

2237 starstar_arg = None

2238 if s.sy == '*':

2239 s.next()

2240 if s.sy == 'IDENT':

2241 star_arg = p_py_arg_decl(s)

2242 if s.sy == ',':

2243 s.next()

2244 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2245 nonempty_declarators = 1, kw_only = 1))

2246 elif s.sy != ')':

2247 s.error("Syntax error in Python function argument list")

2248 if s.sy == '**':

2249 s.next()

2250 starstar_arg = p_py_arg_decl(s)

2251 s.expect(')')

2252 if p_nogil(s):

2253 error(s.pos, "Python function cannot be declared nogil")

2254 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2255 return Nodes.DefNode(pos, name = name, args = args,

2256 star_arg = star_arg, starstar_arg = starstar_arg,

2257 doc = doc, body = body, decorators = decorators)

2258

2259 def p_py_arg_decl(s):

2260 pos = s.position()

2261 name = p_ident(s)

2262 return Nodes.PyArgDeclNode(pos, name = name)

2263

2264 def p_class_statement(s):

2265 # s.sy == 'class'

2266 pos = s.position()

2267 s.next()

2268 class_name = EncodedString( p_ident(s) )

2269 class_name.encoding = s.source_encoding

2270 if s.sy == '(':

2271 s.next()

2272 base_list = p_simple_expr_list(s)

2273 s.expect(')')

2274 else:

2275 base_list = []

2276 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2277 return Nodes.PyClassDefNode(pos,

2278 name = class_name,

2279 bases = ExprNodes.TupleNode(pos, args = base_list),

2280 doc = doc, body = body)

2281

2282 def p_c_class_definition(s, pos, ctx):

2283 # s.sy == 'class'

2284 s.next()

2285 module_path = []

2286 class_name = p_ident(s)

2287 while s.sy == '.':

2288 s.next()

2289 module_path.append(class_name)

2290 class_name = p_ident(s)

2291 if module_path and ctx.visibility != 'extern':

2292 error(pos, "Qualified class name only allowed for 'extern' C class")

2293 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2294 s.next()

2295 as_name = p_ident(s)

2296 else:

2297 as_name = class_name

2298 objstruct_name = None

2299 typeobj_name = None

2300 base_class_module = None

2301 base_class_name = None

2302 if s.sy == '(':

2303 s.next()

2304 base_class_path = [p_ident(s)]

2305 while s.sy == '.':

2306 s.next()

2307 base_class_path.append(p_ident(s))

2308 if s.sy == ',':

2309 s.error("C class may only have one base class")

2310 s.expect(')')

2311 base_class_module = ".".join(base_class_path[:-1])

2312 base_class_name = base_class_path[-1]

2313 if s.sy == '[':

2314 if ctx.visibility not in ('public', 'extern'):

2315 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2316 objstruct_name, typeobj_name = p_c_class_options(s)

2317 if s.sy == ':':

2318 if ctx.level == 'module_pxd':

2319 body_level = 'c_class_pxd'

2320 else:

2321 body_level = 'c_class'

2322 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2323 else:

2324 s.expect_newline("Syntax error in C class definition")

2325 doc = None

2326 body = None

2327 if ctx.visibility == 'extern':

2328 if not module_path:

2329 error(pos, "Module name required for 'extern' C class")

2330 if typeobj_name:

2331 error(pos, "Type object name specification not allowed for 'extern' C class")

2332 elif ctx.visibility == 'public':

2333 if not objstruct_name:

2334 error(pos, "Object struct name specification required for 'public' C class")

2335 if not typeobj_name:

2336 error(pos, "Type object name specification required for 'public' C class")

2337 elif ctx.visibility == 'private':

2338 if ctx.api:

2339 error(pos, "Only 'public' C class can be declared 'api'")

2340 else:

2341 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2342 return Nodes.CClassDefNode(pos,

2343 visibility = ctx.visibility,

2344 typedef_flag = ctx.typedef_flag,

2345 api = ctx.api,

2346 module_name = ".".join(module_path),

2347 class_name = class_name,

2348 as_name = as_name,

2349 base_class_module = base_class_module,

2350 base_class_name = base_class_name,

2351 objstruct_name = objstruct_name,

2352 typeobj_name = typeobj_name,

2353 in_pxd = ctx.level == 'module_pxd',

2354 doc = doc,

2355 body = body)

2356

2357 def p_c_class_options(s):

2358 objstruct_name = None

2359 typeobj_name = None

2360 s.expect('[')

2361 while 1:

2362 if s.sy != 'IDENT':

2363 break

2364 if s.systring == 'object':

2365 s.next()

2366 objstruct_name = p_ident(s)

2367 elif s.systring == 'type':

2368 s.next()

2369 typeobj_name = p_ident(s)

2370 if s.sy != ',':

2371 break

2372 s.next()

2373 s.expect(']', "Expected 'object' or 'type'")

2374 return objstruct_name, typeobj_name

2375

2376 def p_property_decl(s):

2377 pos = s.position()

2378 s.next() # 'property'

2379 name = p_ident(s)

2380 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2381 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2382

2383 def p_doc_string(s):

2384 if s.sy == 'BEGIN_STRING':

2385 pos = s.position()

2386 kind, result = p_cat_string_literal(s)

2387 if s.sy != 'EOF':

2388 s.expect_newline("Syntax error in doc string")

2389 if kind != 'u':

2390 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2391 if kind == 'b':

2392 result.encoding = None # force a unicode string

2393 return result

2394 else:

2395 return None

2396

2397 def p_code(s, level=None):

2398 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2399 if s.sy != 'EOF':

2400 s.error("Syntax error in statement [%s,%s]" % (

2401 repr(s.sy), repr(s.systring)))

2402 return body

2403

2404 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z_]+)\s*=(.*)$")

2405

2406 def p_compiler_directive_comments(s):

2407 result = {}

2408 while s.sy == 'commentline':

2409 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2410 if m:

2411 name = m.group(1)

2412 try:

2413 value = Options.parse_option_value(str(name), str(m.group(2).strip()))

2414 except ValueError, e:

2415 s.error(e.args[0], fatal=False)

2416 if value is not None: # can be False!

2417 result[name] = value

2418 s.next()

2419 return result

2420

2421 def p_module(s, pxd, full_module_name):

2422 pos = s.position()

2423

2424 option_comments = p_compiler_directive_comments(s)

2425 s.parse_comments = False

2426

2427 doc = p_doc_string(s)

2428 if pxd:

2429 level = 'module_pxd'

2430 else:

2431 level = 'module'

2432

2433 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2434 if s.sy != 'EOF':

2435 s.error("Syntax error in statement [%s,%s]" % (

2436 repr(s.sy), repr(s.systring)))

2437 return ModuleNode(pos, doc = doc, body = body,

2438 full_module_name = full_module_name,

2439 option_comments = option_comments)

2440

2441 #----------------------------------------------

2442 #

2443 # Debugging

2444 #

2445 #----------------------------------------------

2446

2447 def print_parse_tree(f, node, level, key = None):

2448 from Nodes import Node

2449 ind = " " * level

2450 if node:

2451 f.write(ind)

2452 if key:

2453 f.write("%s: " % key)

2454 t = type(node)

2455 if t == TupleType:

2456 f.write("(%s @ %s\n" % (node[0], node[1]))

2457 for i in xrange(2, len(node)):

2458 print_parse_tree(f, node[i], level+1)

2459 f.write("%s)\n" % ind)

2460 return

2461 elif isinstance(node, Node):

2462 try:

2463 tag = node.tag

2464 except AttributeError:

2465 tag = node.__class__.__name__

2466 f.write("%s @ %s\n" % (tag, node.pos))

2467 for name, value in node.__dict__.items():

2468 if name != 'tag' and name != 'pos':

2469 print_parse_tree(f, value, level+1, name)

2470 return

2471 elif t == ListType:

2472 f.write("[\n")

2473 for i in xrange(len(node)):

2474 print_parse_tree(f, node[i], level+1)

2475 f.write("%s]\n" % ind)

2476 return

2477 f.write("%s%s\n" % (ind, node))

2478