1 # cython: auto_cpdef=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13 from types import ListType, TupleType

14 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

15 import Nodes

16 import ExprNodes

17 import StringEncoding

18 from StringEncoding import EncodedString, BytesLiteral

19 from ModuleNode import ModuleNode

20 from Errors import error, warning, InternalError

21 from Cython import Utils

22 import Future

23 import Options

24

25 class Ctx(object):

26 # Parsing context

27 level = 'other'

28 visibility = 'private'

29 cdef_flag = 0

30 typedef_flag = 0

31 api = 0

32 overridable = 0

33 nogil = 0

34

35 def __init__(self, **kwds):

36 self.__dict__.update(kwds)

37

38 def __call__(self, **kwds):

39 ctx = Ctx()

40 d = ctx.__dict__

41 d.update(self.__dict__)

42 d.update(kwds)

43 return ctx

44

45 def p_ident(s, message = "Expected an identifier"):

46 if s.sy == 'IDENT':

47 name = s.systring

48 s.next()

49 return name

50 else:

51 s.error(message)

52

53 def p_ident_list(s):

54 names = []

55 while s.sy == 'IDENT':

56 names.append(s.systring)

57 s.next()

58 if s.sy != ',':

59 break

60 s.next()

61 return names

62

63 #------------------------------------------

64 #

65 # Expressions

66 #

67 #------------------------------------------

68

69 def p_binop_expr(s, ops, p_sub_expr):

70 n1 = p_sub_expr(s)

71 while s.sy in ops:

72 op = s.sy

73 pos = s.position()

74 s.next()

75 n2 = p_sub_expr(s)

76 n1 = ExprNodes.binop_node(pos, op, n1, n2)

77 return n1

78

79 #expression: or_test [if or_test else test] | lambda_form

80

81 def p_simple_expr(s):

82 pos = s.position()

83 expr = p_or_test(s)

84 if s.sy == 'if':

85 s.next()

86 test = p_or_test(s)

87 if s.sy == 'else':

88 s.next()

89 other = p_test(s)

90 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

91 else:

92 s.error("Expected 'else'")

93 else:

94 return expr

95

96 #test: or_test | lambda_form

97

98 def p_test(s):

99 return p_or_test(s)

100

101 #or_test: and_test ('or' and_test)*

102

103 def p_or_test(s):

104 return p_rassoc_binop_expr(s, ('or',), p_and_test)

105

106 def p_rassoc_binop_expr(s, ops, p_subexpr):

107 n1 = p_subexpr(s)

108 if s.sy in ops:

109 pos = s.position()

110 op = s.sy

111 s.next()

112 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

113 n1 = ExprNodes.binop_node(pos, op, n1, n2)

114 return n1

115

116 #and_test: not_test ('and' not_test)*

117

118 def p_and_test(s):

119 #return p_binop_expr(s, ('and',), p_not_test)

120 return p_rassoc_binop_expr(s, ('and',), p_not_test)

121

122 #not_test: 'not' not_test | comparison

123

124 def p_not_test(s):

125 if s.sy == 'not':

126 pos = s.position()

127 s.next()

128 return ExprNodes.NotNode(pos, operand = p_not_test(s))

129 else:

130 return p_comparison(s)

131

132 #comparison: expr (comp_op expr)*

133 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

134

135 def p_comparison(s):

136 n1 = p_bit_expr(s)

137 if s.sy in comparison_ops:

138 pos = s.position()

139 op = p_cmp_op(s)

140 n2 = p_bit_expr(s)

141 n1 = ExprNodes.PrimaryCmpNode(pos,

142 operator = op, operand1 = n1, operand2 = n2)

143 if s.sy in comparison_ops:

144 n1.cascade = p_cascaded_cmp(s)

145 return n1

146

147 def p_cascaded_cmp(s):

148 pos = s.position()

149 op = p_cmp_op(s)

150 n2 = p_bit_expr(s)

151 result = ExprNodes.CascadedCmpNode(pos,

152 operator = op, operand2 = n2)

153 if s.sy in comparison_ops:

154 result.cascade = p_cascaded_cmp(s)

155 return result

156

157 def p_cmp_op(s):

158 if s.sy == 'not':

159 s.next()

160 s.expect('in')

161 op = 'not_in'

162 elif s.sy == 'is':

163 s.next()

164 if s.sy == 'not':

165 s.next()

166 op = 'is_not'

167 else:

168 op = 'is'

169 else:

170 op = s.sy

171 s.next()

172 if op == '<>':

173 op = '!='

174 return op

175

176 comparison_ops = (

177 '<', '>', '==', '>=', '<=', '<>', '!=',

178 'in', 'is', 'not'

179 )

180

181 #expr: xor_expr ('|' xor_expr)*

182

183 def p_bit_expr(s):

184 return p_binop_expr(s, ('|',), p_xor_expr)

185

186 #xor_expr: and_expr ('^' and_expr)*

187

188 def p_xor_expr(s):

189 return p_binop_expr(s, ('^',), p_and_expr)

190

191 #and_expr: shift_expr ('&' shift_expr)*

192

193 def p_and_expr(s):

194 return p_binop_expr(s, ('&',), p_shift_expr)

195

196 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

197

198 def p_shift_expr(s):

199 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

200

201 #arith_expr: term (('+'|'-') term)*

202

203 def p_arith_expr(s):

204 return p_binop_expr(s, ('+', '-'), p_term)

205

206 #term: factor (('*'|'/'|'%') factor)*

207

208 def p_term(s):

209 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

210

211 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

212

213 def p_factor(s):

214 sy = s.sy

215 if sy in ('+', '-', '~'):

216 op = s.sy

217 pos = s.position()

218 s.next()

219 return ExprNodes.unop_node(pos, op, p_factor(s))

220 elif sy == '&':

221 pos = s.position()

222 s.next()

223 arg = p_factor(s)

224 return ExprNodes.AmpersandNode(pos, operand = arg)

225 elif sy == "<":

226 return p_typecast(s)

227 elif sy == 'IDENT' and s.systring == "sizeof":

228 return p_sizeof(s)

229 else:

230 return p_power(s)

231

232 def p_typecast(s):

233 # s.sy == "<"

234 pos = s.position()

235 s.next()

236 base_type = p_c_base_type(s)

237 if base_type.name is None:

238 s.error("Unknown type")

239 declarator = p_c_declarator(s, empty = 1)

240 if s.sy == '?':

241 s.next()

242 typecheck = 1

243 else:

244 typecheck = 0

245 s.expect(">")

246 operand = p_factor(s)

247 return ExprNodes.TypecastNode(pos,

248 base_type = base_type,

249 declarator = declarator,

250 operand = operand,

251 typecheck = typecheck)

252

253 def p_sizeof(s):

254 # s.sy == ident "sizeof"

255 pos = s.position()

256 s.next()

257 s.expect('(')

258 # Here we decide if we are looking at an expression or type

259 # If it is actually a type, but parsable as an expression,

260 # we treat it as an expression here.

261 if looking_at_expr(s):

262 operand = p_simple_expr(s)

263 node = ExprNodes.SizeofVarNode(pos, operand = operand)

264 else:

265 base_type = p_c_base_type(s)

266 declarator = p_c_declarator(s, empty = 1)

267 node = ExprNodes.SizeofTypeNode(pos,

268 base_type = base_type, declarator = declarator)

269 s.expect(')')

270 return node

271

272 def p_yield_expression(s):

273 # s.sy == "yield"

274 pos = s.position()

275 s.next()

276 if s.sy not in ('EOF', 'NEWLINE', ')'):

277 expr = p_expr(s)

278 s.error("generators ('yield') are not currently supported")

279 return Nodes.PassStatNode(pos)

280

281 #power: atom trailer* ('**' factor)*

282

283 def p_power(s):

284 n1 = p_atom(s)

285 while s.sy in ('(', '[', '.'):

286 n1 = p_trailer(s, n1)

287 if s.sy == '**':

288 pos = s.position()

289 s.next()

290 n2 = p_factor(s)

291 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

292 return n1

293

294 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

295

296 def p_trailer(s, node1):

297 pos = s.position()

298 if s.sy == '(':

299 return p_call(s, node1)

300 elif s.sy == '[':

301 return p_index(s, node1)

302 else: # s.sy == '.'

303 s.next()

304 name = EncodedString( p_ident(s) )

305 return ExprNodes.AttributeNode(pos,

306 obj = node1, attribute = name)

307

308 # arglist: argument (',' argument)* [',']

309 # argument: [test '='] test # Really [keyword '='] test

310

311 def p_call(s, function):

312 # s.sy == '('

313 pos = s.position()

314 s.next()

315 positional_args = []

316 keyword_args = []

317 star_arg = None

318 starstar_arg = None

319 while s.sy not in ('**', ')'):

320 if s.sy == '*':

321 if star_arg:

322 s.error("only one star-arg parameter allowed",

323 pos = s.position())

324 s.next()

325 star_arg = p_simple_expr(s)

326 else:

327 arg = p_simple_expr(s)

328 if s.sy == '=':

329 s.next()

330 if not arg.is_name:

331 s.error("Expected an identifier before '='",

332 pos = arg.pos)

333 encoded_name = EncodedString(arg.name)

334 keyword = ExprNodes.IdentifierStringNode(arg.pos,

335 value = encoded_name)

336 arg = p_simple_expr(s)

337 keyword_args.append((keyword, arg))

338 else:

339 if keyword_args:

340 s.error("Non-keyword arg following keyword arg",

341 pos = arg.pos)

342 if star_arg:

343 s.error("Non-keyword arg following star-arg",

344 pos = arg.pos)

345 positional_args.append(arg)

346 if s.sy != ',':

347 break

348 s.next()

349

350 if s.sy == '**':

351 s.next()

352 starstar_arg = p_simple_expr(s)

353 if s.sy == ',':

354 s.next()

355 s.expect(')')

356 if not (keyword_args or star_arg or starstar_arg):

357 return ExprNodes.SimpleCallNode(pos,

358 function = function,

359 args = positional_args)

360 else:

361 arg_tuple = None

362 keyword_dict = None

363 if positional_args or not star_arg:

364 arg_tuple = ExprNodes.TupleNode(pos,

365 args = positional_args)

366 if star_arg:

367 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

368 if arg_tuple:

369 arg_tuple = ExprNodes.binop_node(pos,

370 operator = '+', operand1 = arg_tuple,

371 operand2 = star_arg_tuple)

372 else:

373 arg_tuple = star_arg_tuple

374 if keyword_args:

375 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

376 for key, value in keyword_args]

377 keyword_dict = ExprNodes.DictNode(pos,

378 key_value_pairs = keyword_args)

379 return ExprNodes.GeneralCallNode(pos,

380 function = function,

381 positional_args = arg_tuple,

382 keyword_args = keyword_dict,

383 starstar_arg = starstar_arg)

384

385 #lambdef: 'lambda' [varargslist] ':' test

386

387 #subscriptlist: subscript (',' subscript)* [',']

388

389 def p_index(s, base):

390 # s.sy == '['

391 pos = s.position()

392 s.next()

393 subscripts = p_subscript_list(s)

394 if len(subscripts) == 1 and len(subscripts[0]) == 2:

395 start, stop = subscripts[0]

396 result = ExprNodes.SliceIndexNode(pos,

397 base = base, start = start, stop = stop)

398 else:

399 indexes = make_slice_nodes(pos, subscripts)

400 if len(indexes) == 1:

401 index = indexes[0]

402 else:

403 index = ExprNodes.TupleNode(pos, args = indexes)

404 result = ExprNodes.IndexNode(pos,

405 base = base, index = index)

406 s.expect(']')

407 return result

408

409 def p_subscript_list(s):

410 items = [p_subscript(s)]

411 while s.sy == ',':

412 s.next()

413 if s.sy == ']':

414 break

415 items.append(p_subscript(s))

416 return items

417

418 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

419

420 def p_subscript(s):

421 # Parse a subscript and return a list of

422 # 1, 2 or 3 ExprNodes, depending on how

423 # many slice elements were encountered.

424 pos = s.position()

425 if s.sy == '.':

426 expect_ellipsis(s)

427 return [ExprNodes.EllipsisNode(pos)]

428 else:

429 start = p_slice_element(s, (':',))

430 if s.sy != ':':

431 return [start]

432 s.next()

433 stop = p_slice_element(s, (':', ',', ']'))

434 if s.sy != ':':

435 return [start, stop]

436 s.next()

437 step = p_slice_element(s, (':', ',', ']'))

438 return [start, stop, step]

439

440 def p_slice_element(s, follow_set):

441 # Simple expression which may be missing iff

442 # it is followed by something in follow_set.

443 if s.sy not in follow_set:

444 return p_simple_expr(s)

445 else:

446 return None

447

448 def expect_ellipsis(s):

449 s.expect('.')

450 s.expect('.')

451 s.expect('.')

452

453 def make_slice_nodes(pos, subscripts):

454 # Convert a list of subscripts as returned

455 # by p_subscript_list into a list of ExprNodes,

456 # creating SliceNodes for elements with 2 or

457 # more components.

458 result = []

459 for subscript in subscripts:

460 if len(subscript) == 1:

461 result.append(subscript[0])

462 else:

463 result.append(make_slice_node(pos, *subscript))

464 return result

465

466 def make_slice_node(pos, start, stop = None, step = None):

467 if not start:

468 start = ExprNodes.NoneNode(pos)

469 if not stop:

470 stop = ExprNodes.NoneNode(pos)

471 if not step:

472 step = ExprNodes.NoneNode(pos)

473 return ExprNodes.SliceNode(pos,

474 start = start, stop = stop, step = step)

475

476 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

477

478 def p_atom(s):

479 pos = s.position()

480 sy = s.sy

481 if sy == '(':

482 s.next()

483 if s.sy == ')':

484 result = ExprNodes.TupleNode(pos, args = [])

485 elif s.sy == 'yield':

486 result = p_yield_expression(s)

487 else:

488 result = p_expr(s)

489 s.expect(')')

490 return result

491 elif sy == '[':

492 return p_list_maker(s)

493 elif sy == '{':

494 return p_dict_or_set_maker(s)

495 elif sy == '`':

496 return p_backquote_expr(s)

497 elif sy == 'INT':

498 value = s.systring

499 s.next()

500 unsigned = ""

501 longness = ""

502 while value[-1] in "UuLl":

503 if value[-1] in "Ll":

504 longness += "L"

505 else:

506 unsigned += "U"

507 value = value[:-1]

508 return ExprNodes.IntNode(pos,

509 value = value,

510 unsigned = unsigned,

511 longness = longness)

512 elif sy == 'FLOAT':

513 value = s.systring

514 s.next()

515 return ExprNodes.FloatNode(pos, value = value)

516 elif sy == 'IMAG':

517 value = s.systring[:-1]

518 s.next()

519 return ExprNodes.ImagNode(pos, value = value)

520 elif sy == 'BEGIN_STRING':

521 kind, value = p_cat_string_literal(s)

522 if kind == 'c':

523 return ExprNodes.CharNode(pos, value = value)

524 elif kind == 'u':

525 return ExprNodes.UnicodeNode(pos, value = value)

526 else:

527 return ExprNodes.StringNode(pos, value = value)

528 elif sy == 'IDENT':

529 name = EncodedString( s.systring )

530 s.next()

531 if name == "None":

532 return ExprNodes.NoneNode(pos)

533 elif name == "True":

534 return ExprNodes.BoolNode(pos, value=True)

535 elif name == "False":

536 return ExprNodes.BoolNode(pos, value=False)

537 elif name == "NULL":

538 return ExprNodes.NullNode(pos)

539 else:

540 return p_name(s, name)

541 else:

542 s.error("Expected an identifier or literal")

543

544 def p_name(s, name):

545 pos = s.position()

546 if not s.compile_time_expr and name in s.compile_time_env:

547 value = s.compile_time_env.lookup_here(name)

548 rep = repr(value)

549 if isinstance(value, bool):

550 return ExprNodes.BoolNode(pos, value = value)

551 elif isinstance(value, int):

552 return ExprNodes.IntNode(pos, value = rep)

553 elif isinstance(value, long):

554 return ExprNodes.IntNode(pos, value = rep, longness = "L")

555 elif isinstance(value, float):

556 return ExprNodes.FloatNode(pos, value = rep)

557 elif isinstance(value, (str, unicode)):

558 return ExprNodes.StringNode(pos, value = value)

559 else:

560 error(pos, "Invalid type for compile-time constant: %s"

561 % value.__class__.__name__)

562 return ExprNodes.NameNode(pos, name = name)

563

564 def p_cat_string_literal(s):

565 # A sequence of one or more adjacent string literals.

566 # Returns (kind, value) where kind in ('b', 'c', 'u')

567 kind, value = p_string_literal(s)

568 if kind != 'c':

569 strings = [value]

570 while s.sy == 'BEGIN_STRING':

571 next_kind, next_value = p_string_literal(s)

572 if next_kind == 'c':

573 error(s.position(),

574 "Cannot concatenate char literal with another string or char literal")

575 elif next_kind != kind:

576 # we have to switch to unicode now

577 if kind == 'b':

578 # concatenating a unicode string to byte strings

579 strings = [u''.join([s.decode(s.encoding) for s in strings])]

580 elif kind == 'u':

581 # concatenating a byte string to unicode strings

582 strings.append(next_value.decode(next_value.encoding))

583 kind = 'u'

584 else:

585 strings.append(next_value)

586 if kind == 'u':

587 value = EncodedString( u''.join(strings) )

588 else:

589 value = BytesLiteral( ''.join(strings) )

590 value.encoding = s.source_encoding

591 return kind, value

592

593 def p_opt_string_literal(s):

594 if s.sy == 'BEGIN_STRING':

595 return p_string_literal(s)

596 else:

597 return None

598

599 def p_string_literal(s):

600 # A single string or char literal.

601 # Returns (kind, value) where kind in ('b', 'c', 'u')

602 # s.sy == 'BEGIN_STRING'

603 pos = s.position()

604 is_raw = 0

605 kind = s.systring[:1].lower()

606 if kind == 'r':

607 kind = ''

608 is_raw = 1

609 elif kind in 'ub':

610 is_raw = s.systring[1:2].lower() == 'r'

611 elif kind != 'c':

612 kind = ''

613 if Future.unicode_literals in s.context.future_directives:

614 if kind == '':

615 kind = 'u'

616 elif kind == '':

617 kind = 'b'

618 if kind == 'u':

619 chars = StringEncoding.UnicodeLiteralBuilder()

620 else:

621 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

622 while 1:

623 s.next()

624 sy = s.sy

625 #print "p_string_literal: sy =", sy, repr(s.systring) ###

626 if sy == 'CHARS':

627 chars.append(s.systring)

628 elif sy == 'ESCAPE':

629 has_escape = True

630 systr = s.systring

631 if is_raw:

632 if systr == u'\\\n':

633 chars.append(u'\\\n')

634 elif systr == u'\\\"':

635 chars.append(u'"')

636 elif systr == u'\\\'':

637 chars.append(u"'")

638 else:

639 chars.append(systr)

640 else:

641 c = systr[1]

642 if c in u"01234567":

643 chars.append_charval( int(systr[1:], 8) )

644 elif c in u"'\"\\":

645 chars.append(c)

646 elif c in u"abfnrtv":

647 chars.append(

648 StringEncoding.char_from_escape_sequence(systr))

649 elif c == u'\n':

650 pass

651 elif c in u'Uux':

652 if kind == 'u' or c == 'x':

653 chrval = int(systr[2:], 16)

654 if chrval > 1114111: # sys.maxunicode:

655 s.error("Invalid unicode escape '%s'" % systr,

656 pos = pos)

657 elif chrval > 65535:

658 warning(s.position(),

659 "Unicode characters above 65535 are not "

660 "necessarily portable across Python installations", 1)

661 chars.append_charval(chrval)

662 else:

663 # unicode escapes in plain byte strings are not unescaped

664 chars.append(systr)

665 else:

666 chars.append(u'\\' + systr[1:])

667 elif sy == 'NEWLINE':

668 chars.append(u'\n')

669 elif sy == 'END_STRING':

670 break

671 elif sy == 'EOF':

672 s.error("Unclosed string literal", pos = pos)

673 else:

674 s.error(

675 "Unexpected token %r:%r in string literal" %

676 (sy, s.systring))

677 if kind == 'c':

678 value = chars.getchar()

679 if len(value) != 1:

680 error(pos, u"invalid character literal: %r" % value)

681 else:

682 value = chars.getstring()

683 s.next()

684 #print "p_string_literal: value =", repr(value) ###

685 return kind, value

686

687 # list_display ::= "[" [listmaker] "]"

688 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

689 # list_iter ::= list_for | list_if

690 # list_for ::= "for" expression_list "in" testlist [list_iter]

691 # list_if ::= "if" test [list_iter]

692

693 def p_list_maker(s):

694 # s.sy == '['

695 pos = s.position()

696 s.next()

697 if s.sy == ']':

698 s.expect(']')

699 return ExprNodes.ListNode(pos, args = [])

700 expr = p_simple_expr(s)

701 if s.sy == 'for':

702 loop = p_list_for(s)

703 s.expect(']')

704 append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )

705 set_inner_comp_append(loop, append)

706 return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)

707 else:

708 exprs = [expr]

709 if s.sy == ',':

710 s.next()

711 exprs += p_simple_expr_list(s)

712 s.expect(']')

713 return ExprNodes.ListNode(pos, args = exprs)

714

715 def p_list_iter(s):

716 if s.sy == 'for':

717 return p_list_for(s)

718 elif s.sy == 'if':

719 return p_list_if(s)

720 else:

721 return Nodes.PassStatNode(s.position())

722

723 def p_list_for(s):

724 # s.sy == 'for'

725 pos = s.position()

726 s.next()

727 kw = p_for_bounds(s)

728 kw['else_clause'] = None

729 kw['body'] = p_list_iter(s)

730 return Nodes.ForStatNode(pos, **kw)

731

732 def p_list_if(s):

733 # s.sy == 'if'

734 pos = s.position()

735 s.next()

736 test = p_test(s)

737 return Nodes.IfStatNode(pos,

738 if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],

739 else_clause = None )

740

741 def set_inner_comp_append(loop, append):

742 inner_loop = loop

743 while not isinstance(inner_loop.body, Nodes.PassStatNode):

744 inner_loop = inner_loop.body

745 if isinstance(inner_loop, Nodes.IfStatNode):

746 inner_loop = inner_loop.if_clauses[0]

747 inner_loop.body = Nodes.ExprStatNode(append.pos, expr = append)

748

749 #dictmaker: test ':' test (',' test ':' test)* [',']

750

751 def p_dict_or_set_maker(s):

752 # s.sy == '{'

753 pos = s.position()

754 s.next()

755 if s.sy == '}':

756 s.next()

757 return ExprNodes.DictNode(pos, key_value_pairs = [])

758 item = p_simple_expr(s)

759 if s.sy == ',' or s.sy == '}':

760 # set literal

761 values = [item]

762 while s.sy == ',':

763 s.next()

764 values.append( p_simple_expr(s) )

765 s.expect('}')

766 return ExprNodes.SetNode(pos, args=values)

767 elif s.sy == 'for':

768 # set comprehension

769 loop = p_list_for(s)

770 s.expect('}')

771 append = ExprNodes.SetComprehensionAppendNode(item.pos, expr=item)

772 set_inner_comp_append(loop, append)

773 return ExprNodes.SetComprehensionNode(pos, loop=loop, append=append)

774 elif s.sy == ':':

775 # dict literal or comprehension

776 key = item

777 s.next()

778 value = p_simple_expr(s)

779 if s.sy == 'for':

780 # dict comprehension

781 loop = p_list_for(s)

782 s.expect('}')

783 append = ExprNodes.DictComprehensionAppendNode(

784 item.pos, key_expr = key, value_expr = value)

785 set_inner_comp_append(loop, append)

786 return ExprNodes.DictComprehensionNode(pos, loop=loop, append=append)

787 else:

788 # dict literal

789 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

790 while s.sy == ',':

791 s.next()

792 key = p_simple_expr(s)

793 s.expect(':')

794 value = p_simple_expr(s)

795 items.append(

796 ExprNodes.DictItemNode(key.pos, key=key, value=value))

797 s.expect('}')

798 return ExprNodes.DictNode(pos, key_value_pairs=items)

799 else:

800 # raise an error

801 s.expect('}')

802 return ExprNodes.DictNode(pos, key_value_pairs = [])

803

804 def p_backquote_expr(s):

805 # s.sy == '`'

806 pos = s.position()

807 s.next()

808 arg = p_expr(s)

809 s.expect('`')

810 return ExprNodes.BackquoteNode(pos, arg = arg)

811

812 def p_simple_expr_list(s):

813 exprs = []

814 while s.sy not in expr_terminators:

815 exprs.append(p_simple_expr(s))

816 if s.sy != ',':

817 break

818 s.next()

819 return exprs

820

821 def p_expr(s):

822 pos = s.position()

823 expr = p_simple_expr(s)

824 if s.sy == ',':

825 s.next()

826 exprs = [expr] + p_simple_expr_list(s)

827 return ExprNodes.TupleNode(pos, args = exprs)

828 else:

829 return expr

830

831

832 #testlist: test (',' test)* [',']

833 # differs from p_expr only in the fact that it cannot contain conditional expressions

834

835 def p_testlist(s):

836 pos = s.position()

837 expr = p_test(s)

838 if s.sy == ',':

839 exprs = [expr]

840 while s.sy == ',':

841 s.next()

842 exprs.append(p_test(s))

843 return ExprNodes.TupleNode(pos, args = exprs)

844 else:

845 return expr

846

847 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

848

849 #-------------------------------------------------------

850 #

851 # Statements

852 #

853 #-------------------------------------------------------

854

855 def p_global_statement(s):

856 # assume s.sy == 'global'

857 pos = s.position()

858 s.next()

859 names = p_ident_list(s)

860 return Nodes.GlobalNode(pos, names = names)

861

862 def p_expression_or_assignment(s):

863 expr_list = [p_expr(s)]

864 while s.sy == '=':

865 s.next()

866 expr_list.append(p_expr(s))

867 if len(expr_list) == 1:

868 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

869 lhs = expr_list[0]

870 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

871 error(lhs.pos, "Illegal operand for inplace operation.")

872 operator = s.sy[:-1]

873 s.next()

874 rhs = p_expr(s)

875 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

876 expr = expr_list[0]

877 if isinstance(expr, ExprNodes.StringNode):

878 return Nodes.PassStatNode(expr.pos)

879 else:

880 return Nodes.ExprStatNode(expr.pos, expr = expr)

881 else:

882 expr_list_list = []

883 flatten_parallel_assignments(expr_list, expr_list_list)

884 nodes = []

885 for expr_list in expr_list_list:

886 lhs_list = expr_list[:-1]

887 rhs = expr_list[-1]

888 if len(lhs_list) == 1:

889 node = Nodes.SingleAssignmentNode(rhs.pos,

890 lhs = lhs_list[0], rhs = rhs)

891 else:

892 node = Nodes.CascadedAssignmentNode(rhs.pos,

893 lhs_list = lhs_list, rhs = rhs)

894 nodes.append(node)

895 if len(nodes) == 1:

896 return nodes[0]

897 else:

898 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

899

900 def flatten_parallel_assignments(input, output):

901 # The input is a list of expression nodes, representing

902 # the LHSs and RHS of one (possibly cascaded) assignment

903 # statement. If they are all sequence constructors with

904 # the same number of arguments, rearranges them into a

905 # list of equivalent assignments between the individual

906 # elements. This transformation is applied recursively.

907 size = find_parallel_assignment_size(input)

908 if size >= 0:

909 for i in range(size):

910 new_exprs = [expr.args[i] for expr in input]

911 flatten_parallel_assignments(new_exprs, output)

912 else:

913 output.append(input)

914

915 def find_parallel_assignment_size(input):

916 # The input is a list of expression nodes. If

917 # they are all sequence constructors with the same number

918 # of arguments, return that number, else return -1.

919 # Produces an error message if they are all sequence

920 # constructors but not all the same size.

921 for expr in input:

922 if not expr.is_sequence_constructor:

923 return -1

924 rhs = input[-1]

925 rhs_size = len(rhs.args)

926 for lhs in input[:-1]:

927 lhs_size = len(lhs.args)

928 if lhs_size != rhs_size:

929 error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"

930 % (lhs_size, rhs_size))

931 return -1

932 return rhs_size

933

934 def p_print_statement(s):

935 # s.sy == 'print'

936 pos = s.position()

937 s.next()

938 if s.sy == '>>':

939 s.error("'print >>' not yet implemented")

940 args = []

941 ends_with_comma = 0

942 if s.sy not in ('NEWLINE', 'EOF'):

943 args.append(p_simple_expr(s))

944 while s.sy == ',':

945 s.next()

946 if s.sy in ('NEWLINE', 'EOF'):

947 ends_with_comma = 1

948 break

949 args.append(p_simple_expr(s))

950 arg_tuple = ExprNodes.TupleNode(pos, args = args)

951 return Nodes.PrintStatNode(pos,

952 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

953

954 def p_exec_statement(s):

955 # s.sy == 'exec'

956 pos = s.position()

957 s.next()

958 args = [ p_bit_expr(s) ]

959 if s.sy == 'in':

960 s.next()

961 args.append(p_simple_expr(s))

962 if s.sy == ',':

963 s.next()

964 args.append(p_simple_expr(s))

965 else:

966 error(pos, "'exec' currently requires a target mapping (globals/locals)")

967 return Nodes.ExecStatNode(pos, args = args)

968

969 def p_del_statement(s):

970 # s.sy == 'del'

971 pos = s.position()

972 s.next()

973 args = p_simple_expr_list(s)

974 return Nodes.DelStatNode(pos, args = args)

975

976 def p_pass_statement(s, with_newline = 0):

977 pos = s.position()

978 s.expect('pass')

979 if with_newline:

980 s.expect_newline("Expected a newline")

981 return Nodes.PassStatNode(pos)

982

983 def p_break_statement(s):

984 # s.sy == 'break'

985 pos = s.position()

986 s.next()

987 return Nodes.BreakStatNode(pos)

988

989 def p_continue_statement(s):

990 # s.sy == 'continue'

991 pos = s.position()

992 s.next()

993 return Nodes.ContinueStatNode(pos)

994

995 def p_return_statement(s):

996 # s.sy == 'return'

997 pos = s.position()

998 s.next()

999 if s.sy not in statement_terminators:

1000 value = p_expr(s)

1001 else:

1002 value = None

1003 return Nodes.ReturnStatNode(pos, value = value)

1004

1005 def p_raise_statement(s):

1006 # s.sy == 'raise'

1007 pos = s.position()

1008 s.next()

1009 exc_type = None

1010 exc_value = None

1011 exc_tb = None

1012 if s.sy not in statement_terminators:

1013 exc_type = p_simple_expr(s)

1014 if s.sy == ',':

1015 s.next()

1016 exc_value = p_simple_expr(s)

1017 if s.sy == ',':

1018 s.next()

1019 exc_tb = p_simple_expr(s)

1020 if exc_type or exc_value or exc_tb:

1021 return Nodes.RaiseStatNode(pos,

1022 exc_type = exc_type,

1023 exc_value = exc_value,

1024 exc_tb = exc_tb)

1025 else:

1026 return Nodes.ReraiseStatNode(pos)

1027

1028 def p_import_statement(s):

1029 # s.sy in ('import', 'cimport')

1030 pos = s.position()

1031 kind = s.sy

1032 s.next()

1033 items = [p_dotted_name(s, as_allowed = 1)]

1034 while s.sy == ',':

1035 s.next()

1036 items.append(p_dotted_name(s, as_allowed = 1))

1037 stats = []

1038 for pos, target_name, dotted_name, as_name in items:

1039 dotted_name = EncodedString(dotted_name)

1040 if kind == 'cimport':

1041 stat = Nodes.CImportStatNode(pos,

1042 module_name = dotted_name,

1043 as_name = as_name)

1044 else:

1045 if as_name and "." in dotted_name:

1046 name_list = ExprNodes.ListNode(pos, args = [

1047 ExprNodes.IdentifierStringNode(

1048 pos, value = EncodedString("*"))])

1049 else:

1050 name_list = None

1051 stat = Nodes.SingleAssignmentNode(pos,

1052 lhs = ExprNodes.NameNode(pos,

1053 name = as_name or target_name),

1054 rhs = ExprNodes.ImportNode(pos,

1055 module_name = ExprNodes.IdentifierStringNode(

1056 pos, value = dotted_name),

1057 name_list = name_list))

1058 stats.append(stat)

1059 return Nodes.StatListNode(pos, stats = stats)

1060

1061 def p_from_import_statement(s, first_statement = 0):

1062 # s.sy == 'from'

1063 pos = s.position()

1064 s.next()

1065 (dotted_name_pos, _, dotted_name, _) = \

1066 p_dotted_name(s, as_allowed = 0)

1067 if s.sy in ('import', 'cimport'):

1068 kind = s.sy

1069 s.next()

1070 else:

1071 s.error("Expected 'import' or 'cimport'")

1072 is_cimport = kind == 'cimport'

1073 is_parenthesized = False

1074 if s.sy == '*':

1075 imported_names = [(s.position(), "*", None, None)]

1076 s.next()

1077 else:

1078 if s.sy == '(':

1079 is_parenthesized = True

1080 s.next()

1081 imported_names = [p_imported_name(s, is_cimport)]

1082 while s.sy == ',':

1083 s.next()

1084 imported_names.append(p_imported_name(s, is_cimport))

1085 if is_parenthesized:

1086 s.expect(')')

1087 dotted_name = EncodedString(dotted_name)

1088 if dotted_name == '__future__':

1089 if not first_statement:

1090 s.error("from __future__ imports must occur at the beginning of the file")

1091 else:

1092 for (name_pos, name, as_name, kind) in imported_names:

1093 if name == "braces":

1094 s.error("not a chance", name_pos)

1095 break

1096 try:

1097 directive = getattr(Future, name)

1098 except AttributeError:

1099 s.error("future feature %s is not defined" % name, name_pos)

1100 break

1101 s.context.future_directives.add(directive)

1102 return Nodes.PassStatNode(pos)

1103 elif kind == 'cimport':

1104 return Nodes.FromCImportStatNode(pos,

1105 module_name = dotted_name,

1106 imported_names = imported_names)

1107 else:

1108 imported_name_strings = []

1109 items = []

1110 for (name_pos, name, as_name, kind) in imported_names:

1111 encoded_name = EncodedString(name)

1112 imported_name_strings.append(

1113 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1114 items.append(

1115 (name,

1116 ExprNodes.NameNode(name_pos,

1117 name = as_name or name)))

1118 import_list = ExprNodes.ListNode(

1119 imported_names[0][0], args = imported_name_strings)

1120 dotted_name = EncodedString(dotted_name)

1121 return Nodes.FromImportStatNode(pos,

1122 module = ExprNodes.ImportNode(dotted_name_pos,

1123 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1124 name_list = import_list),

1125 items = items)

1126

1127 imported_name_kinds = ('class', 'struct', 'union')

1128

1129 def p_imported_name(s, is_cimport):

1130 pos = s.position()

1131 kind = None

1132 if is_cimport and s.systring in imported_name_kinds:

1133 kind = s.systring

1134 s.next()

1135 name = p_ident(s)

1136 as_name = p_as_name(s)

1137 return (pos, name, as_name, kind)

1138

1139 def p_dotted_name(s, as_allowed):

1140 pos = s.position()

1141 target_name = p_ident(s)

1142 as_name = None

1143 names = [target_name]

1144 while s.sy == '.':

1145 s.next()

1146 names.append(p_ident(s))

1147 if as_allowed:

1148 as_name = p_as_name(s)

1149 return (pos, target_name, u'.'.join(names), as_name)

1150

1151 def p_as_name(s):

1152 if s.sy == 'IDENT' and s.systring == 'as':

1153 s.next()

1154 return p_ident(s)

1155 else:

1156 return None

1157

1158 def p_assert_statement(s):

1159 # s.sy == 'assert'

1160 pos = s.position()

1161 s.next()

1162 cond = p_simple_expr(s)

1163 if s.sy == ',':

1164 s.next()

1165 value = p_simple_expr(s)

1166 else:

1167 value = None

1168 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1169

1170 statement_terminators = (';', 'NEWLINE', 'EOF')

1171

1172 def p_if_statement(s):

1173 # s.sy == 'if'

1174 pos = s.position()

1175 s.next()

1176 if_clauses = [p_if_clause(s)]

1177 while s.sy == 'elif':

1178 s.next()

1179 if_clauses.append(p_if_clause(s))

1180 else_clause = p_else_clause(s)

1181 return Nodes.IfStatNode(pos,

1182 if_clauses = if_clauses, else_clause = else_clause)

1183

1184 def p_if_clause(s):

1185 pos = s.position()

1186 test = p_simple_expr(s)

1187 body = p_suite(s)

1188 return Nodes.IfClauseNode(pos,

1189 condition = test, body = body)

1190

1191 def p_else_clause(s):

1192 if s.sy == 'else':

1193 s.next()

1194 return p_suite(s)

1195 else:

1196 return None

1197

1198 def p_while_statement(s):

1199 # s.sy == 'while'

1200 pos = s.position()

1201 s.next()

1202 test = p_simple_expr(s)

1203 body = p_suite(s)

1204 else_clause = p_else_clause(s)

1205 return Nodes.WhileStatNode(pos,

1206 condition = test, body = body,

1207 else_clause = else_clause)

1208

1209 def p_for_statement(s):

1210 # s.sy == 'for'

1211 pos = s.position()

1212 s.next()

1213 kw = p_for_bounds(s)

1214 kw['body'] = p_suite(s)

1215 kw['else_clause'] = p_else_clause(s)

1216 return Nodes.ForStatNode(pos, **kw)

1217

1218 def p_for_bounds(s):

1219 target = p_for_target(s)

1220 if s.sy == 'in':

1221 s.next()

1222 iterator = p_for_iterator(s)

1223 return { 'target': target, 'iterator': iterator }

1224 else:

1225 if s.sy == 'from':

1226 s.next()

1227 bound1 = p_bit_expr(s)

1228 else:

1229 # Support shorter "for a <= x < b" syntax

1230 bound1, target = target, None

1231 rel1 = p_for_from_relation(s)

1232 name2_pos = s.position()

1233 name2 = p_ident(s)

1234 rel2_pos = s.position()

1235 rel2 = p_for_from_relation(s)

1236 bound2 = p_bit_expr(s)

1237 step = p_for_from_step(s)

1238 if target is None:

1239 target = ExprNodes.NameNode(name2_pos, name = name2)

1240 else:

1241 if not target.is_name:

1242 error(target.pos,

1243 "Target of for-from statement must be a variable name")

1244 elif name2 != target.name:

1245 error(name2_pos,

1246 "Variable name in for-from range does not match target")

1247 if rel1[0] != rel2[0]:

1248 error(rel2_pos,

1249 "Relation directions in for-from do not match")

1250 return {'target': target,

1251 'bound1': bound1,

1252 'relation1': rel1,

1253 'relation2': rel2,

1254 'bound2': bound2,

1255 'step': step }

1256

1257 def p_for_from_relation(s):

1258 if s.sy in inequality_relations:

1259 op = s.sy

1260 s.next()

1261 return op

1262 else:

1263 s.error("Expected one of '<', '<=', '>' '>='")

1264

1265 def p_for_from_step(s):

1266 if s.sy == 'by':

1267 s.next()

1268 step = p_bit_expr(s)

1269 return step

1270 else:

1271 return None

1272

1273 inequality_relations = ('<', '<=', '>', '>=')

1274

1275 def p_target(s, terminator):

1276 pos = s.position()

1277 expr = p_bit_expr(s)

1278 if s.sy == ',':

1279 s.next()

1280 exprs = [expr]

1281 while s.sy != terminator:

1282 exprs.append(p_bit_expr(s))

1283 if s.sy != ',':

1284 break

1285 s.next()

1286 return ExprNodes.TupleNode(pos, args = exprs)

1287 else:

1288 return expr

1289

1290 def p_for_target(s):

1291 return p_target(s, 'in')

1292

1293 def p_for_iterator(s):

1294 pos = s.position()

1295 expr = p_testlist(s)

1296 return ExprNodes.IteratorNode(pos, sequence = expr)

1297

1298 def p_try_statement(s):

1299 # s.sy == 'try'

1300 pos = s.position()

1301 s.next()

1302 body = p_suite(s)

1303 except_clauses = []

1304 else_clause = None

1305 if s.sy in ('except', 'else'):

1306 while s.sy == 'except':

1307 except_clauses.append(p_except_clause(s))

1308 if s.sy == 'else':

1309 s.next()

1310 else_clause = p_suite(s)

1311 body = Nodes.TryExceptStatNode(pos,

1312 body = body, except_clauses = except_clauses,

1313 else_clause = else_clause)

1314 if s.sy != 'finally':

1315 return body

1316 # try-except-finally is equivalent to nested try-except/try-finally

1317 if s.sy == 'finally':

1318 s.next()

1319 finally_clause = p_suite(s)

1320 return Nodes.TryFinallyStatNode(pos,

1321 body = body, finally_clause = finally_clause)

1322 else:

1323 s.error("Expected 'except' or 'finally'")

1324

1325 def p_except_clause(s):

1326 # s.sy == 'except'

1327 pos = s.position()

1328 s.next()

1329 exc_type = None

1330 exc_value = None

1331 if s.sy != ':':

1332 exc_type = p_simple_expr(s)

1333 if s.sy == ',':

1334 s.next()

1335 exc_value = p_simple_expr(s)

1336 body = p_suite(s)

1337 return Nodes.ExceptClauseNode(pos,

1338 pattern = exc_type, target = exc_value, body = body)

1339

1340 def p_include_statement(s, ctx):

1341 pos = s.position()

1342 s.next() # 'include'

1343 _, include_file_name = p_string_literal(s)

1344 s.expect_newline("Syntax error in include statement")

1345 if s.compile_time_eval:

1346 include_file_path = s.context.find_include_file(include_file_name, pos)

1347 if include_file_path:

1348 s.included_files.append(include_file_name)

1349 f = Utils.open_source_file(include_file_path, mode="rU")

1350 source_desc = FileSourceDescriptor(include_file_path)

1351 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1352 try:

1353 tree = p_statement_list(s2, ctx)

1354 finally:

1355 f.close()

1356 return tree

1357 else:

1358 return None

1359 else:

1360 return Nodes.PassStatNode(pos)

1361

1362 def p_with_statement(s):

1363 pos = s.position()

1364 s.next() # 'with'

1365 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1366 if s.sy == 'IDENT' and s.systring == 'nogil':

1367 state = s.systring

1368 s.next()

1369 body = p_suite(s)

1370 return Nodes.GILStatNode(pos, state = state, body = body)

1371 else:

1372 manager = p_expr(s)

1373 target = None

1374 if s.sy == 'IDENT' and s.systring == 'as':

1375 s.next()

1376 allow_multi = (s.sy == '(')

1377 target = p_target(s, ':')

1378 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1379 s.error("Multiple with statement target values not allowed without paranthesis")

1380 body = p_suite(s)

1381 return Nodes.WithStatNode(pos, manager = manager,

1382 target = target, body = body)

1383

1384 def p_simple_statement(s, first_statement = 0):

1385 #print "p_simple_statement:", s.sy, s.systring ###

1386 if s.sy == 'global':

1387 node = p_global_statement(s)

1388 elif s.sy == 'print':

1389 node = p_print_statement(s)

1390 elif s.sy == 'exec':

1391 node = p_exec_statement(s)

1392 elif s.sy == 'del':

1393 node = p_del_statement(s)

1394 elif s.sy == 'break':

1395 node = p_break_statement(s)

1396 elif s.sy == 'continue':

1397 node = p_continue_statement(s)

1398 elif s.sy == 'return':

1399 node = p_return_statement(s)

1400 elif s.sy == 'raise':

1401 node = p_raise_statement(s)

1402 elif s.sy in ('import', 'cimport'):

1403 node = p_import_statement(s)

1404 elif s.sy == 'from':

1405 node = p_from_import_statement(s, first_statement = first_statement)

1406 elif s.sy == 'yield':

1407 node = p_yield_expression(s)

1408 elif s.sy == 'assert':

1409 node = p_assert_statement(s)

1410 elif s.sy == 'pass':

1411 node = p_pass_statement(s)

1412 else:

1413 node = p_expression_or_assignment(s)

1414 return node

1415

1416 def p_simple_statement_list(s, ctx, first_statement = 0):

1417 # Parse a series of simple statements on one line

1418 # separated by semicolons.

1419 stat = p_simple_statement(s, first_statement = first_statement)

1420 if s.sy == ';':

1421 stats = [stat]

1422 while s.sy == ';':

1423 #print "p_simple_statement_list: maybe more to follow" ###

1424 s.next()

1425 if s.sy in ('NEWLINE', 'EOF'):

1426 break

1427 stats.append(p_simple_statement(s))

1428 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1429 s.expect_newline("Syntax error in simple statement list")

1430 return stat

1431

1432 def p_compile_time_expr(s):

1433 old = s.compile_time_expr

1434 s.compile_time_expr = 1

1435 expr = p_expr(s)

1436 s.compile_time_expr = old

1437 return expr

1438

1439 def p_DEF_statement(s):

1440 pos = s.position()

1441 denv = s.compile_time_env

1442 s.next() # 'DEF'

1443 name = p_ident(s)

1444 s.expect('=')

1445 expr = p_compile_time_expr(s)

1446 value = expr.compile_time_value(denv)

1447 #print "p_DEF_statement: %s = %r" % (name, value) ###

1448 denv.declare(name, value)

1449 s.expect_newline()

1450 return Nodes.PassStatNode(pos)

1451

1452 def p_IF_statement(s, ctx):

1453 pos = s.position()

1454 saved_eval = s.compile_time_eval

1455 current_eval = saved_eval

1456 denv = s.compile_time_env

1457 result = None

1458 while 1:

1459 s.next() # 'IF' or 'ELIF'

1460 expr = p_compile_time_expr(s)

1461 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1462 body = p_suite(s, ctx)

1463 if s.compile_time_eval:

1464 result = body

1465 current_eval = 0

1466 if s.sy != 'ELIF':

1467 break

1468 if s.sy == 'ELSE':

1469 s.next()

1470 s.compile_time_eval = current_eval

1471 body = p_suite(s, ctx)

1472 if current_eval:

1473 result = body

1474 if not result:

1475 result = Nodes.PassStatNode(pos)

1476 s.compile_time_eval = saved_eval

1477 return result

1478

1479 def p_statement(s, ctx, first_statement = 0):

1480 cdef_flag = ctx.cdef_flag

1481 if s.sy == 'ctypedef':

1482 if ctx.level not in ('module', 'module_pxd'):

1483 s.error("ctypedef statement not allowed here")

1484 #if ctx.api:

1485 # error(s.position(), "'api' not allowed with 'ctypedef'")

1486 return p_ctypedef_statement(s, ctx)

1487 elif s.sy == 'DEF':

1488 return p_DEF_statement(s)

1489 elif s.sy == 'IF':

1490 return p_IF_statement(s, ctx)

1491 elif s.sy == 'DECORATOR':

1492 if ctx.level not in ('module', 'class', 'c_class', 'property'):

1493 s.error('decorator not allowed here')

1494 s.level = ctx.level

1495 decorators = p_decorators(s)

1496 if s.sy != 'def':

1497 s.error("Decorators can only be followed by functions ")

1498 return p_def_statement(s, decorators)

1499 else:

1500 overridable = 0

1501 if s.sy == 'cdef':

1502 cdef_flag = 1

1503 s.next()

1504 if s.sy == 'cpdef':

1505 cdef_flag = 1

1506 overridable = 1

1507 s.next()

1508 if cdef_flag:

1509 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1510 s.error('cdef statement not allowed here')

1511 s.level = ctx.level

1512 return p_cdef_statement(s, ctx(overridable = overridable))

1513 else:

1514 if ctx.api:

1515 error(s.pos, "'api' not allowed with this statement")

1516 elif s.sy == 'def':

1517 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1518 s.error('def statement not allowed here')

1519 s.level = ctx.level

1520 return p_def_statement(s)

1521 elif s.sy == 'class':

1522 if ctx.level != 'module':

1523 s.error("class definition not allowed here")

1524 return p_class_statement(s)

1525 elif s.sy == 'include':

1526 if ctx.level not in ('module', 'module_pxd'):

1527 s.error("include statement not allowed here")

1528 return p_include_statement(s, ctx)

1529 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1530 return p_property_decl(s)

1531 elif s.sy == 'pass' and ctx.level != 'property':

1532 return p_pass_statement(s, with_newline = 1)

1533 else:

1534 if ctx.level in ('c_class_pxd', 'property'):

1535 s.error("Executable statement not allowed here")

1536 if s.sy == 'if':

1537 return p_if_statement(s)

1538 elif s.sy == 'while':

1539 return p_while_statement(s)

1540 elif s.sy == 'for':

1541 return p_for_statement(s)

1542 elif s.sy == 'try':

1543 return p_try_statement(s)

1544 elif s.sy == 'with':

1545 return p_with_statement(s)

1546 else:

1547 return p_simple_statement_list(

1548 s, ctx, first_statement = first_statement)

1549

1550 def p_statement_list(s, ctx, first_statement = 0):

1551 # Parse a series of statements separated by newlines.

1552 pos = s.position()

1553 stats = []

1554 while s.sy not in ('DEDENT', 'EOF'):

1555 stats.append(p_statement(s, ctx, first_statement = first_statement))

1556 first_statement = 0

1557 if len(stats) == 1:

1558 return stats[0]

1559 else:

1560 return Nodes.StatListNode(pos, stats = stats)

1561

1562 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1563 pos = s.position()

1564 s.expect(':')

1565 doc = None

1566 stmts = []

1567 if s.sy == 'NEWLINE':

1568 s.next()

1569 s.expect_indent()

1570 if with_doc or with_pseudo_doc:

1571 doc = p_doc_string(s)

1572 body = p_statement_list(s, ctx)

1573 s.expect_dedent()

1574 else:

1575 if ctx.api:

1576 error(s.pos, "'api' not allowed with this statement")

1577 if ctx.level in ('module', 'class', 'function', 'other'):

1578 body = p_simple_statement_list(s, ctx)

1579 else:

1580 body = p_pass_statement(s)

1581 s.expect_newline("Syntax error in declarations")

1582 if with_doc:

1583 return doc, body

1584 else:

1585 return body

1586

1587 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1588 """

1589 Parses positional and keyword arguments. end_sy_set

1590 should contain any s.sy that terminate the argument list.

1591 Argument expansion (* and **) are not allowed.

1592

1593 type_positions and type_keywords specifies which argument

1594 positions and/or names which should be interpreted as

1595 types. Other arguments will be treated as expressions.

1596

1597 Returns: (positional_args, keyword_args)

1598 """

1599 positional_args = []

1600 keyword_args = []

1601 pos_idx = 0

1602

1603 while s.sy not in end_sy_set:

1604 if s.sy == '*' or s.sy == '**':

1605 s.error('Argument expansion not allowed here.')

1606

1607 was_keyword = False

1608 parsed_type = False

1609 if s.sy == 'IDENT':

1610 # Since we can have either types or expressions as positional args,

1611 # we use a strategy of looking an extra step forward for a '=' and

1612 # if it is a positional arg we backtrack.

1613 ident = s.systring

1614 s.next()

1615 if s.sy == '=':

1616 s.next()

1617 # Is keyword arg

1618 if ident in type_keywords:

1619 arg = p_c_base_type(s)

1620 parsed_type = True

1621 else:

1622 arg = p_simple_expr(s)

1623 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,

1624 value = EncodedString(ident))

1625 keyword_args.append((keyword_node, arg))

1626 was_keyword = True

1627 else:

1628 s.put_back('IDENT', ident)

1629

1630 if not was_keyword:

1631 if pos_idx in type_positions:

1632 arg = p_c_base_type(s)

1633 parsed_type = True

1634 else:

1635 arg = p_simple_expr(s)

1636 positional_args.append(arg)

1637 pos_idx += 1

1638 if len(keyword_args) > 0:

1639 s.error("Non-keyword arg following keyword arg",

1640 pos = arg.pos)

1641

1642 if s.sy != ',':

1643 if s.sy not in end_sy_set:

1644 if parsed_type:

1645 s.error("Expected: type")

1646 else:

1647 s.error("Expected: expression")

1648 break

1649 s.next()

1650 return positional_args, keyword_args

1651

1652 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1653 # If self_flag is true, this is the base type for the

1654 # self argument of a C method of an extension type.

1655 if s.sy == '(':

1656 return p_c_complex_base_type(s)

1657 else:

1658 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1659

1660 def p_calling_convention(s):

1661 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1662 result = s.systring

1663 s.next()

1664 return result

1665 else:

1666 return ""

1667

1668 calling_convention_words = ("__stdcall", "__cdecl")

1669

1670 def p_c_complex_base_type(s):

1671 # s.sy == '('

1672 pos = s.position()

1673 s.next()

1674 base_type = p_c_base_type(s)

1675 declarator = p_c_declarator(s, empty = 1)

1676 s.expect(')')

1677 return Nodes.CComplexBaseTypeNode(pos,

1678 base_type = base_type, declarator = declarator)

1679

1680 def p_c_simple_base_type(s, self_flag, nonempty):

1681 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1682 is_basic = 0

1683 signed = 1

1684 longness = 0

1685 module_path = []

1686 pos = s.position()

1687 if looking_at_base_type(s):

1688 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1689 is_basic = 1

1690 signed, longness = p_sign_and_longness(s)

1691 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1692 name = s.systring

1693 s.next()

1694 else:

1695 name = 'int'

1696 elif looking_at_dotted_name(s):

1697 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1698 name = s.systring

1699 s.next()

1700 while s.sy == '.':

1701 module_path.append(name)

1702 s.next()

1703 name = p_ident(s)

1704 else:

1705 name = s.systring

1706 s.next()

1707 if nonempty and s.sy != 'IDENT':

1708 # Make sure this is not a declaration of a variable or function.

1709 if s.sy == '(':

1710 s.next()

1711 if s.sy == '*' or s.sy == '**':

1712 s.put_back('(', '(')

1713 else:

1714 s.put_back('(', '(')

1715 s.put_back('IDENT', name)

1716 name = None

1717 elif s.sy not in ('*', '**', '['):

1718 s.put_back('IDENT', name)

1719 name = None

1720

1721 type_node = Nodes.CSimpleBaseTypeNode(pos,

1722 name = name, module_path = module_path,

1723 is_basic_c_type = is_basic, signed = signed,

1724 longness = longness, is_self_arg = self_flag)

1725

1726

1727 # Treat trailing [] on type as buffer access if it appears in a context

1728 # where declarator names are required (so that it cannot mean int[] or

1729 # sizeof(int[SIZE]))...

1730 #

1731 # (This means that buffers cannot occur where there can be empty declarators,

1732 # which is an ok restriction to make.)

1733 if nonempty and s.sy == '[':

1734 return p_buffer_access(s, type_node)

1735 else:

1736 return type_node

1737

1738 def p_buffer_access(s, base_type_node):

1739 # s.sy == '['

1740 pos = s.position()

1741 s.next()

1742 positional_args, keyword_args = (

1743 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1744 )

1745 s.expect(']')

1746

1747 keyword_dict = ExprNodes.DictNode(pos,

1748 key_value_pairs = [

1749 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1750 for key, value in keyword_args

1751 ])

1752

1753 result = Nodes.CBufferAccessTypeNode(pos,

1754 positional_args = positional_args,

1755 keyword_args = keyword_dict,

1756 base_type_node = base_type_node)

1757 return result

1758

1759

1760 def looking_at_name(s):

1761 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1762

1763 def looking_at_expr(s):

1764 if s.systring in base_type_start_words:

1765 return False

1766 elif s.sy == 'IDENT':

1767 is_type = False

1768 name = s.systring

1769 dotted_path = []

1770 s.next()

1771 while s.sy == '.':

1772 s.next()

1773 dotted_path.append(s.systring)

1774 s.expect('IDENT')

1775 saved = s.sy, s.systring

1776 if s.sy == 'IDENT':

1777 is_type = True

1778 elif s.sy == '*' or s.sy == '**':

1779 s.next()

1780 is_type = s.sy == ')'

1781 s.put_back(*saved)

1782 elif s.sy == '(':

1783 s.next()

1784 is_type = s.sy == '*'

1785 s.put_back(*saved)

1786 elif s.sy == '[':

1787 s.next()

1788 is_type = s.sy == ']'

1789 s.put_back(*saved)

1790 dotted_path.reverse()

1791 for p in dotted_path:

1792 s.put_back('IDENT', p)

1793 s.put_back('.', '.')

1794 s.put_back('IDENT', name)

1795 return not is_type

1796 else:

1797 return True

1798

1799 def looking_at_base_type(s):

1800 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1801 return s.sy == 'IDENT' and s.systring in base_type_start_words

1802

1803 def looking_at_dotted_name(s):

1804 if s.sy == 'IDENT':

1805 name = s.systring

1806 s.next()

1807 result = s.sy == '.'

1808 s.put_back('IDENT', name)

1809 return result

1810 else:

1811 return 0

1812

1813 basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")

1814

1815 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1816

1817 base_type_start_words = \

1818 basic_c_type_names + sign_and_longness_words

1819

1820 def p_sign_and_longness(s):

1821 signed = 1

1822 longness = 0

1823 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1824 if s.systring == 'unsigned':

1825 signed = 0

1826 elif s.systring == 'signed':

1827 signed = 2

1828 elif s.systring == 'short':

1829 longness = -1

1830 elif s.systring == 'long':

1831 longness += 1

1832 s.next()

1833 return signed, longness

1834

1835 def p_opt_cname(s):

1836 literal = p_opt_string_literal(s)

1837 if literal:

1838 _, cname = literal

1839 else:

1840 cname = None

1841 return cname

1842

1843 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1844 assignable = 0, nonempty = 0,

1845 calling_convention_allowed = 0):

1846 # If empty is true, the declarator must be empty. If nonempty is true,

1847 # the declarator must be nonempty. Otherwise we don't care.

1848 # If cmethod_flag is true, then if this declarator declares

1849 # a function, it's a C method of an extension type.

1850 pos = s.position()

1851 if s.sy == '(':

1852 s.next()

1853 if s.sy == ')' or looking_at_name(s):

1854 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1855 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1856 else:

1857 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1858 cmethod_flag = cmethod_flag,

1859 nonempty = nonempty,

1860 calling_convention_allowed = 1)

1861 s.expect(')')

1862 else:

1863 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1864 assignable, nonempty)

1865 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1866 error(s.position(), "%s on something that is not a function"

1867 % result.calling_convention)

1868 while s.sy in ('[', '('):

1869 pos = s.position()

1870 if s.sy == '[':

1871 result = p_c_array_declarator(s, result)

1872 else: # sy == '('

1873 s.next()

1874 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1875 cmethod_flag = 0

1876 return result

1877

1878 def p_c_array_declarator(s, base):

1879 pos = s.position()

1880 s.next() # '['

1881 if s.sy != ']':

1882 dim = p_expr(s)

1883 else:

1884 dim = None

1885 s.expect(']')

1886 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1887

1888 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

1889 # Opening paren has already been skipped

1890 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

1891 nonempty_declarators = 0)

1892 ellipsis = p_optional_ellipsis(s)

1893 s.expect(')')

1894 nogil = p_nogil(s)

1895 exc_val, exc_check = p_exception_value_clause(s)

1896 with_gil = p_with_gil(s)

1897 return Nodes.CFuncDeclaratorNode(pos,

1898 base = base, args = args, has_varargs = ellipsis,

1899 exception_value = exc_val, exception_check = exc_check,

1900 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

1901

1902 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1903 assignable, nonempty):

1904 pos = s.position()

1905 calling_convention = p_calling_convention(s)

1906 if s.sy == '*':

1907 s.next()

1908 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1909 cmethod_flag = cmethod_flag,

1910 assignable = assignable, nonempty = nonempty)

1911 result = Nodes.CPtrDeclaratorNode(pos,

1912 base = base)

1913 elif s.sy == '**': # scanner returns this as a single token

1914 s.next()

1915 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1916 cmethod_flag = cmethod_flag,

1917 assignable = assignable, nonempty = nonempty)

1918 result = Nodes.CPtrDeclaratorNode(pos,

1919 base = Nodes.CPtrDeclaratorNode(pos,

1920 base = base))

1921 else:

1922 rhs = None

1923 if s.sy == 'IDENT':

1924 name = EncodedString(s.systring)

1925 if empty:

1926 error(s.position(), "Declarator should be empty")

1927 s.next()

1928 cname = p_opt_cname(s)

1929 if s.sy == '=' and assignable:

1930 s.next()

1931 rhs = p_simple_expr(s)

1932 else:

1933 if nonempty:

1934 error(s.position(), "Empty declarator")

1935 name = ""

1936 cname = None

1937 result = Nodes.CNameDeclaratorNode(pos,

1938 name = name, cname = cname, default = rhs)

1939 result.calling_convention = calling_convention

1940 return result

1941

1942 def p_nogil(s):

1943 if s.sy == 'IDENT' and s.systring == 'nogil':

1944 s.next()

1945 return 1

1946 else:

1947 return 0

1948

1949 def p_with_gil(s):

1950 if s.sy == 'with':

1951 s.next()

1952 s.expect_keyword('gil')

1953 return 1

1954 else:

1955 return 0

1956

1957 def p_exception_value_clause(s):

1958 exc_val = None

1959 exc_check = 0

1960 if s.sy == 'except':

1961 s.next()

1962 if s.sy == '*':

1963 exc_check = 1

1964 s.next()

1965 elif s.sy == '+':

1966 exc_check = '+'

1967 s.next()

1968 if s.sy == 'IDENT':

1969 name = s.systring

1970 s.next()

1971 exc_val = p_name(s, name)

1972 else:

1973 if s.sy == '?':

1974 exc_check = 1

1975 s.next()

1976 exc_val = p_simple_expr(s)

1977 return exc_val, exc_check

1978

1979 c_arg_list_terminators = ('*', '**', '.', ')')

1980

1981 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

1982 nonempty_declarators = 0, kw_only = 0):

1983 # Comma-separated list of C argument declarations, possibly empty.

1984 # May have a trailing comma.

1985 args = []

1986 is_self_arg = cmethod_flag

1987 while s.sy not in c_arg_list_terminators:

1988 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

1989 nonempty = nonempty_declarators, kw_only = kw_only))

1990 if s.sy != ',':

1991 break

1992 s.next()

1993 is_self_arg = 0

1994 return args

1995

1996 def p_optional_ellipsis(s):

1997 if s.sy == '.':

1998 expect_ellipsis(s)

1999 return 1

2000 else:

2001 return 0

2002

2003 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

2004 pos = s.position()

2005 not_none = 0

2006 default = None

2007 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2008 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2009 if s.sy == 'not':

2010 s.next()

2011 if s.sy == 'IDENT' and s.systring == 'None':

2012 s.next()

2013 else:

2014 s.error("Expected 'None'")

2015 if not in_pyfunc:

2016 error(pos, "'not None' only allowed in Python functions")

2017 not_none = 1

2018 if s.sy == '=':

2019 s.next()

2020 if 'pxd' in s.level:

2021 if s.sy not in ['*', '?']:

2022 error(pos, "default values cannot be specified in pxd files, use ? or *")

2023 default = ExprNodes.BoolNode(1)

2024 s.next()

2025 else:

2026 default = p_simple_expr(s)

2027 return Nodes.CArgDeclNode(pos,

2028 base_type = base_type,

2029 declarator = declarator,

2030 not_none = not_none,

2031 default = default,

2032 kw_only = kw_only)

2033

2034 def p_api(s):

2035 if s.sy == 'IDENT' and s.systring == 'api':

2036 s.next()

2037 return 1

2038 else:

2039 return 0

2040

2041 def p_cdef_statement(s, ctx):

2042 pos = s.position()

2043 ctx.visibility = p_visibility(s, ctx.visibility)

2044 ctx.api = ctx.api or p_api(s)

2045 if ctx.api:

2046 if ctx.visibility not in ('private', 'public'):

2047 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2048 if (ctx.visibility == 'extern') and s.sy == 'from':

2049 return p_cdef_extern_block(s, pos, ctx)

2050 elif s.sy == 'import':

2051 s.next()

2052 return p_cdef_extern_block(s, pos, ctx)

2053 if p_nogil(s):

2054 ctx.nogil = 1

2055 if s.sy == ':':

2056 return p_cdef_block(s, ctx)

2057 elif s.sy == 'class':

2058 if ctx.level not in ('module', 'module_pxd'):

2059 error(pos, "Extension type definition not allowed here")

2060 #if ctx.api:

2061 # error(pos, "'api' not allowed with extension class")

2062 return p_c_class_definition(s, pos, ctx)

2063 elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:

2064 if ctx.level not in ('module', 'module_pxd'):

2065 error(pos, "C struct/union/enum definition not allowed here")

2066 #if ctx.visibility == 'public':

2067 # error(pos, "Public struct/union/enum definition not implemented")

2068 #if ctx.api:

2069 # error(pos, "'api' not allowed with '%s'" % s.systring)

2070 if s.systring == "enum":

2071 return p_c_enum_definition(s, pos, ctx)

2072 else:

2073 return p_c_struct_or_union_definition(s, pos, ctx)

2074 elif s.sy == 'pass':

2075 node = p_pass_statement(s)

2076 s.expect_newline('Expected a newline')

2077 return node

2078 else:

2079 return p_c_func_or_var_declaration(s, pos, ctx)

2080

2081 def p_cdef_block(s, ctx):

2082 return p_suite(s, ctx(cdef_flag = 1))

2083

2084 def p_cdef_extern_block(s, pos, ctx):

2085 include_file = None

2086 s.expect('from')

2087 if s.sy == '*':

2088 s.next()

2089 else:

2090 _, include_file = p_string_literal(s)

2091 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2092 if p_nogil(s):

2093 ctx.nogil = 1

2094 body = p_suite(s, ctx)

2095 return Nodes.CDefExternNode(pos,

2096 include_file = include_file,

2097 body = body)

2098

2099 struct_union_or_enum = (

2100 "struct", "union", "enum"

2101 )

2102

2103 def p_c_enum_definition(s, pos, ctx):

2104 # s.sy == ident 'enum'

2105 s.next()

2106 if s.sy == 'IDENT':

2107 name = s.systring

2108 s.next()

2109 cname = p_opt_cname(s)

2110 else:

2111 name = None

2112 cname = None

2113 items = None

2114 s.expect(':')

2115 items = []

2116 if s.sy != 'NEWLINE':

2117 p_c_enum_line(s, items)

2118 else:

2119 s.next() # 'NEWLINE'

2120 s.expect_indent()

2121 while s.sy not in ('DEDENT', 'EOF'):

2122 p_c_enum_line(s, items)

2123 s.expect_dedent()

2124 return Nodes.CEnumDefNode(

2125 pos, name = name, cname = cname, items = items,

2126 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2127 in_pxd = ctx.level == 'module_pxd')

2128

2129 def p_c_enum_line(s, items):

2130 if s.sy != 'pass':

2131 p_c_enum_item(s, items)

2132 while s.sy == ',':

2133 s.next()

2134 if s.sy in ('NEWLINE', 'EOF'):

2135 break

2136 p_c_enum_item(s, items)

2137 else:

2138 s.next()

2139 s.expect_newline("Syntax error in enum item list")

2140

2141 def p_c_enum_item(s, items):

2142 pos = s.position()

2143 name = p_ident(s)

2144 cname = p_opt_cname(s)

2145 value = None

2146 if s.sy == '=':

2147 s.next()

2148 value = p_simple_expr(s)

2149 items.append(Nodes.CEnumDefItemNode(pos,

2150 name = name, cname = cname, value = value))

2151

2152 def p_c_struct_or_union_definition(s, pos, ctx):

2153 # s.sy == ident 'struct' or 'union'

2154 kind = s.systring

2155 s.next()

2156 name = p_ident(s)

2157 cname = p_opt_cname(s)

2158 attributes = None

2159 if s.sy == ':':

2160 s.next()

2161 s.expect('NEWLINE')

2162 s.expect_indent()

2163 attributes = []

2164 body_ctx = Ctx()

2165 while s.sy != 'DEDENT':

2166 if s.sy != 'pass':

2167 attributes.append(

2168 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2169 else:

2170 s.next()

2171 s.expect_newline("Expected a newline")

2172 s.expect_dedent()

2173 else:

2174 s.expect_newline("Syntax error in struct or union definition")

2175 return Nodes.CStructOrUnionDefNode(pos,

2176 name = name, cname = cname, kind = kind, attributes = attributes,

2177 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2178 in_pxd = ctx.level == 'module_pxd')

2179

2180 def p_visibility(s, prev_visibility):

2181 pos = s.position()

2182 visibility = prev_visibility

2183 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2184 visibility = s.systring

2185 if prev_visibility != 'private' and visibility != prev_visibility:

2186 s.error("Conflicting visibility options '%s' and '%s'"

2187 % (prev_visibility, visibility))

2188 s.next()

2189 return visibility

2190

2191 def p_c_modifiers(s):

2192 if s.sy == 'IDENT' and s.systring in ('inline',):

2193 modifier = s.systring

2194 s.next()

2195 return [modifier] + p_c_modifiers(s)

2196 return []

2197

2198 def p_c_func_or_var_declaration(s, pos, ctx):

2199 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2200 modifiers = p_c_modifiers(s)

2201 base_type = p_c_base_type(s, nonempty = 1)

2202 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2203 assignable = 1, nonempty = 1)

2204 declarator.overridable = ctx.overridable

2205 if s.sy == ':':

2206 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):

2207 s.error("C function definition not allowed here")

2208 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2209 result = Nodes.CFuncDefNode(pos,

2210 visibility = ctx.visibility,

2211 base_type = base_type,

2212 declarator = declarator,

2213 body = suite,

2214 doc = doc,

2215 modifiers = modifiers,

2216 api = ctx.api,

2217 overridable = ctx.overridable)

2218 else:

2219 #if api:

2220 # error(s.pos, "'api' not allowed with variable declaration")

2221 declarators = [declarator]

2222 while s.sy == ',':

2223 s.next()

2224 if s.sy == 'NEWLINE':

2225 break

2226 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2227 assignable = 1, nonempty = 1)

2228 declarators.append(declarator)

2229 s.expect_newline("Syntax error in C variable declaration")

2230 result = Nodes.CVarDefNode(pos,

2231 visibility = ctx.visibility,

2232 base_type = base_type,

2233 declarators = declarators,

2234 in_pxd = ctx.level == 'module_pxd',

2235 api = ctx.api,

2236 overridable = ctx.overridable)

2237 return result

2238

2239 def p_ctypedef_statement(s, ctx):

2240 # s.sy == 'ctypedef'

2241 pos = s.position()

2242 s.next()

2243 visibility = p_visibility(s, ctx.visibility)

2244 api = p_api(s)

2245 ctx = ctx(typedef_flag = 1, visibility = visibility)

2246 if api:

2247 ctx.api = 1

2248 if s.sy == 'class':

2249 return p_c_class_definition(s, pos, ctx)

2250 elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):

2251 if s.systring == 'enum':

2252 return p_c_enum_definition(s, pos, ctx)

2253 else:

2254 return p_c_struct_or_union_definition(s, pos, ctx)

2255 else:

2256 base_type = p_c_base_type(s, nonempty = 1)

2257 if base_type.name is None:

2258 s.error("Syntax error in ctypedef statement")

2259 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2260 s.expect_newline("Syntax error in ctypedef statement")

2261 return Nodes.CTypeDefNode(

2262 pos, base_type = base_type,

2263 declarator = declarator, visibility = visibility,

2264 in_pxd = ctx.level == 'module_pxd')

2265

2266 def p_decorators(s):

2267 decorators = []

2268 while s.sy == 'DECORATOR':

2269 pos = s.position()

2270 s.next()

2271 decstring = p_dotted_name(s, as_allowed=0)[2]

2272 names = decstring.split('.')

2273 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2274 for name in names[1:]:

2275 decorator = ExprNodes.AttributeNode(pos,

2276 attribute=EncodedString(name),

2277 obj=decorator)

2278 if s.sy == '(':

2279 decorator = p_call(s, decorator)

2280 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2281 s.expect_newline("Expected a newline after decorator")

2282 return decorators

2283

2284 def p_def_statement(s, decorators=None):

2285 # s.sy == 'def'

2286 pos = s.position()

2287 s.next()

2288 name = EncodedString( p_ident(s) )

2289 #args = []

2290 s.expect('(');

2291 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2292 star_arg = None

2293 starstar_arg = None

2294 if s.sy == '*':

2295 s.next()

2296 if s.sy == 'IDENT':

2297 star_arg = p_py_arg_decl(s)

2298 if s.sy == ',':

2299 s.next()

2300 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2301 nonempty_declarators = 1, kw_only = 1))

2302 elif s.sy != ')':

2303 s.error("Syntax error in Python function argument list")

2304 if s.sy == '**':

2305 s.next()

2306 starstar_arg = p_py_arg_decl(s)

2307 s.expect(')')

2308 if p_nogil(s):

2309 error(s.pos, "Python function cannot be declared nogil")

2310 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2311 return Nodes.DefNode(pos, name = name, args = args,

2312 star_arg = star_arg, starstar_arg = starstar_arg,

2313 doc = doc, body = body, decorators = decorators)

2314

2315 def p_py_arg_decl(s):

2316 pos = s.position()

2317 name = p_ident(s)

2318 return Nodes.PyArgDeclNode(pos, name = name)

2319

2320 def p_class_statement(s):

2321 # s.sy == 'class'

2322 pos = s.position()

2323 s.next()

2324 class_name = EncodedString( p_ident(s) )

2325 class_name.encoding = s.source_encoding

2326 if s.sy == '(':

2327 s.next()

2328 base_list = p_simple_expr_list(s)

2329 s.expect(')')

2330 else:

2331 base_list = []

2332 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2333 return Nodes.PyClassDefNode(pos,

2334 name = class_name,

2335 bases = ExprNodes.TupleNode(pos, args = base_list),

2336 doc = doc, body = body)

2337

2338 def p_c_class_definition(s, pos, ctx):

2339 # s.sy == 'class'

2340 s.next()

2341 module_path = []

2342 class_name = p_ident(s)

2343 while s.sy == '.':

2344 s.next()

2345 module_path.append(class_name)

2346 class_name = p_ident(s)

2347 if module_path and ctx.visibility != 'extern':

2348 error(pos, "Qualified class name only allowed for 'extern' C class")

2349 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2350 s.next()

2351 as_name = p_ident(s)

2352 else:

2353 as_name = class_name

2354 objstruct_name = None

2355 typeobj_name = None

2356 base_class_module = None

2357 base_class_name = None

2358 if s.sy == '(':

2359 s.next()

2360 base_class_path = [p_ident(s)]

2361 while s.sy == '.':

2362 s.next()

2363 base_class_path.append(p_ident(s))

2364 if s.sy == ',':

2365 s.error("C class may only have one base class")

2366 s.expect(')')

2367 base_class_module = ".".join(base_class_path[:-1])

2368 base_class_name = base_class_path[-1]

2369 if s.sy == '[':

2370 if ctx.visibility not in ('public', 'extern'):

2371 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2372 objstruct_name, typeobj_name = p_c_class_options(s)

2373 if s.sy == ':':

2374 if ctx.level == 'module_pxd':

2375 body_level = 'c_class_pxd'

2376 else:

2377 body_level = 'c_class'

2378 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2379 else:

2380 s.expect_newline("Syntax error in C class definition")

2381 doc = None

2382 body = None

2383 if ctx.visibility == 'extern':

2384 if not module_path:

2385 error(pos, "Module name required for 'extern' C class")

2386 if typeobj_name:

2387 error(pos, "Type object name specification not allowed for 'extern' C class")

2388 elif ctx.visibility == 'public':

2389 if not objstruct_name:

2390 error(pos, "Object struct name specification required for 'public' C class")

2391 if not typeobj_name:

2392 error(pos, "Type object name specification required for 'public' C class")

2393 elif ctx.visibility == 'private':

2394 if ctx.api:

2395 error(pos, "Only 'public' C class can be declared 'api'")

2396 else:

2397 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2398 return Nodes.CClassDefNode(pos,

2399 visibility = ctx.visibility,

2400 typedef_flag = ctx.typedef_flag,

2401 api = ctx.api,

2402 module_name = ".".join(module_path),

2403 class_name = class_name,

2404 as_name = as_name,

2405 base_class_module = base_class_module,

2406 base_class_name = base_class_name,

2407 objstruct_name = objstruct_name,

2408 typeobj_name = typeobj_name,

2409 in_pxd = ctx.level == 'module_pxd',

2410 doc = doc,

2411 body = body)

2412

2413 def p_c_class_options(s):

2414 objstruct_name = None

2415 typeobj_name = None

2416 s.expect('[')

2417 while 1:

2418 if s.sy != 'IDENT':

2419 break

2420 if s.systring == 'object':

2421 s.next()

2422 objstruct_name = p_ident(s)

2423 elif s.systring == 'type':

2424 s.next()

2425 typeobj_name = p_ident(s)

2426 if s.sy != ',':

2427 break

2428 s.next()

2429 s.expect(']', "Expected 'object' or 'type'")

2430 return objstruct_name, typeobj_name

2431

2432 def p_property_decl(s):

2433 pos = s.position()

2434 s.next() # 'property'

2435 name = p_ident(s)

2436 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2437 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2438

2439 def p_doc_string(s):

2440 if s.sy == 'BEGIN_STRING':

2441 pos = s.position()

2442 kind, result = p_cat_string_literal(s)

2443 if s.sy != 'EOF':

2444 s.expect_newline("Syntax error in doc string")

2445 if kind != 'u':

2446 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2447 if kind == 'b':

2448 result.encoding = None # force a unicode string

2449 return result

2450 else:

2451 return None

2452

2453 def p_code(s, level=None):

2454 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2455 if s.sy != 'EOF':

2456 s.error("Syntax error in statement [%s,%s]" % (

2457 repr(s.sy), repr(s.systring)))

2458 return body

2459

2460 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*([a-z_]+)\s*=(.*)$")

2461

2462 def p_compiler_directive_comments(s):

2463 result = {}

2464 while s.sy == 'commentline':

2465 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2466 if m:

2467 name = m.group(1)

2468 try:

2469 value = Options.parse_option_value(str(name), str(m.group(2).strip()))

2470 except ValueError, e:

2471 s.error(e.args[0], fatal=False)

2472 if value is not None: # can be False!

2473 result[name] = value

2474 s.next()

2475 return result

2476

2477 def p_module(s, pxd, full_module_name):

2478 pos = s.position()

2479

2480 option_comments = p_compiler_directive_comments(s)

2481 s.parse_comments = False

2482

2483 doc = p_doc_string(s)

2484 if pxd:

2485 level = 'module_pxd'

2486 else:

2487 level = 'module'

2488

2489 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2490 if s.sy != 'EOF':

2491 s.error("Syntax error in statement [%s,%s]" % (

2492 repr(s.sy), repr(s.systring)))

2493 return ModuleNode(pos, doc = doc, body = body,

2494 full_module_name = full_module_name,

2495 option_comments = option_comments)

2496

2497 #----------------------------------------------

2498 #

2499 # Debugging

2500 #

2501 #----------------------------------------------

2502

2503 def print_parse_tree(f, node, level, key = None):

2504 from Nodes import Node

2505 ind = " " * level

2506 if node:

2507 f.write(ind)

2508 if key:

2509 f.write("%s: " % key)

2510 t = type(node)

2511 if t == TupleType:

2512 f.write("(%s @ %s\n" % (node[0], node[1]))

2513 for i in xrange(2, len(node)):

2514 print_parse_tree(f, node[i], level+1)

2515 f.write("%s)\n" % ind)

2516 return

2517 elif isinstance(node, Node):

2518 try:

2519 tag = node.tag

2520 except AttributeError:

2521 tag = node.__class__.__name__

2522 f.write("%s @ %s\n" % (tag, node.pos))

2523 for name, value in node.__dict__.items():

2524 if name != 'tag' and name != 'pos':

2525 print_parse_tree(f, value, level+1, name)

2526 return

2527 elif t == ListType:

2528 f.write("[\n")

2529 for i in xrange(len(node)):

2530 print_parse_tree(f, node[i], level+1)

2531 f.write("%s]\n" % ind)

2532 return

2533 f.write("%s%s\n" % (ind, node))

2534