1 # cython: auto_cpdef=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

14 import Nodes

15 import ExprNodes

16 import StringEncoding

17 from StringEncoding import EncodedString, BytesLiteral

18 from ModuleNode import ModuleNode

19 from Errors import error, warning, InternalError

20 from Cython import Utils

21 import Future

22 import Options

23

24 class Ctx(object):

25 # Parsing context

26 level = 'other'

27 visibility = 'private'

28 cdef_flag = 0

29 typedef_flag = 0

30 api = 0

31 overridable = 0

32 nogil = 0

33

34 def __init__(self, **kwds):

35 self.__dict__.update(kwds)

36

37 def __call__(self, **kwds):

38 ctx = Ctx()

39 d = ctx.__dict__

40 d.update(self.__dict__)

41 d.update(kwds)

42 return ctx

43

44 def p_ident(s, message = "Expected an identifier"):

45 if s.sy == 'IDENT':

46 name = s.systring

47 s.next()

48 return name

49 else:

50 s.error(message)

51

52 def p_ident_list(s):

53 names = []

54 while s.sy == 'IDENT':

55 names.append(s.systring)

56 s.next()

57 if s.sy != ',':

58 break

59 s.next()

60 return names

61

62 #------------------------------------------

63 #

64 # Expressions

65 #

66 #------------------------------------------

67

68 def p_binop_operator(s):

69 pos = s.position()

70 op = s.sy

71 s.next()

72 return op, pos

73

74 def p_binop_expr(s, ops, p_sub_expr):

75 n1 = p_sub_expr(s)

76 while s.sy in ops:

77 op, pos = p_binop_operator(s)

78 n2 = p_sub_expr(s)

79 n1 = ExprNodes.binop_node(pos, op, n1, n2)

80 if op == '/':

81 if Future.division in s.context.future_directives:

82 n1.truedivision = True

83 else:

84 n1.truedivision = None # unknown

85 return n1

86

87 #expression: or_test [if or_test else test] | lambda_form

88

89 def p_simple_expr(s):

90 pos = s.position()

91 expr = p_or_test(s)

92 if s.sy == 'if':

93 s.next()

94 test = p_or_test(s)

95 s.expect('else')

96 other = p_test(s)

97 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

98 else:

99 return expr

100

101 #test: or_test | lambda_form

102

103 def p_test(s):

104 return p_or_test(s)

105

106 #or_test: and_test ('or' and_test)*

107

108 def p_or_test(s):

109 return p_rassoc_binop_expr(s, ('or',), p_and_test)

110

111 def p_rassoc_binop_expr(s, ops, p_subexpr):

112 n1 = p_subexpr(s)

113 if s.sy in ops:

114 pos = s.position()

115 op = s.sy

116 s.next()

117 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

118 n1 = ExprNodes.binop_node(pos, op, n1, n2)

119 return n1

120

121 #and_test: not_test ('and' not_test)*

122

123 def p_and_test(s):

124 #return p_binop_expr(s, ('and',), p_not_test)

125 return p_rassoc_binop_expr(s, ('and',), p_not_test)

126

127 #not_test: 'not' not_test | comparison

128

129 def p_not_test(s):

130 if s.sy == 'not':

131 pos = s.position()

132 s.next()

133 return ExprNodes.NotNode(pos, operand = p_not_test(s))

134 else:

135 return p_comparison(s)

136

137 #comparison: expr (comp_op expr)*

138 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

139

140 def p_comparison(s):

141 n1 = p_starred_expr(s)

142 if s.sy in comparison_ops:

143 pos = s.position()

144 op = p_cmp_op(s)

145 n2 = p_starred_expr(s)

146 n1 = ExprNodes.PrimaryCmpNode(pos,

147 operator = op, operand1 = n1, operand2 = n2)

148 if s.sy in comparison_ops:

149 n1.cascade = p_cascaded_cmp(s)

150 return n1

151

152 def p_starred_expr(s):

153 pos = s.position()

154 if s.sy == '*':

155 starred = True

156 s.next()

157 else:

158 starred = False

159 expr = p_bit_expr(s)

160 if starred:

161 expr = ExprNodes.StarredTargetNode(pos, expr)

162 return expr

163

164 def p_cascaded_cmp(s):

165 pos = s.position()

166 op = p_cmp_op(s)

167 n2 = p_starred_expr(s)

168 result = ExprNodes.CascadedCmpNode(pos,

169 operator = op, operand2 = n2)

170 if s.sy in comparison_ops:

171 result.cascade = p_cascaded_cmp(s)

172 return result

173

174 def p_cmp_op(s):

175 if s.sy == 'not':

176 s.next()

177 s.expect('in')

178 op = 'not_in'

179 elif s.sy == 'is':

180 s.next()

181 if s.sy == 'not':

182 s.next()

183 op = 'is_not'

184 else:

185 op = 'is'

186 else:

187 op = s.sy

188 s.next()

189 if op == '<>':

190 op = '!='

191 return op

192

193 comparison_ops = (

194 '<', '>', '==', '>=', '<=', '<>', '!=',

195 'in', 'is', 'not'

196 )

197

198 #expr: xor_expr ('|' xor_expr)*

199

200 def p_bit_expr(s):

201 return p_binop_expr(s, ('|',), p_xor_expr)

202

203 #xor_expr: and_expr ('^' and_expr)*

204

205 def p_xor_expr(s):

206 return p_binop_expr(s, ('^',), p_and_expr)

207

208 #and_expr: shift_expr ('&' shift_expr)*

209

210 def p_and_expr(s):

211 return p_binop_expr(s, ('&',), p_shift_expr)

212

213 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

214

215 def p_shift_expr(s):

216 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

217

218 #arith_expr: term (('+'|'-') term)*

219

220 def p_arith_expr(s):

221 return p_binop_expr(s, ('+', '-'), p_term)

222

223 #term: factor (('*'|'/'|'%') factor)*

224

225 def p_term(s):

226 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

227

228 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

229

230 def p_factor(s):

231 sy = s.sy

232 if sy in ('+', '-', '~'):

233 op = s.sy

234 pos = s.position()

235 s.next()

236 return ExprNodes.unop_node(pos, op, p_factor(s))

237 elif sy == '&':

238 pos = s.position()

239 s.next()

240 arg = p_factor(s)

241 return ExprNodes.AmpersandNode(pos, operand = arg)

242 elif sy == "<":

243 return p_typecast(s)

244 elif sy == 'IDENT' and s.systring == "sizeof":

245 return p_sizeof(s)

246 else:

247 return p_power(s)

248

249 def p_typecast(s):

250 # s.sy == "<"

251 pos = s.position()

252 s.next()

253 base_type = p_c_base_type(s)

254 if base_type.name is None:

255 s.error("Unknown type")

256 declarator = p_c_declarator(s, empty = 1)

257 if s.sy == '?':

258 s.next()

259 typecheck = 1

260 else:

261 typecheck = 0

262 s.expect(">")

263 operand = p_factor(s)

264 return ExprNodes.TypecastNode(pos,

265 base_type = base_type,

266 declarator = declarator,

267 operand = operand,

268 typecheck = typecheck)

269

270 def p_sizeof(s):

271 # s.sy == ident "sizeof"

272 pos = s.position()

273 s.next()

274 s.expect('(')

275 # Here we decide if we are looking at an expression or type

276 # If it is actually a type, but parsable as an expression,

277 # we treat it as an expression here.

278 if looking_at_expr(s):

279 operand = p_simple_expr(s)

280 node = ExprNodes.SizeofVarNode(pos, operand = operand)

281 else:

282 base_type = p_c_base_type(s)

283 declarator = p_c_declarator(s, empty = 1)

284 node = ExprNodes.SizeofTypeNode(pos,

285 base_type = base_type, declarator = declarator)

286 s.expect(')')

287 return node

288

289 def p_yield_expression(s):

290 # s.sy == "yield"

291 pos = s.position()

292 s.next()

293 if s.sy not in ('EOF', 'NEWLINE', ')'):

294 expr = p_expr(s)

295 s.error("generators ('yield') are not currently supported")

296 return Nodes.PassStatNode(pos)

297

298 #power: atom trailer* ('**' factor)*

299

300 def p_power(s):

301 n1 = p_atom(s)

302 while s.sy in ('(', '[', '.'):

303 n1 = p_trailer(s, n1)

304 if s.sy == '**':

305 pos = s.position()

306 s.next()

307 n2 = p_factor(s)

308 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

309 return n1

310

311 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

312

313 def p_trailer(s, node1):

314 pos = s.position()

315 if s.sy == '(':

316 return p_call(s, node1)

317 elif s.sy == '[':

318 return p_index(s, node1)

319 else: # s.sy == '.'

320 s.next()

321 name = EncodedString( p_ident(s) )

322 return ExprNodes.AttributeNode(pos,

323 obj = node1, attribute = name)

324

325 # arglist: argument (',' argument)* [',']

326 # argument: [test '='] test # Really [keyword '='] test

327

328 def p_call(s, function):

329 # s.sy == '('

330 pos = s.position()

331 s.next()

332 positional_args = []

333 keyword_args = []

334 star_arg = None

335 starstar_arg = None

336 while s.sy not in ('**', ')'):

337 if s.sy == '*':

338 if star_arg:

339 s.error("only one star-arg parameter allowed",

340 pos = s.position())

341 s.next()

342 star_arg = p_simple_expr(s)

343 else:

344 arg = p_simple_expr(s)

345 if s.sy == '=':

346 s.next()

347 if not arg.is_name:

348 s.error("Expected an identifier before '='",

349 pos = arg.pos)

350 encoded_name = EncodedString(arg.name)

351 keyword = ExprNodes.IdentifierStringNode(arg.pos,

352 value = encoded_name)

353 arg = p_simple_expr(s)

354 keyword_args.append((keyword, arg))

355 else:

356 if keyword_args:

357 s.error("Non-keyword arg following keyword arg",

358 pos = arg.pos)

359 if star_arg:

360 s.error("Non-keyword arg following star-arg",

361 pos = arg.pos)

362 positional_args.append(arg)

363 if s.sy != ',':

364 break

365 s.next()

366

367 if s.sy == '**':

368 s.next()

369 starstar_arg = p_simple_expr(s)

370 if s.sy == ',':

371 s.next()

372 s.expect(')')

373 if not (keyword_args or star_arg or starstar_arg):

374 return ExprNodes.SimpleCallNode(pos,

375 function = function,

376 args = positional_args)

377 else:

378 arg_tuple = None

379 keyword_dict = None

380 if positional_args or not star_arg:

381 arg_tuple = ExprNodes.TupleNode(pos,

382 args = positional_args)

383 if star_arg:

384 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

385 if arg_tuple:

386 arg_tuple = ExprNodes.binop_node(pos,

387 operator = '+', operand1 = arg_tuple,

388 operand2 = star_arg_tuple)

389 else:

390 arg_tuple = star_arg_tuple

391 if keyword_args:

392 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

393 for key, value in keyword_args]

394 keyword_dict = ExprNodes.DictNode(pos,

395 key_value_pairs = keyword_args)

396 return ExprNodes.GeneralCallNode(pos,

397 function = function,

398 positional_args = arg_tuple,

399 keyword_args = keyword_dict,

400 starstar_arg = starstar_arg)

401

402 #lambdef: 'lambda' [varargslist] ':' test

403

404 #subscriptlist: subscript (',' subscript)* [',']

405

406 def p_index(s, base):

407 # s.sy == '['

408 pos = s.position()

409 s.next()

410 subscripts = p_subscript_list(s)

411 if len(subscripts) == 1 and len(subscripts[0]) == 2:

412 start, stop = subscripts[0]

413 result = ExprNodes.SliceIndexNode(pos,

414 base = base, start = start, stop = stop)

415 else:

416 indexes = make_slice_nodes(pos, subscripts)

417 if len(indexes) == 1:

418 index = indexes[0]

419 else:

420 index = ExprNodes.TupleNode(pos, args = indexes)

421 result = ExprNodes.IndexNode(pos,

422 base = base, index = index)

423 s.expect(']')

424 return result

425

426 def p_subscript_list(s):

427 items = [p_subscript(s)]

428 while s.sy == ',':

429 s.next()

430 if s.sy == ']':

431 break

432 items.append(p_subscript(s))

433 return items

434

435 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

436

437 def p_subscript(s):

438 # Parse a subscript and return a list of

439 # 1, 2 or 3 ExprNodes, depending on how

440 # many slice elements were encountered.

441 pos = s.position()

442 if s.sy == '.':

443 expect_ellipsis(s)

444 return [ExprNodes.EllipsisNode(pos)]

445 else:

446 start = p_slice_element(s, (':',))

447 if s.sy != ':':

448 return [start]

449 s.next()

450 stop = p_slice_element(s, (':', ',', ']'))

451 if s.sy != ':':

452 return [start, stop]

453 s.next()

454 step = p_slice_element(s, (':', ',', ']'))

455 return [start, stop, step]

456

457 def p_slice_element(s, follow_set):

458 # Simple expression which may be missing iff

459 # it is followed by something in follow_set.

460 if s.sy not in follow_set:

461 return p_simple_expr(s)

462 else:

463 return None

464

465 def expect_ellipsis(s):

466 s.expect('.')

467 s.expect('.')

468 s.expect('.')

469

470 def make_slice_nodes(pos, subscripts):

471 # Convert a list of subscripts as returned

472 # by p_subscript_list into a list of ExprNodes,

473 # creating SliceNodes for elements with 2 or

474 # more components.

475 result = []

476 for subscript in subscripts:

477 if len(subscript) == 1:

478 result.append(subscript[0])

479 else:

480 result.append(make_slice_node(pos, *subscript))

481 return result

482

483 def make_slice_node(pos, start, stop = None, step = None):

484 if not start:

485 start = ExprNodes.NoneNode(pos)

486 if not stop:

487 stop = ExprNodes.NoneNode(pos)

488 if not step:

489 step = ExprNodes.NoneNode(pos)

490 return ExprNodes.SliceNode(pos,

491 start = start, stop = stop, step = step)

492

493 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

494

495 def p_atom(s):

496 pos = s.position()

497 sy = s.sy

498 if sy == '(':

499 s.next()

500 if s.sy == ')':

501 result = ExprNodes.TupleNode(pos, args = [])

502 elif s.sy == 'yield':

503 result = p_yield_expression(s)

504 else:

505 result = p_expr(s)

506 s.expect(')')

507 return result

508 elif sy == '[':

509 return p_list_maker(s)

510 elif sy == '{':

511 return p_dict_or_set_maker(s)

512 elif sy == '`':

513 return p_backquote_expr(s)

514 elif sy == 'INT':

515 value = s.systring

516 s.next()

517 unsigned = ""

518 longness = ""

519 while value[-1] in "UuLl":

520 if value[-1] in "Ll":

521 longness += "L"

522 else:

523 unsigned += "U"

524 value = value[:-1]

525 return ExprNodes.IntNode(pos,

526 value = value,

527 unsigned = unsigned,

528 longness = longness)

529 elif sy == 'FLOAT':

530 value = s.systring

531 s.next()

532 return ExprNodes.FloatNode(pos, value = value)

533 elif sy == 'IMAG':

534 value = s.systring[:-1]

535 s.next()

536 return ExprNodes.ImagNode(pos, value = value)

537 elif sy == 'BEGIN_STRING':

538 kind, value = p_cat_string_literal(s)

539 if kind == 'c':

540 return ExprNodes.CharNode(pos, value = value)

541 elif kind == 'u':

542 return ExprNodes.UnicodeNode(pos, value = value)

543 else:

544 return ExprNodes.StringNode(pos, value = value)

545 elif sy == 'IDENT':

546 name = EncodedString( s.systring )

547 s.next()

548 if name == "None":

549 return ExprNodes.NoneNode(pos)

550 elif name == "True":

551 return ExprNodes.BoolNode(pos, value=True)

552 elif name == "False":

553 return ExprNodes.BoolNode(pos, value=False)

554 elif name == "NULL":

555 return ExprNodes.NullNode(pos)

556 else:

557 return p_name(s, name)

558 else:

559 s.error("Expected an identifier or literal")

560

561 def p_name(s, name):

562 pos = s.position()

563 if not s.compile_time_expr and name in s.compile_time_env:

564 value = s.compile_time_env.lookup_here(name)

565 rep = repr(value)

566 if isinstance(value, bool):

567 return ExprNodes.BoolNode(pos, value = value)

568 elif isinstance(value, int):

569 return ExprNodes.IntNode(pos, value = rep)

570 elif isinstance(value, long):

571 return ExprNodes.IntNode(pos, value = rep, longness = "L")

572 elif isinstance(value, float):

573 return ExprNodes.FloatNode(pos, value = rep)

574 elif isinstance(value, (str, unicode)):

575 return ExprNodes.StringNode(pos, value = value)

576 else:

577 error(pos, "Invalid type for compile-time constant: %s"

578 % value.__class__.__name__)

579 return ExprNodes.NameNode(pos, name = name)

580

581 def p_cat_string_literal(s):

582 # A sequence of one or more adjacent string literals.

583 # Returns (kind, value) where kind in ('b', 'c', 'u')

584 kind, value = p_string_literal(s)

585 if kind != 'c':

586 strings = [value]

587 while s.sy == 'BEGIN_STRING':

588 next_kind, next_value = p_string_literal(s)

589 if next_kind == 'c':

590 error(s.position(),

591 "Cannot concatenate char literal with another string or char literal")

592 elif next_kind != kind:

593 # we have to switch to unicode now

594 if kind == 'b':

595 # concatenating a unicode string to byte strings

596 strings = [u''.join([s.decode(s.encoding) for s in strings])]

597 elif kind == 'u':

598 # concatenating a byte string to unicode strings

599 strings.append(next_value.decode(next_value.encoding))

600 kind = 'u'

601 else:

602 strings.append(next_value)

603 if kind == 'u':

604 value = EncodedString( u''.join(strings) )

605 else:

606 value = BytesLiteral( StringEncoding.join_bytes(strings) )

607 value.encoding = s.source_encoding

608 return kind, value

609

610 def p_opt_string_literal(s):

611 if s.sy == 'BEGIN_STRING':

612 return p_string_literal(s)

613 else:

614 return None

615

616 def p_string_literal(s):

617 # A single string or char literal.

618 # Returns (kind, value) where kind in ('b', 'c', 'u')

619 # s.sy == 'BEGIN_STRING'

620 pos = s.position()

621 is_raw = 0

622 kind = s.systring[:1].lower()

623 if kind == 'r':

624 kind = ''

625 is_raw = 1

626 elif kind in 'ub':

627 is_raw = s.systring[1:2].lower() == 'r'

628 elif kind != 'c':

629 kind = ''

630 if Future.unicode_literals in s.context.future_directives:

631 if kind == '':

632 kind = 'u'

633 elif kind == '':

634 kind = 'b'

635 if kind == 'u':

636 chars = StringEncoding.UnicodeLiteralBuilder()

637 else:

638 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

639 while 1:

640 s.next()

641 sy = s.sy

642 #print "p_string_literal: sy =", sy, repr(s.systring) ###

643 if sy == 'CHARS':

644 chars.append(s.systring)

645 elif sy == 'ESCAPE':

646 has_escape = True

647 systr = s.systring

648 if is_raw:

649 if systr == u'\\\n':

650 chars.append(u'\\\n')

651 elif systr == u'\\\"':

652 chars.append(u'"')

653 elif systr == u'\\\'':

654 chars.append(u"'")

655 else:

656 chars.append(systr)

657 else:

658 c = systr[1]

659 if c in u"01234567":

660 chars.append_charval( int(systr[1:], 8) )

661 elif c in u"'\"\\":

662 chars.append(c)

663 elif c in u"abfnrtv":

664 chars.append(

665 StringEncoding.char_from_escape_sequence(systr))

666 elif c == u'\n':

667 pass

668 elif c in u'Uux':

669 if kind == 'u' or c == 'x':

670 chrval = int(systr[2:], 16)

671 if chrval > 1114111: # sys.maxunicode:

672 s.error("Invalid unicode escape '%s'" % systr,

673 pos = pos)

674 elif chrval > 65535:

675 warning(s.position(),

676 "Unicode characters above 65535 are not "

677 "necessarily portable across Python installations", 1)

678 chars.append_charval(chrval)

679 else:

680 # unicode escapes in plain byte strings are not unescaped

681 chars.append(systr)

682 else:

683 chars.append(u'\\' + systr[1:])

684 elif sy == 'NEWLINE':

685 chars.append(u'\n')

686 elif sy == 'END_STRING':

687 break

688 elif sy == 'EOF':

689 s.error("Unclosed string literal", pos = pos)

690 else:

691 s.error(

692 "Unexpected token %r:%r in string literal" %

693 (sy, s.systring))

694 if kind == 'c':

695 value = chars.getchar()

696 if len(value) != 1:

697 error(pos, u"invalid character literal: %r" % value)

698 else:

699 value = chars.getstring()

700 s.next()

701 #print "p_string_literal: value =", repr(value) ###

702 return kind, value

703

704 # list_display ::= "[" [listmaker] "]"

705 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

706 # list_iter ::= list_for | list_if

707 # list_for ::= "for" expression_list "in" testlist [list_iter]

708 # list_if ::= "if" test [list_iter]

709

710 def p_list_maker(s):

711 # s.sy == '['

712 pos = s.position()

713 s.next()

714 if s.sy == ']':

715 s.expect(']')

716 return ExprNodes.ListNode(pos, args = [])

717 expr = p_simple_expr(s)

718 if s.sy == 'for':

719 target = ExprNodes.ListNode(pos, args = [])

720 append = ExprNodes.ComprehensionAppendNode(

721 pos, expr=expr, target=ExprNodes.CloneNode(target))

722 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

723 s.expect(']')

724 return ExprNodes.ComprehensionNode(

725 pos, loop=loop, append=append, target=target)

726 else:

727 exprs = [expr]

728 if s.sy == ',':

729 s.next()

730 exprs += p_simple_expr_list(s)

731 s.expect(']')

732 return ExprNodes.ListNode(pos, args = exprs)

733

734 def p_list_iter(s, body):

735 if s.sy == 'for':

736 return p_list_for(s, body)

737 elif s.sy == 'if':

738 return p_list_if(s, body)

739 else:

740 # insert the 'append' operation into the loop

741 return body

742

743 def p_list_for(s, body):

744 # s.sy == 'for'

745 pos = s.position()

746 s.next()

747 kw = p_for_bounds(s)

748 kw['else_clause'] = None

749 kw['body'] = p_list_iter(s, body)

750 return Nodes.ForStatNode(pos, **kw)

751

752 def p_list_if(s, body):

753 # s.sy == 'if'

754 pos = s.position()

755 s.next()

756 test = p_test(s)

757 return Nodes.IfStatNode(pos,

758 if_clauses = [Nodes.IfClauseNode(pos, condition = test,

759 body = p_list_iter(s, body))],

760 else_clause = None )

761

762 #dictmaker: test ':' test (',' test ':' test)* [',']

763

764 def p_dict_or_set_maker(s):

765 # s.sy == '{'

766 pos = s.position()

767 s.next()

768 if s.sy == '}':

769 s.next()

770 return ExprNodes.DictNode(pos, key_value_pairs = [])

771 item = p_simple_expr(s)

772 if s.sy == ',' or s.sy == '}':

773 # set literal

774 values = [item]

775 while s.sy == ',':

776 s.next()

777 if s.sy == '}':

778 break

779 values.append( p_simple_expr(s) )

780 s.expect('}')

781 return ExprNodes.SetNode(pos, args=values)

782 elif s.sy == 'for':

783 # set comprehension

784 target = ExprNodes.SetNode(pos, args=[])

785 append = ExprNodes.ComprehensionAppendNode(

786 item.pos, expr=item, target=ExprNodes.CloneNode(target))

787 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

788 s.expect('}')

789 return ExprNodes.ComprehensionNode(

790 pos, loop=loop, append=append, target=target)

791 elif s.sy == ':':

792 # dict literal or comprehension

793 key = item

794 s.next()

795 value = p_simple_expr(s)

796 if s.sy == 'for':

797 # dict comprehension

798 target = ExprNodes.DictNode(pos, key_value_pairs = [])

799 append = ExprNodes.DictComprehensionAppendNode(

800 item.pos, key_expr=key, value_expr=value,

801 target=ExprNodes.CloneNode(target))

802 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

803 s.expect('}')

804 return ExprNodes.ComprehensionNode(

805 pos, loop=loop, append=append, target=target)

806 else:

807 # dict literal

808 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

809 while s.sy == ',':

810 s.next()

811 if s.sy == '}':

812 break

813 key = p_simple_expr(s)

814 s.expect(':')

815 value = p_simple_expr(s)

816 items.append(

817 ExprNodes.DictItemNode(key.pos, key=key, value=value))

818 s.expect('}')

819 return ExprNodes.DictNode(pos, key_value_pairs=items)

820 else:

821 # raise an error

822 s.expect('}')

823 return ExprNodes.DictNode(pos, key_value_pairs = [])

824

825 def p_backquote_expr(s):

826 # s.sy == '`'

827 pos = s.position()

828 s.next()

829 arg = p_expr(s)

830 s.expect('`')

831 return ExprNodes.BackquoteNode(pos, arg = arg)

832

833 def p_simple_expr_list(s):

834 exprs = []

835 while s.sy not in expr_terminators:

836 expr = p_simple_expr(s)

837 exprs.append(expr)

838 if s.sy != ',':

839 break

840 s.next()

841 return exprs

842

843 def p_expr(s):

844 pos = s.position()

845 expr = p_simple_expr(s)

846 if s.sy == ',':

847 s.next()

848 exprs = [expr] + p_simple_expr_list(s)

849 return ExprNodes.TupleNode(pos, args = exprs)

850 else:

851 return expr

852

853

854 #testlist: test (',' test)* [',']

855 # differs from p_expr only in the fact that it cannot contain conditional expressions

856

857 def p_testlist(s):

858 pos = s.position()

859 expr = p_test(s)

860 if s.sy == ',':

861 exprs = [expr]

862 while s.sy == ',':

863 s.next()

864 exprs.append(p_test(s))

865 return ExprNodes.TupleNode(pos, args = exprs)

866 else:

867 return expr

868

869 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

870

871 #-------------------------------------------------------

872 #

873 # Statements

874 #

875 #-------------------------------------------------------

876

877 def p_global_statement(s):

878 # assume s.sy == 'global'

879 pos = s.position()

880 s.next()

881 names = p_ident_list(s)

882 return Nodes.GlobalNode(pos, names = names)

883

884 def p_expression_or_assignment(s):

885 expr_list = [p_expr(s)]

886 while s.sy == '=':

887 s.next()

888 expr_list.append(p_expr(s))

889 if len(expr_list) == 1:

890 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

891 lhs = expr_list[0]

892 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

893 error(lhs.pos, "Illegal operand for inplace operation.")

894 operator = s.sy[:-1]

895 s.next()

896 rhs = p_expr(s)

897 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

898 expr = expr_list[0]

899 if isinstance(expr, ExprNodes.StringNode):

900 return Nodes.PassStatNode(expr.pos)

901 else:

902 return Nodes.ExprStatNode(expr.pos, expr = expr)

903 else:

904 expr_list_list = []

905 flatten_parallel_assignments(expr_list, expr_list_list)

906 nodes = []

907 for expr_list in expr_list_list:

908 lhs_list = expr_list[:-1]

909 rhs = expr_list[-1]

910 if len(lhs_list) == 1:

911 node = Nodes.SingleAssignmentNode(rhs.pos,

912 lhs = lhs_list[0], rhs = rhs)

913 else:

914 node = Nodes.CascadedAssignmentNode(rhs.pos,

915 lhs_list = lhs_list, rhs = rhs)

916 nodes.append(node)

917 if len(nodes) == 1:

918 return nodes[0]

919 else:

920 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

921

922 def flatten_parallel_assignments(input, output):

923 # The input is a list of expression nodes, representing the LHSs

924 # and RHS of one (possibly cascaded) assignment statement. For

925 # sequence constructors, rearranges the matching parts of both

926 # sides into a list of equivalent assignments between the

927 # individual elements. This transformation is applied

928 # recursively, so that nested structures get matched as well.

929 rhs = input[-1]

930 if not rhs.is_sequence_constructor:

931 output.append(input)

932 return

933

934 rhs_size = len(rhs.args)

935 lhs_targets = [ [] for _ in range(rhs_size) ]

936 starred_assignments = []

937 for lhs in input[:-1]:

938 if not lhs.is_sequence_constructor:

939 if lhs.is_starred:

940 error(lhs.pos, "starred assignment target must be in a list or tuple")

941 output.append([lhs,rhs])

942 continue

943 lhs_size = len(lhs.args)

944 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])

945 if starred_targets:

946 if starred_targets > 1:

947 error(lhs.pos, "more than 1 starred expression in assignment")

948 output.append([lhs,rhs])

949 continue

950 elif lhs_size - starred_targets > rhs_size:

951 error(lhs.pos, "need more than %d value%s to unpack"

952 % (rhs_size, (rhs_size != 1) and 's' or ''))

953 output.append([lhs,rhs])

954 continue

955 map_starred_assignment(lhs_targets, starred_assignments,

956 lhs.args, rhs.args)

957 else:

958 if lhs_size > rhs_size:

959 error(lhs.pos, "need more than %d value%s to unpack"

960 % (rhs_size, (rhs_size != 1) and 's' or ''))

961 output.append([lhs,rhs])

962 continue

963 elif lhs_size < rhs_size:

964 error(lhs.pos, "too many values to unpack (expected %d, got %d)"

965 % (lhs_size, rhs_size))

966 output.append([lhs,rhs])

967 continue

968 else:

969 for targets, expr in zip(lhs_targets, lhs.args):

970 targets.append(expr)

971

972 # recursively flatten partial assignments

973 for cascade, rhs in zip(lhs_targets, rhs.args):

974 if cascade:

975 cascade.append(rhs)

976 flatten_parallel_assignments(cascade, output)

977

978 # recursively flatten starred assignments

979 for cascade in starred_assignments:

980 if cascade[0].is_sequence_constructor:

981 flatten_parallel_assignments(cascade, output)

982 else:

983 output.append(cascade)

984

985 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):

986 # Appends the fixed-position LHS targets to the target list that

987 # appear left and right of the starred argument.

988 #

989 # The starred_assignments list receives a new tuple

990 # (lhs_target, rhs_values_list) that maps the remaining arguments

991 # (those that match the starred target) to a list.

992

993 # left side of the starred target

994 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):

995 if expr.is_starred:

996 starred = i

997 lhs_remaining = len(lhs_args) - i - 1

998 break

999 targets.append(expr)

1000 else:

1001 raise InternalError("no starred arg found when splitting starred assignment")

1002

1003 # right side of the starred target

1004 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],

1005 lhs_args[-lhs_remaining:])):

1006 targets.append(expr)

1007

1008 # the starred target itself, must be assigned a (potentially empty) list

1009 target = lhs_args[starred].target # unpack starred node

1010 starred_rhs = rhs_args[starred:]

1011 if lhs_remaining:

1012 starred_rhs = starred_rhs[:-lhs_remaining]

1013 if starred_rhs:

1014 pos = starred_rhs[0].pos

1015 else:

1016 pos = target.pos

1017 starred_assignments.append([

1018 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])

1019

1020

1021 def p_print_statement(s):

1022 # s.sy == 'print'

1023 pos = s.position()

1024 s.next()

1025 if s.sy == '>>':

1026 s.error("'print >>' not yet implemented")

1027 args = []

1028 ends_with_comma = 0

1029 if s.sy not in ('NEWLINE', 'EOF'):

1030 args.append(p_simple_expr(s))

1031 while s.sy == ',':

1032 s.next()

1033 if s.sy in ('NEWLINE', 'EOF'):

1034 ends_with_comma = 1

1035 break

1036 args.append(p_simple_expr(s))

1037 arg_tuple = ExprNodes.TupleNode(pos, args = args)

1038 return Nodes.PrintStatNode(pos,

1039 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

1040

1041 def p_exec_statement(s):

1042 # s.sy == 'exec'

1043 pos = s.position()

1044 s.next()

1045 args = [ p_bit_expr(s) ]

1046 if s.sy == 'in':

1047 s.next()

1048 args.append(p_simple_expr(s))

1049 if s.sy == ',':

1050 s.next()

1051 args.append(p_simple_expr(s))

1052 else:

1053 error(pos, "'exec' currently requires a target mapping (globals/locals)")

1054 return Nodes.ExecStatNode(pos, args = args)

1055

1056 def p_del_statement(s):

1057 # s.sy == 'del'

1058 pos = s.position()

1059 s.next()

1060 args = p_simple_expr_list(s)

1061 return Nodes.DelStatNode(pos, args = args)

1062

1063 def p_pass_statement(s, with_newline = 0):

1064 pos = s.position()

1065 s.expect('pass')

1066 if with_newline:

1067 s.expect_newline("Expected a newline")

1068 return Nodes.PassStatNode(pos)

1069

1070 def p_break_statement(s):

1071 # s.sy == 'break'

1072 pos = s.position()

1073 s.next()

1074 return Nodes.BreakStatNode(pos)

1075

1076 def p_continue_statement(s):

1077 # s.sy == 'continue'

1078 pos = s.position()

1079 s.next()

1080 return Nodes.ContinueStatNode(pos)

1081

1082 def p_return_statement(s):

1083 # s.sy == 'return'

1084 pos = s.position()

1085 s.next()

1086 if s.sy not in statement_terminators:

1087 value = p_expr(s)

1088 else:

1089 value = None

1090 return Nodes.ReturnStatNode(pos, value = value)

1091

1092 def p_raise_statement(s):

1093 # s.sy == 'raise'

1094 pos = s.position()

1095 s.next()

1096 exc_type = None

1097 exc_value = None

1098 exc_tb = None

1099 if s.sy not in statement_terminators:

1100 exc_type = p_simple_expr(s)

1101 if s.sy == ',':

1102 s.next()

1103 exc_value = p_simple_expr(s)

1104 if s.sy == ',':

1105 s.next()

1106 exc_tb = p_simple_expr(s)

1107 if exc_type or exc_value or exc_tb:

1108 return Nodes.RaiseStatNode(pos,

1109 exc_type = exc_type,

1110 exc_value = exc_value,

1111 exc_tb = exc_tb)

1112 else:

1113 return Nodes.ReraiseStatNode(pos)

1114

1115 def p_import_statement(s):

1116 # s.sy in ('import', 'cimport')

1117 pos = s.position()

1118 kind = s.sy

1119 s.next()

1120 items = [p_dotted_name(s, as_allowed = 1)]

1121 while s.sy == ',':

1122 s.next()

1123 items.append(p_dotted_name(s, as_allowed = 1))

1124 stats = []

1125 for pos, target_name, dotted_name, as_name in items:

1126 dotted_name = EncodedString(dotted_name)

1127 if kind == 'cimport':

1128 stat = Nodes.CImportStatNode(pos,

1129 module_name = dotted_name,

1130 as_name = as_name)

1131 else:

1132 if as_name and "." in dotted_name:

1133 name_list = ExprNodes.ListNode(pos, args = [

1134 ExprNodes.IdentifierStringNode(

1135 pos, value = EncodedString("*"))])

1136 else:

1137 name_list = None

1138 stat = Nodes.SingleAssignmentNode(pos,

1139 lhs = ExprNodes.NameNode(pos,

1140 name = as_name or target_name),

1141 rhs = ExprNodes.ImportNode(pos,

1142 module_name = ExprNodes.IdentifierStringNode(

1143 pos, value = dotted_name),

1144 name_list = name_list))

1145 stats.append(stat)

1146 return Nodes.StatListNode(pos, stats = stats)

1147

1148 def p_from_import_statement(s, first_statement = 0):

1149 # s.sy == 'from'

1150 pos = s.position()

1151 s.next()

1152 (dotted_name_pos, _, dotted_name, _) = \

1153 p_dotted_name(s, as_allowed = 0)

1154 if s.sy in ('import', 'cimport'):

1155 kind = s.sy

1156 s.next()

1157 else:

1158 s.error("Expected 'import' or 'cimport'")

1159 is_cimport = kind == 'cimport'

1160 is_parenthesized = False

1161 if s.sy == '*':

1162 imported_names = [(s.position(), "*", None, None)]

1163 s.next()

1164 else:

1165 if s.sy == '(':

1166 is_parenthesized = True

1167 s.next()

1168 imported_names = [p_imported_name(s, is_cimport)]

1169 while s.sy == ',':

1170 s.next()

1171 imported_names.append(p_imported_name(s, is_cimport))

1172 if is_parenthesized:

1173 s.expect(')')

1174 dotted_name = EncodedString(dotted_name)

1175 if dotted_name == '__future__':

1176 if not first_statement:

1177 s.error("from __future__ imports must occur at the beginning of the file")

1178 else:

1179 for (name_pos, name, as_name, kind) in imported_names:

1180 if name == "braces":

1181 s.error("not a chance", name_pos)

1182 break

1183 try:

1184 directive = getattr(Future, name)

1185 except AttributeError:

1186 s.error("future feature %s is not defined" % name, name_pos)

1187 break

1188 s.context.future_directives.add(directive)

1189 return Nodes.PassStatNode(pos)

1190 elif kind == 'cimport':

1191 return Nodes.FromCImportStatNode(pos,

1192 module_name = dotted_name,

1193 imported_names = imported_names)

1194 else:

1195 imported_name_strings = []

1196 items = []

1197 for (name_pos, name, as_name, kind) in imported_names:

1198 encoded_name = EncodedString(name)

1199 imported_name_strings.append(

1200 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1201 items.append(

1202 (name,

1203 ExprNodes.NameNode(name_pos,

1204 name = as_name or name)))

1205 import_list = ExprNodes.ListNode(

1206 imported_names[0][0], args = imported_name_strings)

1207 dotted_name = EncodedString(dotted_name)

1208 return Nodes.FromImportStatNode(pos,

1209 module = ExprNodes.ImportNode(dotted_name_pos,

1210 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1211 name_list = import_list),

1212 items = items)

1213

1214 imported_name_kinds = ('class', 'struct', 'union')

1215

1216 def p_imported_name(s, is_cimport):

1217 pos = s.position()

1218 kind = None

1219 if is_cimport and s.systring in imported_name_kinds:

1220 kind = s.systring

1221 s.next()

1222 name = p_ident(s)

1223 as_name = p_as_name(s)

1224 return (pos, name, as_name, kind)

1225

1226 def p_dotted_name(s, as_allowed):

1227 pos = s.position()

1228 target_name = p_ident(s)

1229 as_name = None

1230 names = [target_name]

1231 while s.sy == '.':

1232 s.next()

1233 names.append(p_ident(s))

1234 if as_allowed:

1235 as_name = p_as_name(s)

1236 return (pos, target_name, u'.'.join(names), as_name)

1237

1238 def p_as_name(s):

1239 if s.sy == 'IDENT' and s.systring == 'as':

1240 s.next()

1241 return p_ident(s)

1242 else:

1243 return None

1244

1245 def p_assert_statement(s):

1246 # s.sy == 'assert'

1247 pos = s.position()

1248 s.next()

1249 cond = p_simple_expr(s)

1250 if s.sy == ',':

1251 s.next()

1252 value = p_simple_expr(s)

1253 else:

1254 value = None

1255 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1256

1257 statement_terminators = (';', 'NEWLINE', 'EOF')

1258

1259 def p_if_statement(s):

1260 # s.sy == 'if'

1261 pos = s.position()

1262 s.next()

1263 if_clauses = [p_if_clause(s)]

1264 while s.sy == 'elif':

1265 s.next()

1266 if_clauses.append(p_if_clause(s))

1267 else_clause = p_else_clause(s)

1268 return Nodes.IfStatNode(pos,

1269 if_clauses = if_clauses, else_clause = else_clause)

1270

1271 def p_if_clause(s):

1272 pos = s.position()

1273 test = p_simple_expr(s)

1274 body = p_suite(s)

1275 return Nodes.IfClauseNode(pos,

1276 condition = test, body = body)

1277

1278 def p_else_clause(s):

1279 if s.sy == 'else':

1280 s.next()

1281 return p_suite(s)

1282 else:

1283 return None

1284

1285 def p_while_statement(s):

1286 # s.sy == 'while'

1287 pos = s.position()

1288 s.next()

1289 test = p_simple_expr(s)

1290 body = p_suite(s)

1291 else_clause = p_else_clause(s)

1292 return Nodes.WhileStatNode(pos,

1293 condition = test, body = body,

1294 else_clause = else_clause)

1295

1296 def p_for_statement(s):

1297 # s.sy == 'for'

1298 pos = s.position()

1299 s.next()

1300 kw = p_for_bounds(s)

1301 kw['body'] = p_suite(s)

1302 kw['else_clause'] = p_else_clause(s)

1303 return Nodes.ForStatNode(pos, **kw)

1304

1305 def p_for_bounds(s):

1306 target = p_for_target(s)

1307 if s.sy == 'in':

1308 s.next()

1309 iterator = p_for_iterator(s)

1310 return { 'target': target, 'iterator': iterator }

1311 else:

1312 if s.sy == 'from':

1313 s.next()

1314 bound1 = p_bit_expr(s)

1315 else:

1316 # Support shorter "for a <= x < b" syntax

1317 bound1, target = target, None

1318 rel1 = p_for_from_relation(s)

1319 name2_pos = s.position()

1320 name2 = p_ident(s)

1321 rel2_pos = s.position()

1322 rel2 = p_for_from_relation(s)

1323 bound2 = p_bit_expr(s)

1324 step = p_for_from_step(s)

1325 if target is None:

1326 target = ExprNodes.NameNode(name2_pos, name = name2)

1327 else:

1328 if not target.is_name:

1329 error(target.pos,

1330 "Target of for-from statement must be a variable name")

1331 elif name2 != target.name:

1332 error(name2_pos,

1333 "Variable name in for-from range does not match target")

1334 if rel1[0] != rel2[0]:

1335 error(rel2_pos,

1336 "Relation directions in for-from do not match")

1337 return {'target': target,

1338 'bound1': bound1,

1339 'relation1': rel1,

1340 'relation2': rel2,

1341 'bound2': bound2,

1342 'step': step }

1343

1344 def p_for_from_relation(s):

1345 if s.sy in inequality_relations:

1346 op = s.sy

1347 s.next()

1348 return op

1349 else:

1350 s.error("Expected one of '<', '<=', '>' '>='")

1351

1352 def p_for_from_step(s):

1353 if s.sy == 'by':

1354 s.next()

1355 step = p_bit_expr(s)

1356 return step

1357 else:

1358 return None

1359

1360 inequality_relations = ('<', '<=', '>', '>=')

1361

1362 def p_target(s, terminator):

1363 pos = s.position()

1364 expr = p_starred_expr(s)

1365 if s.sy == ',':

1366 s.next()

1367 exprs = [expr]

1368 while s.sy != terminator:

1369 exprs.append(p_starred_expr(s))

1370 if s.sy != ',':

1371 break

1372 s.next()

1373 return ExprNodes.TupleNode(pos, args = exprs)

1374 else:

1375 return expr

1376

1377 def p_for_target(s):

1378 return p_target(s, 'in')

1379

1380 def p_for_iterator(s):

1381 pos = s.position()

1382 expr = p_testlist(s)

1383 return ExprNodes.IteratorNode(pos, sequence = expr)

1384

1385 def p_try_statement(s):

1386 # s.sy == 'try'

1387 pos = s.position()

1388 s.next()

1389 body = p_suite(s)

1390 except_clauses = []

1391 else_clause = None

1392 if s.sy in ('except', 'else'):

1393 while s.sy == 'except':

1394 except_clauses.append(p_except_clause(s))

1395 if s.sy == 'else':

1396 s.next()

1397 else_clause = p_suite(s)

1398 body = Nodes.TryExceptStatNode(pos,

1399 body = body, except_clauses = except_clauses,

1400 else_clause = else_clause)

1401 if s.sy != 'finally':

1402 return body

1403 # try-except-finally is equivalent to nested try-except/try-finally

1404 if s.sy == 'finally':

1405 s.next()

1406 finally_clause = p_suite(s)

1407 return Nodes.TryFinallyStatNode(pos,

1408 body = body, finally_clause = finally_clause)

1409 else:

1410 s.error("Expected 'except' or 'finally'")

1411

1412 def p_except_clause(s):

1413 # s.sy == 'except'

1414 pos = s.position()

1415 s.next()

1416 exc_type = None

1417 exc_value = None

1418 if s.sy != ':':

1419 exc_type = p_simple_expr(s)

1420 if s.sy == ',':

1421 s.next()

1422 exc_value = p_simple_expr(s)

1423 body = p_suite(s)

1424 return Nodes.ExceptClauseNode(pos,

1425 pattern = exc_type, target = exc_value, body = body)

1426

1427 def p_include_statement(s, ctx):

1428 pos = s.position()

1429 s.next() # 'include'

1430 _, include_file_name = p_string_literal(s)

1431 s.expect_newline("Syntax error in include statement")

1432 if s.compile_time_eval:

1433 include_file_path = s.context.find_include_file(include_file_name, pos)

1434 if include_file_path:

1435 s.included_files.append(include_file_name)

1436 f = Utils.open_source_file(include_file_path, mode="rU")

1437 source_desc = FileSourceDescriptor(include_file_path)

1438 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1439 try:

1440 tree = p_statement_list(s2, ctx)

1441 finally:

1442 f.close()

1443 return tree

1444 else:

1445 return None

1446 else:

1447 return Nodes.PassStatNode(pos)

1448

1449 def p_with_statement(s):

1450 pos = s.position()

1451 s.next() # 'with'

1452 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1453 if s.sy == 'IDENT' and s.systring == 'nogil':

1454 state = s.systring

1455 s.next()

1456 body = p_suite(s)

1457 return Nodes.GILStatNode(pos, state = state, body = body)

1458 else:

1459 manager = p_expr(s)

1460 target = None

1461 if s.sy == 'IDENT' and s.systring == 'as':

1462 s.next()

1463 allow_multi = (s.sy == '(')

1464 target = p_target(s, ':')

1465 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1466 s.error("Multiple with statement target values not allowed without paranthesis")

1467 body = p_suite(s)

1468 return Nodes.WithStatNode(pos, manager = manager,

1469 target = target, body = body)

1470

1471 def p_simple_statement(s, first_statement = 0):

1472 #print "p_simple_statement:", s.sy, s.systring ###

1473 if s.sy == 'global':

1474 node = p_global_statement(s)

1475 elif s.sy == 'print':

1476 node = p_print_statement(s)

1477 elif s.sy == 'exec':

1478 node = p_exec_statement(s)

1479 elif s.sy == 'del':

1480 node = p_del_statement(s)

1481 elif s.sy == 'break':

1482 node = p_break_statement(s)

1483 elif s.sy == 'continue':

1484 node = p_continue_statement(s)

1485 elif s.sy == 'return':

1486 node = p_return_statement(s)

1487 elif s.sy == 'raise':

1488 node = p_raise_statement(s)

1489 elif s.sy in ('import', 'cimport'):

1490 node = p_import_statement(s)

1491 elif s.sy == 'from':

1492 node = p_from_import_statement(s, first_statement = first_statement)

1493 elif s.sy == 'yield':

1494 node = p_yield_expression(s)

1495 elif s.sy == 'assert':

1496 node = p_assert_statement(s)

1497 elif s.sy == 'pass':

1498 node = p_pass_statement(s)

1499 else:

1500 node = p_expression_or_assignment(s)

1501 return node

1502

1503 def p_simple_statement_list(s, ctx, first_statement = 0):

1504 # Parse a series of simple statements on one line

1505 # separated by semicolons.

1506 stat = p_simple_statement(s, first_statement = first_statement)

1507 if s.sy == ';':

1508 stats = [stat]

1509 while s.sy == ';':

1510 #print "p_simple_statement_list: maybe more to follow" ###

1511 s.next()

1512 if s.sy in ('NEWLINE', 'EOF'):

1513 break

1514 stats.append(p_simple_statement(s))

1515 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1516 s.expect_newline("Syntax error in simple statement list")

1517 return stat

1518

1519 def p_compile_time_expr(s):

1520 old = s.compile_time_expr

1521 s.compile_time_expr = 1

1522 expr = p_expr(s)

1523 s.compile_time_expr = old

1524 return expr

1525

1526 def p_DEF_statement(s):

1527 pos = s.position()

1528 denv = s.compile_time_env

1529 s.next() # 'DEF'

1530 name = p_ident(s)

1531 s.expect('=')

1532 expr = p_compile_time_expr(s)

1533 value = expr.compile_time_value(denv)

1534 #print "p_DEF_statement: %s = %r" % (name, value) ###

1535 denv.declare(name, value)

1536 s.expect_newline()

1537 return Nodes.PassStatNode(pos)

1538

1539 def p_IF_statement(s, ctx):

1540 pos = s.position()

1541 saved_eval = s.compile_time_eval

1542 current_eval = saved_eval

1543 denv = s.compile_time_env

1544 result = None

1545 while 1:

1546 s.next() # 'IF' or 'ELIF'

1547 expr = p_compile_time_expr(s)

1548 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1549 body = p_suite(s, ctx)

1550 if s.compile_time_eval:

1551 result = body

1552 current_eval = 0

1553 if s.sy != 'ELIF':

1554 break

1555 if s.sy == 'ELSE':

1556 s.next()

1557 s.compile_time_eval = current_eval

1558 body = p_suite(s, ctx)

1559 if current_eval:

1560 result = body

1561 if not result:

1562 result = Nodes.PassStatNode(pos)

1563 s.compile_time_eval = saved_eval

1564 return result

1565

1566 def p_statement(s, ctx, first_statement = 0):

1567 cdef_flag = ctx.cdef_flag

1568 decorators = None

1569 if s.sy == 'ctypedef':

1570 if ctx.level not in ('module', 'module_pxd'):

1571 s.error("ctypedef statement not allowed here")

1572 #if ctx.api:

1573 # error(s.position(), "'api' not allowed with 'ctypedef'")

1574 return p_ctypedef_statement(s, ctx)

1575 elif s.sy == 'DEF':

1576 return p_DEF_statement(s)

1577 elif s.sy == 'IF':

1578 return p_IF_statement(s, ctx)

1579 elif s.sy == 'DECORATOR':

1580 if ctx.level not in ('module', 'class', 'c_class', 'property', 'module_pxd', 'c_class_pxd'):

1581 print ctx.level

1582 s.error('decorator not allowed here')

1583 s.level = ctx.level

1584 decorators = p_decorators(s)

1585 if s.sy not in ('def', 'cdef', 'cpdef', 'class'):

1586 s.error("Decorators can only be followed by functions or classes")

1587 elif s.sy == 'pass' and cdef_flag:

1588 # empty cdef block

1589 return p_pass_statement(s, with_newline = 1)

1590

1591 overridable = 0

1592 if s.sy == 'cdef':

1593 cdef_flag = 1

1594 s.next()

1595 elif s.sy == 'cpdef':

1596 cdef_flag = 1

1597 overridable = 1

1598 s.next()

1599 if cdef_flag:

1600 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1601 s.error('cdef statement not allowed here')

1602 s.level = ctx.level

1603 node = p_cdef_statement(s, ctx(overridable = overridable))

1604 if decorators is not None:

1605 if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):

1606 s.error("Decorators can only be followed by functions or Python classes")

1607 node.decorators = decorators

1608 return node

1609 else:

1610 if ctx.api:

1611 error(s.pos, "'api' not allowed with this statement")

1612 elif s.sy == 'def':

1613 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1614 s.error('def statement not allowed here')

1615 s.level = ctx.level

1616 return p_def_statement(s, decorators)

1617 elif s.sy == 'class':

1618 if ctx.level != 'module':

1619 s.error("class definition not allowed here")

1620 return p_class_statement(s, decorators)

1621 elif s.sy == 'include':

1622 if ctx.level not in ('module', 'module_pxd'):

1623 s.error("include statement not allowed here")

1624 return p_include_statement(s, ctx)

1625 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1626 return p_property_decl(s)

1627 elif s.sy == 'pass' and ctx.level != 'property':

1628 return p_pass_statement(s, with_newline = 1)

1629 else:

1630 if ctx.level in ('c_class_pxd', 'property'):

1631 s.error("Executable statement not allowed here")

1632 if s.sy == 'if':

1633 return p_if_statement(s)

1634 elif s.sy == 'while':

1635 return p_while_statement(s)

1636 elif s.sy == 'for':

1637 return p_for_statement(s)

1638 elif s.sy == 'try':

1639 return p_try_statement(s)

1640 elif s.sy == 'with':

1641 return p_with_statement(s)

1642 else:

1643 return p_simple_statement_list(

1644 s, ctx, first_statement = first_statement)

1645

1646 def p_statement_list(s, ctx, first_statement = 0):

1647 # Parse a series of statements separated by newlines.

1648 pos = s.position()

1649 stats = []

1650 while s.sy not in ('DEDENT', 'EOF'):

1651 stats.append(p_statement(s, ctx, first_statement = first_statement))

1652 first_statement = 0

1653 if len(stats) == 1:

1654 return stats[0]

1655 else:

1656 return Nodes.StatListNode(pos, stats = stats)

1657

1658 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1659 pos = s.position()

1660 s.expect(':')

1661 doc = None

1662 stmts = []

1663 if s.sy == 'NEWLINE':

1664 s.next()

1665 s.expect_indent()

1666 if with_doc or with_pseudo_doc:

1667 doc = p_doc_string(s)

1668 body = p_statement_list(s, ctx)

1669 s.expect_dedent()

1670 else:

1671 if ctx.api:

1672 error(s.pos, "'api' not allowed with this statement")

1673 if ctx.level in ('module', 'class', 'function', 'other'):

1674 body = p_simple_statement_list(s, ctx)

1675 else:

1676 body = p_pass_statement(s)

1677 s.expect_newline("Syntax error in declarations")

1678 if with_doc:

1679 return doc, body

1680 else:

1681 return body

1682

1683 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1684 """

1685 Parses positional and keyword arguments. end_sy_set

1686 should contain any s.sy that terminate the argument list.

1687 Argument expansion (* and **) are not allowed.

1688

1689 type_positions and type_keywords specifies which argument

1690 positions and/or names which should be interpreted as

1691 types. Other arguments will be treated as expressions.

1692

1693 Returns: (positional_args, keyword_args)

1694 """

1695 positional_args = []

1696 keyword_args = []

1697 pos_idx = 0

1698

1699 while s.sy not in end_sy_set:

1700 if s.sy == '*' or s.sy == '**':

1701 s.error('Argument expansion not allowed here.')

1702

1703 was_keyword = False

1704 parsed_type = False

1705 if s.sy == 'IDENT':

1706 # Since we can have either types or expressions as positional args,

1707 # we use a strategy of looking an extra step forward for a '=' and

1708 # if it is a positional arg we backtrack.

1709 ident = s.systring

1710 s.next()

1711 if s.sy == '=':

1712 s.next()

1713 # Is keyword arg

1714 if ident in type_keywords:

1715 arg = p_c_base_type(s)

1716 parsed_type = True

1717 else:

1718 arg = p_simple_expr(s)

1719 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,

1720 value = EncodedString(ident))

1721 keyword_args.append((keyword_node, arg))

1722 was_keyword = True

1723 else:

1724 s.put_back('IDENT', ident)

1725

1726 if not was_keyword:

1727 if pos_idx in type_positions:

1728 arg = p_c_base_type(s)

1729 parsed_type = True

1730 else:

1731 arg = p_simple_expr(s)

1732 positional_args.append(arg)

1733 pos_idx += 1

1734 if len(keyword_args) > 0:

1735 s.error("Non-keyword arg following keyword arg",

1736 pos = arg.pos)

1737

1738 if s.sy != ',':

1739 if s.sy not in end_sy_set:

1740 if parsed_type:

1741 s.error("Expected: type")

1742 else:

1743 s.error("Expected: expression")

1744 break

1745 s.next()

1746 return positional_args, keyword_args

1747

1748 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1749 # If self_flag is true, this is the base type for the

1750 # self argument of a C method of an extension type.

1751 if s.sy == '(':

1752 return p_c_complex_base_type(s)

1753 else:

1754 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1755

1756 def p_calling_convention(s):

1757 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1758 result = s.systring

1759 s.next()

1760 return result

1761 else:

1762 return ""

1763

1764 calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

1765

1766 def p_c_complex_base_type(s):

1767 # s.sy == '('

1768 pos = s.position()

1769 s.next()

1770 base_type = p_c_base_type(s)

1771 declarator = p_c_declarator(s, empty = 1)

1772 s.expect(')')

1773 return Nodes.CComplexBaseTypeNode(pos,

1774 base_type = base_type, declarator = declarator)

1775

1776 def p_c_simple_base_type(s, self_flag, nonempty):

1777 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1778 is_basic = 0

1779 signed = 1

1780 longness = 0

1781 complex = 0

1782 module_path = []

1783 pos = s.position()

1784 if not s.sy == 'IDENT':

1785 error(pos, "Expected an identifier, found '%s'" % s.sy)

1786 if looking_at_base_type(s):

1787 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1788 is_basic = 1

1789 if s.sy == 'IDENT' and s.systring in special_basic_c_types:

1790 signed, longness = special_basic_c_types[s.systring]

1791 name = s.systring

1792 s.next()

1793 else:

1794 signed, longness = p_sign_and_longness(s)

1795 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1796 name = s.systring

1797 s.next()

1798 else:

1799 name = 'int'

1800 if s.sy == 'IDENT' and s.systring == 'complex':

1801 complex = 1

1802 s.next()

1803 elif looking_at_dotted_name(s):

1804 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1805 name = s.systring

1806 s.next()

1807 while s.sy == '.':

1808 module_path.append(name)

1809 s.next()

1810 name = p_ident(s)

1811 else:

1812 name = s.systring

1813 s.next()

1814 if nonempty and s.sy != 'IDENT':

1815 # Make sure this is not a declaration of a variable or function.

1816 if s.sy == '(':

1817 s.next()

1818 if s.sy == '*' or s.sy == '**':

1819 s.put_back('(', '(')

1820 else:

1821 s.put_back('(', '(')

1822 s.put_back('IDENT', name)

1823 name = None

1824 elif s.sy not in ('*', '**', '['):

1825 s.put_back('IDENT', name)

1826 name = None

1827

1828 type_node = Nodes.CSimpleBaseTypeNode(pos,

1829 name = name, module_path = module_path,

1830 is_basic_c_type = is_basic, signed = signed,

1831 complex = complex, longness = longness,

1832 is_self_arg = self_flag)

1833

1834

1835 # Treat trailing [] on type as buffer access if it appears in a context

1836 # where declarator names are required (so that it cannot mean int[] or

1837 # sizeof(int[SIZE]))...

1838 #

1839 # (This means that buffers cannot occur where there can be empty declarators,

1840 # which is an ok restriction to make.)

1841 if nonempty and s.sy == '[':

1842 return p_buffer_access(s, type_node)

1843 else:

1844 return type_node

1845

1846 def p_buffer_access(s, base_type_node):

1847 # s.sy == '['

1848 pos = s.position()

1849 s.next()

1850 positional_args, keyword_args = (

1851 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1852 )

1853 s.expect(']')

1854

1855 keyword_dict = ExprNodes.DictNode(pos,

1856 key_value_pairs = [

1857 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1858 for key, value in keyword_args

1859 ])

1860

1861 result = Nodes.CBufferAccessTypeNode(pos,

1862 positional_args = positional_args,

1863 keyword_args = keyword_dict,

1864 base_type_node = base_type_node)

1865 return result

1866

1867

1868 def looking_at_name(s):

1869 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1870

1871 def looking_at_expr(s):

1872 if s.systring in base_type_start_words:

1873 return False

1874 elif s.sy == 'IDENT':

1875 is_type = False

1876 name = s.systring

1877 dotted_path = []

1878 s.next()

1879 while s.sy == '.':

1880 s.next()

1881 dotted_path.append(s.systring)

1882 s.expect('IDENT')

1883 saved = s.sy, s.systring

1884 if s.sy == 'IDENT':

1885 is_type = True

1886 elif s.sy == '*' or s.sy == '**':

1887 s.next()

1888 is_type = s.sy == ')'

1889 s.put_back(*saved)

1890 elif s.sy == '(':

1891 s.next()

1892 is_type = s.sy == '*'

1893 s.put_back(*saved)

1894 elif s.sy == '[':

1895 s.next()

1896 is_type = s.sy == ']'

1897 s.put_back(*saved)

1898 dotted_path.reverse()

1899 for p in dotted_path:

1900 s.put_back('IDENT', p)

1901 s.put_back('.', '.')

1902 s.put_back('IDENT', name)

1903 return not is_type

1904 else:

1905 return True

1906

1907 def looking_at_base_type(s):

1908 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1909 return s.sy == 'IDENT' and s.systring in base_type_start_words

1910

1911 def looking_at_dotted_name(s):

1912 if s.sy == 'IDENT':

1913 name = s.systring

1914 s.next()

1915 result = s.sy == '.'

1916 s.put_back('IDENT', name)

1917 return result

1918 else:

1919 return 0

1920

1921 basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

1922

1923 special_basic_c_types = {

1924 # name : (signed, longness)

1925 "Py_ssize_t" : (2, 0),

1926 "size_t" : (0, 0),

1927 }

1928

1929 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1930

1931 base_type_start_words = \

1932 basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)

1933

1934 def p_sign_and_longness(s):

1935 signed = 1

1936 longness = 0

1937 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1938 if s.systring == 'unsigned':

1939 signed = 0

1940 elif s.systring == 'signed':

1941 signed = 2

1942 elif s.systring == 'short':

1943 longness = -1

1944 elif s.systring == 'long':

1945 longness += 1

1946 s.next()

1947 return signed, longness

1948

1949 def p_opt_cname(s):

1950 literal = p_opt_string_literal(s)

1951 if literal:

1952 _, cname = literal

1953 else:

1954 cname = None

1955 return cname

1956

1957 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1958 assignable = 0, nonempty = 0,

1959 calling_convention_allowed = 0):

1960 # If empty is true, the declarator must be empty. If nonempty is true,

1961 # the declarator must be nonempty. Otherwise we don't care.

1962 # If cmethod_flag is true, then if this declarator declares

1963 # a function, it's a C method of an extension type.

1964 pos = s.position()

1965 if s.sy == '(':

1966 s.next()

1967 if s.sy == ')' or looking_at_name(s):

1968 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1969 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1970 else:

1971 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1972 cmethod_flag = cmethod_flag,

1973 nonempty = nonempty,

1974 calling_convention_allowed = 1)

1975 s.expect(')')

1976 else:

1977 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1978 assignable, nonempty)

1979 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1980 error(s.position(), "%s on something that is not a function"

1981 % result.calling_convention)

1982 while s.sy in ('[', '('):

1983 pos = s.position()

1984 if s.sy == '[':

1985 result = p_c_array_declarator(s, result)

1986 else: # sy == '('

1987 s.next()

1988 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1989 cmethod_flag = 0

1990 return result

1991

1992 def p_c_array_declarator(s, base):

1993 pos = s.position()

1994 s.next() # '['

1995 if s.sy != ']':

1996 dim = p_expr(s)

1997 else:

1998 dim = None

1999 s.expect(']')

2000 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

2001

2002 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

2003 # Opening paren has already been skipped

2004 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

2005 nonempty_declarators = 0)

2006 ellipsis = p_optional_ellipsis(s)

2007 s.expect(')')

2008 nogil = p_nogil(s)

2009 exc_val, exc_check = p_exception_value_clause(s)

2010 with_gil = p_with_gil(s)

2011 return Nodes.CFuncDeclaratorNode(pos,

2012 base = base, args = args, has_varargs = ellipsis,

2013 exception_value = exc_val, exception_check = exc_check,

2014 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

2015

2016 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

2017 assignable, nonempty):

2018 pos = s.position()

2019 calling_convention = p_calling_convention(s)

2020 if s.sy == '*':

2021 s.next()

2022 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2023 cmethod_flag = cmethod_flag,

2024 assignable = assignable, nonempty = nonempty)

2025 result = Nodes.CPtrDeclaratorNode(pos,

2026 base = base)

2027 elif s.sy == '**': # scanner returns this as a single token

2028 s.next()

2029 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2030 cmethod_flag = cmethod_flag,

2031 assignable = assignable, nonempty = nonempty)

2032 result = Nodes.CPtrDeclaratorNode(pos,

2033 base = Nodes.CPtrDeclaratorNode(pos,

2034 base = base))

2035 else:

2036 rhs = None

2037 if s.sy == 'IDENT':

2038 name = EncodedString(s.systring)

2039 if empty:

2040 error(s.position(), "Declarator should be empty")

2041 s.next()

2042 cname = p_opt_cname(s)

2043 if s.sy == '=' and assignable:

2044 s.next()

2045 rhs = p_simple_expr(s)

2046 else:

2047 if nonempty:

2048 error(s.position(), "Empty declarator")

2049 name = ""

2050 cname = None

2051 result = Nodes.CNameDeclaratorNode(pos,

2052 name = name, cname = cname, default = rhs)

2053 result.calling_convention = calling_convention

2054 return result

2055

2056 def p_nogil(s):

2057 if s.sy == 'IDENT' and s.systring == 'nogil':

2058 s.next()

2059 return 1

2060 else:

2061 return 0

2062

2063 def p_with_gil(s):

2064 if s.sy == 'with':

2065 s.next()

2066 s.expect_keyword('gil')

2067 return 1

2068 else:

2069 return 0

2070

2071 def p_exception_value_clause(s):

2072 exc_val = None

2073 exc_check = 0

2074 if s.sy == 'except':

2075 s.next()

2076 if s.sy == '*':

2077 exc_check = 1

2078 s.next()

2079 elif s.sy == '+':

2080 exc_check = '+'

2081 s.next()

2082 if s.sy == 'IDENT':

2083 name = s.systring

2084 s.next()

2085 exc_val = p_name(s, name)

2086 else:

2087 if s.sy == '?':

2088 exc_check = 1

2089 s.next()

2090 exc_val = p_simple_expr(s)

2091 return exc_val, exc_check

2092

2093 c_arg_list_terminators = ('*', '**', '.', ')')

2094

2095 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

2096 nonempty_declarators = 0, kw_only = 0):

2097 # Comma-separated list of C argument declarations, possibly empty.

2098 # May have a trailing comma.

2099 args = []

2100 is_self_arg = cmethod_flag

2101 while s.sy not in c_arg_list_terminators:

2102 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

2103 nonempty = nonempty_declarators, kw_only = kw_only))

2104 if s.sy != ',':

2105 break

2106 s.next()

2107 is_self_arg = 0

2108 return args

2109

2110 def p_optional_ellipsis(s):

2111 if s.sy == '.':

2112 expect_ellipsis(s)

2113 return 1

2114 else:

2115 return 0

2116

2117 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

2118 pos = s.position()

2119 not_none = 0

2120 default = None

2121 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2122 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2123 if s.sy == 'not':

2124 s.next()

2125 if s.sy == 'IDENT' and s.systring == 'None':

2126 s.next()

2127 else:

2128 s.error("Expected 'None'")

2129 if not in_pyfunc:

2130 error(pos, "'not None' only allowed in Python functions")

2131 not_none = 1

2132 if s.sy == '=':

2133 s.next()

2134 if 'pxd' in s.level:

2135 if s.sy not in ['*', '?']:

2136 error(pos, "default values cannot be specified in pxd files, use ? or *")

2137 default = ExprNodes.BoolNode(1)

2138 s.next()

2139 else:

2140 default = p_simple_expr(s)

2141 return Nodes.CArgDeclNode(pos,

2142 base_type = base_type,

2143 declarator = declarator,

2144 not_none = not_none,

2145 default = default,

2146 kw_only = kw_only)

2147

2148 def p_api(s):

2149 if s.sy == 'IDENT' and s.systring == 'api':

2150 s.next()

2151 return 1

2152 else:

2153 return 0

2154

2155 def p_cdef_statement(s, ctx):

2156 pos = s.position()

2157 ctx.visibility = p_visibility(s, ctx.visibility)

2158 ctx.api = ctx.api or p_api(s)

2159 if ctx.api:

2160 if ctx.visibility not in ('private', 'public'):

2161 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2162 if (ctx.visibility == 'extern') and s.sy == 'from':

2163 return p_cdef_extern_block(s, pos, ctx)

2164 elif s.sy == 'import':

2165 s.next()

2166 return p_cdef_extern_block(s, pos, ctx)

2167 elif p_nogil(s):

2168 ctx.nogil = 1

2169 if ctx.overridable:

2170 error(pos, "cdef blocks cannot be declared cpdef")

2171 return p_cdef_block(s, ctx)

2172 elif s.sy == ':':

2173 if ctx.overridable:

2174 error(pos, "cdef blocks cannot be declared cpdef")

2175 return p_cdef_block(s, ctx)

2176 elif s.sy == 'class':

2177 if ctx.level not in ('module', 'module_pxd'):

2178 error(pos, "Extension type definition not allowed here")

2179 if ctx.overridable:

2180 error(pos, "Extension types cannot be declared cpdef")

2181 return p_c_class_definition(s, pos, ctx)

2182 elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):

2183 if ctx.level not in ('module', 'module_pxd'):

2184 error(pos, "C struct/union/enum definition not allowed here")

2185 if ctx.overridable:

2186 error(pos, "C struct/union/enum cannot be declared cpdef")

2187 if s.systring == "enum":

2188 return p_c_enum_definition(s, pos, ctx)

2189 else:

2190 return p_c_struct_or_union_definition(s, pos, ctx)

2191 else:

2192 return p_c_func_or_var_declaration(s, pos, ctx)

2193

2194 def p_cdef_block(s, ctx):

2195 return p_suite(s, ctx(cdef_flag = 1))

2196

2197 def p_cdef_extern_block(s, pos, ctx):

2198 if ctx.overridable:

2199 error(pos, "cdef extern blocks cannot be declared cpdef")

2200 include_file = None

2201 s.expect('from')

2202 if s.sy == '*':

2203 s.next()

2204 else:

2205 _, include_file = p_string_literal(s)

2206 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2207 if p_nogil(s):

2208 ctx.nogil = 1

2209 body = p_suite(s, ctx)

2210 return Nodes.CDefExternNode(pos,

2211 include_file = include_file,

2212 body = body)

2213

2214 def p_c_enum_definition(s, pos, ctx):

2215 # s.sy == ident 'enum'

2216 s.next()

2217 if s.sy == 'IDENT':

2218 name = s.systring

2219 s.next()

2220 cname = p_opt_cname(s)

2221 else:

2222 name = None

2223 cname = None

2224 items = None

2225 s.expect(':')

2226 items = []

2227 if s.sy != 'NEWLINE':

2228 p_c_enum_line(s, items)

2229 else:

2230 s.next() # 'NEWLINE'

2231 s.expect_indent()

2232 while s.sy not in ('DEDENT', 'EOF'):

2233 p_c_enum_line(s, items)

2234 s.expect_dedent()

2235 return Nodes.CEnumDefNode(

2236 pos, name = name, cname = cname, items = items,

2237 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2238 in_pxd = ctx.level == 'module_pxd')

2239

2240 def p_c_enum_line(s, items):

2241 if s.sy != 'pass':

2242 p_c_enum_item(s, items)

2243 while s.sy == ',':

2244 s.next()

2245 if s.sy in ('NEWLINE', 'EOF'):

2246 break

2247 p_c_enum_item(s, items)

2248 else:

2249 s.next()

2250 s.expect_newline("Syntax error in enum item list")

2251

2252 def p_c_enum_item(s, items):

2253 pos = s.position()

2254 name = p_ident(s)

2255 cname = p_opt_cname(s)

2256 value = None

2257 if s.sy == '=':

2258 s.next()

2259 value = p_simple_expr(s)

2260 items.append(Nodes.CEnumDefItemNode(pos,

2261 name = name, cname = cname, value = value))

2262

2263 def p_c_struct_or_union_definition(s, pos, ctx):

2264 packed = False

2265 if s.systring == 'packed':

2266 packed = True

2267 s.next()

2268 if s.sy != 'IDENT' or s.systring != 'struct':

2269 s.expected('struct')

2270 # s.sy == ident 'struct' or 'union'

2271 kind = s.systring

2272 s.next()

2273 name = p_ident(s)

2274 cname = p_opt_cname(s)

2275 attributes = None

2276 if s.sy == ':':

2277 s.next()

2278 s.expect('NEWLINE')

2279 s.expect_indent()

2280 attributes = []

2281 body_ctx = Ctx()

2282 while s.sy != 'DEDENT':

2283 if s.sy != 'pass':

2284 attributes.append(

2285 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2286 else:

2287 s.next()

2288 s.expect_newline("Expected a newline")

2289 s.expect_dedent()

2290 else:

2291 s.expect_newline("Syntax error in struct or union definition")

2292 return Nodes.CStructOrUnionDefNode(pos,

2293 name = name, cname = cname, kind = kind, attributes = attributes,

2294 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2295 in_pxd = ctx.level == 'module_pxd', packed = packed)

2296

2297 def p_visibility(s, prev_visibility):

2298 pos = s.position()

2299 visibility = prev_visibility

2300 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2301 visibility = s.systring

2302 if prev_visibility != 'private' and visibility != prev_visibility:

2303 s.error("Conflicting visibility options '%s' and '%s'"

2304 % (prev_visibility, visibility))

2305 s.next()

2306 return visibility

2307

2308 def p_c_modifiers(s):

2309 if s.sy == 'IDENT' and s.systring in ('inline',):

2310 modifier = s.systring

2311 s.next()

2312 return [modifier] + p_c_modifiers(s)

2313 return []

2314

2315 def p_c_func_or_var_declaration(s, pos, ctx):

2316 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2317 modifiers = p_c_modifiers(s)

2318 base_type = p_c_base_type(s, nonempty = 1)

2319 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2320 assignable = 1, nonempty = 1)

2321 declarator.overridable = ctx.overridable

2322 if s.sy == ':':

2323 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):

2324 s.error("C function definition not allowed here")

2325 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2326 result = Nodes.CFuncDefNode(pos,

2327 visibility = ctx.visibility,

2328 base_type = base_type,

2329 declarator = declarator,

2330 body = suite,

2331 doc = doc,

2332 modifiers = modifiers,

2333 api = ctx.api,

2334 overridable = ctx.overridable)

2335 else:

2336 #if api:

2337 # error(s.pos, "'api' not allowed with variable declaration")

2338 declarators = [declarator]

2339 while s.sy == ',':

2340 s.next()

2341 if s.sy == 'NEWLINE':

2342 break

2343 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2344 assignable = 1, nonempty = 1)

2345 declarators.append(declarator)

2346 s.expect_newline("Syntax error in C variable declaration")

2347 result = Nodes.CVarDefNode(pos,

2348 visibility = ctx.visibility,

2349 base_type = base_type,

2350 declarators = declarators,

2351 in_pxd = ctx.level == 'module_pxd',

2352 api = ctx.api,

2353 overridable = ctx.overridable)

2354 return result

2355

2356 def p_ctypedef_statement(s, ctx):

2357 # s.sy == 'ctypedef'

2358 pos = s.position()

2359 s.next()

2360 visibility = p_visibility(s, ctx.visibility)

2361 api = p_api(s)

2362 ctx = ctx(typedef_flag = 1, visibility = visibility)

2363 if api:

2364 ctx.api = 1

2365 if s.sy == 'class':

2366 return p_c_class_definition(s, pos, ctx)

2367 elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):

2368 if s.systring == 'enum':

2369 return p_c_enum_definition(s, pos, ctx)

2370 else:

2371 return p_c_struct_or_union_definition(s, pos, ctx)

2372 else:

2373 base_type = p_c_base_type(s, nonempty = 1)

2374 if base_type.name is None:

2375 s.error("Syntax error in ctypedef statement")

2376 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2377 s.expect_newline("Syntax error in ctypedef statement")

2378 return Nodes.CTypeDefNode(

2379 pos, base_type = base_type,

2380 declarator = declarator, visibility = visibility,

2381 in_pxd = ctx.level == 'module_pxd')

2382

2383 def p_decorators(s):

2384 decorators = []

2385 while s.sy == 'DECORATOR':

2386 pos = s.position()

2387 s.next()

2388 decstring = p_dotted_name(s, as_allowed=0)[2]

2389 names = decstring.split('.')

2390 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2391 for name in names[1:]:

2392 decorator = ExprNodes.AttributeNode(pos,

2393 attribute=EncodedString(name),

2394 obj=decorator)

2395 if s.sy == '(':

2396 decorator = p_call(s, decorator)

2397 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2398 s.expect_newline("Expected a newline after decorator")

2399 return decorators

2400

2401 def p_def_statement(s, decorators=None):

2402 # s.sy == 'def'

2403 pos = s.position()

2404 s.next()

2405 name = EncodedString( p_ident(s) )

2406 #args = []

2407 s.expect('(');

2408 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2409 star_arg = None

2410 starstar_arg = None

2411 if s.sy == '*':

2412 s.next()

2413 if s.sy == 'IDENT':

2414 star_arg = p_py_arg_decl(s)

2415 if s.sy == ',':

2416 s.next()

2417 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2418 nonempty_declarators = 1, kw_only = 1))

2419 elif s.sy != ')':

2420 s.error("Syntax error in Python function argument list")

2421 if s.sy == '**':

2422 s.next()

2423 starstar_arg = p_py_arg_decl(s)

2424 s.expect(')')

2425 if p_nogil(s):

2426 error(s.pos, "Python function cannot be declared nogil")

2427 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2428 return Nodes.DefNode(pos, name = name, args = args,

2429 star_arg = star_arg, starstar_arg = starstar_arg,

2430 doc = doc, body = body, decorators = decorators)

2431

2432 def p_py_arg_decl(s):

2433 pos = s.position()

2434 name = p_ident(s)

2435 return Nodes.PyArgDeclNode(pos, name = name)

2436

2437 def p_class_statement(s, decorators):

2438 # s.sy == 'class'

2439 pos = s.position()

2440 s.next()

2441 class_name = EncodedString( p_ident(s) )

2442 class_name.encoding = s.source_encoding

2443 if s.sy == '(':

2444 s.next()

2445 base_list = p_simple_expr_list(s)

2446 s.expect(')')

2447 else:

2448 base_list = []

2449 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2450 return Nodes.PyClassDefNode(pos,

2451 name = class_name,

2452 bases = ExprNodes.TupleNode(pos, args = base_list),

2453 doc = doc, body = body, decorators = decorators)

2454

2455 def p_c_class_definition(s, pos, ctx):

2456 # s.sy == 'class'

2457 s.next()

2458 module_path = []

2459 class_name = p_ident(s)

2460 while s.sy == '.':

2461 s.next()

2462 module_path.append(class_name)

2463 class_name = p_ident(s)

2464 if module_path and ctx.visibility != 'extern':

2465 error(pos, "Qualified class name only allowed for 'extern' C class")

2466 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2467 s.next()

2468 as_name = p_ident(s)

2469 else:

2470 as_name = class_name

2471 objstruct_name = None

2472 typeobj_name = None

2473 base_class_module = None

2474 base_class_name = None

2475 if s.sy == '(':

2476 s.next()

2477 base_class_path = [p_ident(s)]

2478 while s.sy == '.':

2479 s.next()

2480 base_class_path.append(p_ident(s))

2481 if s.sy == ',':

2482 s.error("C class may only have one base class")

2483 s.expect(')')

2484 base_class_module = ".".join(base_class_path[:-1])

2485 base_class_name = base_class_path[-1]

2486 if s.sy == '[':

2487 if ctx.visibility not in ('public', 'extern'):

2488 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2489 objstruct_name, typeobj_name = p_c_class_options(s)

2490 if s.sy == ':':

2491 if ctx.level == 'module_pxd':

2492 body_level = 'c_class_pxd'

2493 else:

2494 body_level = 'c_class'

2495 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2496 else:

2497 s.expect_newline("Syntax error in C class definition")

2498 doc = None

2499 body = None

2500 if ctx.visibility == 'extern':

2501 if not module_path:

2502 error(pos, "Module name required for 'extern' C class")

2503 if typeobj_name:

2504 error(pos, "Type object name specification not allowed for 'extern' C class")

2505 elif ctx.visibility == 'public':

2506 if not objstruct_name:

2507 error(pos, "Object struct name specification required for 'public' C class")

2508 if not typeobj_name:

2509 error(pos, "Type object name specification required for 'public' C class")

2510 elif ctx.visibility == 'private':

2511 if ctx.api:

2512 error(pos, "Only 'public' C class can be declared 'api'")

2513 else:

2514 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2515 return Nodes.CClassDefNode(pos,

2516 visibility = ctx.visibility,

2517 typedef_flag = ctx.typedef_flag,

2518 api = ctx.api,

2519 module_name = ".".join(module_path),

2520 class_name = class_name,

2521 as_name = as_name,

2522 base_class_module = base_class_module,

2523 base_class_name = base_class_name,

2524 objstruct_name = objstruct_name,

2525 typeobj_name = typeobj_name,

2526 in_pxd = ctx.level == 'module_pxd',

2527 doc = doc,

2528 body = body)

2529

2530 def p_c_class_options(s):

2531 objstruct_name = None

2532 typeobj_name = None

2533 s.expect('[')

2534 while 1:

2535 if s.sy != 'IDENT':

2536 break

2537 if s.systring == 'object':

2538 s.next()

2539 objstruct_name = p_ident(s)

2540 elif s.systring == 'type':

2541 s.next()

2542 typeobj_name = p_ident(s)

2543 if s.sy != ',':

2544 break

2545 s.next()

2546 s.expect(']', "Expected 'object' or 'type'")

2547 return objstruct_name, typeobj_name

2548

2549 def p_property_decl(s):

2550 pos = s.position()

2551 s.next() # 'property'

2552 name = p_ident(s)

2553 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2554 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2555

2556 def p_doc_string(s):

2557 if s.sy == 'BEGIN_STRING':

2558 pos = s.position()

2559 kind, result = p_cat_string_literal(s)

2560 if s.sy != 'EOF':

2561 s.expect_newline("Syntax error in doc string")

2562 if kind != 'u':

2563 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2564 if kind == 'b':

2565 result.encoding = None # force a unicode string

2566 return result

2567 else:

2568 return None

2569

2570 def p_code(s, level=None):

2571 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2572 if s.sy != 'EOF':

2573 s.error("Syntax error in statement [%s,%s]" % (

2574 repr(s.sy), repr(s.systring)))

2575 return body

2576

2577 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*(\w+)\s*=(.*)$")

2578

2579 def p_compiler_directive_comments(s):

2580 result = {}

2581 while s.sy == 'commentline':

2582 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2583 if m:

2584 name = m.group(1)

2585 try:

2586 value = Options.parse_option_value(str(name), str(m.group(2).strip()))

2587 if value is not None: # can be False!

2588 result[name] = value

2589 except ValueError, e:

2590 s.error(e.args[0], fatal=False)

2591 s.next()

2592 return result

2593

2594 def p_module(s, pxd, full_module_name):

2595 pos = s.position()

2596

2597 option_comments = p_compiler_directive_comments(s)

2598 s.parse_comments = False

2599

2600 doc = p_doc_string(s)

2601 if pxd:

2602 level = 'module_pxd'

2603 else:

2604 level = 'module'

2605

2606 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2607 if s.sy != 'EOF':

2608 s.error("Syntax error in statement [%s,%s]" % (

2609 repr(s.sy), repr(s.systring)))

2610 return ModuleNode(pos, doc = doc, body = body,

2611 full_module_name = full_module_name,

2612 option_comments = option_comments)

2613

2614 #----------------------------------------------

2615 #

2616 # Debugging

2617 #

2618 #----------------------------------------------

2619

2620 def print_parse_tree(f, node, level, key = None):

2621 from Nodes import Node

2622 ind = " " * level

2623 if node:

2624 f.write(ind)

2625 if key:

2626 f.write("%s: " % key)

2627 t = type(node)

2628 if t is tuple:

2629 f.write("(%s @ %s\n" % (node[0], node[1]))

2630 for i in xrange(2, len(node)):

2631 print_parse_tree(f, node[i], level+1)

2632 f.write("%s)\n" % ind)

2633 return

2634 elif isinstance(node, Node):

2635 try:

2636 tag = node.tag

2637 except AttributeError:

2638 tag = node.__class__.__name__

2639 f.write("%s @ %s\n" % (tag, node.pos))

2640 for name, value in node.__dict__.items():

2641 if name != 'tag' and name != 'pos':

2642 print_parse_tree(f, value, level+1, name)

2643 return

2644 elif t is list:

2645 f.write("[\n")

2646 for i in xrange(len(node)):

2647 print_parse_tree(f, node[i], level+1)

2648 f.write("%s]\n" % ind)

2649 return

2650 f.write("%s%s\n" % (ind, node))

2651