1 # cython: auto_cpdef=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

14 import Nodes

15 import ExprNodes

16 import StringEncoding

17 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes

18 from ModuleNode import ModuleNode

19 from Errors import error, warning, InternalError

20 from Cython import Utils

21 import Future

22 import Options

23

24 class Ctx(object):

25 # Parsing context

26 level = 'other'

27 visibility = 'private'

28 cdef_flag = 0

29 typedef_flag = 0

30 api = 0

31 overridable = 0

32 nogil = 0

33

34 def __init__(self, **kwds):

35 self.__dict__.update(kwds)

36

37 def __call__(self, **kwds):

38 ctx = Ctx()

39 d = ctx.__dict__

40 d.update(self.__dict__)

41 d.update(kwds)

42 return ctx

43

44 def p_ident(s, message = "Expected an identifier"):

45 if s.sy == 'IDENT':

46 name = s.systring

47 s.next()

48 return name

49 else:

50 s.error(message)

51

52 def p_ident_list(s):

53 names = []

54 while s.sy == 'IDENT':

55 names.append(s.systring)

56 s.next()

57 if s.sy != ',':

58 break

59 s.next()

60 return names

61

62 #------------------------------------------

63 #

64 # Expressions

65 #

66 #------------------------------------------

67

68 def p_binop_operator(s):

69 pos = s.position()

70 op = s.sy

71 s.next()

72 return op, pos

73

74 def p_binop_expr(s, ops, p_sub_expr):

75 n1 = p_sub_expr(s)

76 while s.sy in ops:

77 op, pos = p_binop_operator(s)

78 n2 = p_sub_expr(s)

79 n1 = ExprNodes.binop_node(pos, op, n1, n2)

80 if op == '/':

81 if Future.division in s.context.future_directives:

82 n1.truedivision = True

83 else:

84 n1.truedivision = None # unknown

85 return n1

86

87 #expression: or_test [if or_test else test] | lambda_form

88

89 def p_simple_expr(s):

90 pos = s.position()

91 expr = p_or_test(s)

92 if s.sy == 'if':

93 s.next()

94 test = p_or_test(s)

95 s.expect('else')

96 other = p_test(s)

97 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

98 else:

99 return expr

100

101 #test: or_test | lambda_form

102

103 def p_test(s):

104 return p_or_test(s)

105

106 #or_test: and_test ('or' and_test)*

107

108 def p_or_test(s):

109 return p_rassoc_binop_expr(s, ('or',), p_and_test)

110

111 def p_rassoc_binop_expr(s, ops, p_subexpr):

112 n1 = p_subexpr(s)

113 if s.sy in ops:

114 pos = s.position()

115 op = s.sy

116 s.next()

117 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

118 n1 = ExprNodes.binop_node(pos, op, n1, n2)

119 return n1

120

121 #and_test: not_test ('and' not_test)*

122

123 def p_and_test(s):

124 #return p_binop_expr(s, ('and',), p_not_test)

125 return p_rassoc_binop_expr(s, ('and',), p_not_test)

126

127 #not_test: 'not' not_test | comparison

128

129 def p_not_test(s):

130 if s.sy == 'not':

131 pos = s.position()

132 s.next()

133 return ExprNodes.NotNode(pos, operand = p_not_test(s))

134 else:

135 return p_comparison(s)

136

137 #comparison: expr (comp_op expr)*

138 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

139

140 def p_comparison(s):

141 n1 = p_starred_expr(s)

142 if s.sy in comparison_ops:

143 pos = s.position()

144 op = p_cmp_op(s)

145 n2 = p_starred_expr(s)

146 n1 = ExprNodes.PrimaryCmpNode(pos,

147 operator = op, operand1 = n1, operand2 = n2)

148 if s.sy in comparison_ops:

149 n1.cascade = p_cascaded_cmp(s)

150 return n1

151

152 def p_starred_expr(s):

153 pos = s.position()

154 if s.sy == '*':

155 starred = True

156 s.next()

157 else:

158 starred = False

159 expr = p_bit_expr(s)

160 if starred:

161 expr = ExprNodes.StarredTargetNode(pos, expr)

162 return expr

163

164 def p_cascaded_cmp(s):

165 pos = s.position()

166 op = p_cmp_op(s)

167 n2 = p_starred_expr(s)

168 result = ExprNodes.CascadedCmpNode(pos,

169 operator = op, operand2 = n2)

170 if s.sy in comparison_ops:

171 result.cascade = p_cascaded_cmp(s)

172 return result

173

174 def p_cmp_op(s):

175 if s.sy == 'not':

176 s.next()

177 s.expect('in')

178 op = 'not_in'

179 elif s.sy == 'is':

180 s.next()

181 if s.sy == 'not':

182 s.next()

183 op = 'is_not'

184 else:

185 op = 'is'

186 else:

187 op = s.sy

188 s.next()

189 if op == '<>':

190 op = '!='

191 return op

192

193 comparison_ops = (

194 '<', '>', '==', '>=', '<=', '<>', '!=',

195 'in', 'is', 'not'

196 )

197

198 #expr: xor_expr ('|' xor_expr)*

199

200 def p_bit_expr(s):

201 return p_binop_expr(s, ('|',), p_xor_expr)

202

203 #xor_expr: and_expr ('^' and_expr)*

204

205 def p_xor_expr(s):

206 return p_binop_expr(s, ('^',), p_and_expr)

207

208 #and_expr: shift_expr ('&' shift_expr)*

209

210 def p_and_expr(s):

211 return p_binop_expr(s, ('&',), p_shift_expr)

212

213 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

214

215 def p_shift_expr(s):

216 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

217

218 #arith_expr: term (('+'|'-') term)*

219

220 def p_arith_expr(s):

221 return p_binop_expr(s, ('+', '-'), p_term)

222

223 #term: factor (('*'|'/'|'%') factor)*

224

225 def p_term(s):

226 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

227

228 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

229

230 def p_factor(s):

231 sy = s.sy

232 if sy in ('+', '-', '~'):

233 op = s.sy

234 pos = s.position()

235 s.next()

236 return ExprNodes.unop_node(pos, op, p_factor(s))

237 elif sy == '&':

238 pos = s.position()

239 s.next()

240 arg = p_factor(s)

241 return ExprNodes.AmpersandNode(pos, operand = arg)

242 elif sy == "<":

243 return p_typecast(s)

244 elif sy == 'IDENT' and s.systring == "sizeof":

245 return p_sizeof(s)

246 else:

247 return p_power(s)

248

249 def p_typecast(s):

250 # s.sy == "<"

251 pos = s.position()

252 s.next()

253 base_type = p_c_base_type(s)

254 if base_type.name is None:

255 s.error("Unknown type")

256 declarator = p_c_declarator(s, empty = 1)

257 if s.sy == '?':

258 s.next()

259 typecheck = 1

260 else:

261 typecheck = 0

262 s.expect(">")

263 operand = p_factor(s)

264 return ExprNodes.TypecastNode(pos,

265 base_type = base_type,

266 declarator = declarator,

267 operand = operand,

268 typecheck = typecheck)

269

270 def p_sizeof(s):

271 # s.sy == ident "sizeof"

272 pos = s.position()

273 s.next()

274 s.expect('(')

275 # Here we decide if we are looking at an expression or type

276 # If it is actually a type, but parsable as an expression,

277 # we treat it as an expression here.

278 if looking_at_expr(s):

279 operand = p_simple_expr(s)

280 node = ExprNodes.SizeofVarNode(pos, operand = operand)

281 else:

282 base_type = p_c_base_type(s)

283 declarator = p_c_declarator(s, empty = 1)

284 node = ExprNodes.SizeofTypeNode(pos,

285 base_type = base_type, declarator = declarator)

286 s.expect(')')

287 return node

288

289 def p_yield_expression(s):

290 # s.sy == "yield"

291 pos = s.position()

292 s.next()

293 if s.sy not in ('EOF', 'NEWLINE', ')'):

294 expr = p_expr(s)

295 s.error("generators ('yield') are not currently supported")

296 return Nodes.PassStatNode(pos)

297

298 #power: atom trailer* ('**' factor)*

299

300 def p_power(s):

301 n1 = p_atom(s)

302 while s.sy in ('(', '[', '.'):

303 n1 = p_trailer(s, n1)

304 if s.sy == '**':

305 pos = s.position()

306 s.next()

307 n2 = p_factor(s)

308 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

309 return n1

310

311 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

312

313 def p_trailer(s, node1):

314 pos = s.position()

315 if s.sy == '(':

316 return p_call(s, node1)

317 elif s.sy == '[':

318 return p_index(s, node1)

319 else: # s.sy == '.'

320 s.next()

321 name = EncodedString( p_ident(s) )

322 return ExprNodes.AttributeNode(pos,

323 obj = node1, attribute = name)

324

325 # arglist: argument (',' argument)* [',']

326 # argument: [test '='] test # Really [keyword '='] test

327

328 def p_call(s, function):

329 # s.sy == '('

330 pos = s.position()

331 s.next()

332 positional_args = []

333 keyword_args = []

334 star_arg = None

335 starstar_arg = None

336 while s.sy not in ('**', ')'):

337 if s.sy == '*':

338 if star_arg:

339 s.error("only one star-arg parameter allowed",

340 pos = s.position())

341 s.next()

342 star_arg = p_simple_expr(s)

343 else:

344 arg = p_simple_expr(s)

345 if s.sy == '=':

346 s.next()

347 if not arg.is_name:

348 s.error("Expected an identifier before '='",

349 pos = arg.pos)

350 encoded_name = EncodedString(arg.name)

351 keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)

352 arg = p_simple_expr(s)

353 keyword_args.append((keyword, arg))

354 else:

355 if keyword_args:

356 s.error("Non-keyword arg following keyword arg",

357 pos = arg.pos)

358 if star_arg:

359 s.error("Non-keyword arg following star-arg",

360 pos = arg.pos)

361 positional_args.append(arg)

362 if s.sy != ',':

363 break

364 s.next()

365

366 if s.sy == '**':

367 s.next()

368 starstar_arg = p_simple_expr(s)

369 if s.sy == ',':

370 s.next()

371 s.expect(')')

372 if not (keyword_args or star_arg or starstar_arg):

373 return ExprNodes.SimpleCallNode(pos,

374 function = function,

375 args = positional_args)

376 else:

377 arg_tuple = None

378 keyword_dict = None

379 if positional_args or not star_arg:

380 arg_tuple = ExprNodes.TupleNode(pos,

381 args = positional_args)

382 if star_arg:

383 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

384 if arg_tuple:

385 arg_tuple = ExprNodes.binop_node(pos,

386 operator = '+', operand1 = arg_tuple,

387 operand2 = star_arg_tuple)

388 else:

389 arg_tuple = star_arg_tuple

390 if keyword_args:

391 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

392 for key, value in keyword_args]

393 keyword_dict = ExprNodes.DictNode(pos,

394 key_value_pairs = keyword_args)

395 return ExprNodes.GeneralCallNode(pos,

396 function = function,

397 positional_args = arg_tuple,

398 keyword_args = keyword_dict,

399 starstar_arg = starstar_arg)

400

401 #lambdef: 'lambda' [varargslist] ':' test

402

403 #subscriptlist: subscript (',' subscript)* [',']

404

405 def p_index(s, base):

406 # s.sy == '['

407 pos = s.position()

408 s.next()

409 subscripts = p_subscript_list(s)

410 if len(subscripts) == 1 and len(subscripts[0]) == 2:

411 start, stop = subscripts[0]

412 result = ExprNodes.SliceIndexNode(pos,

413 base = base, start = start, stop = stop)

414 else:

415 indexes = make_slice_nodes(pos, subscripts)

416 if len(indexes) == 1:

417 index = indexes[0]

418 else:

419 index = ExprNodes.TupleNode(pos, args = indexes)

420 result = ExprNodes.IndexNode(pos,

421 base = base, index = index)

422 s.expect(']')

423 return result

424

425 def p_subscript_list(s):

426 items = [p_subscript(s)]

427 while s.sy == ',':

428 s.next()

429 if s.sy == ']':

430 break

431 items.append(p_subscript(s))

432 return items

433

434 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

435

436 def p_subscript(s):

437 # Parse a subscript and return a list of

438 # 1, 2 or 3 ExprNodes, depending on how

439 # many slice elements were encountered.

440 pos = s.position()

441 if s.sy == '.':

442 expect_ellipsis(s)

443 return [ExprNodes.EllipsisNode(pos)]

444 else:

445 start = p_slice_element(s, (':',))

446 if s.sy != ':':

447 return [start]

448 s.next()

449 stop = p_slice_element(s, (':', ',', ']'))

450 if s.sy != ':':

451 return [start, stop]

452 s.next()

453 step = p_slice_element(s, (':', ',', ']'))

454 return [start, stop, step]

455

456 def p_slice_element(s, follow_set):

457 # Simple expression which may be missing iff

458 # it is followed by something in follow_set.

459 if s.sy not in follow_set:

460 return p_simple_expr(s)

461 else:

462 return None

463

464 def expect_ellipsis(s):

465 s.expect('.')

466 s.expect('.')

467 s.expect('.')

468

469 def make_slice_nodes(pos, subscripts):

470 # Convert a list of subscripts as returned

471 # by p_subscript_list into a list of ExprNodes,

472 # creating SliceNodes for elements with 2 or

473 # more components.

474 result = []

475 for subscript in subscripts:

476 if len(subscript) == 1:

477 result.append(subscript[0])

478 else:

479 result.append(make_slice_node(pos, *subscript))

480 return result

481

482 def make_slice_node(pos, start, stop = None, step = None):

483 if not start:

484 start = ExprNodes.NoneNode(pos)

485 if not stop:

486 stop = ExprNodes.NoneNode(pos)

487 if not step:

488 step = ExprNodes.NoneNode(pos)

489 return ExprNodes.SliceNode(pos,

490 start = start, stop = stop, step = step)

491

492 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

493

494 def p_atom(s):

495 pos = s.position()

496 sy = s.sy

497 if sy == '(':

498 s.next()

499 if s.sy == ')':

500 result = ExprNodes.TupleNode(pos, args = [])

501 elif s.sy == 'yield':

502 result = p_yield_expression(s)

503 else:

504 result = p_expr(s)

505 s.expect(')')

506 return result

507 elif sy == '[':

508 return p_list_maker(s)

509 elif sy == '{':

510 return p_dict_or_set_maker(s)

511 elif sy == '`':

512 return p_backquote_expr(s)

513 elif sy == 'INT':

514 value = s.systring

515 s.next()

516 unsigned = ""

517 longness = ""

518 while value[-1] in "UuLl":

519 if value[-1] in "Ll":

520 longness += "L"

521 else:

522 unsigned += "U"

523 value = value[:-1]

524 return ExprNodes.IntNode(pos,

525 value = value,

526 unsigned = unsigned,

527 longness = longness)

528 elif sy == 'FLOAT':

529 value = s.systring

530 s.next()

531 return ExprNodes.FloatNode(pos, value = value)

532 elif sy == 'IMAG':

533 value = s.systring[:-1]

534 s.next()

535 return ExprNodes.ImagNode(pos, value = value)

536 elif sy == 'BEGIN_STRING':

537 kind, value = p_cat_string_literal(s)

538 if kind == 'c':

539 return ExprNodes.CharNode(pos, value = value)

540 elif kind == 'u':

541 return ExprNodes.UnicodeNode(pos, value = value)

542 elif kind == 'b':

543 return ExprNodes.BytesNode(pos, value = value)

544 else:

545 return ExprNodes.StringNode(pos, value = value)

546 elif sy == 'IDENT':

547 name = EncodedString( s.systring )

548 s.next()

549 if name == "None":

550 return ExprNodes.NoneNode(pos)

551 elif name == "True":

552 return ExprNodes.BoolNode(pos, value=True)

553 elif name == "False":

554 return ExprNodes.BoolNode(pos, value=False)

555 elif name == "NULL":

556 return ExprNodes.NullNode(pos)

557 else:

558 return p_name(s, name)

559 else:

560 s.error("Expected an identifier or literal")

561

562 def p_name(s, name):

563 pos = s.position()

564 if not s.compile_time_expr and name in s.compile_time_env:

565 value = s.compile_time_env.lookup_here(name)

566 rep = repr(value)

567 if isinstance(value, bool):

568 return ExprNodes.BoolNode(pos, value = value)

569 elif isinstance(value, int):

570 return ExprNodes.IntNode(pos, value = rep)

571 elif isinstance(value, long):

572 return ExprNodes.IntNode(pos, value = rep, longness = "L")

573 elif isinstance(value, float):

574 return ExprNodes.FloatNode(pos, value = rep)

575 elif isinstance(value, _unicode):

576 return ExprNodes.UnicodeNode(pos, value = value)

577 elif isinstance(value, _bytes):

578 return ExprNodes.BytesNode(pos, value = value)

579 else:

580 error(pos, "Invalid type for compile-time constant: %s"

581 % value.__class__.__name__)

582 return ExprNodes.NameNode(pos, name = name)

583

584 def p_cat_string_literal(s):

585 # A sequence of one or more adjacent string literals.

586 # Returns (kind, value) where kind in ('b', 'c', 'u', '')

587 kind, value = p_string_literal(s)

588 if s.sy != 'BEGIN_STRING':

589 return kind, value

590 if kind != 'c':

591 strings = [value]

592 while s.sy == 'BEGIN_STRING':

593 pos = s.position()

594 next_kind, next_value = p_string_literal(s)

595 if next_kind == 'c':

596 error(pos, "Cannot concatenate char literal with another string or char literal")

597 elif next_kind != kind:

598 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %

599 (kind, next_kind))

600 else:

601 strings.append(next_value)

602 if kind == 'u':

603 value = EncodedString( u''.join(strings) )

604 else:

605 value = BytesLiteral( StringEncoding.join_bytes(strings) )

606 value.encoding = s.source_encoding

607 return kind, value

608

609 def p_opt_string_literal(s):

610 if s.sy == 'BEGIN_STRING':

611 return p_string_literal(s)

612 else:

613 return None

614

615 def p_string_literal(s):

616 # A single string or char literal.

617 # Returns (kind, value) where kind in ('b', 'c', 'u')

618 # s.sy == 'BEGIN_STRING'

619 pos = s.position()

620 is_raw = 0

621 kind = s.systring[:1].lower()

622 if kind == 'r':

623 kind = ''

624 is_raw = 1

625 elif kind in 'ub':

626 is_raw = s.systring[1:2].lower() == 'r'

627 elif kind != 'c':

628 kind = ''

629 if Future.unicode_literals in s.context.future_directives:

630 if kind == '':

631 kind = 'u'

632 if kind == 'u':

633 chars = StringEncoding.UnicodeLiteralBuilder()

634 else:

635 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

636 while 1:

637 s.next()

638 sy = s.sy

639 #print "p_string_literal: sy =", sy, repr(s.systring) ###

640 if sy == 'CHARS':

641 chars.append(s.systring)

642 elif sy == 'ESCAPE':

643 has_escape = True

644 systr = s.systring

645 if is_raw:

646 if systr == u'\\\n':

647 chars.append(u'\\\n')

648 elif systr == u'\\\"':

649 chars.append(u'"')

650 elif systr == u'\\\'':

651 chars.append(u"'")

652 else:

653 chars.append(systr)

654 else:

655 c = systr[1]

656 if c in u"01234567":

657 chars.append_charval( int(systr[1:], 8) )

658 elif c in u"'\"\\":

659 chars.append(c)

660 elif c in u"abfnrtv":

661 chars.append(

662 StringEncoding.char_from_escape_sequence(systr))

663 elif c == u'\n':

664 pass

665 elif c in u'Uux':

666 if kind == 'u' or c == 'x':

667 chrval = int(systr[2:], 16)

668 if chrval > 1114111: # sys.maxunicode:

669 s.error("Invalid unicode escape '%s'" % systr,

670 pos = pos)

671 elif chrval > 65535:

672 warning(s.position(),

673 "Unicode characters above 65535 are not "

674 "necessarily portable across Python installations", 1)

675 chars.append_charval(chrval)

676 else:

677 # unicode escapes in plain byte strings are not unescaped

678 chars.append(systr)

679 else:

680 chars.append(u'\\' + systr[1:])

681 elif sy == 'NEWLINE':

682 chars.append(u'\n')

683 elif sy == 'END_STRING':

684 break

685 elif sy == 'EOF':

686 s.error("Unclosed string literal", pos = pos)

687 else:

688 s.error(

689 "Unexpected token %r:%r in string literal" %

690 (sy, s.systring))

691 if kind == 'c':

692 value = chars.getchar()

693 if len(value) != 1:

694 error(pos, u"invalid character literal: %r" % value)

695 else:

696 value = chars.getstring()

697 s.next()

698 #print "p_string_literal: value =", repr(value) ###

699 return kind, value

700

701 # list_display ::= "[" [listmaker] "]"

702 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

703 # list_iter ::= list_for | list_if

704 # list_for ::= "for" expression_list "in" testlist [list_iter]

705 # list_if ::= "if" test [list_iter]

706

707 def p_list_maker(s):

708 # s.sy == '['

709 pos = s.position()

710 s.next()

711 if s.sy == ']':

712 s.expect(']')

713 return ExprNodes.ListNode(pos, args = [])

714 expr = p_simple_expr(s)

715 if s.sy == 'for':

716 target = ExprNodes.ListNode(pos, args = [])

717 append = ExprNodes.ComprehensionAppendNode(

718 pos, expr=expr, target=ExprNodes.CloneNode(target))

719 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

720 s.expect(']')

721 return ExprNodes.ComprehensionNode(

722 pos, loop=loop, append=append, target=target)

723 else:

724 exprs = [expr]

725 if s.sy == ',':

726 s.next()

727 exprs += p_simple_expr_list(s)

728 s.expect(']')

729 return ExprNodes.ListNode(pos, args = exprs)

730

731 def p_list_iter(s, body):

732 if s.sy == 'for':

733 return p_list_for(s, body)

734 elif s.sy == 'if':

735 return p_list_if(s, body)

736 else:

737 # insert the 'append' operation into the loop

738 return body

739

740 def p_list_for(s, body):

741 # s.sy == 'for'

742 pos = s.position()

743 s.next()

744 kw = p_for_bounds(s)

745 kw['else_clause'] = None

746 kw['body'] = p_list_iter(s, body)

747 return Nodes.ForStatNode(pos, **kw)

748

749 def p_list_if(s, body):

750 # s.sy == 'if'

751 pos = s.position()

752 s.next()

753 test = p_test(s)

754 return Nodes.IfStatNode(pos,

755 if_clauses = [Nodes.IfClauseNode(pos, condition = test,

756 body = p_list_iter(s, body))],

757 else_clause = None )

758

759 #dictmaker: test ':' test (',' test ':' test)* [',']

760

761 def p_dict_or_set_maker(s):

762 # s.sy == '{'

763 pos = s.position()

764 s.next()

765 if s.sy == '}':

766 s.next()

767 return ExprNodes.DictNode(pos, key_value_pairs = [])

768 item = p_simple_expr(s)

769 if s.sy == ',' or s.sy == '}':

770 # set literal

771 values = [item]

772 while s.sy == ',':

773 s.next()

774 if s.sy == '}':

775 break

776 values.append( p_simple_expr(s) )

777 s.expect('}')

778 return ExprNodes.SetNode(pos, args=values)

779 elif s.sy == 'for':

780 # set comprehension

781 target = ExprNodes.SetNode(pos, args=[])

782 append = ExprNodes.ComprehensionAppendNode(

783 item.pos, expr=item, target=ExprNodes.CloneNode(target))

784 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

785 s.expect('}')

786 return ExprNodes.ComprehensionNode(

787 pos, loop=loop, append=append, target=target)

788 elif s.sy == ':':

789 # dict literal or comprehension

790 key = item

791 s.next()

792 value = p_simple_expr(s)

793 if s.sy == 'for':

794 # dict comprehension

795 target = ExprNodes.DictNode(pos, key_value_pairs = [])

796 append = ExprNodes.DictComprehensionAppendNode(

797 item.pos, key_expr=key, value_expr=value,

798 target=ExprNodes.CloneNode(target))

799 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

800 s.expect('}')

801 return ExprNodes.ComprehensionNode(

802 pos, loop=loop, append=append, target=target)

803 else:

804 # dict literal

805 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

806 while s.sy == ',':

807 s.next()

808 if s.sy == '}':

809 break

810 key = p_simple_expr(s)

811 s.expect(':')

812 value = p_simple_expr(s)

813 items.append(

814 ExprNodes.DictItemNode(key.pos, key=key, value=value))

815 s.expect('}')

816 return ExprNodes.DictNode(pos, key_value_pairs=items)

817 else:

818 # raise an error

819 s.expect('}')

820 return ExprNodes.DictNode(pos, key_value_pairs = [])

821

822 def p_backquote_expr(s):

823 # s.sy == '`'

824 pos = s.position()

825 s.next()

826 arg = p_expr(s)

827 s.expect('`')

828 return ExprNodes.BackquoteNode(pos, arg = arg)

829

830 def p_simple_expr_list(s):

831 exprs = []

832 while s.sy not in expr_terminators:

833 expr = p_simple_expr(s)

834 exprs.append(expr)

835 if s.sy != ',':

836 break

837 s.next()

838 return exprs

839

840 def p_expr(s):

841 pos = s.position()

842 expr = p_simple_expr(s)

843 if s.sy == ',':

844 s.next()

845 exprs = [expr] + p_simple_expr_list(s)

846 return ExprNodes.TupleNode(pos, args = exprs)

847 else:

848 return expr

849

850

851 #testlist: test (',' test)* [',']

852 # differs from p_expr only in the fact that it cannot contain conditional expressions

853

854 def p_testlist(s):

855 pos = s.position()

856 expr = p_test(s)

857 if s.sy == ',':

858 exprs = [expr]

859 while s.sy == ',':

860 s.next()

861 exprs.append(p_test(s))

862 return ExprNodes.TupleNode(pos, args = exprs)

863 else:

864 return expr

865

866 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

867

868 #-------------------------------------------------------

869 #

870 # Statements

871 #

872 #-------------------------------------------------------

873

874 def p_global_statement(s):

875 # assume s.sy == 'global'

876 pos = s.position()

877 s.next()

878 names = p_ident_list(s)

879 return Nodes.GlobalNode(pos, names = names)

880

881 def p_expression_or_assignment(s):

882 expr_list = [p_expr(s)]

883 while s.sy == '=':

884 s.next()

885 expr_list.append(p_expr(s))

886 if len(expr_list) == 1:

887 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

888 lhs = expr_list[0]

889 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

890 error(lhs.pos, "Illegal operand for inplace operation.")

891 operator = s.sy[:-1]

892 s.next()

893 rhs = p_expr(s)

894 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

895 expr = expr_list[0]

896 if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):

897 return Nodes.PassStatNode(expr.pos)

898 else:

899 return Nodes.ExprStatNode(expr.pos, expr = expr)

900 else:

901 expr_list_list = []

902 flatten_parallel_assignments(expr_list, expr_list_list)

903 nodes = []

904 for expr_list in expr_list_list:

905 lhs_list = expr_list[:-1]

906 rhs = expr_list[-1]

907 if len(lhs_list) == 1:

908 node = Nodes.SingleAssignmentNode(rhs.pos,

909 lhs = lhs_list[0], rhs = rhs)

910 else:

911 node = Nodes.CascadedAssignmentNode(rhs.pos,

912 lhs_list = lhs_list, rhs = rhs)

913 nodes.append(node)

914 if len(nodes) == 1:

915 return nodes[0]

916 else:

917 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

918

919 def flatten_parallel_assignments(input, output):

920 # The input is a list of expression nodes, representing the LHSs

921 # and RHS of one (possibly cascaded) assignment statement. For

922 # sequence constructors, rearranges the matching parts of both

923 # sides into a list of equivalent assignments between the

924 # individual elements. This transformation is applied

925 # recursively, so that nested structures get matched as well.

926 rhs = input[-1]

927 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):

928 output.append(input)

929 return

930

931 complete_assignments = [rhs]

932

933 rhs_size = len(rhs.args)

934 lhs_targets = [ [] for _ in range(rhs_size) ]

935 starred_assignments = []

936 for lhs in input[:-1]:

937 if not lhs.is_sequence_constructor:

938 if lhs.is_starred:

939 error(lhs.pos, "starred assignment target must be in a list or tuple")

940 complete_assignments.append(lhs)

941 continue

942 lhs_size = len(lhs.args)

943 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])

944 if starred_targets:

945 if starred_targets > 1:

946 error(lhs.pos, "more than 1 starred expression in assignment")

947 output.append([lhs,rhs])

948 continue

949 elif lhs_size - starred_targets > rhs_size:

950 error(lhs.pos, "need more than %d value%s to unpack"

951 % (rhs_size, (rhs_size != 1) and 's' or ''))

952 output.append([lhs,rhs])

953 continue

954 map_starred_assignment(lhs_targets, starred_assignments,

955 lhs.args, rhs.args)

956 else:

957 if lhs_size > rhs_size:

958 error(lhs.pos, "need more than %d value%s to unpack"

959 % (rhs_size, (rhs_size != 1) and 's' or ''))

960 output.append([lhs,rhs])

961 continue

962 elif lhs_size < rhs_size:

963 error(lhs.pos, "too many values to unpack (expected %d, got %d)"

964 % (lhs_size, rhs_size))

965 output.append([lhs,rhs])

966 continue

967 else:

968 for targets, expr in zip(lhs_targets, lhs.args):

969 targets.append(expr)

970

971 if len(complete_assignments) > 1:

972 output.append(complete_assignments[::-1])

973

974 # recursively flatten partial assignments

975 for cascade, rhs in zip(lhs_targets, rhs.args):

976 if cascade:

977 cascade.append(rhs)

978 flatten_parallel_assignments(cascade, output)

979

980 # recursively flatten starred assignments

981 for cascade in starred_assignments:

982 if cascade[0].is_sequence_constructor:

983 flatten_parallel_assignments(cascade, output)

984 else:

985 output.append(cascade)

986

987 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):

988 # Appends the fixed-position LHS targets to the target list that

989 # appear left and right of the starred argument.

990 #

991 # The starred_assignments list receives a new tuple

992 # (lhs_target, rhs_values_list) that maps the remaining arguments

993 # (those that match the starred target) to a list.

994

995 # left side of the starred target

996 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):

997 if expr.is_starred:

998 starred = i

999 lhs_remaining = len(lhs_args) - i - 1

1000 break

1001 targets.append(expr)

1002 else:

1003 raise InternalError("no starred arg found when splitting starred assignment")

1004

1005 # right side of the starred target

1006 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],

1007 lhs_args[-lhs_remaining:])):

1008 targets.append(expr)

1009

1010 # the starred target itself, must be assigned a (potentially empty) list

1011 target = lhs_args[starred].target # unpack starred node

1012 starred_rhs = rhs_args[starred:]

1013 if lhs_remaining:

1014 starred_rhs = starred_rhs[:-lhs_remaining]

1015 if starred_rhs:

1016 pos = starred_rhs[0].pos

1017 else:

1018 pos = target.pos

1019 starred_assignments.append([

1020 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])

1021

1022

1023 def p_print_statement(s):

1024 # s.sy == 'print'

1025 pos = s.position()

1026 s.next()

1027 if s.sy == '>>':

1028 s.error("'print >>' not yet implemented")

1029 args = []

1030 ends_with_comma = 0

1031 if s.sy not in ('NEWLINE', 'EOF'):

1032 args.append(p_simple_expr(s))

1033 while s.sy == ',':

1034 s.next()

1035 if s.sy in ('NEWLINE', 'EOF'):

1036 ends_with_comma = 1

1037 break

1038 args.append(p_simple_expr(s))

1039 arg_tuple = ExprNodes.TupleNode(pos, args = args)

1040 return Nodes.PrintStatNode(pos,

1041 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

1042

1043 def p_exec_statement(s):

1044 # s.sy == 'exec'

1045 pos = s.position()

1046 s.next()

1047 args = [ p_bit_expr(s) ]

1048 if s.sy == 'in':

1049 s.next()

1050 args.append(p_simple_expr(s))

1051 if s.sy == ',':

1052 s.next()

1053 args.append(p_simple_expr(s))

1054 else:

1055 error(pos, "'exec' currently requires a target mapping (globals/locals)")

1056 return Nodes.ExecStatNode(pos, args = args)

1057

1058 def p_del_statement(s):

1059 # s.sy == 'del'

1060 pos = s.position()

1061 s.next()

1062 args = p_simple_expr_list(s)

1063 return Nodes.DelStatNode(pos, args = args)

1064

1065 def p_pass_statement(s, with_newline = 0):

1066 pos = s.position()

1067 s.expect('pass')

1068 if with_newline:

1069 s.expect_newline("Expected a newline")

1070 return Nodes.PassStatNode(pos)

1071

1072 def p_break_statement(s):

1073 # s.sy == 'break'

1074 pos = s.position()

1075 s.next()

1076 return Nodes.BreakStatNode(pos)

1077

1078 def p_continue_statement(s):

1079 # s.sy == 'continue'

1080 pos = s.position()

1081 s.next()

1082 return Nodes.ContinueStatNode(pos)

1083

1084 def p_return_statement(s):

1085 # s.sy == 'return'

1086 pos = s.position()

1087 s.next()

1088 if s.sy not in statement_terminators:

1089 value = p_expr(s)

1090 else:

1091 value = None

1092 return Nodes.ReturnStatNode(pos, value = value)

1093

1094 def p_raise_statement(s):

1095 # s.sy == 'raise'

1096 pos = s.position()

1097 s.next()

1098 exc_type = None

1099 exc_value = None

1100 exc_tb = None

1101 if s.sy not in statement_terminators:

1102 exc_type = p_simple_expr(s)

1103 if s.sy == ',':

1104 s.next()

1105 exc_value = p_simple_expr(s)

1106 if s.sy == ',':

1107 s.next()

1108 exc_tb = p_simple_expr(s)

1109 if exc_type or exc_value or exc_tb:

1110 return Nodes.RaiseStatNode(pos,

1111 exc_type = exc_type,

1112 exc_value = exc_value,

1113 exc_tb = exc_tb)

1114 else:

1115 return Nodes.ReraiseStatNode(pos)

1116

1117 def p_import_statement(s):

1118 # s.sy in ('import', 'cimport')

1119 pos = s.position()

1120 kind = s.sy

1121 s.next()

1122 items = [p_dotted_name(s, as_allowed = 1)]

1123 while s.sy == ',':

1124 s.next()

1125 items.append(p_dotted_name(s, as_allowed = 1))

1126 stats = []

1127 for pos, target_name, dotted_name, as_name in items:

1128 dotted_name = EncodedString(dotted_name)

1129 if kind == 'cimport':

1130 stat = Nodes.CImportStatNode(pos,

1131 module_name = dotted_name,

1132 as_name = as_name)

1133 else:

1134 if as_name and "." in dotted_name:

1135 name_list = ExprNodes.ListNode(pos, args = [

1136 ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])

1137 else:

1138 name_list = None

1139 stat = Nodes.SingleAssignmentNode(pos,

1140 lhs = ExprNodes.NameNode(pos,

1141 name = as_name or target_name),

1142 rhs = ExprNodes.ImportNode(pos,

1143 module_name = ExprNodes.IdentifierStringNode(

1144 pos, value = dotted_name),

1145 name_list = name_list))

1146 stats.append(stat)

1147 return Nodes.StatListNode(pos, stats = stats)

1148

1149 def p_from_import_statement(s, first_statement = 0):

1150 # s.sy == 'from'

1151 pos = s.position()

1152 s.next()

1153 (dotted_name_pos, _, dotted_name, _) = \

1154 p_dotted_name(s, as_allowed = 0)

1155 if s.sy in ('import', 'cimport'):

1156 kind = s.sy

1157 s.next()

1158 else:

1159 s.error("Expected 'import' or 'cimport'")

1160 is_cimport = kind == 'cimport'

1161 is_parenthesized = False

1162 if s.sy == '*':

1163 imported_names = [(s.position(), "*", None, None)]

1164 s.next()

1165 else:

1166 if s.sy == '(':

1167 is_parenthesized = True

1168 s.next()

1169 imported_names = [p_imported_name(s, is_cimport)]

1170 while s.sy == ',':

1171 s.next()

1172 imported_names.append(p_imported_name(s, is_cimport))

1173 if is_parenthesized:

1174 s.expect(')')

1175 dotted_name = EncodedString(dotted_name)

1176 if dotted_name == '__future__':

1177 if not first_statement:

1178 s.error("from __future__ imports must occur at the beginning of the file")

1179 else:

1180 for (name_pos, name, as_name, kind) in imported_names:

1181 if name == "braces":

1182 s.error("not a chance", name_pos)

1183 break

1184 try:

1185 directive = getattr(Future, name)

1186 except AttributeError:

1187 s.error("future feature %s is not defined" % name, name_pos)

1188 break

1189 s.context.future_directives.add(directive)

1190 return Nodes.PassStatNode(pos)

1191 elif kind == 'cimport':

1192 return Nodes.FromCImportStatNode(pos,

1193 module_name = dotted_name,

1194 imported_names = imported_names)

1195 else:

1196 imported_name_strings = []

1197 items = []

1198 for (name_pos, name, as_name, kind) in imported_names:

1199 encoded_name = EncodedString(name)

1200 imported_name_strings.append(

1201 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1202 items.append(

1203 (name,

1204 ExprNodes.NameNode(name_pos,

1205 name = as_name or name)))

1206 import_list = ExprNodes.ListNode(

1207 imported_names[0][0], args = imported_name_strings)

1208 dotted_name = EncodedString(dotted_name)

1209 return Nodes.FromImportStatNode(pos,

1210 module = ExprNodes.ImportNode(dotted_name_pos,

1211 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1212 name_list = import_list),

1213 items = items)

1214

1215 imported_name_kinds = ('class', 'struct', 'union')

1216

1217 def p_imported_name(s, is_cimport):

1218 pos = s.position()

1219 kind = None

1220 if is_cimport and s.systring in imported_name_kinds:

1221 kind = s.systring

1222 s.next()

1223 name = p_ident(s)

1224 as_name = p_as_name(s)

1225 return (pos, name, as_name, kind)

1226

1227 def p_dotted_name(s, as_allowed):

1228 pos = s.position()

1229 target_name = p_ident(s)

1230 as_name = None

1231 names = [target_name]

1232 while s.sy == '.':

1233 s.next()

1234 names.append(p_ident(s))

1235 if as_allowed:

1236 as_name = p_as_name(s)

1237 return (pos, target_name, u'.'.join(names), as_name)

1238

1239 def p_as_name(s):

1240 if s.sy == 'IDENT' and s.systring == 'as':

1241 s.next()

1242 return p_ident(s)

1243 else:

1244 return None

1245

1246 def p_assert_statement(s):

1247 # s.sy == 'assert'

1248 pos = s.position()

1249 s.next()

1250 cond = p_simple_expr(s)

1251 if s.sy == ',':

1252 s.next()

1253 value = p_simple_expr(s)

1254 else:

1255 value = None

1256 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1257

1258 statement_terminators = (';', 'NEWLINE', 'EOF')

1259

1260 def p_if_statement(s):

1261 # s.sy == 'if'

1262 pos = s.position()

1263 s.next()

1264 if_clauses = [p_if_clause(s)]

1265 while s.sy == 'elif':

1266 s.next()

1267 if_clauses.append(p_if_clause(s))

1268 else_clause = p_else_clause(s)

1269 return Nodes.IfStatNode(pos,

1270 if_clauses = if_clauses, else_clause = else_clause)

1271

1272 def p_if_clause(s):

1273 pos = s.position()

1274 test = p_simple_expr(s)

1275 body = p_suite(s)

1276 return Nodes.IfClauseNode(pos,

1277 condition = test, body = body)

1278

1279 def p_else_clause(s):

1280 if s.sy == 'else':

1281 s.next()

1282 return p_suite(s)

1283 else:

1284 return None

1285

1286 def p_while_statement(s):

1287 # s.sy == 'while'

1288 pos = s.position()

1289 s.next()

1290 test = p_simple_expr(s)

1291 body = p_suite(s)

1292 else_clause = p_else_clause(s)

1293 return Nodes.WhileStatNode(pos,

1294 condition = test, body = body,

1295 else_clause = else_clause)

1296

1297 def p_for_statement(s):

1298 # s.sy == 'for'

1299 pos = s.position()

1300 s.next()

1301 kw = p_for_bounds(s)

1302 kw['body'] = p_suite(s)

1303 kw['else_clause'] = p_else_clause(s)

1304 return Nodes.ForStatNode(pos, **kw)

1305

1306 def p_for_bounds(s):

1307 target = p_for_target(s)

1308 if s.sy == 'in':

1309 s.next()

1310 iterator = p_for_iterator(s)

1311 return { 'target': target, 'iterator': iterator }

1312 else:

1313 if s.sy == 'from':

1314 s.next()

1315 bound1 = p_bit_expr(s)

1316 else:

1317 # Support shorter "for a <= x < b" syntax

1318 bound1, target = target, None

1319 rel1 = p_for_from_relation(s)

1320 name2_pos = s.position()

1321 name2 = p_ident(s)

1322 rel2_pos = s.position()

1323 rel2 = p_for_from_relation(s)

1324 bound2 = p_bit_expr(s)

1325 step = p_for_from_step(s)

1326 if target is None:

1327 target = ExprNodes.NameNode(name2_pos, name = name2)

1328 else:

1329 if not target.is_name:

1330 error(target.pos,

1331 "Target of for-from statement must be a variable name")

1332 elif name2 != target.name:

1333 error(name2_pos,

1334 "Variable name in for-from range does not match target")

1335 if rel1[0] != rel2[0]:

1336 error(rel2_pos,

1337 "Relation directions in for-from do not match")

1338 return {'target': target,

1339 'bound1': bound1,

1340 'relation1': rel1,

1341 'relation2': rel2,

1342 'bound2': bound2,

1343 'step': step }

1344

1345 def p_for_from_relation(s):

1346 if s.sy in inequality_relations:

1347 op = s.sy

1348 s.next()

1349 return op

1350 else:

1351 s.error("Expected one of '<', '<=', '>' '>='")

1352

1353 def p_for_from_step(s):

1354 if s.sy == 'by':

1355 s.next()

1356 step = p_bit_expr(s)

1357 return step

1358 else:

1359 return None

1360

1361 inequality_relations = ('<', '<=', '>', '>=')

1362

1363 def p_target(s, terminator):

1364 pos = s.position()

1365 expr = p_starred_expr(s)

1366 if s.sy == ',':

1367 s.next()

1368 exprs = [expr]

1369 while s.sy != terminator:

1370 exprs.append(p_starred_expr(s))

1371 if s.sy != ',':

1372 break

1373 s.next()

1374 return ExprNodes.TupleNode(pos, args = exprs)

1375 else:

1376 return expr

1377

1378 def p_for_target(s):

1379 return p_target(s, 'in')

1380

1381 def p_for_iterator(s):

1382 pos = s.position()

1383 expr = p_testlist(s)

1384 return ExprNodes.IteratorNode(pos, sequence = expr)

1385

1386 def p_try_statement(s):

1387 # s.sy == 'try'

1388 pos = s.position()

1389 s.next()

1390 body = p_suite(s)

1391 except_clauses = []

1392 else_clause = None

1393 if s.sy in ('except', 'else'):

1394 while s.sy == 'except':

1395 except_clauses.append(p_except_clause(s))

1396 if s.sy == 'else':

1397 s.next()

1398 else_clause = p_suite(s)

1399 body = Nodes.TryExceptStatNode(pos,

1400 body = body, except_clauses = except_clauses,

1401 else_clause = else_clause)

1402 if s.sy != 'finally':

1403 return body

1404 # try-except-finally is equivalent to nested try-except/try-finally

1405 if s.sy == 'finally':

1406 s.next()

1407 finally_clause = p_suite(s)

1408 return Nodes.TryFinallyStatNode(pos,

1409 body = body, finally_clause = finally_clause)

1410 else:

1411 s.error("Expected 'except' or 'finally'")

1412

1413 def p_except_clause(s):

1414 # s.sy == 'except'

1415 pos = s.position()

1416 s.next()

1417 exc_type = None

1418 exc_value = None

1419 if s.sy != ':':

1420 exc_type = p_simple_expr(s)

1421 if s.sy == ',':

1422 s.next()

1423 exc_value = p_simple_expr(s)

1424 body = p_suite(s)

1425 return Nodes.ExceptClauseNode(pos,

1426 pattern = exc_type, target = exc_value, body = body)

1427

1428 def p_include_statement(s, ctx):

1429 pos = s.position()

1430 s.next() # 'include'

1431 _, include_file_name = p_string_literal(s)

1432 s.expect_newline("Syntax error in include statement")

1433 if s.compile_time_eval:

1434 include_file_name = include_file_name.decode(s.source_encoding)

1435 include_file_path = s.context.find_include_file(include_file_name, pos)

1436 if include_file_path:

1437 s.included_files.append(include_file_name)

1438 f = Utils.open_source_file(include_file_path, mode="rU")

1439 source_desc = FileSourceDescriptor(include_file_path)

1440 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1441 try:

1442 tree = p_statement_list(s2, ctx)

1443 finally:

1444 f.close()

1445 return tree

1446 else:

1447 return None

1448 else:

1449 return Nodes.PassStatNode(pos)

1450

1451 def p_with_statement(s):

1452 pos = s.position()

1453 s.next() # 'with'

1454 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1455 if s.sy == 'IDENT' and s.systring == 'nogil':

1456 state = s.systring

1457 s.next()

1458 body = p_suite(s)

1459 return Nodes.GILStatNode(pos, state = state, body = body)

1460 else:

1461 manager = p_expr(s)

1462 target = None

1463 if s.sy == 'IDENT' and s.systring == 'as':

1464 s.next()

1465 allow_multi = (s.sy == '(')

1466 target = p_target(s, ':')

1467 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1468 s.error("Multiple with statement target values not allowed without paranthesis")

1469 body = p_suite(s)

1470 return Nodes.WithStatNode(pos, manager = manager,

1471 target = target, body = body)

1472

1473 def p_simple_statement(s, first_statement = 0):

1474 #print "p_simple_statement:", s.sy, s.systring ###

1475 if s.sy == 'global':

1476 node = p_global_statement(s)

1477 elif s.sy == 'print':

1478 node = p_print_statement(s)

1479 elif s.sy == 'exec':

1480 node = p_exec_statement(s)

1481 elif s.sy == 'del':

1482 node = p_del_statement(s)

1483 elif s.sy == 'break':

1484 node = p_break_statement(s)

1485 elif s.sy == 'continue':

1486 node = p_continue_statement(s)

1487 elif s.sy == 'return':

1488 node = p_return_statement(s)

1489 elif s.sy == 'raise':

1490 node = p_raise_statement(s)

1491 elif s.sy in ('import', 'cimport'):

1492 node = p_import_statement(s)

1493 elif s.sy == 'from':

1494 node = p_from_import_statement(s, first_statement = first_statement)

1495 elif s.sy == 'yield':

1496 node = p_yield_expression(s)

1497 elif s.sy == 'assert':

1498 node = p_assert_statement(s)

1499 elif s.sy == 'pass':

1500 node = p_pass_statement(s)

1501 else:

1502 node = p_expression_or_assignment(s)

1503 return node

1504

1505 def p_simple_statement_list(s, ctx, first_statement = 0):

1506 # Parse a series of simple statements on one line

1507 # separated by semicolons.

1508 stat = p_simple_statement(s, first_statement = first_statement)

1509 if s.sy == ';':

1510 stats = [stat]

1511 while s.sy == ';':

1512 #print "p_simple_statement_list: maybe more to follow" ###

1513 s.next()

1514 if s.sy in ('NEWLINE', 'EOF'):

1515 break

1516 stats.append(p_simple_statement(s))

1517 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1518 s.expect_newline("Syntax error in simple statement list")

1519 return stat

1520

1521 def p_compile_time_expr(s):

1522 old = s.compile_time_expr

1523 s.compile_time_expr = 1

1524 expr = p_expr(s)

1525 s.compile_time_expr = old

1526 return expr

1527

1528 def p_DEF_statement(s):

1529 pos = s.position()

1530 denv = s.compile_time_env

1531 s.next() # 'DEF'

1532 name = p_ident(s)

1533 s.expect('=')

1534 expr = p_compile_time_expr(s)

1535 value = expr.compile_time_value(denv)

1536 #print "p_DEF_statement: %s = %r" % (name, value) ###

1537 denv.declare(name, value)

1538 s.expect_newline()

1539 return Nodes.PassStatNode(pos)

1540

1541 def p_IF_statement(s, ctx):

1542 pos = s.position()

1543 saved_eval = s.compile_time_eval

1544 current_eval = saved_eval

1545 denv = s.compile_time_env

1546 result = None

1547 while 1:

1548 s.next() # 'IF' or 'ELIF'

1549 expr = p_compile_time_expr(s)

1550 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1551 body = p_suite(s, ctx)

1552 if s.compile_time_eval:

1553 result = body

1554 current_eval = 0

1555 if s.sy != 'ELIF':

1556 break

1557 if s.sy == 'ELSE':

1558 s.next()

1559 s.compile_time_eval = current_eval

1560 body = p_suite(s, ctx)

1561 if current_eval:

1562 result = body

1563 if not result:

1564 result = Nodes.PassStatNode(pos)

1565 s.compile_time_eval = saved_eval

1566 return result

1567

1568 def p_statement(s, ctx, first_statement = 0):

1569 cdef_flag = ctx.cdef_flag

1570 decorators = None

1571 if s.sy == 'ctypedef':

1572 if ctx.level not in ('module', 'module_pxd'):

1573 s.error("ctypedef statement not allowed here")

1574 #if ctx.api:

1575 # error(s.position(), "'api' not allowed with 'ctypedef'")

1576 return p_ctypedef_statement(s, ctx)

1577 elif s.sy == 'DEF':

1578 return p_DEF_statement(s)

1579 elif s.sy == 'IF':

1580 return p_IF_statement(s, ctx)

1581 elif s.sy == 'DECORATOR':

1582 if ctx.level not in ('module', 'class', 'c_class', 'property', 'module_pxd', 'c_class_pxd'):

1583 print ctx.level

1584 s.error('decorator not allowed here')

1585 s.level = ctx.level

1586 decorators = p_decorators(s)

1587 if s.sy not in ('def', 'cdef', 'cpdef', 'class'):

1588 s.error("Decorators can only be followed by functions or classes")

1589 elif s.sy == 'pass' and cdef_flag:

1590 # empty cdef block

1591 return p_pass_statement(s, with_newline = 1)

1592

1593 overridable = 0

1594 if s.sy == 'cdef':

1595 cdef_flag = 1

1596 s.next()

1597 elif s.sy == 'cpdef':

1598 cdef_flag = 1

1599 overridable = 1

1600 s.next()

1601 if cdef_flag:

1602 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1603 s.error('cdef statement not allowed here')

1604 s.level = ctx.level

1605 node = p_cdef_statement(s, ctx(overridable = overridable))

1606 if decorators is not None:

1607 if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):

1608 s.error("Decorators can only be followed by functions or Python classes")

1609 node.decorators = decorators

1610 return node

1611 else:

1612 if ctx.api:

1613 error(s.pos, "'api' not allowed with this statement")

1614 elif s.sy == 'def':

1615 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1616 s.error('def statement not allowed here')

1617 s.level = ctx.level

1618 return p_def_statement(s, decorators)

1619 elif s.sy == 'class':

1620 if ctx.level != 'module':

1621 s.error("class definition not allowed here")

1622 return p_class_statement(s, decorators)

1623 elif s.sy == 'include':

1624 if ctx.level not in ('module', 'module_pxd'):

1625 s.error("include statement not allowed here")

1626 return p_include_statement(s, ctx)

1627 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1628 return p_property_decl(s)

1629 elif s.sy == 'pass' and ctx.level != 'property':

1630 return p_pass_statement(s, with_newline = 1)

1631 else:

1632 if ctx.level in ('c_class_pxd', 'property'):

1633 s.error("Executable statement not allowed here")

1634 if s.sy == 'if':

1635 return p_if_statement(s)

1636 elif s.sy == 'while':

1637 return p_while_statement(s)

1638 elif s.sy == 'for':

1639 return p_for_statement(s)

1640 elif s.sy == 'try':

1641 return p_try_statement(s)

1642 elif s.sy == 'with':

1643 return p_with_statement(s)

1644 else:

1645 return p_simple_statement_list(

1646 s, ctx, first_statement = first_statement)

1647

1648 def p_statement_list(s, ctx, first_statement = 0):

1649 # Parse a series of statements separated by newlines.

1650 pos = s.position()

1651 stats = []

1652 while s.sy not in ('DEDENT', 'EOF'):

1653 stats.append(p_statement(s, ctx, first_statement = first_statement))

1654 first_statement = 0

1655 if len(stats) == 1:

1656 return stats[0]

1657 else:

1658 return Nodes.StatListNode(pos, stats = stats)

1659

1660 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1661 pos = s.position()

1662 s.expect(':')

1663 doc = None

1664 stmts = []

1665 if s.sy == 'NEWLINE':

1666 s.next()

1667 s.expect_indent()

1668 if with_doc or with_pseudo_doc:

1669 doc = p_doc_string(s)

1670 body = p_statement_list(s, ctx)

1671 s.expect_dedent()

1672 else:

1673 if ctx.api:

1674 error(s.pos, "'api' not allowed with this statement")

1675 if ctx.level in ('module', 'class', 'function', 'other'):

1676 body = p_simple_statement_list(s, ctx)

1677 else:

1678 body = p_pass_statement(s)

1679 s.expect_newline("Syntax error in declarations")

1680 if with_doc:

1681 return doc, body

1682 else:

1683 return body

1684

1685 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1686 """

1687 Parses positional and keyword arguments. end_sy_set

1688 should contain any s.sy that terminate the argument list.

1689 Argument expansion (* and **) are not allowed.

1690

1691 type_positions and type_keywords specifies which argument

1692 positions and/or names which should be interpreted as

1693 types. Other arguments will be treated as expressions.

1694

1695 Returns: (positional_args, keyword_args)

1696 """

1697 positional_args = []

1698 keyword_args = []

1699 pos_idx = 0

1700

1701 while s.sy not in end_sy_set:

1702 if s.sy == '*' or s.sy == '**':

1703 s.error('Argument expansion not allowed here.')

1704

1705 was_keyword = False

1706 parsed_type = False

1707 if s.sy == 'IDENT':

1708 # Since we can have either types or expressions as positional args,

1709 # we use a strategy of looking an extra step forward for a '=' and

1710 # if it is a positional arg we backtrack.

1711 ident = s.systring

1712 s.next()

1713 if s.sy == '=':

1714 s.next()

1715 # Is keyword arg

1716 if ident in type_keywords:

1717 arg = p_c_base_type(s)

1718 parsed_type = True

1719 else:

1720 arg = p_simple_expr(s)

1721 keyword_node = ExprNodes.IdentifierStringNode(

1722 arg.pos, value = EncodedString(ident))

1723 keyword_args.append((keyword_node, arg))

1724 was_keyword = True

1725 else:

1726 s.put_back('IDENT', ident)

1727

1728 if not was_keyword:

1729 if pos_idx in type_positions:

1730 arg = p_c_base_type(s)

1731 parsed_type = True

1732 else:

1733 arg = p_simple_expr(s)

1734 positional_args.append(arg)

1735 pos_idx += 1

1736 if len(keyword_args) > 0:

1737 s.error("Non-keyword arg following keyword arg",

1738 pos = arg.pos)

1739

1740 if s.sy != ',':

1741 if s.sy not in end_sy_set:

1742 if parsed_type:

1743 s.error("Expected: type")

1744 else:

1745 s.error("Expected: expression")

1746 break

1747 s.next()

1748 return positional_args, keyword_args

1749

1750 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1751 # If self_flag is true, this is the base type for the

1752 # self argument of a C method of an extension type.

1753 if s.sy == '(':

1754 return p_c_complex_base_type(s)

1755 else:

1756 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1757

1758 def p_calling_convention(s):

1759 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1760 result = s.systring

1761 s.next()

1762 return result

1763 else:

1764 return ""

1765

1766 calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

1767

1768 def p_c_complex_base_type(s):

1769 # s.sy == '('

1770 pos = s.position()

1771 s.next()

1772 base_type = p_c_base_type(s)

1773 declarator = p_c_declarator(s, empty = 1)

1774 s.expect(')')

1775 return Nodes.CComplexBaseTypeNode(pos,

1776 base_type = base_type, declarator = declarator)

1777

1778 def p_c_simple_base_type(s, self_flag, nonempty):

1779 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1780 is_basic = 0

1781 signed = 1

1782 longness = 0

1783 complex = 0

1784 module_path = []

1785 pos = s.position()

1786 if not s.sy == 'IDENT':

1787 error(pos, "Expected an identifier, found '%s'" % s.sy)

1788 if looking_at_base_type(s):

1789 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1790 is_basic = 1

1791 if s.sy == 'IDENT' and s.systring in special_basic_c_types:

1792 signed, longness = special_basic_c_types[s.systring]

1793 name = s.systring

1794 s.next()

1795 else:

1796 signed, longness = p_sign_and_longness(s)

1797 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1798 name = s.systring

1799 s.next()

1800 else:

1801 name = 'int'

1802 if s.sy == 'IDENT' and s.systring == 'complex':

1803 complex = 1

1804 s.next()

1805 elif looking_at_dotted_name(s):

1806 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1807 name = s.systring

1808 s.next()

1809 while s.sy == '.':

1810 module_path.append(name)

1811 s.next()

1812 name = p_ident(s)

1813 else:

1814 name = s.systring

1815 s.next()

1816 if nonempty and s.sy != 'IDENT':

1817 # Make sure this is not a declaration of a variable or function.

1818 if s.sy == '(':

1819 s.next()

1820 if s.sy == '*' or s.sy == '**':

1821 s.put_back('(', '(')

1822 else:

1823 s.put_back('(', '(')

1824 s.put_back('IDENT', name)

1825 name = None

1826 elif s.sy not in ('*', '**', '['):

1827 s.put_back('IDENT', name)

1828 name = None

1829

1830 type_node = Nodes.CSimpleBaseTypeNode(pos,

1831 name = name, module_path = module_path,

1832 is_basic_c_type = is_basic, signed = signed,

1833 complex = complex, longness = longness,

1834 is_self_arg = self_flag)

1835

1836

1837 # Treat trailing [] on type as buffer access if it appears in a context

1838 # where declarator names are required (so that it cannot mean int[] or

1839 # sizeof(int[SIZE]))...

1840 #

1841 # (This means that buffers cannot occur where there can be empty declarators,

1842 # which is an ok restriction to make.)

1843 if nonempty and s.sy == '[':

1844 return p_buffer_access(s, type_node)

1845 else:

1846 return type_node

1847

1848 def p_buffer_access(s, base_type_node):

1849 # s.sy == '['

1850 pos = s.position()

1851 s.next()

1852 positional_args, keyword_args = (

1853 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1854 )

1855 s.expect(']')

1856

1857 keyword_dict = ExprNodes.DictNode(pos,

1858 key_value_pairs = [

1859 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1860 for key, value in keyword_args

1861 ])

1862

1863 result = Nodes.CBufferAccessTypeNode(pos,

1864 positional_args = positional_args,

1865 keyword_args = keyword_dict,

1866 base_type_node = base_type_node)

1867 return result

1868

1869

1870 def looking_at_name(s):

1871 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1872

1873 def looking_at_expr(s):

1874 if s.systring in base_type_start_words:

1875 return False

1876 elif s.sy == 'IDENT':

1877 is_type = False

1878 name = s.systring

1879 dotted_path = []

1880 s.next()

1881 while s.sy == '.':

1882 s.next()

1883 dotted_path.append(s.systring)

1884 s.expect('IDENT')

1885 saved = s.sy, s.systring

1886 if s.sy == 'IDENT':

1887 is_type = True

1888 elif s.sy == '*' or s.sy == '**':

1889 s.next()

1890 is_type = s.sy == ')'

1891 s.put_back(*saved)

1892 elif s.sy == '(':

1893 s.next()

1894 is_type = s.sy == '*'

1895 s.put_back(*saved)

1896 elif s.sy == '[':

1897 s.next()

1898 is_type = s.sy == ']'

1899 s.put_back(*saved)

1900 dotted_path.reverse()

1901 for p in dotted_path:

1902 s.put_back('IDENT', p)

1903 s.put_back('.', '.')

1904 s.put_back('IDENT', name)

1905 return not is_type

1906 else:

1907 return True

1908

1909 def looking_at_base_type(s):

1910 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1911 return s.sy == 'IDENT' and s.systring in base_type_start_words

1912

1913 def looking_at_dotted_name(s):

1914 if s.sy == 'IDENT':

1915 name = s.systring

1916 s.next()

1917 result = s.sy == '.'

1918 s.put_back('IDENT', name)

1919 return result

1920 else:

1921 return 0

1922

1923 basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

1924

1925 special_basic_c_types = {

1926 # name : (signed, longness)

1927 "Py_ssize_t" : (2, 0),

1928 "size_t" : (0, 0),

1929 }

1930

1931 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1932

1933 base_type_start_words = \

1934 basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)

1935

1936 def p_sign_and_longness(s):

1937 signed = 1

1938 longness = 0

1939 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1940 if s.systring == 'unsigned':

1941 signed = 0

1942 elif s.systring == 'signed':

1943 signed = 2

1944 elif s.systring == 'short':

1945 longness = -1

1946 elif s.systring == 'long':

1947 longness += 1

1948 s.next()

1949 return signed, longness

1950

1951 def p_opt_cname(s):

1952 literal = p_opt_string_literal(s)

1953 if literal:

1954 _, cname = literal

1955 cname = EncodedString(cname)

1956 cname.encoding = s.source_encoding

1957 else:

1958 cname = None

1959 return cname

1960

1961 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1962 assignable = 0, nonempty = 0,

1963 calling_convention_allowed = 0):

1964 # If empty is true, the declarator must be empty. If nonempty is true,

1965 # the declarator must be nonempty. Otherwise we don't care.

1966 # If cmethod_flag is true, then if this declarator declares

1967 # a function, it's a C method of an extension type.

1968 pos = s.position()

1969 if s.sy == '(':

1970 s.next()

1971 if s.sy == ')' or looking_at_name(s):

1972 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1973 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1974 else:

1975 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1976 cmethod_flag = cmethod_flag,

1977 nonempty = nonempty,

1978 calling_convention_allowed = 1)

1979 s.expect(')')

1980 else:

1981 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1982 assignable, nonempty)

1983 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1984 error(s.position(), "%s on something that is not a function"

1985 % result.calling_convention)

1986 while s.sy in ('[', '('):

1987 pos = s.position()

1988 if s.sy == '[':

1989 result = p_c_array_declarator(s, result)

1990 else: # sy == '('

1991 s.next()

1992 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1993 cmethod_flag = 0

1994 return result

1995

1996 def p_c_array_declarator(s, base):

1997 pos = s.position()

1998 s.next() # '['

1999 if s.sy != ']':

2000 dim = p_expr(s)

2001 else:

2002 dim = None

2003 s.expect(']')

2004 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

2005

2006 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

2007 # Opening paren has already been skipped

2008 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

2009 nonempty_declarators = 0)

2010 ellipsis = p_optional_ellipsis(s)

2011 s.expect(')')

2012 nogil = p_nogil(s)

2013 exc_val, exc_check = p_exception_value_clause(s)

2014 with_gil = p_with_gil(s)

2015 return Nodes.CFuncDeclaratorNode(pos,

2016 base = base, args = args, has_varargs = ellipsis,

2017 exception_value = exc_val, exception_check = exc_check,

2018 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

2019

2020 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

2021 assignable, nonempty):

2022 pos = s.position()

2023 calling_convention = p_calling_convention(s)

2024 if s.sy == '*':

2025 s.next()

2026 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2027 cmethod_flag = cmethod_flag,

2028 assignable = assignable, nonempty = nonempty)

2029 result = Nodes.CPtrDeclaratorNode(pos,

2030 base = base)

2031 elif s.sy == '**': # scanner returns this as a single token

2032 s.next()

2033 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2034 cmethod_flag = cmethod_flag,

2035 assignable = assignable, nonempty = nonempty)

2036 result = Nodes.CPtrDeclaratorNode(pos,

2037 base = Nodes.CPtrDeclaratorNode(pos,

2038 base = base))

2039 else:

2040 rhs = None

2041 if s.sy == 'IDENT':

2042 name = EncodedString(s.systring)

2043 if empty:

2044 error(s.position(), "Declarator should be empty")

2045 s.next()

2046 cname = p_opt_cname(s)

2047 if s.sy == '=' and assignable:

2048 s.next()

2049 rhs = p_simple_expr(s)

2050 else:

2051 if nonempty:

2052 error(s.position(), "Empty declarator")

2053 name = ""

2054 cname = None

2055 result = Nodes.CNameDeclaratorNode(pos,

2056 name = name, cname = cname, default = rhs)

2057 result.calling_convention = calling_convention

2058 return result

2059

2060 def p_nogil(s):

2061 if s.sy == 'IDENT' and s.systring == 'nogil':

2062 s.next()

2063 return 1

2064 else:

2065 return 0

2066

2067 def p_with_gil(s):

2068 if s.sy == 'with':

2069 s.next()

2070 s.expect_keyword('gil')

2071 return 1

2072 else:

2073 return 0

2074

2075 def p_exception_value_clause(s):

2076 exc_val = None

2077 exc_check = 0

2078 if s.sy == 'except':

2079 s.next()

2080 if s.sy == '*':

2081 exc_check = 1

2082 s.next()

2083 elif s.sy == '+':

2084 exc_check = '+'

2085 s.next()

2086 if s.sy == 'IDENT':

2087 name = s.systring

2088 s.next()

2089 exc_val = p_name(s, name)

2090 else:

2091 if s.sy == '?':

2092 exc_check = 1

2093 s.next()

2094 exc_val = p_simple_expr(s)

2095 return exc_val, exc_check

2096

2097 c_arg_list_terminators = ('*', '**', '.', ')')

2098

2099 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

2100 nonempty_declarators = 0, kw_only = 0):

2101 # Comma-separated list of C argument declarations, possibly empty.

2102 # May have a trailing comma.

2103 args = []

2104 is_self_arg = cmethod_flag

2105 while s.sy not in c_arg_list_terminators:

2106 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

2107 nonempty = nonempty_declarators, kw_only = kw_only))

2108 if s.sy != ',':

2109 break

2110 s.next()

2111 is_self_arg = 0

2112 return args

2113

2114 def p_optional_ellipsis(s):

2115 if s.sy == '.':

2116 expect_ellipsis(s)

2117 return 1

2118 else:

2119 return 0

2120

2121 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

2122 pos = s.position()

2123 not_none = 0

2124 default = None

2125 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2126 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2127 if s.sy == 'not':

2128 s.next()

2129 if s.sy == 'IDENT' and s.systring == 'None':

2130 s.next()

2131 else:

2132 s.error("Expected 'None'")

2133 if not in_pyfunc:

2134 error(pos, "'not None' only allowed in Python functions")

2135 not_none = 1

2136 if s.sy == '=':

2137 s.next()

2138 if 'pxd' in s.level:

2139 if s.sy not in ['*', '?']:

2140 error(pos, "default values cannot be specified in pxd files, use ? or *")

2141 default = ExprNodes.BoolNode(1)

2142 s.next()

2143 else:

2144 default = p_simple_expr(s)

2145 return Nodes.CArgDeclNode(pos,

2146 base_type = base_type,

2147 declarator = declarator,

2148 not_none = not_none,

2149 default = default,

2150 kw_only = kw_only)

2151

2152 def p_api(s):

2153 if s.sy == 'IDENT' and s.systring == 'api':

2154 s.next()

2155 return 1

2156 else:

2157 return 0

2158

2159 def p_cdef_statement(s, ctx):

2160 pos = s.position()

2161 ctx.visibility = p_visibility(s, ctx.visibility)

2162 ctx.api = ctx.api or p_api(s)

2163 if ctx.api:

2164 if ctx.visibility not in ('private', 'public'):

2165 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2166 if (ctx.visibility == 'extern') and s.sy == 'from':

2167 return p_cdef_extern_block(s, pos, ctx)

2168 elif s.sy == 'import':

2169 s.next()

2170 return p_cdef_extern_block(s, pos, ctx)

2171 elif p_nogil(s):

2172 ctx.nogil = 1

2173 if ctx.overridable:

2174 error(pos, "cdef blocks cannot be declared cpdef")

2175 return p_cdef_block(s, ctx)

2176 elif s.sy == ':':

2177 if ctx.overridable:

2178 error(pos, "cdef blocks cannot be declared cpdef")

2179 return p_cdef_block(s, ctx)

2180 elif s.sy == 'class':

2181 if ctx.level not in ('module', 'module_pxd'):

2182 error(pos, "Extension type definition not allowed here")

2183 if ctx.overridable:

2184 error(pos, "Extension types cannot be declared cpdef")

2185 return p_c_class_definition(s, pos, ctx)

2186 elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):

2187 if ctx.level not in ('module', 'module_pxd'):

2188 error(pos, "C struct/union/enum definition not allowed here")

2189 if ctx.overridable:

2190 error(pos, "C struct/union/enum cannot be declared cpdef")

2191 if s.systring == "enum":

2192 return p_c_enum_definition(s, pos, ctx)

2193 else:

2194 return p_c_struct_or_union_definition(s, pos, ctx)

2195 else:

2196 return p_c_func_or_var_declaration(s, pos, ctx)

2197

2198 def p_cdef_block(s, ctx):

2199 return p_suite(s, ctx(cdef_flag = 1))

2200

2201 def p_cdef_extern_block(s, pos, ctx):

2202 if ctx.overridable:

2203 error(pos, "cdef extern blocks cannot be declared cpdef")

2204 include_file = None

2205 s.expect('from')

2206 if s.sy == '*':

2207 s.next()

2208 else:

2209 _, include_file = p_string_literal(s)

2210 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2211 if p_nogil(s):

2212 ctx.nogil = 1

2213 body = p_suite(s, ctx)

2214 return Nodes.CDefExternNode(pos,

2215 include_file = include_file,

2216 body = body)

2217

2218 def p_c_enum_definition(s, pos, ctx):

2219 # s.sy == ident 'enum'

2220 s.next()

2221 if s.sy == 'IDENT':

2222 name = s.systring

2223 s.next()

2224 cname = p_opt_cname(s)

2225 else:

2226 name = None

2227 cname = None

2228 items = None

2229 s.expect(':')

2230 items = []

2231 if s.sy != 'NEWLINE':

2232 p_c_enum_line(s, items)

2233 else:

2234 s.next() # 'NEWLINE'

2235 s.expect_indent()

2236 while s.sy not in ('DEDENT', 'EOF'):

2237 p_c_enum_line(s, items)

2238 s.expect_dedent()

2239 return Nodes.CEnumDefNode(

2240 pos, name = name, cname = cname, items = items,

2241 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2242 in_pxd = ctx.level == 'module_pxd')

2243

2244 def p_c_enum_line(s, items):

2245 if s.sy != 'pass':

2246 p_c_enum_item(s, items)

2247 while s.sy == ',':

2248 s.next()

2249 if s.sy in ('NEWLINE', 'EOF'):

2250 break

2251 p_c_enum_item(s, items)

2252 else:

2253 s.next()

2254 s.expect_newline("Syntax error in enum item list")

2255

2256 def p_c_enum_item(s, items):

2257 pos = s.position()

2258 name = p_ident(s)

2259 cname = p_opt_cname(s)

2260 value = None

2261 if s.sy == '=':

2262 s.next()

2263 value = p_simple_expr(s)

2264 items.append(Nodes.CEnumDefItemNode(pos,

2265 name = name, cname = cname, value = value))

2266

2267 def p_c_struct_or_union_definition(s, pos, ctx):

2268 packed = False

2269 if s.systring == 'packed':

2270 packed = True

2271 s.next()

2272 if s.sy != 'IDENT' or s.systring != 'struct':

2273 s.expected('struct')

2274 # s.sy == ident 'struct' or 'union'

2275 kind = s.systring

2276 s.next()

2277 name = p_ident(s)

2278 cname = p_opt_cname(s)

2279 attributes = None

2280 if s.sy == ':':

2281 s.next()

2282 s.expect('NEWLINE')

2283 s.expect_indent()

2284 attributes = []

2285 body_ctx = Ctx()

2286 while s.sy != 'DEDENT':

2287 if s.sy != 'pass':

2288 attributes.append(

2289 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2290 else:

2291 s.next()

2292 s.expect_newline("Expected a newline")

2293 s.expect_dedent()

2294 else:

2295 s.expect_newline("Syntax error in struct or union definition")

2296 return Nodes.CStructOrUnionDefNode(pos,

2297 name = name, cname = cname, kind = kind, attributes = attributes,

2298 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2299 in_pxd = ctx.level == 'module_pxd', packed = packed)

2300

2301 def p_visibility(s, prev_visibility):

2302 pos = s.position()

2303 visibility = prev_visibility

2304 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2305 visibility = s.systring

2306 if prev_visibility != 'private' and visibility != prev_visibility:

2307 s.error("Conflicting visibility options '%s' and '%s'"

2308 % (prev_visibility, visibility))

2309 s.next()

2310 return visibility

2311

2312 def p_c_modifiers(s):

2313 if s.sy == 'IDENT' and s.systring in ('inline',):

2314 modifier = s.systring

2315 s.next()

2316 return [modifier] + p_c_modifiers(s)

2317 return []

2318

2319 def p_c_func_or_var_declaration(s, pos, ctx):

2320 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2321 modifiers = p_c_modifiers(s)

2322 base_type = p_c_base_type(s, nonempty = 1)

2323 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2324 assignable = 1, nonempty = 1)

2325 declarator.overridable = ctx.overridable

2326 if s.sy == ':':

2327 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):

2328 s.error("C function definition not allowed here")

2329 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2330 result = Nodes.CFuncDefNode(pos,

2331 visibility = ctx.visibility,

2332 base_type = base_type,

2333 declarator = declarator,

2334 body = suite,

2335 doc = doc,

2336 modifiers = modifiers,

2337 api = ctx.api,

2338 overridable = ctx.overridable)

2339 else:

2340 #if api:

2341 # error(s.pos, "'api' not allowed with variable declaration")

2342 declarators = [declarator]

2343 while s.sy == ',':

2344 s.next()

2345 if s.sy == 'NEWLINE':

2346 break

2347 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2348 assignable = 1, nonempty = 1)

2349 declarators.append(declarator)

2350 s.expect_newline("Syntax error in C variable declaration")

2351 result = Nodes.CVarDefNode(pos,

2352 visibility = ctx.visibility,

2353 base_type = base_type,

2354 declarators = declarators,

2355 in_pxd = ctx.level == 'module_pxd',

2356 api = ctx.api,

2357 overridable = ctx.overridable)

2358 return result

2359

2360 def p_ctypedef_statement(s, ctx):

2361 # s.sy == 'ctypedef'

2362 pos = s.position()

2363 s.next()

2364 visibility = p_visibility(s, ctx.visibility)

2365 api = p_api(s)

2366 ctx = ctx(typedef_flag = 1, visibility = visibility)

2367 if api:

2368 ctx.api = 1

2369 if s.sy == 'class':

2370 return p_c_class_definition(s, pos, ctx)

2371 elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):

2372 if s.systring == 'enum':

2373 return p_c_enum_definition(s, pos, ctx)

2374 else:

2375 return p_c_struct_or_union_definition(s, pos, ctx)

2376 else:

2377 base_type = p_c_base_type(s, nonempty = 1)

2378 if base_type.name is None:

2379 s.error("Syntax error in ctypedef statement")

2380 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2381 s.expect_newline("Syntax error in ctypedef statement")

2382 return Nodes.CTypeDefNode(

2383 pos, base_type = base_type,

2384 declarator = declarator, visibility = visibility,

2385 in_pxd = ctx.level == 'module_pxd')

2386

2387 def p_decorators(s):

2388 decorators = []

2389 while s.sy == 'DECORATOR':

2390 pos = s.position()

2391 s.next()

2392 decstring = p_dotted_name(s, as_allowed=0)[2]

2393 names = decstring.split('.')

2394 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2395 for name in names[1:]:

2396 decorator = ExprNodes.AttributeNode(pos,

2397 attribute=EncodedString(name),

2398 obj=decorator)

2399 if s.sy == '(':

2400 decorator = p_call(s, decorator)

2401 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2402 s.expect_newline("Expected a newline after decorator")

2403 return decorators

2404

2405 def p_def_statement(s, decorators=None):

2406 # s.sy == 'def'

2407 pos = s.position()

2408 s.next()

2409 name = EncodedString( p_ident(s) )

2410 #args = []

2411 s.expect('(');

2412 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2413 star_arg = None

2414 starstar_arg = None

2415 if s.sy == '*':

2416 s.next()

2417 if s.sy == 'IDENT':

2418 star_arg = p_py_arg_decl(s)

2419 if s.sy == ',':

2420 s.next()

2421 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2422 nonempty_declarators = 1, kw_only = 1))

2423 elif s.sy != ')':

2424 s.error("Syntax error in Python function argument list")

2425 if s.sy == '**':

2426 s.next()

2427 starstar_arg = p_py_arg_decl(s)

2428 s.expect(')')

2429 if p_nogil(s):

2430 error(s.pos, "Python function cannot be declared nogil")

2431 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2432 return Nodes.DefNode(pos, name = name, args = args,

2433 star_arg = star_arg, starstar_arg = starstar_arg,

2434 doc = doc, body = body, decorators = decorators)

2435

2436 def p_py_arg_decl(s):

2437 pos = s.position()

2438 name = p_ident(s)

2439 return Nodes.PyArgDeclNode(pos, name = name)

2440

2441 def p_class_statement(s, decorators):

2442 # s.sy == 'class'

2443 pos = s.position()

2444 s.next()

2445 class_name = EncodedString( p_ident(s) )

2446 class_name.encoding = s.source_encoding

2447 if s.sy == '(':

2448 s.next()

2449 base_list = p_simple_expr_list(s)

2450 s.expect(')')

2451 else:

2452 base_list = []

2453 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2454 return Nodes.PyClassDefNode(pos,

2455 name = class_name,

2456 bases = ExprNodes.TupleNode(pos, args = base_list),

2457 doc = doc, body = body, decorators = decorators)

2458

2459 def p_c_class_definition(s, pos, ctx):

2460 # s.sy == 'class'

2461 s.next()

2462 module_path = []

2463 class_name = p_ident(s)

2464 while s.sy == '.':

2465 s.next()

2466 module_path.append(class_name)

2467 class_name = p_ident(s)

2468 if module_path and ctx.visibility != 'extern':

2469 error(pos, "Qualified class name only allowed for 'extern' C class")

2470 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2471 s.next()

2472 as_name = p_ident(s)

2473 else:

2474 as_name = class_name

2475 objstruct_name = None

2476 typeobj_name = None

2477 base_class_module = None

2478 base_class_name = None

2479 if s.sy == '(':

2480 s.next()

2481 base_class_path = [p_ident(s)]

2482 while s.sy == '.':

2483 s.next()

2484 base_class_path.append(p_ident(s))

2485 if s.sy == ',':

2486 s.error("C class may only have one base class")

2487 s.expect(')')

2488 base_class_module = ".".join(base_class_path[:-1])

2489 base_class_name = base_class_path[-1]

2490 if s.sy == '[':

2491 if ctx.visibility not in ('public', 'extern'):

2492 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2493 objstruct_name, typeobj_name = p_c_class_options(s)

2494 if s.sy == ':':

2495 if ctx.level == 'module_pxd':

2496 body_level = 'c_class_pxd'

2497 else:

2498 body_level = 'c_class'

2499 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2500 else:

2501 s.expect_newline("Syntax error in C class definition")

2502 doc = None

2503 body = None

2504 if ctx.visibility == 'extern':

2505 if not module_path:

2506 error(pos, "Module name required for 'extern' C class")

2507 if typeobj_name:

2508 error(pos, "Type object name specification not allowed for 'extern' C class")

2509 elif ctx.visibility == 'public':

2510 if not objstruct_name:

2511 error(pos, "Object struct name specification required for 'public' C class")

2512 if not typeobj_name:

2513 error(pos, "Type object name specification required for 'public' C class")

2514 elif ctx.visibility == 'private':

2515 if ctx.api:

2516 error(pos, "Only 'public' C class can be declared 'api'")

2517 else:

2518 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2519 return Nodes.CClassDefNode(pos,

2520 visibility = ctx.visibility,

2521 typedef_flag = ctx.typedef_flag,

2522 api = ctx.api,

2523 module_name = ".".join(module_path),

2524 class_name = class_name,

2525 as_name = as_name,

2526 base_class_module = base_class_module,

2527 base_class_name = base_class_name,

2528 objstruct_name = objstruct_name,

2529 typeobj_name = typeobj_name,

2530 in_pxd = ctx.level == 'module_pxd',

2531 doc = doc,

2532 body = body)

2533

2534 def p_c_class_options(s):

2535 objstruct_name = None

2536 typeobj_name = None

2537 s.expect('[')

2538 while 1:

2539 if s.sy != 'IDENT':

2540 break

2541 if s.systring == 'object':

2542 s.next()

2543 objstruct_name = p_ident(s)

2544 elif s.systring == 'type':

2545 s.next()

2546 typeobj_name = p_ident(s)

2547 if s.sy != ',':

2548 break

2549 s.next()

2550 s.expect(']', "Expected 'object' or 'type'")

2551 return objstruct_name, typeobj_name

2552

2553 def p_property_decl(s):

2554 pos = s.position()

2555 s.next() # 'property'

2556 name = p_ident(s)

2557 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2558 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2559

2560 def p_doc_string(s):

2561 if s.sy == 'BEGIN_STRING':

2562 pos = s.position()

2563 kind, result = p_cat_string_literal(s)

2564 if s.sy != 'EOF':

2565 s.expect_newline("Syntax error in doc string")

2566 if kind != 'u':

2567 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2568 if kind == 'b':

2569 result.encoding = None # force a unicode string

2570 return result

2571 else:

2572 return None

2573

2574 def p_code(s, level=None):

2575 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2576 if s.sy != 'EOF':

2577 s.error("Syntax error in statement [%s,%s]" % (

2578 repr(s.sy), repr(s.systring)))

2579 return body

2580

2581 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*(\w+)\s*=(.*)$")

2582

2583 def p_compiler_directive_comments(s):

2584 result = {}

2585 while s.sy == 'commentline':

2586 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2587 if m:

2588 name = m.group(1)

2589 try:

2590 value = Options.parse_directive_value(str(name), str(m.group(2).strip()))

2591 if value is not None: # can be False!

2592 result[name] = value

2593 except ValueError, e:

2594 s.error(e.args[0], fatal=False)

2595 s.next()

2596 return result

2597

2598 def p_module(s, pxd, full_module_name):

2599 pos = s.position()

2600

2601 directive_comments = p_compiler_directive_comments(s)

2602 s.parse_comments = False

2603

2604 doc = p_doc_string(s)

2605 if pxd:

2606 level = 'module_pxd'

2607 else:

2608 level = 'module'

2609

2610 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2611 if s.sy != 'EOF':

2612 s.error("Syntax error in statement [%s,%s]" % (

2613 repr(s.sy), repr(s.systring)))

2614 return ModuleNode(pos, doc = doc, body = body,

2615 full_module_name = full_module_name,

2616 directive_comments = directive_comments)

2617

2618 #----------------------------------------------

2619 #

2620 # Debugging

2621 #

2622 #----------------------------------------------

2623

2624 def print_parse_tree(f, node, level, key = None):

2625 from Nodes import Node

2626 ind = " " * level

2627 if node:

2628 f.write(ind)

2629 if key:

2630 f.write("%s: " % key)

2631 t = type(node)

2632 if t is tuple:

2633 f.write("(%s @ %s\n" % (node[0], node[1]))

2634 for i in xrange(2, len(node)):

2635 print_parse_tree(f, node[i], level+1)

2636 f.write("%s)\n" % ind)

2637 return

2638 elif isinstance(node, Node):

2639 try:

2640 tag = node.tag

2641 except AttributeError:

2642 tag = node.__class__.__name__

2643 f.write("%s @ %s\n" % (tag, node.pos))

2644 for name, value in node.__dict__.items():

2645 if name != 'tag' and name != 'pos':

2646 print_parse_tree(f, value, level+1, name)

2647 return

2648 elif t is list:

2649 f.write("[\n")

2650 for i in xrange(len(node)):

2651 print_parse_tree(f, node[i], level+1)

2652 f.write("%s]\n" % ind)

2653 return

2654 f.write("%s%s\n" % (ind, node))

2655