cython-devel: b254c4dbc089 Cython/Compiler/Parsing.py

Cython has moved to github.

cython-devel

view Cython/Compiler/Parsing.py @ 3484:b254c4dbc089

support for some Python 3 (or 2.6+) syntax features (found by test_grammar.py in Py3.1.1):
- oct/bin notation: 0o12345, 0b10101
- function annotations (only pure syntax support, not currently used)
also: allow decorators on inner functions

author	Stefan Behnel <scoder@users.berlios.de>
date	Thu Jan 28 23:05:39 2010 +0100 (2 years ago)
parents	28bf4771649d
children	3e25233bbcc7

line source

1 # cython: auto_cpdef=True

2 #

3 # Pyrex Parser

4 #

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

10 import os

11 import re

12 import sys

13 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

14 import Nodes

15 import ExprNodes

16 import StringEncoding

17 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes

18 from ModuleNode import ModuleNode

19 from Errors import error, warning, InternalError

20 from Cython import Utils

21 import Future

22 import Options

24 class Ctx(object):

25 # Parsing context

26 level = 'other'

27 visibility = 'private'

28 cdef_flag = 0

29 typedef_flag = 0

30 api = 0

31 overridable = 0

32 nogil = 0

34 def __init__(self, **kwds):

35 self.__dict__.update(kwds)

37 def __call__(self, **kwds):

38 ctx = Ctx()

39 d = ctx.__dict__

40 d.update(self.__dict__)

41 d.update(kwds)

42 return ctx

44 def p_ident(s, message = "Expected an identifier"):

45 if s.sy == 'IDENT':

46 name = s.systring

47 s.next()

48 return name

49 else:

50 s.error(message)

52 def p_ident_list(s):

53 names = []

54 while s.sy == 'IDENT':

55 names.append(s.systring)

56 s.next()

57 if s.sy != ',':

58 break

59 s.next()

60 return names

62 #------------------------------------------

63 #

64 # Expressions

65 #

66 #------------------------------------------

68 def p_binop_operator(s):

69 pos = s.position()

70 op = s.sy

71 s.next()

72 return op, pos

74 def p_binop_expr(s, ops, p_sub_expr):

75 n1 = p_sub_expr(s)

76 while s.sy in ops:

77 op, pos = p_binop_operator(s)

78 n2 = p_sub_expr(s)

79 n1 = ExprNodes.binop_node(pos, op, n1, n2)

80 if op == '/':

81 if Future.division in s.context.future_directives:

82 n1.truedivision = True

83 else:

84 n1.truedivision = None # unknown

85 return n1

87 #expression: or_test [if or_test else test] | lambda_form

89 # actually:

90 #test: or_test ['if' or_test 'else' test] | lambdef

92 def p_simple_expr(s):

93 if s.sy == 'lambda':

94 return p_lambdef(s)

95 pos = s.position()

96 expr = p_or_test(s)

97 if s.sy == 'if':

98 s.next()

99 test = p_or_test(s)

100 s.expect('else')

101 other = p_test(s)

102 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

103 else:

104 return expr

105

106 #lambdef: 'lambda' [varargslist] ':' test

107

108 def p_lambdef(s, allow_conditional=True):

109 # s.sy == 'lambda'

110 pos = s.position()

111 s.next()

112 if s.sy == ':':

113 args = []

114 star_arg = starstar_arg = None

115 else:

116 args, star_arg, starstar_arg = p_varargslist(

117 s, terminator=':', annotated=False)

118 s.expect(':')

119 if allow_conditional:

120 expr = p_simple_expr(s)

121 else:

122 expr = p_test_nocond(s)

123 return ExprNodes.LambdaNode(

124 pos, args = args,

125 star_arg = star_arg, starstar_arg = starstar_arg,

126 result_expr = expr)

127

128 #lambdef_nocond: 'lambda' [varargslist] ':' test_nocond

129

130 def p_lambdef_nocond(s):

131 return p_lambdef(s, allow_conditional=False)

132

133 #test: or_test | lambdef

134

135 def p_test(s):

136 return p_or_test(s)

137

138 #test_nocond: or_test | lambdef_nocond

139

140 def p_test_nocond(s):

141 if s.sy == 'lambda':

142 return p_lambdef_nocond(s)

143 else:

144 return p_or_test(s)

145

146 #or_test: and_test ('or' and_test)*

147

148 def p_or_test(s):

149 return p_rassoc_binop_expr(s, ('or',), p_and_test)

150

151 def p_rassoc_binop_expr(s, ops, p_subexpr):

152 n1 = p_subexpr(s)

153 if s.sy in ops:

154 pos = s.position()

155 op = s.sy

156 s.next()

157 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

158 n1 = ExprNodes.binop_node(pos, op, n1, n2)

159 return n1

160

161 #and_test: not_test ('and' not_test)*

162

163 def p_and_test(s):

164 #return p_binop_expr(s, ('and',), p_not_test)

165 return p_rassoc_binop_expr(s, ('and',), p_not_test)

166

167 #not_test: 'not' not_test | comparison

168

169 def p_not_test(s):

170 if s.sy == 'not':

171 pos = s.position()

172 s.next()

173 return ExprNodes.NotNode(pos, operand = p_not_test(s))

174 else:

175 return p_comparison(s)

176

177 #comparison: expr (comp_op expr)*

178 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

179

180 def p_comparison(s):

181 n1 = p_starred_expr(s)

182 if s.sy in comparison_ops:

183 pos = s.position()

184 op = p_cmp_op(s)

185 n2 = p_starred_expr(s)

186 n1 = ExprNodes.PrimaryCmpNode(pos,

187 operator = op, operand1 = n1, operand2 = n2)

188 if s.sy in comparison_ops:

189 n1.cascade = p_cascaded_cmp(s)

190 return n1

191

192 def p_starred_expr(s):

193 pos = s.position()

194 if s.sy == '*':

195 starred = True

196 s.next()

197 else:

198 starred = False

199 expr = p_bit_expr(s)

200 if starred:

201 expr = ExprNodes.StarredTargetNode(pos, expr)

202 return expr

203

204 def p_cascaded_cmp(s):

205 pos = s.position()

206 op = p_cmp_op(s)

207 n2 = p_starred_expr(s)

208 result = ExprNodes.CascadedCmpNode(pos,

209 operator = op, operand2 = n2)

210 if s.sy in comparison_ops:

211 result.cascade = p_cascaded_cmp(s)

212 return result

213

214 def p_cmp_op(s):

215 if s.sy == 'not':

216 s.next()

217 s.expect('in')

218 op = 'not_in'

219 elif s.sy == 'is':

220 s.next()

221 if s.sy == 'not':

222 s.next()

223 op = 'is_not'

224 else:

225 op = 'is'

226 else:

227 op = s.sy

228 s.next()

229 if op == '<>':

230 op = '!='

231 return op

232

233 comparison_ops = (

234 '<', '>', '==', '>=', '<=', '<>', '!=',

235 'in', 'is', 'not'

236 )

237

238 #expr: xor_expr ('|' xor_expr)*

239

240 def p_bit_expr(s):

241 return p_binop_expr(s, ('|',), p_xor_expr)

242

243 #xor_expr: and_expr ('^' and_expr)*

244

245 def p_xor_expr(s):

246 return p_binop_expr(s, ('^',), p_and_expr)

247

248 #and_expr: shift_expr ('&' shift_expr)*

249

250 def p_and_expr(s):

251 return p_binop_expr(s, ('&',), p_shift_expr)

252

253 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

254

255 def p_shift_expr(s):

256 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

257

258 #arith_expr: term (('+'|'-') term)*

259

260 def p_arith_expr(s):

261 return p_binop_expr(s, ('+', '-'), p_term)

262

263 #term: factor (('*'|'/'|'%') factor)*

264

265 def p_term(s):

266 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

267

268 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

269

270 def p_factor(s):

271 sy = s.sy

272 if sy in ('+', '-', '~'):

273 op = s.sy

274 pos = s.position()

275 s.next()

276 return ExprNodes.unop_node(pos, op, p_factor(s))

277 elif sy == '&':

278 pos = s.position()

279 s.next()

280 arg = p_factor(s)

281 return ExprNodes.AmpersandNode(pos, operand = arg)

282 elif sy == "<":

283 return p_typecast(s)

284 elif sy == 'IDENT' and s.systring == "sizeof":

285 return p_sizeof(s)

286 else:

287 return p_power(s)

288

289 def p_typecast(s):

290 # s.sy == "<"

291 pos = s.position()

292 s.next()

293 base_type = p_c_base_type(s)

294 if base_type.name is None:

295 s.error("Unknown type")

296 declarator = p_c_declarator(s, empty = 1)

297 if s.sy == '?':

298 s.next()

299 typecheck = 1

300 else:

301 typecheck = 0

302 s.expect(">")

303 operand = p_factor(s)

304 return ExprNodes.TypecastNode(pos,

305 base_type = base_type,

306 declarator = declarator,

307 operand = operand,

308 typecheck = typecheck)

309

310 def p_sizeof(s):

311 # s.sy == ident "sizeof"

312 pos = s.position()

313 s.next()

314 s.expect('(')

315 # Here we decide if we are looking at an expression or type

316 # If it is actually a type, but parsable as an expression,

317 # we treat it as an expression here.

318 if looking_at_expr(s):

319 operand = p_simple_expr(s)

320 node = ExprNodes.SizeofVarNode(pos, operand = operand)

321 else:

322 base_type = p_c_base_type(s)

323 declarator = p_c_declarator(s, empty = 1)

324 node = ExprNodes.SizeofTypeNode(pos,

325 base_type = base_type, declarator = declarator)

326 s.expect(')')

327 return node

328

329 def p_yield_expression(s):

330 # s.sy == "yield"

331 pos = s.position()

332 s.next()

333 if s.sy != ')' and s.sy not in statement_terminators:

334 arg = p_expr(s)

335 else:

336 arg = None

337 return ExprNodes.YieldExprNode(pos, arg=arg)

338

339 def p_yield_statement(s):

340 # s.sy == "yield"

341 yield_expr = p_yield_expression(s)

342 return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)

343

344 #power: atom trailer* ('**' factor)*

345

346 def p_power(s):

347 n1 = p_atom(s)

348 while s.sy in ('(', '[', '.'):

349 n1 = p_trailer(s, n1)

350 if s.sy == '**':

351 pos = s.position()

352 s.next()

353 n2 = p_factor(s)

354 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

355 return n1

356

357 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

358

359 def p_trailer(s, node1):

360 pos = s.position()

361 if s.sy == '(':

362 return p_call(s, node1)

363 elif s.sy == '[':

364 return p_index(s, node1)

365 else: # s.sy == '.'

366 s.next()

367 name = EncodedString( p_ident(s) )

368 return ExprNodes.AttributeNode(pos,

369 obj = node1, attribute = name)

370

371 # arglist: argument (',' argument)* [',']

372 # argument: [test '='] test # Really [keyword '='] test

373

374 def p_call(s, function):

375 # s.sy == '('

376 pos = s.position()

377 s.next()

378 positional_args = []

379 keyword_args = []

380 star_arg = None

381 starstar_arg = None

382 while s.sy not in ('**', ')'):

383 if s.sy == '*':

384 if star_arg:

385 s.error("only one star-arg parameter allowed",

386 pos = s.position())

387 s.next()

388 star_arg = p_simple_expr(s)

389 else:

390 arg = p_simple_expr(s)

391 if s.sy == '=':

392 s.next()

393 if not arg.is_name:

394 s.error("Expected an identifier before '='",

395 pos = arg.pos)

396 encoded_name = EncodedString(arg.name)

397 keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)

398 arg = p_simple_expr(s)

399 keyword_args.append((keyword, arg))

400 else:

401 if keyword_args:

402 s.error("Non-keyword arg following keyword arg",

403 pos = arg.pos)

404 if star_arg:

405 s.error("Non-keyword arg following star-arg",

406 pos = arg.pos)

407 positional_args.append(arg)

408 if s.sy != ',':

409 break

410 s.next()

411

412 if s.sy == '**':

413 s.next()

414 starstar_arg = p_simple_expr(s)

415 if s.sy == ',':

416 s.next()

417 s.expect(')')

418 if not (keyword_args or star_arg or starstar_arg):

419 return ExprNodes.SimpleCallNode(pos,

420 function = function,

421 args = positional_args)

422 else:

423 arg_tuple = None

424 keyword_dict = None

425 if positional_args or not star_arg:

426 arg_tuple = ExprNodes.TupleNode(pos,

427 args = positional_args)

428 if star_arg:

429 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

430 if arg_tuple:

431 arg_tuple = ExprNodes.binop_node(pos,

432 operator = '+', operand1 = arg_tuple,

433 operand2 = star_arg_tuple)

434 else:

435 arg_tuple = star_arg_tuple

436 if keyword_args:

437 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

438 for key, value in keyword_args]

439 keyword_dict = ExprNodes.DictNode(pos,

440 key_value_pairs = keyword_args)

441 return ExprNodes.GeneralCallNode(pos,

442 function = function,

443 positional_args = arg_tuple,

444 keyword_args = keyword_dict,

445 starstar_arg = starstar_arg)

446

447 #lambdef: 'lambda' [varargslist] ':' test

448

449 #subscriptlist: subscript (',' subscript)* [',']

450

451 def p_index(s, base):

452 # s.sy == '['

453 pos = s.position()

454 s.next()

455 subscripts = p_subscript_list(s)

456 if len(subscripts) == 1 and len(subscripts[0]) == 2:

457 start, stop = subscripts[0]

458 result = ExprNodes.SliceIndexNode(pos,

459 base = base, start = start, stop = stop)

460 else:

461 indexes = make_slice_nodes(pos, subscripts)

462 if len(indexes) == 1:

463 index = indexes[0]

464 else:

465 index = ExprNodes.TupleNode(pos, args = indexes)

466 result = ExprNodes.IndexNode(pos,

467 base = base, index = index)

468 s.expect(']')

469 return result

470

471 def p_subscript_list(s):

472 items = [p_subscript(s)]

473 while s.sy == ',':

474 s.next()

475 if s.sy == ']':

476 break

477 items.append(p_subscript(s))

478 return items

479

480 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

481

482 def p_subscript(s):

483 # Parse a subscript and return a list of

484 # 1, 2 or 3 ExprNodes, depending on how

485 # many slice elements were encountered.

486 pos = s.position()

487 if s.sy == '.':

488 expect_ellipsis(s)

489 return [ExprNodes.EllipsisNode(pos)]

490 else:

491 start = p_slice_element(s, (':',))

492 if s.sy != ':':

493 return [start]

494 s.next()

495 stop = p_slice_element(s, (':', ',', ']'))

496 if s.sy != ':':

497 return [start, stop]

498 s.next()

499 step = p_slice_element(s, (':', ',', ']'))

500 return [start, stop, step]

501

502 def p_slice_element(s, follow_set):

503 # Simple expression which may be missing iff

504 # it is followed by something in follow_set.

505 if s.sy not in follow_set:

506 return p_simple_expr(s)

507 else:

508 return None

509

510 def expect_ellipsis(s):

511 s.expect('.')

512 s.expect('.')

513 s.expect('.')

514

515 def make_slice_nodes(pos, subscripts):

516 # Convert a list of subscripts as returned

517 # by p_subscript_list into a list of ExprNodes,

518 # creating SliceNodes for elements with 2 or

519 # more components.

520 result = []

521 for subscript in subscripts:

522 if len(subscript) == 1:

523 result.append(subscript[0])

524 else:

525 result.append(make_slice_node(pos, *subscript))

526 return result

527

528 def make_slice_node(pos, start, stop = None, step = None):

529 if not start:

530 start = ExprNodes.NoneNode(pos)

531 if not stop:

532 stop = ExprNodes.NoneNode(pos)

533 if not step:

534 step = ExprNodes.NoneNode(pos)

535 return ExprNodes.SliceNode(pos,

536 start = start, stop = stop, step = step)

537

538 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

539

540 def p_atom(s):

541 pos = s.position()

542 sy = s.sy

543 if sy == '(':

544 s.next()

545 if s.sy == ')':

546 result = ExprNodes.TupleNode(pos, args = [])

547 elif s.sy == 'yield':

548 result = p_yield_expression(s)

549 else:

550 result = p_expr(s)

551 s.expect(')')

552 return result

553 elif sy == '[':

554 return p_list_maker(s)

555 elif sy == '{':

556 return p_dict_or_set_maker(s)

557 elif sy == '`':

558 return p_backquote_expr(s)

559 elif sy == 'INT':

560 value = s.systring

561 s.next()

562 unsigned = ""

563 longness = ""

564 while value[-1] in "UuLl":

565 if value[-1] in "Ll":

566 longness += "L"

567 else:

568 unsigned += "U"

569 value = value[:-1]

570 return ExprNodes.IntNode(pos,

571 value = value,

572 unsigned = unsigned,

573 longness = longness)

574 elif sy == 'FLOAT':

575 value = s.systring

576 s.next()

577 return ExprNodes.FloatNode(pos, value = value)

578 elif sy == 'IMAG':

579 value = s.systring[:-1]

580 s.next()

581 return ExprNodes.ImagNode(pos, value = value)

582 elif sy == 'BEGIN_STRING':

583 kind, value = p_cat_string_literal(s)

584 if kind == 'c':

585 return ExprNodes.CharNode(pos, value = value)

586 elif kind == 'u':

587 return ExprNodes.UnicodeNode(pos, value = value)

588 elif kind == 'b':

589 return ExprNodes.BytesNode(pos, value = value)

590 else:

591 return ExprNodes.StringNode(pos, value = value)

592 elif sy == 'IDENT':

593 name = EncodedString( s.systring )

594 s.next()

595 if name == "None":

596 return ExprNodes.NoneNode(pos)

597 elif name == "True":

598 return ExprNodes.BoolNode(pos, value=True)

599 elif name == "False":

600 return ExprNodes.BoolNode(pos, value=False)

601 elif name == "NULL":

602 return ExprNodes.NullNode(pos)

603 else:

604 return p_name(s, name)

605 else:

606 s.error("Expected an identifier or literal")

607

608 def p_name(s, name):

609 pos = s.position()

610 if not s.compile_time_expr and name in s.compile_time_env:

611 value = s.compile_time_env.lookup_here(name)

612 rep = repr(value)

613 if isinstance(value, bool):

614 return ExprNodes.BoolNode(pos, value = value)

615 elif isinstance(value, int):

616 return ExprNodes.IntNode(pos, value = rep)

617 elif isinstance(value, long):

618 return ExprNodes.IntNode(pos, value = rep, longness = "L")

619 elif isinstance(value, float):

620 return ExprNodes.FloatNode(pos, value = rep)

621 elif isinstance(value, _unicode):

622 return ExprNodes.UnicodeNode(pos, value = value)

623 elif isinstance(value, _bytes):

624 return ExprNodes.BytesNode(pos, value = value)

625 else:

626 error(pos, "Invalid type for compile-time constant: %s"

627 % value.__class__.__name__)

628 return ExprNodes.NameNode(pos, name = name)

629

630 def p_cat_string_literal(s):

631 # A sequence of one or more adjacent string literals.

632 # Returns (kind, value) where kind in ('b', 'c', 'u', '')

633 kind, value = p_string_literal(s)

634 if s.sy != 'BEGIN_STRING':

635 return kind, value

636 if kind != 'c':

637 strings = [value]

638 while s.sy == 'BEGIN_STRING':

639 pos = s.position()

640 next_kind, next_value = p_string_literal(s)

641 if next_kind == 'c':

642 error(pos, "Cannot concatenate char literal with another string or char literal")

643 elif next_kind != kind:

644 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %

645 (kind, next_kind))

646 else:

647 strings.append(next_value)

648 if kind == 'u':

649 value = EncodedString( u''.join(strings) )

650 else:

651 value = BytesLiteral( StringEncoding.join_bytes(strings) )

652 value.encoding = s.source_encoding

653 return kind, value

654

655 def p_opt_string_literal(s):

656 if s.sy == 'BEGIN_STRING':

657 return p_string_literal(s)

658 else:

659 return None

660

661 def p_string_literal(s):

662 # A single string or char literal.

663 # Returns (kind, value) where kind in ('b', 'c', 'u')

664 # s.sy == 'BEGIN_STRING'

665 pos = s.position()

666 is_raw = 0

667 kind = s.systring[:1].lower()

668 if kind == 'r':

669 kind = ''

670 is_raw = 1

671 elif kind in 'ub':

672 is_raw = s.systring[1:2].lower() == 'r'

673 elif kind != 'c':

674 kind = ''

675 if Future.unicode_literals in s.context.future_directives:

676 if kind == '':

677 kind = 'u'

678 if kind == 'u':

679 chars = StringEncoding.UnicodeLiteralBuilder()

680 else:

681 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

682 while 1:

683 s.next()

684 sy = s.sy

685 #print "p_string_literal: sy =", sy, repr(s.systring) ###

686 if sy == 'CHARS':

687 chars.append(s.systring)

688 elif sy == 'ESCAPE':

689 has_escape = True

690 systr = s.systring

691 if is_raw:

692 if systr == u'\\\n':

693 chars.append(u'\\\n')

694 elif systr == u'\\\"':

695 chars.append(u'"')

696 elif systr == u'\\\'':

697 chars.append(u"'")

698 else:

699 chars.append(systr)

700 else:

701 c = systr[1]

702 if c in u"01234567":

703 chars.append_charval( int(systr[1:], 8) )

704 elif c in u"'\"\\":

705 chars.append(c)

706 elif c in u"abfnrtv":

707 chars.append(

708 StringEncoding.char_from_escape_sequence(systr))

709 elif c == u'\n':

710 pass

711 elif c in u'Uux':

712 if kind == 'u' or c == 'x':

713 chrval = int(systr[2:], 16)

714 if chrval > 1114111: # sys.maxunicode:

715 s.error("Invalid unicode escape '%s'" % systr,

716 pos = pos)

717 elif chrval > 65535:

718 warning(s.position(),

719 "Unicode characters above 65535 are not "

720 "necessarily portable across Python installations", 1)

721 chars.append_charval(chrval)

722 else:

723 # unicode escapes in plain byte strings are not unescaped

724 chars.append(systr)

725 else:

726 chars.append(u'\\' + systr[1:])

727 elif sy == 'NEWLINE':

728 chars.append(u'\n')

729 elif sy == 'END_STRING':

730 break

731 elif sy == 'EOF':

732 s.error("Unclosed string literal", pos = pos)

733 else:

734 s.error(

735 "Unexpected token %r:%r in string literal" %

736 (sy, s.systring))

737 if kind == 'c':

738 value = chars.getchar()

739 if len(value) != 1:

740 error(pos, u"invalid character literal: %r" % value)

741 else:

742 value = chars.getstring()

743 s.next()

744 #print "p_string_literal: value =", repr(value) ###

745 return kind, value

746

747 # list_display ::= "[" [listmaker] "]"

748 # listmaker ::= expression ( comp_for | ( "," expression )* [","] )

749 # comp_iter ::= comp_for | comp_if

750 # comp_for ::= "for" expression_list "in" testlist [comp_iter]

751 # comp_if ::= "if" test [comp_iter]

752

753 def p_list_maker(s):

754 # s.sy == '['

755 pos = s.position()

756 s.next()

757 if s.sy == ']':

758 s.expect(']')

759 return ExprNodes.ListNode(pos, args = [])

760 expr = p_simple_expr(s)

761 if s.sy == 'for':

762 target = ExprNodes.ListNode(pos, args = [])

763 append = ExprNodes.ComprehensionAppendNode(

764 pos, expr=expr, target=ExprNodes.CloneNode(target))

765 loop = p_comp_for(s, Nodes.ExprStatNode(append.pos, expr=append))

766 s.expect(']')

767 return ExprNodes.ComprehensionNode(

768 pos, loop=loop, append=append, target=target)

769 else:

770 exprs = [expr]

771 if s.sy == ',':

772 s.next()

773 exprs += p_simple_expr_list(s)

774 s.expect(']')

775 return ExprNodes.ListNode(pos, args = exprs)

776

777 def p_comp_iter(s, body):

778 if s.sy == 'for':

779 return p_comp_for(s, body)

780 elif s.sy == 'if':

781 return p_comp_if(s, body)

782 else:

783 # insert the 'append' operation into the loop

784 return body

785

786 def p_comp_for(s, body):

787 # s.sy == 'for'

788 pos = s.position()

789 s.next()

790 kw = p_for_bounds(s)

791 kw['else_clause'] = None

792 kw['body'] = p_comp_iter(s, body)

793 return Nodes.ForStatNode(pos, **kw)

794

795 def p_comp_if(s, body):

796 # s.sy == 'if'

797 pos = s.position()

798 s.next()

799 test = p_test_nocond(s)

800 return Nodes.IfStatNode(pos,

801 if_clauses = [Nodes.IfClauseNode(pos, condition = test,

802 body = p_comp_iter(s, body))],

803 else_clause = None )

804

805 #dictmaker: test ':' test (',' test ':' test)* [',']

806

807 def p_dict_or_set_maker(s):

808 # s.sy == '{'

809 pos = s.position()

810 s.next()

811 if s.sy == '}':

812 s.next()

813 return ExprNodes.DictNode(pos, key_value_pairs = [])

814 item = p_simple_expr(s)

815 if s.sy == ',' or s.sy == '}':

816 # set literal

817 values = [item]

818 while s.sy == ',':

819 s.next()

820 if s.sy == '}':

821 break

822 values.append( p_simple_expr(s) )

823 s.expect('}')

824 return ExprNodes.SetNode(pos, args=values)

825 elif s.sy == 'for':

826 # set comprehension

827 target = ExprNodes.SetNode(pos, args=[])

828 append = ExprNodes.ComprehensionAppendNode(

829 item.pos, expr=item, target=ExprNodes.CloneNode(target))

830 loop = p_comp_for(s, Nodes.ExprStatNode(append.pos, expr=append))

831 s.expect('}')

832 return ExprNodes.ComprehensionNode(

833 pos, loop=loop, append=append, target=target)

834 elif s.sy == ':':

835 # dict literal or comprehension

836 key = item

837 s.next()

838 value = p_simple_expr(s)

839 if s.sy == 'for':

840 # dict comprehension

841 target = ExprNodes.DictNode(pos, key_value_pairs = [])

842 append = ExprNodes.DictComprehensionAppendNode(

843 item.pos, key_expr=key, value_expr=value,

844 target=ExprNodes.CloneNode(target))

845 loop = p_comp_for(s, Nodes.ExprStatNode(append.pos, expr=append))

846 s.expect('}')

847 return ExprNodes.ComprehensionNode(

848 pos, loop=loop, append=append, target=target)

849 else:

850 # dict literal

851 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

852 while s.sy == ',':

853 s.next()

854 if s.sy == '}':

855 break

856 key = p_simple_expr(s)

857 s.expect(':')

858 value = p_simple_expr(s)

859 items.append(

860 ExprNodes.DictItemNode(key.pos, key=key, value=value))

861 s.expect('}')

862 return ExprNodes.DictNode(pos, key_value_pairs=items)

863 else:

864 # raise an error

865 s.expect('}')

866 return ExprNodes.DictNode(pos, key_value_pairs = [])

867

868 def p_backquote_expr(s):

869 # s.sy == '`'

870 pos = s.position()

871 s.next()

872 arg = p_expr(s)

873 s.expect('`')

874 return ExprNodes.BackquoteNode(pos, arg = arg)

875

876 def p_simple_expr_list(s):

877 exprs = []

878 while s.sy not in expr_terminators:

879 expr = p_simple_expr(s)

880 exprs.append(expr)

881 if s.sy != ',':

882 break

883 s.next()

884 return exprs

885

886 def p_expr(s):

887 pos = s.position()

888 expr = p_simple_expr(s)

889 if s.sy == ',':

890 s.next()

891 exprs = [expr] + p_simple_expr_list(s)

892 return ExprNodes.TupleNode(pos, args = exprs)

893 else:

894 return expr

895

896

897 #testlist: test (',' test)* [',']

898 # differs from p_expr only in the fact that it cannot contain conditional expressions

899

900 def p_testlist(s):

901 pos = s.position()

902 expr = p_test(s)

903 if s.sy == ',':

904 exprs = [expr]

905 while s.sy == ',':

906 s.next()

907 exprs.append(p_test(s))

908 return ExprNodes.TupleNode(pos, args = exprs)

909 else:

910 return expr

911

912 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

913

914 #-------------------------------------------------------

915 #

916 # Statements

917 #

918 #-------------------------------------------------------

919

920 def p_global_statement(s):

921 # assume s.sy == 'global'

922 pos = s.position()

923 s.next()

924 names = p_ident_list(s)

925 return Nodes.GlobalNode(pos, names = names)

926

927 def p_expression_or_assignment(s):

928 expr_list = [p_expr(s)]

929 while s.sy == '=':

930 s.next()

931 expr_list.append(p_expr(s))

932 if len(expr_list) == 1:

933 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

934 lhs = expr_list[0]

935 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

936 error(lhs.pos, "Illegal operand for inplace operation.")

937 operator = s.sy[:-1]

938 s.next()

939 rhs = p_expr(s)

940 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

941 expr = expr_list[0]

942 if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):

943 return Nodes.PassStatNode(expr.pos)

944 else:

945 return Nodes.ExprStatNode(expr.pos, expr = expr)

946 else:

947 expr_list_list = []

948 flatten_parallel_assignments(expr_list, expr_list_list)

949 nodes = []

950 for expr_list in expr_list_list:

951 lhs_list = expr_list[:-1]

952 rhs = expr_list[-1]

953 if len(lhs_list) == 1:

954 node = Nodes.SingleAssignmentNode(rhs.pos,

955 lhs = lhs_list[0], rhs = rhs)

956 else:

957 node = Nodes.CascadedAssignmentNode(rhs.pos,

958 lhs_list = lhs_list, rhs = rhs)

959 nodes.append(node)

960 if len(nodes) == 1:

961 return nodes[0]

962 else:

963 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

964

965 def flatten_parallel_assignments(input, output):

966 # The input is a list of expression nodes, representing the LHSs

967 # and RHS of one (possibly cascaded) assignment statement. For

968 # sequence constructors, rearranges the matching parts of both

969 # sides into a list of equivalent assignments between the

970 # individual elements. This transformation is applied

971 # recursively, so that nested structures get matched as well.

972 rhs = input[-1]

973 if not rhs.is_sequence_constructor or not sum([lhs.is_sequence_constructor for lhs in input[:-1]]):

974 output.append(input)

975 return

976

977 complete_assignments = []

978

979 rhs_size = len(rhs.args)

980 lhs_targets = [ [] for _ in range(rhs_size) ]

981 starred_assignments = []

982 for lhs in input[:-1]:

983 if not lhs.is_sequence_constructor:

984 if lhs.is_starred:

985 error(lhs.pos, "starred assignment target must be in a list or tuple")

986 complete_assignments.append(lhs)

987 continue

988 lhs_size = len(lhs.args)

989 starred_targets = sum([1 for expr in lhs.args if expr.is_starred])

990 if starred_targets:

991 if starred_targets > 1:

992 error(lhs.pos, "more than 1 starred expression in assignment")

993 output.append([lhs,rhs])

994 continue

995 elif lhs_size - starred_targets > rhs_size:

996 error(lhs.pos, "need more than %d value%s to unpack"

997 % (rhs_size, (rhs_size != 1) and 's' or ''))

998 output.append([lhs,rhs])

999 continue

1000 map_starred_assignment(lhs_targets, starred_assignments,

1001 lhs.args, rhs.args)

1002 else:

1003 if lhs_size > rhs_size:

1004 error(lhs.pos, "need more than %d value%s to unpack"

1005 % (rhs_size, (rhs_size != 1) and 's' or ''))

1006 output.append([lhs,rhs])

1007 continue

1008 elif lhs_size < rhs_size:

1009 error(lhs.pos, "too many values to unpack (expected %d, got %d)"

1010 % (lhs_size, rhs_size))

1011 output.append([lhs,rhs])

1012 continue

1013 else:

1014 for targets, expr in zip(lhs_targets, lhs.args):

1015 targets.append(expr)

1016

1017 if complete_assignments:

1018 complete_assignments.append(rhs)

1019 output.append(complete_assignments)

1020

1021 # recursively flatten partial assignments

1022 for cascade, rhs in zip(lhs_targets, rhs.args):

1023 if cascade:

1024 cascade.append(rhs)

1025 flatten_parallel_assignments(cascade, output)

1026

1027 # recursively flatten starred assignments

1028 for cascade in starred_assignments:

1029 if cascade[0].is_sequence_constructor:

1030 flatten_parallel_assignments(cascade, output)

1031 else:

1032 output.append(cascade)

1033

1034 def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):

1035 # Appends the fixed-position LHS targets to the target list that

1036 # appear left and right of the starred argument.

1037 #

1038 # The starred_assignments list receives a new tuple

1039 # (lhs_target, rhs_values_list) that maps the remaining arguments

1040 # (those that match the starred target) to a list.

1041

1042 # left side of the starred target

1043 for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):

1044 if expr.is_starred:

1045 starred = i

1046 lhs_remaining = len(lhs_args) - i - 1

1047 break

1048 targets.append(expr)

1049 else:

1050 raise InternalError("no starred arg found when splitting starred assignment")

1051

1052 # right side of the starred target

1053 for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],

1054 lhs_args[-lhs_remaining:])):

1055 targets.append(expr)

1056

1057 # the starred target itself, must be assigned a (potentially empty) list

1058 target = lhs_args[starred].target # unpack starred node

1059 starred_rhs = rhs_args[starred:]

1060 if lhs_remaining:

1061 starred_rhs = starred_rhs[:-lhs_remaining]

1062 if starred_rhs:

1063 pos = starred_rhs[0].pos

1064 else:

1065 pos = target.pos

1066 starred_assignments.append([

1067 target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])

1068

1069

1070 def p_print_statement(s):

1071 # s.sy == 'print'

1072 pos = s.position()

1073 s.next()

1074 if s.sy == '>>':

1075 s.error("'print >>' not yet implemented")

1076 args = []

1077 ends_with_comma = 0

1078 if s.sy not in ('NEWLINE', 'EOF'):

1079 args.append(p_simple_expr(s))

1080 while s.sy == ',':

1081 s.next()

1082 if s.sy in ('NEWLINE', 'EOF'):

1083 ends_with_comma = 1

1084 break

1085 args.append(p_simple_expr(s))

1086 arg_tuple = ExprNodes.TupleNode(pos, args = args)

1087 return Nodes.PrintStatNode(pos,

1088 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

1089

1090 def p_exec_statement(s):

1091 # s.sy == 'exec'

1092 pos = s.position()

1093 s.next()

1094 args = [ p_bit_expr(s) ]

1095 if s.sy == 'in':

1096 s.next()

1097 args.append(p_simple_expr(s))

1098 if s.sy == ',':

1099 s.next()

1100 args.append(p_simple_expr(s))

1101 else:

1102 error(pos, "'exec' currently requires a target mapping (globals/locals)")

1103 return Nodes.ExecStatNode(pos, args = args)

1104

1105 def p_del_statement(s):

1106 # s.sy == 'del'

1107 pos = s.position()

1108 s.next()

1109 args = p_simple_expr_list(s)

1110 return Nodes.DelStatNode(pos, args = args)

1111

1112 def p_pass_statement(s, with_newline = 0):

1113 pos = s.position()

1114 s.expect('pass')

1115 if with_newline:

1116 s.expect_newline("Expected a newline")

1117 return Nodes.PassStatNode(pos)

1118

1119 def p_break_statement(s):

1120 # s.sy == 'break'

1121 pos = s.position()

1122 s.next()

1123 return Nodes.BreakStatNode(pos)

1124

1125 def p_continue_statement(s):

1126 # s.sy == 'continue'

1127 pos = s.position()

1128 s.next()

1129 return Nodes.ContinueStatNode(pos)

1130

1131 def p_return_statement(s):

1132 # s.sy == 'return'

1133 pos = s.position()

1134 s.next()

1135 if s.sy not in statement_terminators:

1136 value = p_expr(s)

1137 else:

1138 value = None

1139 return Nodes.ReturnStatNode(pos, value = value)

1140

1141 def p_raise_statement(s):

1142 # s.sy == 'raise'

1143 pos = s.position()

1144 s.next()

1145 exc_type = None

1146 exc_value = None

1147 exc_tb = None

1148 if s.sy not in statement_terminators:

1149 exc_type = p_simple_expr(s)

1150 if s.sy == ',':

1151 s.next()

1152 exc_value = p_simple_expr(s)

1153 if s.sy == ',':

1154 s.next()

1155 exc_tb = p_simple_expr(s)

1156 if exc_type or exc_value or exc_tb:

1157 return Nodes.RaiseStatNode(pos,

1158 exc_type = exc_type,

1159 exc_value = exc_value,

1160 exc_tb = exc_tb)

1161 else:

1162 return Nodes.ReraiseStatNode(pos)

1163

1164 def p_import_statement(s):

1165 # s.sy in ('import', 'cimport')

1166 pos = s.position()

1167 kind = s.sy

1168 s.next()

1169 items = [p_dotted_name(s, as_allowed = 1)]

1170 while s.sy == ',':

1171 s.next()

1172 items.append(p_dotted_name(s, as_allowed = 1))

1173 stats = []

1174 for pos, target_name, dotted_name, as_name in items:

1175 dotted_name = EncodedString(dotted_name)

1176 if kind == 'cimport':

1177 stat = Nodes.CImportStatNode(pos,

1178 module_name = dotted_name,

1179 as_name = as_name)

1180 else:

1181 if as_name and "." in dotted_name:

1182 name_list = ExprNodes.ListNode(pos, args = [

1183 ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])

1184 else:

1185 name_list = None

1186 stat = Nodes.SingleAssignmentNode(pos,

1187 lhs = ExprNodes.NameNode(pos,

1188 name = as_name or target_name),

1189 rhs = ExprNodes.ImportNode(pos,

1190 module_name = ExprNodes.IdentifierStringNode(

1191 pos, value = dotted_name),

1192 name_list = name_list))

1193 stats.append(stat)

1194 return Nodes.StatListNode(pos, stats = stats)

1195

1196 def p_from_import_statement(s, first_statement = 0):

1197 # s.sy == 'from'

1198 pos = s.position()

1199 s.next()

1200 (dotted_name_pos, _, dotted_name, _) = \

1201 p_dotted_name(s, as_allowed = 0)

1202 if s.sy in ('import', 'cimport'):

1203 kind = s.sy

1204 s.next()

1205 else:

1206 s.error("Expected 'import' or 'cimport'")

1207 is_cimport = kind == 'cimport'

1208 is_parenthesized = False

1209 if s.sy == '*':

1210 imported_names = [(s.position(), "*", None, None)]

1211 s.next()

1212 else:

1213 if s.sy == '(':

1214 is_parenthesized = True

1215 s.next()

1216 imported_names = [p_imported_name(s, is_cimport)]

1217 while s.sy == ',':

1218 s.next()

1219 if is_parenthesized and s.sy == ')':

1220 break

1221 imported_names.append(p_imported_name(s, is_cimport))

1222 if is_parenthesized:

1223 s.expect(')')

1224 dotted_name = EncodedString(dotted_name)

1225 if dotted_name == '__future__':

1226 if not first_statement:

1227 s.error("from __future__ imports must occur at the beginning of the file")

1228 else:

1229 for (name_pos, name, as_name, kind) in imported_names:

1230 if name == "braces":

1231 s.error("not a chance", name_pos)

1232 break

1233 try:

1234 directive = getattr(Future, name)

1235 except AttributeError:

1236 s.error("future feature %s is not defined" % name, name_pos)

1237 break

1238 s.context.future_directives.add(directive)

1239 return Nodes.PassStatNode(pos)

1240 elif kind == 'cimport':

1241 return Nodes.FromCImportStatNode(pos,

1242 module_name = dotted_name,

1243 imported_names = imported_names)

1244 else:

1245 imported_name_strings = []

1246 items = []

1247 for (name_pos, name, as_name, kind) in imported_names:

1248 encoded_name = EncodedString(name)

1249 imported_name_strings.append(

1250 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1251 items.append(

1252 (name,

1253 ExprNodes.NameNode(name_pos,

1254 name = as_name or name)))

1255 import_list = ExprNodes.ListNode(

1256 imported_names[0][0], args = imported_name_strings)

1257 dotted_name = EncodedString(dotted_name)

1258 return Nodes.FromImportStatNode(pos,

1259 module = ExprNodes.ImportNode(dotted_name_pos,

1260 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1261 name_list = import_list),

1262 items = items)

1263

1264 imported_name_kinds = ('class', 'struct', 'union')

1265

1266 def p_imported_name(s, is_cimport):

1267 pos = s.position()

1268 kind = None

1269 if is_cimport and s.systring in imported_name_kinds:

1270 kind = s.systring

1271 s.next()

1272 name = p_ident(s)

1273 as_name = p_as_name(s)

1274 return (pos, name, as_name, kind)

1275

1276 def p_dotted_name(s, as_allowed):

1277 pos = s.position()

1278 target_name = p_ident(s)

1279 as_name = None

1280 names = [target_name]

1281 while s.sy == '.':

1282 s.next()

1283 names.append(p_ident(s))

1284 if as_allowed:

1285 as_name = p_as_name(s)

1286 return (pos, target_name, u'.'.join(names), as_name)

1287

1288 def p_as_name(s):

1289 if s.sy == 'IDENT' and s.systring == 'as':

1290 s.next()

1291 return p_ident(s)

1292 else:

1293 return None

1294

1295 def p_assert_statement(s):

1296 # s.sy == 'assert'

1297 pos = s.position()

1298 s.next()

1299 cond = p_simple_expr(s)

1300 if s.sy == ',':

1301 s.next()

1302 value = p_simple_expr(s)

1303 else:

1304 value = None

1305 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1306

1307 statement_terminators = (';', 'NEWLINE', 'EOF')

1308

1309 def p_if_statement(s):

1310 # s.sy == 'if'

1311 pos = s.position()

1312 s.next()

1313 if_clauses = [p_if_clause(s)]

1314 while s.sy == 'elif':

1315 s.next()

1316 if_clauses.append(p_if_clause(s))

1317 else_clause = p_else_clause(s)

1318 return Nodes.IfStatNode(pos,

1319 if_clauses = if_clauses, else_clause = else_clause)

1320

1321 def p_if_clause(s):

1322 pos = s.position()

1323 test = p_simple_expr(s)

1324 body = p_suite(s)

1325 return Nodes.IfClauseNode(pos,

1326 condition = test, body = body)

1327

1328 def p_else_clause(s):

1329 if s.sy == 'else':

1330 s.next()

1331 return p_suite(s)

1332 else:

1333 return None

1334

1335 def p_while_statement(s):

1336 # s.sy == 'while'

1337 pos = s.position()

1338 s.next()

1339 test = p_simple_expr(s)

1340 body = p_suite(s)

1341 else_clause = p_else_clause(s)

1342 return Nodes.WhileStatNode(pos,

1343 condition = test, body = body,

1344 else_clause = else_clause)

1345

1346 def p_for_statement(s):

1347 # s.sy == 'for'

1348 pos = s.position()

1349 s.next()

1350 kw = p_for_bounds(s)

1351 kw['body'] = p_suite(s)

1352 kw['else_clause'] = p_else_clause(s)

1353 return Nodes.ForStatNode(pos, **kw)

1354

1355 def p_for_bounds(s):

1356 target = p_for_target(s)

1357 if s.sy == 'in':

1358 s.next()

1359 iterator = p_for_iterator(s)

1360 return { 'target': target, 'iterator': iterator }

1361 else:

1362 if s.sy == 'from':

1363 s.next()

1364 bound1 = p_bit_expr(s)

1365 else:

1366 # Support shorter "for a <= x < b" syntax

1367 bound1, target = target, None

1368 rel1 = p_for_from_relation(s)

1369 name2_pos = s.position()

1370 name2 = p_ident(s)

1371 rel2_pos = s.position()

1372 rel2 = p_for_from_relation(s)

1373 bound2 = p_bit_expr(s)

1374 step = p_for_from_step(s)

1375 if target is None:

1376 target = ExprNodes.NameNode(name2_pos, name = name2)

1377 else:

1378 if not target.is_name:

1379 error(target.pos,

1380 "Target of for-from statement must be a variable name")

1381 elif name2 != target.name:

1382 error(name2_pos,

1383 "Variable name in for-from range does not match target")

1384 if rel1[0] != rel2[0]:

1385 error(rel2_pos,

1386 "Relation directions in for-from do not match")

1387 return {'target': target,

1388 'bound1': bound1,

1389 'relation1': rel1,

1390 'relation2': rel2,

1391 'bound2': bound2,

1392 'step': step }

1393

1394 def p_for_from_relation(s):

1395 if s.sy in inequality_relations:

1396 op = s.sy

1397 s.next()

1398 return op

1399 else:

1400 s.error("Expected one of '<', '<=', '>' '>='")

1401

1402 def p_for_from_step(s):

1403 if s.sy == 'by':

1404 s.next()

1405 step = p_bit_expr(s)

1406 return step

1407 else:

1408 return None

1409

1410 inequality_relations = ('<', '<=', '>', '>=')

1411

1412 def p_target(s, terminator):

1413 pos = s.position()

1414 expr = p_starred_expr(s)

1415 if s.sy == ',':

1416 s.next()

1417 exprs = [expr]

1418 while s.sy != terminator:

1419 exprs.append(p_starred_expr(s))

1420 if s.sy != ',':

1421 break

1422 s.next()

1423 return ExprNodes.TupleNode(pos, args = exprs)

1424 else:

1425 return expr

1426

1427 def p_for_target(s):

1428 return p_target(s, 'in')

1429

1430 def p_for_iterator(s):

1431 pos = s.position()

1432 expr = p_testlist(s)

1433 return ExprNodes.IteratorNode(pos, sequence = expr)

1434

1435 def p_try_statement(s):

1436 # s.sy == 'try'

1437 pos = s.position()

1438 s.next()

1439 body = p_suite(s)

1440 except_clauses = []

1441 else_clause = None

1442 if s.sy in ('except', 'else'):

1443 while s.sy == 'except':

1444 except_clauses.append(p_except_clause(s))

1445 if s.sy == 'else':

1446 s.next()

1447 else_clause = p_suite(s)

1448 body = Nodes.TryExceptStatNode(pos,

1449 body = body, except_clauses = except_clauses,

1450 else_clause = else_clause)

1451 if s.sy != 'finally':

1452 return body

1453 # try-except-finally is equivalent to nested try-except/try-finally

1454 if s.sy == 'finally':

1455 s.next()

1456 finally_clause = p_suite(s)

1457 return Nodes.TryFinallyStatNode(pos,

1458 body = body, finally_clause = finally_clause)

1459 else:

1460 s.error("Expected 'except' or 'finally'")

1461

1462 def p_except_clause(s):

1463 # s.sy == 'except'

1464 pos = s.position()

1465 s.next()

1466 exc_type = None

1467 exc_value = None

1468 if s.sy != ':':

1469 exc_type = p_simple_expr(s)

1470 if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'):

1471 s.next()

1472 exc_value = p_simple_expr(s)

1473 body = p_suite(s)

1474 return Nodes.ExceptClauseNode(pos,

1475 pattern = exc_type, target = exc_value, body = body)

1476

1477 def p_include_statement(s, ctx):

1478 pos = s.position()

1479 s.next() # 'include'

1480 _, include_file_name = p_string_literal(s)

1481 s.expect_newline("Syntax error in include statement")

1482 if s.compile_time_eval:

1483 include_file_name = include_file_name.decode(s.source_encoding)

1484 include_file_path = s.context.find_include_file(include_file_name, pos)

1485 if include_file_path:

1486 s.included_files.append(include_file_name)

1487 f = Utils.open_source_file(include_file_path, mode="rU")

1488 source_desc = FileSourceDescriptor(include_file_path)

1489 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1490 try:

1491 tree = p_statement_list(s2, ctx)

1492 finally:

1493 f.close()

1494 return tree

1495 else:

1496 return None

1497 else:

1498 return Nodes.PassStatNode(pos)

1499

1500 def p_with_statement(s):

1501 pos = s.position()

1502 s.next() # 'with'

1503 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1504 if s.sy == 'IDENT' and s.systring == 'nogil':

1505 state = s.systring

1506 s.next()

1507 body = p_suite(s)

1508 return Nodes.GILStatNode(pos, state = state, body = body)

1509 else:

1510 manager = p_expr(s)

1511 target = None

1512 if s.sy == 'IDENT' and s.systring == 'as':

1513 s.next()

1514 allow_multi = (s.sy == '(')

1515 target = p_target(s, ':')

1516 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1517 s.error("Multiple with statement target values not allowed without paranthesis")

1518 body = p_suite(s)

1519 return Nodes.WithStatNode(pos, manager = manager,

1520 target = target, body = body)

1521

1522 def p_simple_statement(s, first_statement = 0):

1523 #print "p_simple_statement:", s.sy, s.systring ###

1524 if s.sy == 'global':

1525 node = p_global_statement(s)

1526 elif s.sy == 'print':

1527 node = p_print_statement(s)

1528 elif s.sy == 'exec':

1529 node = p_exec_statement(s)

1530 elif s.sy == 'del':

1531 node = p_del_statement(s)

1532 elif s.sy == 'break':

1533 node = p_break_statement(s)

1534 elif s.sy == 'continue':

1535 node = p_continue_statement(s)

1536 elif s.sy == 'return':

1537 node = p_return_statement(s)

1538 elif s.sy == 'raise':

1539 node = p_raise_statement(s)

1540 elif s.sy in ('import', 'cimport'):

1541 node = p_import_statement(s)

1542 elif s.sy == 'from':

1543 node = p_from_import_statement(s, first_statement = first_statement)

1544 elif s.sy == 'yield':

1545 node = p_yield_statement(s)

1546 elif s.sy == 'assert':

1547 node = p_assert_statement(s)

1548 elif s.sy == 'pass':

1549 node = p_pass_statement(s)

1550 else:

1551 node = p_expression_or_assignment(s)

1552 return node

1553

1554 def p_simple_statement_list(s, ctx, first_statement = 0):

1555 # Parse a series of simple statements on one line

1556 # separated by semicolons.

1557 stat = p_simple_statement(s, first_statement = first_statement)

1558 if s.sy == ';':

1559 stats = [stat]

1560 while s.sy == ';':

1561 #print "p_simple_statement_list: maybe more to follow" ###

1562 s.next()

1563 if s.sy in ('NEWLINE', 'EOF'):

1564 break

1565 stats.append(p_simple_statement(s))

1566 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1567 s.expect_newline("Syntax error in simple statement list")

1568 return stat

1569

1570 def p_compile_time_expr(s):

1571 old = s.compile_time_expr

1572 s.compile_time_expr = 1

1573 expr = p_expr(s)

1574 s.compile_time_expr = old

1575 return expr

1576

1577 def p_DEF_statement(s):

1578 pos = s.position()

1579 denv = s.compile_time_env

1580 s.next() # 'DEF'

1581 name = p_ident(s)

1582 s.expect('=')

1583 expr = p_compile_time_expr(s)

1584 value = expr.compile_time_value(denv)

1585 #print "p_DEF_statement: %s = %r" % (name, value) ###

1586 denv.declare(name, value)

1587 s.expect_newline()

1588 return Nodes.PassStatNode(pos)

1589

1590 def p_IF_statement(s, ctx):

1591 pos = s.position()

1592 saved_eval = s.compile_time_eval

1593 current_eval = saved_eval

1594 denv = s.compile_time_env

1595 result = None

1596 while 1:

1597 s.next() # 'IF' or 'ELIF'

1598 expr = p_compile_time_expr(s)

1599 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1600 body = p_suite(s, ctx)

1601 if s.compile_time_eval:

1602 result = body

1603 current_eval = 0

1604 if s.sy != 'ELIF':

1605 break

1606 if s.sy == 'ELSE':

1607 s.next()

1608 s.compile_time_eval = current_eval

1609 body = p_suite(s, ctx)

1610 if current_eval:

1611 result = body

1612 if not result:

1613 result = Nodes.PassStatNode(pos)

1614 s.compile_time_eval = saved_eval

1615 return result

1616

1617 def p_statement(s, ctx, first_statement = 0):

1618 cdef_flag = ctx.cdef_flag

1619 decorators = None

1620 if s.sy == 'ctypedef':

1621 if ctx.level not in ('module', 'module_pxd'):

1622 s.error("ctypedef statement not allowed here")

1623 #if ctx.api:

1624 # error(s.position(), "'api' not allowed with 'ctypedef'")

1625 return p_ctypedef_statement(s, ctx)

1626 elif s.sy == 'DEF':

1627 return p_DEF_statement(s)

1628 elif s.sy == 'IF':

1629 return p_IF_statement(s, ctx)

1630 elif s.sy == 'DECORATOR':

1631 if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd'):

1632 print ctx.level

1633 s.error('decorator not allowed here')

1634 s.level = ctx.level

1635 decorators = p_decorators(s)

1636 if s.sy not in ('def', 'cdef', 'cpdef', 'class'):

1637 s.error("Decorators can only be followed by functions or classes")

1638 elif s.sy == 'pass' and cdef_flag:

1639 # empty cdef block

1640 return p_pass_statement(s, with_newline = 1)

1641

1642 overridable = 0

1643 if s.sy == 'cdef':

1644 cdef_flag = 1

1645 s.next()

1646 elif s.sy == 'cpdef':

1647 cdef_flag = 1

1648 overridable = 1

1649 s.next()

1650 if cdef_flag:

1651 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1652 s.error('cdef statement not allowed here')

1653 s.level = ctx.level

1654 node = p_cdef_statement(s, ctx(overridable = overridable))

1655 if decorators is not None:

1656 if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):

1657 s.error("Decorators can only be followed by functions or Python classes")

1658 node.decorators = decorators

1659 return node

1660 else:

1661 if ctx.api:

1662 error(s.pos, "'api' not allowed with this statement")

1663 elif s.sy == 'def':

1664 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property', 'function'):

1665 s.error('def statement not allowed here')

1666 s.level = ctx.level

1667 return p_def_statement(s, decorators)

1668 elif s.sy == 'class':

1669 if ctx.level != 'module':

1670 s.error("class definition not allowed here")

1671 return p_class_statement(s, decorators)

1672 elif s.sy == 'include':

1673 if ctx.level not in ('module', 'module_pxd'):

1674 s.error("include statement not allowed here")

1675 return p_include_statement(s, ctx)

1676 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1677 return p_property_decl(s)

1678 elif s.sy == 'pass' and ctx.level != 'property':

1679 return p_pass_statement(s, with_newline = 1)

1680 else:

1681 if ctx.level in ('c_class_pxd', 'property'):

1682 s.error("Executable statement not allowed here")

1683 if s.sy == 'if':

1684 return p_if_statement(s)

1685 elif s.sy == 'while':

1686 return p_while_statement(s)

1687 elif s.sy == 'for':

1688 return p_for_statement(s)

1689 elif s.sy == 'try':

1690 return p_try_statement(s)

1691 elif s.sy == 'with':

1692 return p_with_statement(s)

1693 else:

1694 return p_simple_statement_list(

1695 s, ctx, first_statement = first_statement)

1696

1697 def p_statement_list(s, ctx, first_statement = 0):

1698 # Parse a series of statements separated by newlines.

1699 pos = s.position()

1700 stats = []

1701 while s.sy not in ('DEDENT', 'EOF'):

1702 stats.append(p_statement(s, ctx, first_statement = first_statement))

1703 first_statement = 0

1704 if len(stats) == 1:

1705 return stats[0]

1706 else:

1707 return Nodes.StatListNode(pos, stats = stats)

1708

1709 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1710 pos = s.position()

1711 s.expect(':')

1712 doc = None

1713 stmts = []

1714 if s.sy == 'NEWLINE':

1715 s.next()

1716 s.expect_indent()

1717 if with_doc or with_pseudo_doc:

1718 doc = p_doc_string(s)

1719 body = p_statement_list(s, ctx)

1720 s.expect_dedent()

1721 else:

1722 if ctx.api:

1723 error(s.pos, "'api' not allowed with this statement")

1724 if ctx.level in ('module', 'class', 'function', 'other'):

1725 body = p_simple_statement_list(s, ctx)

1726 else:

1727 body = p_pass_statement(s)

1728 s.expect_newline("Syntax error in declarations")

1729 if with_doc:

1730 return doc, body

1731 else:

1732 return body

1733

1734 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1735 """

1736 Parses positional and keyword arguments. end_sy_set

1737 should contain any s.sy that terminate the argument list.

1738 Argument expansion (* and **) are not allowed.

1739

1740 type_positions and type_keywords specifies which argument

1741 positions and/or names which should be interpreted as

1742 types. Other arguments will be treated as expressions.

1743

1744 Returns: (positional_args, keyword_args)

1745 """

1746 positional_args = []

1747 keyword_args = []

1748 pos_idx = 0

1749

1750 while s.sy not in end_sy_set:

1751 if s.sy == '*' or s.sy == '**':

1752 s.error('Argument expansion not allowed here.')

1753

1754 was_keyword = False

1755 parsed_type = False

1756 if s.sy == 'IDENT':

1757 # Since we can have either types or expressions as positional args,

1758 # we use a strategy of looking an extra step forward for a '=' and

1759 # if it is a positional arg we backtrack.

1760 ident = s.systring

1761 s.next()

1762 if s.sy == '=':

1763 s.next()

1764 # Is keyword arg

1765 if ident in type_keywords:

1766 arg = p_c_base_type(s)

1767 parsed_type = True

1768 else:

1769 arg = p_simple_expr(s)

1770 keyword_node = ExprNodes.IdentifierStringNode(

1771 arg.pos, value = EncodedString(ident))

1772 keyword_args.append((keyword_node, arg))

1773 was_keyword = True

1774 else:

1775 s.put_back('IDENT', ident)

1776

1777 if not was_keyword:

1778 if pos_idx in type_positions:

1779 arg = p_c_base_type(s)

1780 parsed_type = True

1781 else:

1782 arg = p_simple_expr(s)

1783 positional_args.append(arg)

1784 pos_idx += 1

1785 if len(keyword_args) > 0:

1786 s.error("Non-keyword arg following keyword arg",

1787 pos = arg.pos)

1788

1789 if s.sy != ',':

1790 if s.sy not in end_sy_set:

1791 if parsed_type:

1792 s.error("Expected: type")

1793 else:

1794 s.error("Expected: expression")

1795 break

1796 s.next()

1797 return positional_args, keyword_args

1798

1799 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1800 # If self_flag is true, this is the base type for the

1801 # self argument of a C method of an extension type.

1802 if s.sy == '(':

1803 return p_c_complex_base_type(s)

1804 else:

1805 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1806

1807 def p_calling_convention(s):

1808 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1809 result = s.systring

1810 s.next()

1811 return result

1812 else:

1813 return ""

1814

1815 calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

1816

1817 def p_c_complex_base_type(s):

1818 # s.sy == '('

1819 pos = s.position()

1820 s.next()

1821 base_type = p_c_base_type(s)

1822 declarator = p_c_declarator(s, empty = 1)

1823 s.expect(')')

1824 return Nodes.CComplexBaseTypeNode(pos,

1825 base_type = base_type, declarator = declarator)

1826

1827 def p_c_simple_base_type(s, self_flag, nonempty):

1828 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1829 is_basic = 0

1830 signed = 1

1831 longness = 0

1832 complex = 0

1833 module_path = []

1834 pos = s.position()

1835 if not s.sy == 'IDENT':

1836 error(pos, "Expected an identifier, found '%s'" % s.sy)

1837 if looking_at_base_type(s):

1838 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1839 is_basic = 1

1840 if s.sy == 'IDENT' and s.systring in special_basic_c_types:

1841 signed, longness = special_basic_c_types[s.systring]

1842 name = s.systring

1843 s.next()

1844 else:

1845 signed, longness = p_sign_and_longness(s)

1846 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1847 name = s.systring

1848 s.next()

1849 else:

1850 name = 'int'

1851 if s.sy == 'IDENT' and s.systring == 'complex':

1852 complex = 1

1853 s.next()

1854 elif looking_at_dotted_name(s):

1855 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1856 name = s.systring

1857 s.next()

1858 while s.sy == '.':

1859 module_path.append(name)

1860 s.next()

1861 name = p_ident(s)

1862 else:

1863 name = s.systring

1864 s.next()

1865 if nonempty and s.sy != 'IDENT':

1866 # Make sure this is not a declaration of a variable or function.

1867 if s.sy == '(':

1868 s.next()

1869 if s.sy == '*' or s.sy == '**':

1870 s.put_back('(', '(')

1871 else:

1872 s.put_back('(', '(')

1873 s.put_back('IDENT', name)

1874 name = None

1875 elif s.sy not in ('*', '**', '['):

1876 s.put_back('IDENT', name)

1877 name = None

1878

1879 type_node = Nodes.CSimpleBaseTypeNode(pos,

1880 name = name, module_path = module_path,

1881 is_basic_c_type = is_basic, signed = signed,

1882 complex = complex, longness = longness,

1883 is_self_arg = self_flag)

1884

1885

1886 # Treat trailing [] on type as buffer access if it appears in a context

1887 # where declarator names are required (so that it cannot mean int[] or

1888 # sizeof(int[SIZE]))...

1889 #

1890 # (This means that buffers cannot occur where there can be empty declarators,

1891 # which is an ok restriction to make.)

1892 if nonempty and s.sy == '[':

1893 return p_buffer_access(s, type_node)

1894 else:

1895 return type_node

1896

1897 def p_buffer_access(s, base_type_node):

1898 # s.sy == '['

1899 pos = s.position()

1900 s.next()

1901 positional_args, keyword_args = (

1902 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1903 )

1904 s.expect(']')

1905

1906 keyword_dict = ExprNodes.DictNode(pos,

1907 key_value_pairs = [

1908 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1909 for key, value in keyword_args

1910 ])

1911

1912 result = Nodes.CBufferAccessTypeNode(pos,

1913 positional_args = positional_args,

1914 keyword_args = keyword_dict,

1915 base_type_node = base_type_node)

1916 return result

1917

1918

1919 def looking_at_name(s):

1920 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1921

1922 def looking_at_expr(s):

1923 if s.systring in base_type_start_words:

1924 return False

1925 elif s.sy == 'IDENT':

1926 is_type = False

1927 name = s.systring

1928 dotted_path = []

1929 s.next()

1930 while s.sy == '.':

1931 s.next()

1932 dotted_path.append(s.systring)

1933 s.expect('IDENT')

1934 saved = s.sy, s.systring

1935 if s.sy == 'IDENT':

1936 is_type = True

1937 elif s.sy == '*' or s.sy == '**':

1938 s.next()

1939 is_type = s.sy == ')'

1940 s.put_back(*saved)

1941 elif s.sy == '(':

1942 s.next()

1943 is_type = s.sy == '*'

1944 s.put_back(*saved)

1945 elif s.sy == '[':

1946 s.next()

1947 is_type = s.sy == ']'

1948 s.put_back(*saved)

1949 dotted_path.reverse()

1950 for p in dotted_path:

1951 s.put_back('IDENT', p)

1952 s.put_back('.', '.')

1953 s.put_back('IDENT', name)

1954 return not is_type

1955 else:

1956 return True

1957

1958 def looking_at_base_type(s):

1959 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1960 return s.sy == 'IDENT' and s.systring in base_type_start_words

1961

1962 def looking_at_dotted_name(s):

1963 if s.sy == 'IDENT':

1964 name = s.systring

1965 s.next()

1966 result = s.sy == '.'

1967 s.put_back('IDENT', name)

1968 return result

1969 else:

1970 return 0

1971

1972 basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

1973

1974 special_basic_c_types = {

1975 # name : (signed, longness)

1976 "Py_ssize_t" : (2, 0),

1977 "size_t" : (0, 0),

1978 }

1979

1980 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1981

1982 base_type_start_words = \

1983 basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)

1984

1985 def p_sign_and_longness(s):

1986 signed = 1

1987 longness = 0

1988 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1989 if s.systring == 'unsigned':

1990 signed = 0

1991 elif s.systring == 'signed':

1992 signed = 2

1993 elif s.systring == 'short':

1994 longness = -1

1995 elif s.systring == 'long':

1996 longness += 1

1997 s.next()

1998 return signed, longness

1999

2000 def p_opt_cname(s):

2001 literal = p_opt_string_literal(s)

2002 if literal:

2003 _, cname = literal

2004 cname = EncodedString(cname)

2005 cname.encoding = s.source_encoding

2006 else:

2007 cname = None

2008 return cname

2009

2010 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

2011 assignable = 0, nonempty = 0,

2012 calling_convention_allowed = 0):

2013 # If empty is true, the declarator must be empty. If nonempty is true,

2014 # the declarator must be nonempty. Otherwise we don't care.

2015 # If cmethod_flag is true, then if this declarator declares

2016 # a function, it's a C method of an extension type.

2017 pos = s.position()

2018 if s.sy == '(':

2019 s.next()

2020 if s.sy == ')' or looking_at_name(s):

2021 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

2022 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

2023 else:

2024 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2025 cmethod_flag = cmethod_flag,

2026 nonempty = nonempty,

2027 calling_convention_allowed = 1)

2028 s.expect(')')

2029 else:

2030 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

2031 assignable, nonempty)

2032 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

2033 error(s.position(), "%s on something that is not a function"

2034 % result.calling_convention)

2035 while s.sy in ('[', '('):

2036 pos = s.position()

2037 if s.sy == '[':

2038 result = p_c_array_declarator(s, result)

2039 else: # sy == '('

2040 s.next()

2041 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

2042 cmethod_flag = 0

2043 return result

2044

2045 def p_c_array_declarator(s, base):

2046 pos = s.position()

2047 s.next() # '['

2048 if s.sy != ']':

2049 dim = p_expr(s)

2050 else:

2051 dim = None

2052 s.expect(']')

2053 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

2054

2055 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

2056 # Opening paren has already been skipped

2057 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

2058 nonempty_declarators = 0)

2059 ellipsis = p_optional_ellipsis(s)

2060 s.expect(')')

2061 nogil = p_nogil(s)

2062 exc_val, exc_check = p_exception_value_clause(s)

2063 with_gil = p_with_gil(s)

2064 return Nodes.CFuncDeclaratorNode(pos,

2065 base = base, args = args, has_varargs = ellipsis,

2066 exception_value = exc_val, exception_check = exc_check,

2067 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

2068

2069 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

2070 assignable, nonempty):

2071 pos = s.position()

2072 calling_convention = p_calling_convention(s)

2073 if s.sy == '*':

2074 s.next()

2075 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2076 cmethod_flag = cmethod_flag,

2077 assignable = assignable, nonempty = nonempty)

2078 result = Nodes.CPtrDeclaratorNode(pos,

2079 base = base)

2080 elif s.sy == '**': # scanner returns this as a single token

2081 s.next()

2082 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

2083 cmethod_flag = cmethod_flag,

2084 assignable = assignable, nonempty = nonempty)

2085 result = Nodes.CPtrDeclaratorNode(pos,

2086 base = Nodes.CPtrDeclaratorNode(pos,

2087 base = base))

2088 else:

2089 rhs = None

2090 if s.sy == 'IDENT':

2091 name = EncodedString(s.systring)

2092 if empty:

2093 error(s.position(), "Declarator should be empty")

2094 s.next()

2095 cname = p_opt_cname(s)

2096 if s.sy == '=' and assignable:

2097 s.next()

2098 rhs = p_simple_expr(s)

2099 else:

2100 if nonempty:

2101 error(s.position(), "Empty declarator")

2102 name = ""

2103 cname = None

2104 result = Nodes.CNameDeclaratorNode(pos,

2105 name = name, cname = cname, default = rhs)

2106 result.calling_convention = calling_convention

2107 return result

2108

2109 def p_nogil(s):

2110 if s.sy == 'IDENT' and s.systring == 'nogil':

2111 s.next()

2112 return 1

2113 else:

2114 return 0

2115

2116 def p_with_gil(s):

2117 if s.sy == 'with':

2118 s.next()

2119 s.expect_keyword('gil')

2120 return 1

2121 else:

2122 return 0

2123

2124 def p_exception_value_clause(s):

2125 exc_val = None

2126 exc_check = 0

2127 if s.sy == 'except':

2128 s.next()

2129 if s.sy == '*':

2130 exc_check = 1

2131 s.next()

2132 elif s.sy == '+':

2133 exc_check = '+'

2134 s.next()

2135 if s.sy == 'IDENT':

2136 name = s.systring

2137 s.next()

2138 exc_val = p_name(s, name)

2139 else:

2140 if s.sy == '?':

2141 exc_check = 1

2142 s.next()

2143 exc_val = p_simple_expr(s)

2144 return exc_val, exc_check

2145

2146 c_arg_list_terminators = ('*', '**', '.', ')')

2147

2148 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

2149 nonempty_declarators = 0, kw_only = 0, annotated = 1):

2150 # Comma-separated list of C argument declarations, possibly empty.

2151 # May have a trailing comma.

2152 args = []

2153 is_self_arg = cmethod_flag

2154 while s.sy not in c_arg_list_terminators:

2155 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

2156 nonempty = nonempty_declarators, kw_only = kw_only,

2157 annotated = annotated))

2158 if s.sy != ',':

2159 break

2160 s.next()

2161 is_self_arg = 0

2162 return args

2163

2164 def p_optional_ellipsis(s):

2165 if s.sy == '.':

2166 expect_ellipsis(s)

2167 return 1

2168 else:

2169 return 0

2170

2171 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,

2172 kw_only = 0, annotated = 1):

2173 pos = s.position()

2174 not_none = 0

2175 default = None

2176 annotation = None

2177 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2178 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2179 if s.sy == 'not':

2180 s.next()

2181 if s.sy == 'IDENT' and s.systring == 'None':

2182 s.next()

2183 else:

2184 s.error("Expected 'None'")

2185 if not in_pyfunc:

2186 error(pos, "'not None' only allowed in Python functions")

2187 not_none = 1

2188 if annotated and s.sy == ':':

2189 s.next()

2190 annotation = p_simple_expr(s)

2191 if s.sy == '=':

2192 s.next()

2193 if 'pxd' in s.level:

2194 if s.sy not in ['*', '?']:

2195 error(pos, "default values cannot be specified in pxd files, use ? or *")

2196 default = ExprNodes.BoolNode(1)

2197 s.next()

2198 else:

2199 default = p_simple_expr(s)

2200 return Nodes.CArgDeclNode(pos,

2201 base_type = base_type,

2202 declarator = declarator,

2203 not_none = not_none,

2204 default = default,

2205 annotation = annotation,

2206 kw_only = kw_only)

2207

2208 def p_api(s):

2209 if s.sy == 'IDENT' and s.systring == 'api':

2210 s.next()

2211 return 1

2212 else:

2213 return 0

2214

2215 def p_cdef_statement(s, ctx):

2216 pos = s.position()

2217 ctx.visibility = p_visibility(s, ctx.visibility)

2218 ctx.api = ctx.api or p_api(s)

2219 if ctx.api:

2220 if ctx.visibility not in ('private', 'public'):

2221 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2222 if (ctx.visibility == 'extern') and s.sy == 'from':

2223 return p_cdef_extern_block(s, pos, ctx)

2224 elif s.sy == 'import':

2225 s.next()

2226 return p_cdef_extern_block(s, pos, ctx)

2227 elif p_nogil(s):

2228 ctx.nogil = 1

2229 if ctx.overridable:

2230 error(pos, "cdef blocks cannot be declared cpdef")

2231 return p_cdef_block(s, ctx)

2232 elif s.sy == ':':

2233 if ctx.overridable:

2234 error(pos, "cdef blocks cannot be declared cpdef")

2235 return p_cdef_block(s, ctx)

2236 elif s.sy == 'class':

2237 if ctx.level not in ('module', 'module_pxd'):

2238 error(pos, "Extension type definition not allowed here")

2239 if ctx.overridable:

2240 error(pos, "Extension types cannot be declared cpdef")

2241 return p_c_class_definition(s, pos, ctx)

2242 elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):

2243 if ctx.level not in ('module', 'module_pxd'):

2244 error(pos, "C struct/union/enum definition not allowed here")

2245 if ctx.overridable:

2246 error(pos, "C struct/union/enum cannot be declared cpdef")

2247 if s.systring == "enum":

2248 return p_c_enum_definition(s, pos, ctx)

2249 else:

2250 return p_c_struct_or_union_definition(s, pos, ctx)

2251 else:

2252 return p_c_func_or_var_declaration(s, pos, ctx)

2253

2254 def p_cdef_block(s, ctx):

2255 return p_suite(s, ctx(cdef_flag = 1))

2256

2257 def p_cdef_extern_block(s, pos, ctx):

2258 if ctx.overridable:

2259 error(pos, "cdef extern blocks cannot be declared cpdef")

2260 include_file = None

2261 s.expect('from')

2262 if s.sy == '*':

2263 s.next()

2264 else:

2265 _, include_file = p_string_literal(s)

2266 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2267 if p_nogil(s):

2268 ctx.nogil = 1

2269 body = p_suite(s, ctx)

2270 return Nodes.CDefExternNode(pos,

2271 include_file = include_file,

2272 body = body)

2273

2274 def p_c_enum_definition(s, pos, ctx):

2275 # s.sy == ident 'enum'

2276 s.next()

2277 if s.sy == 'IDENT':

2278 name = s.systring

2279 s.next()

2280 cname = p_opt_cname(s)

2281 else:

2282 name = None

2283 cname = None

2284 items = None

2285 s.expect(':')

2286 items = []

2287 if s.sy != 'NEWLINE':

2288 p_c_enum_line(s, items)

2289 else:

2290 s.next() # 'NEWLINE'

2291 s.expect_indent()

2292 while s.sy not in ('DEDENT', 'EOF'):

2293 p_c_enum_line(s, items)

2294 s.expect_dedent()

2295 return Nodes.CEnumDefNode(

2296 pos, name = name, cname = cname, items = items,

2297 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2298 in_pxd = ctx.level == 'module_pxd')

2299

2300 def p_c_enum_line(s, items):

2301 if s.sy != 'pass':

2302 p_c_enum_item(s, items)

2303 while s.sy == ',':

2304 s.next()

2305 if s.sy in ('NEWLINE', 'EOF'):

2306 break

2307 p_c_enum_item(s, items)

2308 else:

2309 s.next()

2310 s.expect_newline("Syntax error in enum item list")

2311

2312 def p_c_enum_item(s, items):

2313 pos = s.position()

2314 name = p_ident(s)

2315 cname = p_opt_cname(s)

2316 value = None

2317 if s.sy == '=':

2318 s.next()

2319 value = p_simple_expr(s)

2320 items.append(Nodes.CEnumDefItemNode(pos,

2321 name = name, cname = cname, value = value))

2322

2323 def p_c_struct_or_union_definition(s, pos, ctx):

2324 packed = False

2325 if s.systring == 'packed':

2326 packed = True

2327 s.next()

2328 if s.sy != 'IDENT' or s.systring != 'struct':

2329 s.expected('struct')

2330 # s.sy == ident 'struct' or 'union'

2331 kind = s.systring

2332 s.next()

2333 name = p_ident(s)

2334 cname = p_opt_cname(s)

2335 attributes = None

2336 if s.sy == ':':

2337 s.next()

2338 s.expect('NEWLINE')

2339 s.expect_indent()

2340 attributes = []

2341 body_ctx = Ctx()

2342 while s.sy != 'DEDENT':

2343 if s.sy != 'pass':

2344 attributes.append(

2345 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2346 else:

2347 s.next()

2348 s.expect_newline("Expected a newline")

2349 s.expect_dedent()

2350 else:

2351 s.expect_newline("Syntax error in struct or union definition")

2352 return Nodes.CStructOrUnionDefNode(pos,

2353 name = name, cname = cname, kind = kind, attributes = attributes,

2354 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2355 in_pxd = ctx.level == 'module_pxd', packed = packed)

2356

2357 def p_visibility(s, prev_visibility):

2358 pos = s.position()

2359 visibility = prev_visibility

2360 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2361 visibility = s.systring

2362 if prev_visibility != 'private' and visibility != prev_visibility:

2363 s.error("Conflicting visibility options '%s' and '%s'"

2364 % (prev_visibility, visibility))

2365 s.next()

2366 return visibility

2367

2368 def p_c_modifiers(s):

2369 if s.sy == 'IDENT' and s.systring in ('inline',):

2370 modifier = s.systring

2371 s.next()

2372 return [modifier] + p_c_modifiers(s)

2373 return []

2374

2375 def p_c_func_or_var_declaration(s, pos, ctx):

2376 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2377 modifiers = p_c_modifiers(s)

2378 base_type = p_c_base_type(s, nonempty = 1)

2379 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2380 assignable = 1, nonempty = 1)

2381 declarator.overridable = ctx.overridable

2382 if s.sy == ':':

2383 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd'):

2384 s.error("C function definition not allowed here")

2385 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2386 result = Nodes.CFuncDefNode(pos,

2387 visibility = ctx.visibility,

2388 base_type = base_type,

2389 declarator = declarator,

2390 body = suite,

2391 doc = doc,

2392 modifiers = modifiers,

2393 api = ctx.api,

2394 overridable = ctx.overridable)

2395 else:

2396 #if api:

2397 # error(s.pos, "'api' not allowed with variable declaration")

2398 declarators = [declarator]

2399 while s.sy == ',':

2400 s.next()

2401 if s.sy == 'NEWLINE':

2402 break

2403 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2404 assignable = 1, nonempty = 1)

2405 declarators.append(declarator)

2406 s.expect_newline("Syntax error in C variable declaration")

2407 result = Nodes.CVarDefNode(pos,

2408 visibility = ctx.visibility,

2409 base_type = base_type,

2410 declarators = declarators,

2411 in_pxd = ctx.level == 'module_pxd',

2412 api = ctx.api,

2413 overridable = ctx.overridable)

2414 return result

2415

2416 def p_ctypedef_statement(s, ctx):

2417 # s.sy == 'ctypedef'

2418 pos = s.position()

2419 s.next()

2420 visibility = p_visibility(s, ctx.visibility)

2421 api = p_api(s)

2422 ctx = ctx(typedef_flag = 1, visibility = visibility)

2423 if api:

2424 ctx.api = 1

2425 if s.sy == 'class':

2426 return p_c_class_definition(s, pos, ctx)

2427 elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):

2428 if s.systring == 'enum':

2429 return p_c_enum_definition(s, pos, ctx)

2430 else:

2431 return p_c_struct_or_union_definition(s, pos, ctx)

2432 else:

2433 base_type = p_c_base_type(s, nonempty = 1)

2434 if base_type.name is None:

2435 s.error("Syntax error in ctypedef statement")

2436 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2437 s.expect_newline("Syntax error in ctypedef statement")

2438 return Nodes.CTypeDefNode(

2439 pos, base_type = base_type,

2440 declarator = declarator, visibility = visibility,

2441 in_pxd = ctx.level == 'module_pxd')

2442

2443 def p_decorators(s):

2444 decorators = []

2445 while s.sy == 'DECORATOR':

2446 pos = s.position()

2447 s.next()

2448 decstring = p_dotted_name(s, as_allowed=0)[2]

2449 names = decstring.split('.')

2450 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2451 for name in names[1:]:

2452 decorator = ExprNodes.AttributeNode(pos,

2453 attribute=EncodedString(name),

2454 obj=decorator)

2455 if s.sy == '(':

2456 decorator = p_call(s, decorator)

2457 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2458 s.expect_newline("Expected a newline after decorator")

2459 return decorators

2460

2461 def p_def_statement(s, decorators=None):

2462 # s.sy == 'def'

2463 pos = s.position()

2464 s.next()

2465 name = EncodedString( p_ident(s) )

2466 s.expect('(');

2467 args, star_arg, starstar_arg = p_varargslist(s, terminator=')')

2468 s.expect(')')

2469 if p_nogil(s):

2470 error(s.pos, "Python function cannot be declared nogil")

2471 return_type_annotation = None

2472 if s.sy == '->':

2473 s.next()

2474 return_type_annotation = p_simple_expr(s)

2475 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2476 return Nodes.DefNode(pos, name = name, args = args,

2477 star_arg = star_arg, starstar_arg = starstar_arg,

2478 doc = doc, body = body, decorators = decorators,

2479 return_type_annotation = return_type_annotation)

2480

2481 def p_varargslist(s, terminator=')', annotated=1):

2482 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,

2483 annotated = annotated)

2484 star_arg = None

2485 starstar_arg = None

2486 if s.sy == '*':

2487 s.next()

2488 if s.sy == 'IDENT':

2489 star_arg = p_py_arg_decl(s)

2490 if s.sy == ',':

2491 s.next()

2492 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2493 nonempty_declarators = 1, kw_only = 1))

2494 elif s.sy != terminator:

2495 s.error("Syntax error in Python function argument list")

2496 if s.sy == '**':

2497 s.next()

2498 starstar_arg = p_py_arg_decl(s)

2499 return (args, star_arg, starstar_arg)

2500

2501 def p_py_arg_decl(s):

2502 pos = s.position()

2503 name = p_ident(s)

2504 annotation = None

2505 if s.sy == ':':

2506 s.next()

2507 annotation = p_simple_expr(s)

2508 return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)

2509

2510 def p_class_statement(s, decorators):

2511 # s.sy == 'class'

2512 pos = s.position()

2513 s.next()

2514 class_name = EncodedString( p_ident(s) )

2515 class_name.encoding = s.source_encoding

2516 if s.sy == '(':

2517 s.next()

2518 base_list = p_simple_expr_list(s)

2519 s.expect(')')

2520 else:

2521 base_list = []

2522 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2523 return Nodes.PyClassDefNode(pos,

2524 name = class_name,

2525 bases = ExprNodes.TupleNode(pos, args = base_list),

2526 doc = doc, body = body, decorators = decorators)

2527

2528 def p_c_class_definition(s, pos, ctx):

2529 # s.sy == 'class'

2530 s.next()

2531 module_path = []

2532 class_name = p_ident(s)

2533 while s.sy == '.':

2534 s.next()

2535 module_path.append(class_name)

2536 class_name = p_ident(s)

2537 if module_path and ctx.visibility != 'extern':

2538 error(pos, "Qualified class name only allowed for 'extern' C class")

2539 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2540 s.next()

2541 as_name = p_ident(s)

2542 else:

2543 as_name = class_name

2544 objstruct_name = None

2545 typeobj_name = None

2546 base_class_module = None

2547 base_class_name = None

2548 if s.sy == '(':

2549 s.next()

2550 base_class_path = [p_ident(s)]

2551 while s.sy == '.':

2552 s.next()

2553 base_class_path.append(p_ident(s))

2554 if s.sy == ',':

2555 s.error("C class may only have one base class")

2556 s.expect(')')

2557 base_class_module = ".".join(base_class_path[:-1])

2558 base_class_name = base_class_path[-1]

2559 if s.sy == '[':

2560 if ctx.visibility not in ('public', 'extern'):

2561 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2562 objstruct_name, typeobj_name = p_c_class_options(s)

2563 if s.sy == ':':

2564 if ctx.level == 'module_pxd':

2565 body_level = 'c_class_pxd'

2566 else:

2567 body_level = 'c_class'

2568 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2569 else:

2570 s.expect_newline("Syntax error in C class definition")

2571 doc = None

2572 body = None

2573 if ctx.visibility == 'extern':

2574 if not module_path:

2575 error(pos, "Module name required for 'extern' C class")

2576 if typeobj_name:

2577 error(pos, "Type object name specification not allowed for 'extern' C class")

2578 elif ctx.visibility == 'public':

2579 if not objstruct_name:

2580 error(pos, "Object struct name specification required for 'public' C class")

2581 if not typeobj_name:

2582 error(pos, "Type object name specification required for 'public' C class")

2583 elif ctx.visibility == 'private':

2584 if ctx.api:

2585 error(pos, "Only 'public' C class can be declared 'api'")

2586 else:

2587 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2588 return Nodes.CClassDefNode(pos,

2589 visibility = ctx.visibility,

2590 typedef_flag = ctx.typedef_flag,

2591 api = ctx.api,

2592 module_name = ".".join(module_path),

2593 class_name = class_name,

2594 as_name = as_name,

2595 base_class_module = base_class_module,

2596 base_class_name = base_class_name,

2597 objstruct_name = objstruct_name,

2598 typeobj_name = typeobj_name,

2599 in_pxd = ctx.level == 'module_pxd',

2600 doc = doc,

2601 body = body)

2602

2603 def p_c_class_options(s):

2604 objstruct_name = None

2605 typeobj_name = None

2606 s.expect('[')

2607 while 1:

2608 if s.sy != 'IDENT':

2609 break

2610 if s.systring == 'object':

2611 s.next()

2612 objstruct_name = p_ident(s)

2613 elif s.systring == 'type':

2614 s.next()

2615 typeobj_name = p_ident(s)

2616 if s.sy != ',':

2617 break

2618 s.next()

2619 s.expect(']', "Expected 'object' or 'type'")

2620 return objstruct_name, typeobj_name

2621

2622 def p_property_decl(s):

2623 pos = s.position()

2624 s.next() # 'property'

2625 name = p_ident(s)

2626 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2627 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2628

2629 def p_doc_string(s):

2630 if s.sy == 'BEGIN_STRING':

2631 pos = s.position()

2632 kind, result = p_cat_string_literal(s)

2633 if s.sy != 'EOF':

2634 s.expect_newline("Syntax error in doc string")

2635 if kind != 'u':

2636 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2637 if kind == 'b':

2638 result.encoding = None # force a unicode string

2639 return result

2640 else:

2641 return None

2642

2643 def p_code(s, level=None):

2644 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2645 if s.sy != 'EOF':

2646 s.error("Syntax error in statement [%s,%s]" % (

2647 repr(s.sy), repr(s.systring)))

2648 return body

2649

2650 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*(\w+)\s*=(.*)$")

2651

2652 def p_compiler_directive_comments(s):

2653 result = {}

2654 while s.sy == 'commentline':

2655 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2656 if m:

2657 name = m.group(1)

2658 try:

2659 value = Options.parse_directive_value(str(name), str(m.group(2).strip()))

2660 if value is not None: # can be False!

2661 result[name] = value

2662 except ValueError, e:

2663 s.error(e.args[0], fatal=False)

2664 s.next()

2665 return result

2666

2667 def p_module(s, pxd, full_module_name):

2668 pos = s.position()

2669

2670 directive_comments = p_compiler_directive_comments(s)

2671 s.parse_comments = False

2672

2673 doc = p_doc_string(s)

2674 if pxd:

2675 level = 'module_pxd'

2676 else:

2677 level = 'module'

2678

2679 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2680 if s.sy != 'EOF':

2681 s.error("Syntax error in statement [%s,%s]" % (

2682 repr(s.sy), repr(s.systring)))

2683 return ModuleNode(pos, doc = doc, body = body,

2684 full_module_name = full_module_name,

2685 directive_comments = directive_comments)

2686

2687 #----------------------------------------------

2688 #

2689 # Debugging

2690 #

2691 #----------------------------------------------

2692

2693 def print_parse_tree(f, node, level, key = None):

2694 from types import ListType, TupleType

2695 from Nodes import Node

2696 ind = " " * level

2697 if node:

2698 f.write(ind)

2699 if key:

2700 f.write("%s: " % key)

2701 t = type(node)

2702 if t is tuple:

2703 f.write("(%s @ %s\n" % (node[0], node[1]))

2704 for i in xrange(2, len(node)):

2705 print_parse_tree(f, node[i], level+1)

2706 f.write("%s)\n" % ind)

2707 return

2708 elif isinstance(node, Node):

2709 try:

2710 tag = node.tag

2711 except AttributeError:

2712 tag = node.__class__.__name__

2713 f.write("%s @ %s\n" % (tag, node.pos))

2714 for name, value in node.__dict__.items():

2715 if name != 'tag' and name != 'pos':

2716 print_parse_tree(f, value, level+1, name)

2717 return

2718 elif t is list:

2719 f.write("[\n")

2720 for i in xrange(len(node)):

2721 print_parse_tree(f, node[i], level+1)

2722 f.write("%s]\n" % ind)

2723 return

2724 f.write("%s%s\n" % (ind, node))

2725