1 # cython: auto_cpdef=True, infer_types=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13

14 try:

15 from __builtin__ import set

16 except ImportError:

17 try:

18 from builtins import set

19 except ImportError:

20 from sets import Set as set

21

22 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

23 import Nodes

24 import ExprNodes

25 import StringEncoding

26 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes

27 from ModuleNode import ModuleNode

28 from Errors import error, warning, InternalError

29 from Cython import Utils

30 import Future

31 import Options

32

33 class Ctx(object):

34 # Parsing context

35 level = 'other'

36 visibility = 'private'

37 cdef_flag = 0

38 typedef_flag = 0

39 api = 0

40 overridable = 0

41 nogil = 0

42 namespace = None

43 templates = None

44

45 def __init__(self, **kwds):

46 self.__dict__.update(kwds)

47

48 def __call__(self, **kwds):

49 ctx = Ctx()

50 d = ctx.__dict__

51 d.update(self.__dict__)

52 d.update(kwds)

53 return ctx

54

55 def p_ident(s, message = "Expected an identifier"):

56 if s.sy == 'IDENT':

57 name = s.systring

58 s.next()

59 return name

60 else:

61 s.error(message)

62

63 def p_ident_list(s):

64 names = []

65 while s.sy == 'IDENT':

66 names.append(s.systring)

67 s.next()

68 if s.sy != ',':

69 break

70 s.next()

71 return names

72

73 #------------------------------------------

74 #

75 # Expressions

76 #

77 #------------------------------------------

78

79 def p_binop_operator(s):

80 pos = s.position()

81 op = s.sy

82 s.next()

83 return op, pos

84

85 def p_binop_expr(s, ops, p_sub_expr):

86 n1 = p_sub_expr(s)

87 while s.sy in ops:

88 op, pos = p_binop_operator(s)

89 n2 = p_sub_expr(s)

90 n1 = ExprNodes.binop_node(pos, op, n1, n2)

91 if op == '/':

92 if Future.division in s.context.future_directives:

93 n1.truedivision = True

94 else:

95 n1.truedivision = None # unknown

96 return n1

97

98 #expression: or_test [if or_test else test] | lambda_form

99

100 def p_simple_expr(s):

101 pos = s.position()

102 expr = p_or_test(s)

103 if s.sy == 'if':

104 s.next()

105 test = p_or_test(s)

106 s.expect('else')

107 other = p_test(s)

108 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

109 else:

110 return expr

111

112 #test: or_test | lambda_form

113

114 def p_test(s):

115 return p_or_test(s)

116

117 #or_test: and_test ('or' and_test)*

118

119 def p_or_test(s):

120 return p_rassoc_binop_expr(s, ('or',), p_and_test)

121

122 def p_rassoc_binop_expr(s, ops, p_subexpr):

123 n1 = p_subexpr(s)

124 if s.sy in ops:

125 pos = s.position()

126 op = s.sy

127 s.next()

128 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

129 n1 = ExprNodes.binop_node(pos, op, n1, n2)

130 return n1

131

132 #and_test: not_test ('and' not_test)*

133

134 def p_and_test(s):

135 #return p_binop_expr(s, ('and',), p_not_test)

136 return p_rassoc_binop_expr(s, ('and',), p_not_test)

137

138 #not_test: 'not' not_test | comparison

139

140 def p_not_test(s):

141 if s.sy == 'not':

142 pos = s.position()

143 s.next()

144 return ExprNodes.NotNode(pos, operand = p_not_test(s))

145 else:

146 return p_comparison(s)

147

148 #comparison: expr (comp_op expr)*

149 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

150

151 def p_comparison(s):

152 n1 = p_starred_expr(s)

153 if s.sy in comparison_ops:

154 pos = s.position()

155 op = p_cmp_op(s)

156 n2 = p_starred_expr(s)

157 n1 = ExprNodes.PrimaryCmpNode(pos,

158 operator = op, operand1 = n1, operand2 = n2)

159 if s.sy in comparison_ops:

160 n1.cascade = p_cascaded_cmp(s)

161 return n1

162

163 def p_starred_expr(s):

164 pos = s.position()

165 if s.sy == '*':

166 starred = True

167 s.next()

168 else:

169 starred = False

170 expr = p_bit_expr(s)

171 if starred:

172 expr = ExprNodes.StarredTargetNode(pos, expr)

173 return expr

174

175 def p_cascaded_cmp(s):

176 pos = s.position()

177 op = p_cmp_op(s)

178 n2 = p_starred_expr(s)

179 result = ExprNodes.CascadedCmpNode(pos,

180 operator = op, operand2 = n2)

181 if s.sy in comparison_ops:

182 result.cascade = p_cascaded_cmp(s)

183 return result

184

185 def p_cmp_op(s):

186 if s.sy == 'not':

187 s.next()

188 s.expect('in')

189 op = 'not_in'

190 elif s.sy == 'is':

191 s.next()

192 if s.sy == 'not':

193 s.next()

194 op = 'is_not'

195 else:

196 op = 'is'

197 else:

198 op = s.sy

199 s.next()

200 if op == '<>':

201 op = '!='

202 return op

203

204 comparison_ops = (

205 '<', '>', '==', '>=', '<=', '<>', '!=',

206 'in', 'is', 'not'

207 )

208

209 #expr: xor_expr ('|' xor_expr)*

210

211 def p_bit_expr(s):

212 return p_binop_expr(s, ('|',), p_xor_expr)

213

214 #xor_expr: and_expr ('^' and_expr)*

215

216 def p_xor_expr(s):

217 return p_binop_expr(s, ('^',), p_and_expr)

218

219 #and_expr: shift_expr ('&' shift_expr)*

220

221 def p_and_expr(s):

222 return p_binop_expr(s, ('&',), p_shift_expr)

223

224 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

225

226 def p_shift_expr(s):

227 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

228

229 #arith_expr: term (('+'|'-') term)*

230

231 def p_arith_expr(s):

232 return p_binop_expr(s, ('+', '-'), p_term)

233

234 #term: factor (('*'|'/'|'%') factor)*

235

236 def p_term(s):

237 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

238

239 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

240

241 def p_factor(s):

242 sy = s.sy

243 if sy in ('+', '-', '~'):

244 op = s.sy

245 pos = s.position()

246 s.next()

247 return ExprNodes.unop_node(pos, op, p_factor(s))

248 elif sy == '&':

249 pos = s.position()

250 s.next()

251 arg = p_factor(s)

252 return ExprNodes.AmpersandNode(pos, operand = arg)

253 elif sy == "<":

254 return p_typecast(s)

255 elif sy == 'IDENT' and s.systring == "sizeof":

256 return p_sizeof(s)

257 else:

258 return p_power(s)

259

260 def p_typecast(s):

261 # s.sy == "<"

262 pos = s.position()

263 s.next()

264 base_type = p_c_base_type(s)

265 if base_type.name is None:

266 s.error("Unknown type")

267 declarator = p_c_declarator(s, empty = 1)

268 if s.sy == '?':

269 s.next()

270 typecheck = 1

271 else:

272 typecheck = 0

273 s.expect(">")

274 operand = p_factor(s)

275 return ExprNodes.TypecastNode(pos,

276 base_type = base_type,

277 declarator = declarator,

278 operand = operand,

279 typecheck = typecheck)

280

281 def p_sizeof(s):

282 # s.sy == ident "sizeof"

283 pos = s.position()

284 s.next()

285 s.expect('(')

286 # Here we decide if we are looking at an expression or type

287 # If it is actually a type, but parsable as an expression,

288 # we treat it as an expression here.

289 if looking_at_expr(s):

290 operand = p_simple_expr(s)

291 node = ExprNodes.SizeofVarNode(pos, operand = operand)

292 else:

293 base_type = p_c_base_type(s)

294 declarator = p_c_declarator(s, empty = 1)

295 node = ExprNodes.SizeofTypeNode(pos,

296 base_type = base_type, declarator = declarator)

297 s.expect(')')

298 return node

299

300 def p_yield_expression(s):

301 # s.sy == "yield"

302 pos = s.position()

303 s.next()

304 if s.sy not in ('EOF', 'NEWLINE', ')'):

305 expr = p_expr(s)

306 s.error("generators ('yield') are not currently supported")

307 return Nodes.PassStatNode(pos)

308

309 #power: atom trailer* ('**' factor)*

310

311 def p_power(s):

312 if s.systring == 'new' and s.peek()[0] == 'IDENT':

313 return p_new_expr(s)

314 n1 = p_atom(s)

315 while s.sy in ('(', '[', '.'):

316 n1 = p_trailer(s, n1)

317 if s.sy == '**':

318 pos = s.position()

319 s.next()

320 n2 = p_factor(s)

321 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

322 return n1

323

324 def p_new_expr(s):

325 # s.systring == 'new'.

326 pos = s.position()

327 s.next()

328 cppclass = p_c_base_type(s)

329 return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))

330

331 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

332

333 def p_trailer(s, node1):

334 pos = s.position()

335 if s.sy == '(':

336 return p_call(s, node1)

337 elif s.sy == '[':

338 return p_index(s, node1)

339 else: # s.sy == '.'

340 s.next()

341 name = EncodedString( p_ident(s) )

342 return ExprNodes.AttributeNode(pos,

343 obj = node1, attribute = name)

344

345 # arglist: argument (',' argument)* [',']

346 # argument: [test '='] test # Really [keyword '='] test

347

348 def p_call(s, function):

349 # s.sy == '('

350 pos = s.position()

351 s.next()

352 positional_args = []

353 keyword_args = []

354 star_arg = None

355 starstar_arg = None

356 while s.sy not in ('**', ')'):

357 if s.sy == '*':

358 if star_arg:

359 s.error("only one star-arg parameter allowed",

360 pos = s.position())

361 s.next()

362 star_arg = p_simple_expr(s)

363 else:

364 arg = p_simple_expr(s)

365 if s.sy == '=':

366 s.next()

367 if not arg.is_name:

368 s.error("Expected an identifier before '='",

369 pos = arg.pos)

370 encoded_name = EncodedString(arg.name)

371 keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)

372 arg = p_simple_expr(s)

373 keyword_args.append((keyword, arg))

374 else:

375 if keyword_args:

376 s.error("Non-keyword arg following keyword arg",

377 pos = arg.pos)

378 if star_arg:

379 s.error("Non-keyword arg following star-arg",

380 pos = arg.pos)

381 positional_args.append(arg)

382 if s.sy != ',':

383 break

384 s.next()

385

386 if s.sy == '**':

387 s.next()

388 starstar_arg = p_simple_expr(s)

389 if s.sy == ',':

390 s.next()

391 s.expect(')')

392 if not (keyword_args or star_arg or starstar_arg):

393 return ExprNodes.SimpleCallNode(pos,

394 function = function,

395 args = positional_args)

396 else:

397 arg_tuple = None

398 keyword_dict = None

399 if positional_args or not star_arg:

400 arg_tuple = ExprNodes.TupleNode(pos,

401 args = positional_args)

402 if star_arg:

403 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

404 if arg_tuple:

405 arg_tuple = ExprNodes.binop_node(pos,

406 operator = '+', operand1 = arg_tuple,

407 operand2 = star_arg_tuple)

408 else:

409 arg_tuple = star_arg_tuple

410 if keyword_args:

411 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

412 for key, value in keyword_args]

413 keyword_dict = ExprNodes.DictNode(pos,

414 key_value_pairs = keyword_args)

415 return ExprNodes.GeneralCallNode(pos,

416 function = function,

417 positional_args = arg_tuple,

418 keyword_args = keyword_dict,

419 starstar_arg = starstar_arg)

420

421 #lambdef: 'lambda' [varargslist] ':' test

422

423 #subscriptlist: subscript (',' subscript)* [',']

424

425 def p_index(s, base):

426 # s.sy == '['

427 pos = s.position()

428 s.next()

429 subscripts = p_subscript_list(s)

430 if len(subscripts) == 1 and len(subscripts[0]) == 2:

431 start, stop = subscripts[0]

432 result = ExprNodes.SliceIndexNode(pos,

433 base = base, start = start, stop = stop)

434 else:

435 indexes = make_slice_nodes(pos, subscripts)

436 if len(indexes) == 1:

437 index = indexes[0]

438 else:

439 index = ExprNodes.TupleNode(pos, args = indexes)

440 result = ExprNodes.IndexNode(pos,

441 base = base, index = index)

442 s.expect(']')

443 return result

444

445 def p_subscript_list(s):

446 items = [p_subscript(s)]

447 while s.sy == ',':

448 s.next()

449 if s.sy == ']':

450 break

451 items.append(p_subscript(s))

452 return items

453

454 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

455

456 def p_subscript(s):

457 # Parse a subscript and return a list of

458 # 1, 2 or 3 ExprNodes, depending on how

459 # many slice elements were encountered.

460 pos = s.position()

461 if s.sy == '.':

462 expect_ellipsis(s)

463 return [ExprNodes.EllipsisNode(pos)]

464 else:

465 start = p_slice_element(s, (':',))

466 if s.sy != ':':

467 return [start]

468 s.next()

469 stop = p_slice_element(s, (':', ',', ']'))

470 if s.sy != ':':

471 return [start, stop]

472 s.next()

473 step = p_slice_element(s, (':', ',', ']'))

474 return [start, stop, step]

475

476 def p_slice_element(s, follow_set):

477 # Simple expression which may be missing iff

478 # it is followed by something in follow_set.

479 if s.sy not in follow_set:

480 return p_simple_expr(s)

481 else:

482 return None

483

484 def expect_ellipsis(s):

485 s.expect('.')

486 s.expect('.')

487 s.expect('.')

488

489 def make_slice_nodes(pos, subscripts):

490 # Convert a list of subscripts as returned

491 # by p_subscript_list into a list of ExprNodes,

492 # creating SliceNodes for elements with 2 or

493 # more components.

494 result = []

495 for subscript in subscripts:

496 if len(subscript) == 1:

497 result.append(subscript[0])

498 else:

499 result.append(make_slice_node(pos, *subscript))

500 return result

501

502 def make_slice_node(pos, start, stop = None, step = None):

503 if not start:

504 start = ExprNodes.NoneNode(pos)

505 if not stop:

506 stop = ExprNodes.NoneNode(pos)

507 if not step:

508 step = ExprNodes.NoneNode(pos)

509 return ExprNodes.SliceNode(pos,

510 start = start, stop = stop, step = step)

511

512 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

513

514 def p_atom(s):

515 pos = s.position()

516 sy = s.sy

517 if sy == '(':

518 s.next()

519 if s.sy == ')':

520 result = ExprNodes.TupleNode(pos, args = [])

521 elif s.sy == 'yield':

522 result = p_yield_expression(s)

523 else:

524 result = p_expr(s)

525 s.expect(')')

526 return result

527 elif sy == '[':

528 return p_list_maker(s)

529 elif sy == '{':

530 return p_dict_or_set_maker(s)

531 elif sy == '`':

532 return p_backquote_expr(s)

533 elif sy == 'INT':

534 value = s.systring

535 s.next()

536 unsigned = ""

537 longness = ""

538 while value[-1] in "UuLl":

539 if value[-1] in "Ll":

540 longness += "L"

541 else:

542 unsigned += "U"

543 value = value[:-1]

544 return ExprNodes.IntNode(pos,

545 value = value,

546 unsigned = unsigned,

547 longness = longness)

548 elif sy == 'FLOAT':

549 value = s.systring

550 s.next()

551 return ExprNodes.FloatNode(pos, value = value)

552 elif sy == 'IMAG':

553 value = s.systring[:-1]

554 s.next()

555 return ExprNodes.ImagNode(pos, value = value)

556 elif sy == 'BEGIN_STRING':

557 kind, value = p_cat_string_literal(s)

558 if kind == 'c':

559 return ExprNodes.CharNode(pos, value = value)

560 elif kind == 'u':

561 return ExprNodes.UnicodeNode(pos, value = value)

562 elif kind == 'b':

563 return ExprNodes.BytesNode(pos, value = value)

564 else:

565 return ExprNodes.StringNode(pos, value = value)

566 elif sy == 'IDENT':

567 name = EncodedString( s.systring )

568 s.next()

569 if name == "None":

570 return ExprNodes.NoneNode(pos)

571 elif name == "True":

572 return ExprNodes.BoolNode(pos, value=True)

573 elif name == "False":

574 return ExprNodes.BoolNode(pos, value=False)

575 elif name == "NULL":

576 return ExprNodes.NullNode(pos)

577 else:

578 return p_name(s, name)

579 else:

580 s.error("Expected an identifier or literal")

581

582 def p_name(s, name):

583 pos = s.position()

584 if not s.compile_time_expr and name in s.compile_time_env:

585 value = s.compile_time_env.lookup_here(name)

586 rep = repr(value)

587 if isinstance(value, bool):

588 return ExprNodes.BoolNode(pos, value = value)

589 elif isinstance(value, int):

590 return ExprNodes.IntNode(pos, value = rep)

591 elif isinstance(value, long):

592 return ExprNodes.IntNode(pos, value = rep, longness = "L")

593 elif isinstance(value, float):

594 return ExprNodes.FloatNode(pos, value = rep)

595 elif isinstance(value, _unicode):

596 return ExprNodes.UnicodeNode(pos, value = value)

597 elif isinstance(value, _bytes):

598 return ExprNodes.BytesNode(pos, value = value)

599 else:

600 error(pos, "Invalid type for compile-time constant: %s"

601 % value.__class__.__name__)

602 return ExprNodes.NameNode(pos, name = name)

603

604 def p_cat_string_literal(s):

605 # A sequence of one or more adjacent string literals.

606 # Returns (kind, value) where kind in ('b', 'c', 'u', '')

607 kind, value = p_string_literal(s)

608 if s.sy != 'BEGIN_STRING':

609 return kind, value

610 if kind != 'c':

611 strings = [value]

612 while s.sy == 'BEGIN_STRING':

613 pos = s.position()

614 next_kind, next_value = p_string_literal(s)

615 if next_kind == 'c':

616 error(pos, "Cannot concatenate char literal with another string or char literal")

617 elif next_kind != kind:

618 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %

619 (kind, next_kind))

620 else:

621 strings.append(next_value)

622 if kind == 'u':

623 value = EncodedString( u''.join(strings) )

624 else:

625 value = BytesLiteral( StringEncoding.join_bytes(strings) )

626 value.encoding = s.source_encoding

627 return kind, value

628

629 def p_opt_string_literal(s):

630 if s.sy == 'BEGIN_STRING':

631 return p_string_literal(s)

632 else:

633 return None

634

635 def p_string_literal(s):

636 # A single string or char literal.

637 # Returns (kind, value) where kind in ('b', 'c', 'u')

638 # s.sy == 'BEGIN_STRING'

639 pos = s.position()

640 is_raw = 0

641 kind = s.systring[:1].lower()

642 if kind == 'r':

643 kind = ''

644 is_raw = 1

645 elif kind in 'ub':

646 is_raw = s.systring[1:2].lower() == 'r'

647 elif kind != 'c':

648 kind = ''

649 if Future.unicode_literals in s.context.future_directives:

650 if kind == '':

651 kind = 'u'

652 if kind == 'u':

653 chars = StringEncoding.UnicodeLiteralBuilder()

654 else:

655 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

656 while 1:

657 s.next()

658 sy = s.sy

659 #print "p_string_literal: sy =", sy, repr(s.systring) ###

660 if sy == 'CHARS':

661 chars.append(s.systring)

662 elif sy == 'ESCAPE':

663 has_escape = True

664 systr = s.systring

665 if is_raw:

666 if systr == u'\\\n':

667 chars.append(u'\\\n')

668 elif systr == u'\\\"':

669 chars.append(u'"')

670 elif systr == u'\\\'':

671 chars.append(u"'")

672 else:

673 chars.append(systr)

674 else:

675 c = systr[1]

676 if c in u"01234567":

677 chars.append_charval( int(systr[1:], 8) )

678 elif c in u"'\"\\":

679 chars.append(c)

680 elif c in u"abfnrtv":

681 chars.append(

682 StringEncoding.char_from_escape_sequence(systr))

683 elif c == u'\n':

684 pass

685 elif c in u'Uux':

686 if kind == 'u' or c == 'x':

687 chrval = int(systr[2:], 16)

688 if chrval > 1114111: # sys.maxunicode:

689 s.error("Invalid unicode escape '%s'" % systr,

690 pos = pos)

691 elif chrval > 65535:

692 warning(s.position(),

693 "Unicode characters above 65535 are not "

694 "necessarily portable across Python installations", 1)

695 chars.append_charval(chrval)

696 else:

697 # unicode escapes in plain byte strings are not unescaped

698 chars.append(systr)

699 else:

700 chars.append(u'\\' + systr[1:])

701 elif sy == 'NEWLINE':

702 chars.append(u'\n')

703 elif sy == 'END_STRING':

704 break

705 elif sy == 'EOF':

706 s.error("Unclosed string literal", pos = pos)

707 else:

708 s.error(

709 "Unexpected token %r:%r in string literal" %

710 (sy, s.systring))

711 if kind == 'c':

712 value = chars.getchar()

713 if len(value) != 1:

714 error(pos, u"invalid character literal: %r" % value)

715 else:

716 value = chars.getstring()

717 s.next()

718 #print "p_string_literal: value =", repr(value) ###

719 return kind, value

720

721 # list_display ::= "[" [listmaker] "]"

722 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

723 # list_iter ::= list_for | list_if

724 # list_for ::= "for" expression_list "in" testlist [list_iter]

725 # list_if ::= "if" test [list_iter]

726

727 def p_list_maker(s):

728 # s.sy == '['

729 pos = s.position()

730 s.next()

731 if s.sy == ']':

732 s.expect(']')

733 return ExprNodes.ListNode(pos, args = [])

734 expr = p_simple_expr(s)

735 if s.sy == 'for':

736 target = ExprNodes.ListNode(pos, args = [])

737 append = ExprNodes.ComprehensionAppendNode(

738 pos, expr=expr, target=ExprNodes.CloneNode(target))

739 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

740 s.expect(']')

741 return ExprNodes.ComprehensionNode(

742 pos, loop=loop, append=append, target=target)

743 else:

744 exprs = [expr]

745 if s.sy == ',':

746 s.next()

747 exprs += p_simple_expr_list(s)

748 s.expect(']')

749 return ExprNodes.ListNode(pos, args = exprs)

750

751 def p_list_iter(s, body):

752 if s.sy == 'for':

753 return p_list_for(s, body)

754 elif s.sy == 'if':

755 return p_list_if(s, body)

756 else:

757 # insert the 'append' operation into the loop

758 return body

759

760 def p_list_for(s, body):

761 # s.sy == 'for'

762 pos = s.position()

763 s.next()

764 kw = p_for_bounds(s)

765 kw['else_clause'] = None

766 kw['body'] = p_list_iter(s, body)

767 return Nodes.ForStatNode(pos, **kw)

768

769 def p_list_if(s, body):

770 # s.sy == 'if'

771 pos = s.position()

772 s.next()

773 test = p_test(s)

774 return Nodes.IfStatNode(pos,

775 if_clauses = [Nodes.IfClauseNode(pos, condition = test,

776 body = p_list_iter(s, body))],

777 else_clause = None )

778

779 #dictmaker: test ':' test (',' test ':' test)* [',']

780

781 def p_dict_or_set_maker(s):

782 # s.sy == '{'

783 pos = s.position()

784 s.next()

785 if s.sy == '}':

786 s.next()

787 return ExprNodes.DictNode(pos, key_value_pairs = [])

788 item = p_simple_expr(s)

789 if s.sy == ',' or s.sy == '}':

790 # set literal

791 values = [item]

792 while s.sy == ',':

793 s.next()

794 if s.sy == '}':

795 break

796 values.append( p_simple_expr(s) )

797 s.expect('}')

798 return ExprNodes.SetNode(pos, args=values)

799 elif s.sy == 'for':

800 # set comprehension

801 target = ExprNodes.SetNode(pos, args=[])

802 append = ExprNodes.ComprehensionAppendNode(

803 item.pos, expr=item, target=ExprNodes.CloneNode(target))

804 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

805 s.expect('}')

806 return ExprNodes.ComprehensionNode(

807 pos, loop=loop, append=append, target=target)

808 elif s.sy == ':':

809 # dict literal or comprehension

810 key = item

811 s.next()

812 value = p_simple_expr(s)

813 if s.sy == 'for':

814 # dict comprehension

815 target = ExprNodes.DictNode(pos, key_value_pairs = [])

816 append = ExprNodes.DictComprehensionAppendNode(

817 item.pos, key_expr=key, value_expr=value,

818 target=ExprNodes.CloneNode(target))

819 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

820 s.expect('}')

821 return ExprNodes.ComprehensionNode(

822 pos, loop=loop, append=append, target=target)

823 else:

824 # dict literal

825 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

826 while s.sy == ',':

827 s.next()

828 if s.sy == '}':

829 break

830 key = p_simple_expr(s)

831 s.expect(':')

832 value = p_simple_expr(s)

833 items.append(

834 ExprNodes.DictItemNode(key.pos, key=key, value=value))

835 s.expect('}')

836 return ExprNodes.DictNode(pos, key_value_pairs=items)

837 else:

838 # raise an error

839 s.expect('}')

840 return ExprNodes.DictNode(pos, key_value_pairs = [])

841

842 def p_backquote_expr(s):

843 # s.sy == '`'

844 pos = s.position()

845 s.next()

846 arg = p_expr(s)

847 s.expect('`')

848 return ExprNodes.BackquoteNode(pos, arg = arg)

849

850 def p_simple_expr_list(s):

851 exprs = []

852 while s.sy not in expr_terminators:

853 expr = p_simple_expr(s)

854 exprs.append(expr)

855 if s.sy != ',':

856 break

857 s.next()

858 return exprs

859

860 def p_expr(s):

861 pos = s.position()

862 expr = p_simple_expr(s)

863 if s.sy == ',':

864 s.next()

865 exprs = [expr] + p_simple_expr_list(s)

866 return ExprNodes.TupleNode(pos, args = exprs)

867 else:

868 return expr

869

870

871 #testlist: test (',' test)* [',']

872 # differs from p_expr only in the fact that it cannot contain conditional expressions

873

874 def p_testlist(s):

875 pos = s.position()

876 expr = p_test(s)

877 if s.sy == ',':

878 exprs = [expr]

879 while s.sy == ',':

880 s.next()

881 exprs.append(p_test(s))

882 return ExprNodes.TupleNode(pos, args = exprs)

883 else:

884 return expr

885

886 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

887

888 #-------------------------------------------------------

889 #

890 # Statements

891 #

892 #-------------------------------------------------------

893

894 def p_global_statement(s):

895 # assume s.sy == 'global'

896 pos = s.position()

897 s.next()

898 names = p_ident_list(s)

899 return Nodes.GlobalNode(pos, names = names)

900

901 def p_expression_or_assignment(s):

902 expr_list = [p_expr(s)]

903 while s.sy == '=':

904 s.next()

905 expr_list.append(p_expr(s))

906 if len(expr_list) == 1:

907 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

908 lhs = expr_list[0]

909 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

910 error(lhs.pos, "Illegal operand for inplace operation.")

911 operator = s.sy[:-1]

912 s.next()

913 rhs = p_expr(s)

914 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

915 expr = expr_list[0]

916 if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):

917 return Nodes.PassStatNode(expr.pos)

918 else:

919 return Nodes.ExprStatNode(expr.pos, expr = expr)

920

921 rhs = expr_list[-1]

922 if len(expr_list) == 2:

923 return Nodes.SingleAssignmentNode(rhs.pos,

924 lhs = expr_list[0], rhs = rhs)

925 else:

926 return Nodes.CascadedAssignmentNode(rhs.pos,

927 lhs_list = expr_list[:-1], rhs = rhs)

928

929 def p_print_statement(s):

930 # s.sy == 'print'

931 pos = s.position()

932 ends_with_comma = 0

933 s.next()

934 if s.sy == '>>':

935 s.next()

936 stream = p_simple_expr(s)

937 if s.sy == ',':

938 s.next()

939 ends_with_comma = s.sy in ('NEWLINE', 'EOF')

940 else:

941 stream = None

942 args = []

943 if s.sy not in ('NEWLINE', 'EOF'):

944 args.append(p_simple_expr(s))

945 while s.sy == ',':

946 s.next()

947 if s.sy in ('NEWLINE', 'EOF'):

948 ends_with_comma = 1

949 break

950 args.append(p_simple_expr(s))

951 arg_tuple = ExprNodes.TupleNode(pos, args = args)

952 return Nodes.PrintStatNode(pos,

953 arg_tuple = arg_tuple, stream = stream,

954 append_newline = not ends_with_comma)

955

956 def p_exec_statement(s):

957 # s.sy == 'exec'

958 pos = s.position()

959 s.next()

960 args = [ p_bit_expr(s) ]

961 if s.sy == 'in':

962 s.next()

963 args.append(p_simple_expr(s))

964 if s.sy == ',':

965 s.next()

966 args.append(p_simple_expr(s))

967 else:

968 error(pos, "'exec' currently requires a target mapping (globals/locals)")

969 return Nodes.ExecStatNode(pos, args = args)

970

971 def p_del_statement(s):

972 # s.sy == 'del'

973 pos = s.position()

974 s.next()

975 args = p_simple_expr_list(s)

976 return Nodes.DelStatNode(pos, args = args)

977

978 def p_pass_statement(s, with_newline = 0):

979 pos = s.position()

980 s.expect('pass')

981 if with_newline:

982 s.expect_newline("Expected a newline")

983 return Nodes.PassStatNode(pos)

984

985 def p_break_statement(s):

986 # s.sy == 'break'

987 pos = s.position()

988 s.next()

989 return Nodes.BreakStatNode(pos)

990

991 def p_continue_statement(s):

992 # s.sy == 'continue'

993 pos = s.position()

994 s.next()

995 return Nodes.ContinueStatNode(pos)

996

997 def p_return_statement(s):

998 # s.sy == 'return'

999 pos = s.position()

1000 s.next()

1001 if s.sy not in statement_terminators:

1002 value = p_expr(s)

1003 else:

1004 value = None

1005 return Nodes.ReturnStatNode(pos, value = value)

1006

1007 def p_raise_statement(s):

1008 # s.sy == 'raise'

1009 pos = s.position()

1010 s.next()

1011 exc_type = None

1012 exc_value = None

1013 exc_tb = None

1014 if s.sy not in statement_terminators:

1015 exc_type = p_simple_expr(s)

1016 if s.sy == ',':

1017 s.next()

1018 exc_value = p_simple_expr(s)

1019 if s.sy == ',':

1020 s.next()

1021 exc_tb = p_simple_expr(s)

1022 if exc_type or exc_value or exc_tb:

1023 return Nodes.RaiseStatNode(pos,

1024 exc_type = exc_type,

1025 exc_value = exc_value,

1026 exc_tb = exc_tb)

1027 else:

1028 return Nodes.ReraiseStatNode(pos)

1029

1030 def p_import_statement(s):

1031 # s.sy in ('import', 'cimport')

1032 pos = s.position()

1033 kind = s.sy

1034 s.next()

1035 items = [p_dotted_name(s, as_allowed = 1)]

1036 while s.sy == ',':

1037 s.next()

1038 items.append(p_dotted_name(s, as_allowed = 1))

1039 stats = []

1040 for pos, target_name, dotted_name, as_name in items:

1041 dotted_name = EncodedString(dotted_name)

1042 if kind == 'cimport':

1043 stat = Nodes.CImportStatNode(pos,

1044 module_name = dotted_name,

1045 as_name = as_name)

1046 else:

1047 if as_name and "." in dotted_name:

1048 name_list = ExprNodes.ListNode(pos, args = [

1049 ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])

1050 else:

1051 name_list = None

1052 stat = Nodes.SingleAssignmentNode(pos,

1053 lhs = ExprNodes.NameNode(pos,

1054 name = as_name or target_name),

1055 rhs = ExprNodes.ImportNode(pos,

1056 module_name = ExprNodes.IdentifierStringNode(

1057 pos, value = dotted_name),

1058 name_list = name_list))

1059 stats.append(stat)

1060 return Nodes.StatListNode(pos, stats = stats)

1061

1062 def p_from_import_statement(s, first_statement = 0):

1063 # s.sy == 'from'

1064 pos = s.position()

1065 s.next()

1066 (dotted_name_pos, _, dotted_name, _) = \

1067 p_dotted_name(s, as_allowed = 0)

1068 if s.sy in ('import', 'cimport'):

1069 kind = s.sy

1070 s.next()

1071 else:

1072 s.error("Expected 'import' or 'cimport'")

1073 is_cimport = kind == 'cimport'

1074 is_parenthesized = False

1075 if s.sy == '*':

1076 imported_names = [(s.position(), "*", None, None)]

1077 s.next()

1078 else:

1079 if s.sy == '(':

1080 is_parenthesized = True

1081 s.next()

1082 imported_names = [p_imported_name(s, is_cimport)]

1083 while s.sy == ',':

1084 s.next()

1085 imported_names.append(p_imported_name(s, is_cimport))

1086 if is_parenthesized:

1087 s.expect(')')

1088 dotted_name = EncodedString(dotted_name)

1089 if dotted_name == '__future__':

1090 if not first_statement:

1091 s.error("from __future__ imports must occur at the beginning of the file")

1092 else:

1093 for (name_pos, name, as_name, kind) in imported_names:

1094 if name == "braces":

1095 s.error("not a chance", name_pos)

1096 break

1097 try:

1098 directive = getattr(Future, name)

1099 except AttributeError:

1100 s.error("future feature %s is not defined" % name, name_pos)

1101 break

1102 s.context.future_directives.add(directive)

1103 return Nodes.PassStatNode(pos)

1104 elif kind == 'cimport':

1105 return Nodes.FromCImportStatNode(pos,

1106 module_name = dotted_name,

1107 imported_names = imported_names)

1108 else:

1109 imported_name_strings = []

1110 items = []

1111 for (name_pos, name, as_name, kind) in imported_names:

1112 encoded_name = EncodedString(name)

1113 imported_name_strings.append(

1114 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1115 items.append(

1116 (name,

1117 ExprNodes.NameNode(name_pos,

1118 name = as_name or name)))

1119 import_list = ExprNodes.ListNode(

1120 imported_names[0][0], args = imported_name_strings)

1121 dotted_name = EncodedString(dotted_name)

1122 return Nodes.FromImportStatNode(pos,

1123 module = ExprNodes.ImportNode(dotted_name_pos,

1124 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1125 name_list = import_list),

1126 items = items)

1127

1128 imported_name_kinds = ('class', 'struct', 'union')

1129

1130 def p_imported_name(s, is_cimport):

1131 pos = s.position()

1132 kind = None

1133 if is_cimport and s.systring in imported_name_kinds:

1134 kind = s.systring

1135 s.next()

1136 name = p_ident(s)

1137 as_name = p_as_name(s)

1138 return (pos, name, as_name, kind)

1139

1140 def p_dotted_name(s, as_allowed):

1141 pos = s.position()

1142 target_name = p_ident(s)

1143 as_name = None

1144 names = [target_name]

1145 while s.sy == '.':

1146 s.next()

1147 names.append(p_ident(s))

1148 if as_allowed:

1149 as_name = p_as_name(s)

1150 return (pos, target_name, u'.'.join(names), as_name)

1151

1152 def p_as_name(s):

1153 if s.sy == 'IDENT' and s.systring == 'as':

1154 s.next()

1155 return p_ident(s)

1156 else:

1157 return None

1158

1159 def p_assert_statement(s):

1160 # s.sy == 'assert'

1161 pos = s.position()

1162 s.next()

1163 cond = p_simple_expr(s)

1164 if s.sy == ',':

1165 s.next()

1166 value = p_simple_expr(s)

1167 else:

1168 value = None

1169 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1170

1171 statement_terminators = (';', 'NEWLINE', 'EOF')

1172

1173 def p_if_statement(s):

1174 # s.sy == 'if'

1175 pos = s.position()

1176 s.next()

1177 if_clauses = [p_if_clause(s)]

1178 while s.sy == 'elif':

1179 s.next()

1180 if_clauses.append(p_if_clause(s))

1181 else_clause = p_else_clause(s)

1182 return Nodes.IfStatNode(pos,

1183 if_clauses = if_clauses, else_clause = else_clause)

1184

1185 def p_if_clause(s):

1186 pos = s.position()

1187 test = p_simple_expr(s)

1188 body = p_suite(s)

1189 return Nodes.IfClauseNode(pos,

1190 condition = test, body = body)

1191

1192 def p_else_clause(s):

1193 if s.sy == 'else':

1194 s.next()

1195 return p_suite(s)

1196 else:

1197 return None

1198

1199 def p_while_statement(s):

1200 # s.sy == 'while'

1201 pos = s.position()

1202 s.next()

1203 test = p_simple_expr(s)

1204 body = p_suite(s)

1205 else_clause = p_else_clause(s)

1206 return Nodes.WhileStatNode(pos,

1207 condition = test, body = body,

1208 else_clause = else_clause)

1209

1210 def p_for_statement(s):

1211 # s.sy == 'for'

1212 pos = s.position()

1213 s.next()

1214 kw = p_for_bounds(s)

1215 kw['body'] = p_suite(s)

1216 kw['else_clause'] = p_else_clause(s)

1217 return Nodes.ForStatNode(pos, **kw)

1218

1219 def p_for_bounds(s):

1220 target = p_for_target(s)

1221 if s.sy == 'in':

1222 s.next()

1223 iterator = p_for_iterator(s)

1224 return { 'target': target, 'iterator': iterator }

1225 else:

1226 if s.sy == 'from':

1227 s.next()

1228 bound1 = p_bit_expr(s)

1229 else:

1230 # Support shorter "for a <= x < b" syntax

1231 bound1, target = target, None

1232 rel1 = p_for_from_relation(s)

1233 name2_pos = s.position()

1234 name2 = p_ident(s)

1235 rel2_pos = s.position()

1236 rel2 = p_for_from_relation(s)

1237 bound2 = p_bit_expr(s)

1238 step = p_for_from_step(s)

1239 if target is None:

1240 target = ExprNodes.NameNode(name2_pos, name = name2)

1241 else:

1242 if not target.is_name:

1243 error(target.pos,

1244 "Target of for-from statement must be a variable name")

1245 elif name2 != target.name:

1246 error(name2_pos,

1247 "Variable name in for-from range does not match target")

1248 if rel1[0] != rel2[0]:

1249 error(rel2_pos,

1250 "Relation directions in for-from do not match")

1251 return {'target': target,

1252 'bound1': bound1,

1253 'relation1': rel1,

1254 'relation2': rel2,

1255 'bound2': bound2,

1256 'step': step }

1257

1258 def p_for_from_relation(s):

1259 if s.sy in inequality_relations:

1260 op = s.sy

1261 s.next()

1262 return op

1263 else:

1264 s.error("Expected one of '<', '<=', '>' '>='")

1265

1266 def p_for_from_step(s):

1267 if s.sy == 'by':

1268 s.next()

1269 step = p_bit_expr(s)

1270 return step

1271 else:

1272 return None

1273

1274 inequality_relations = ('<', '<=', '>', '>=')

1275

1276 def p_target(s, terminator):

1277 pos = s.position()

1278 expr = p_starred_expr(s)

1279 if s.sy == ',':

1280 s.next()

1281 exprs = [expr]

1282 while s.sy != terminator:

1283 exprs.append(p_starred_expr(s))

1284 if s.sy != ',':

1285 break

1286 s.next()

1287 return ExprNodes.TupleNode(pos, args = exprs)

1288 else:

1289 return expr

1290

1291 def p_for_target(s):

1292 return p_target(s, 'in')

1293

1294 def p_for_iterator(s):

1295 pos = s.position()

1296 expr = p_testlist(s)

1297 return ExprNodes.IteratorNode(pos, sequence = expr)

1298

1299 def p_try_statement(s):

1300 # s.sy == 'try'

1301 pos = s.position()

1302 s.next()

1303 body = p_suite(s)

1304 except_clauses = []

1305 else_clause = None

1306 if s.sy in ('except', 'else'):

1307 while s.sy == 'except':

1308 except_clauses.append(p_except_clause(s))

1309 if s.sy == 'else':

1310 s.next()

1311 else_clause = p_suite(s)

1312 body = Nodes.TryExceptStatNode(pos,

1313 body = body, except_clauses = except_clauses,

1314 else_clause = else_clause)

1315 if s.sy != 'finally':

1316 return body

1317 # try-except-finally is equivalent to nested try-except/try-finally

1318 if s.sy == 'finally':

1319 s.next()

1320 finally_clause = p_suite(s)

1321 return Nodes.TryFinallyStatNode(pos,

1322 body = body, finally_clause = finally_clause)

1323 else:

1324 s.error("Expected 'except' or 'finally'")

1325

1326 def p_except_clause(s):

1327 # s.sy == 'except'

1328 pos = s.position()

1329 s.next()

1330 exc_type = None

1331 exc_value = None

1332 if s.sy != ':':

1333 exc_type = p_simple_expr(s)

1334 if s.sy == ',':

1335 s.next()

1336 exc_value = p_simple_expr(s)

1337 body = p_suite(s)

1338 return Nodes.ExceptClauseNode(pos,

1339 pattern = exc_type, target = exc_value, body = body)

1340

1341 def p_include_statement(s, ctx):

1342 pos = s.position()

1343 s.next() # 'include'

1344 _, include_file_name = p_string_literal(s)

1345 s.expect_newline("Syntax error in include statement")

1346 if s.compile_time_eval:

1347 include_file_name = include_file_name.decode(s.source_encoding)

1348 include_file_path = s.context.find_include_file(include_file_name, pos)

1349 if include_file_path:

1350 s.included_files.append(include_file_name)

1351 f = Utils.open_source_file(include_file_path, mode="rU")

1352 source_desc = FileSourceDescriptor(include_file_path)

1353 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1354 try:

1355 tree = p_statement_list(s2, ctx)

1356 finally:

1357 f.close()

1358 return tree

1359 else:

1360 return None

1361 else:

1362 return Nodes.PassStatNode(pos)

1363

1364 def p_with_statement(s):

1365 pos = s.position()

1366 s.next() # 'with'

1367 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1368 if s.sy == 'IDENT' and s.systring == 'nogil':

1369 state = s.systring

1370 s.next()

1371 body = p_suite(s)

1372 return Nodes.GILStatNode(pos, state = state, body = body)

1373 elif s.systring == 'template':

1374 templates = []

1375 s.next()

1376 s.expect('[')

1377 #s.next()

1378 templates.append(s.systring)

1379 s.next()

1380 while s.systring == ',':

1381 s.next()

1382 templates.append(s.systring)

1383 s.next()

1384 s.expect(']')

1385 if s.sy == ':':

1386 s.next()

1387 s.expect_newline("Syntax error in template function declaration")

1388 s.expect_indent()

1389 body_ctx = Ctx()

1390 body_ctx.templates = templates

1391 func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)

1392 s.expect_dedent()

1393 return func_or_var

1394 else:

1395 error(pos, "Syntax error in template function declaration")

1396 else:

1397 manager = p_expr(s)

1398 target = None

1399 if s.sy == 'IDENT' and s.systring == 'as':

1400 s.next()

1401 allow_multi = (s.sy == '(')

1402 target = p_target(s, ':')

1403 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1404 s.error("Multiple with statement target values not allowed without paranthesis")

1405 body = p_suite(s)

1406 return Nodes.WithStatNode(pos, manager = manager,

1407 target = target, body = body)

1408

1409 def p_simple_statement(s, first_statement = 0):

1410 #print "p_simple_statement:", s.sy, s.systring ###

1411 if s.sy == 'global':

1412 node = p_global_statement(s)

1413 elif s.sy == 'print':

1414 node = p_print_statement(s)

1415 elif s.sy == 'exec':

1416 node = p_exec_statement(s)

1417 elif s.sy == 'del':

1418 node = p_del_statement(s)

1419 elif s.sy == 'break':

1420 node = p_break_statement(s)

1421 elif s.sy == 'continue':

1422 node = p_continue_statement(s)

1423 elif s.sy == 'return':

1424 node = p_return_statement(s)

1425 elif s.sy == 'raise':

1426 node = p_raise_statement(s)

1427 elif s.sy in ('import', 'cimport'):

1428 node = p_import_statement(s)

1429 elif s.sy == 'from':

1430 node = p_from_import_statement(s, first_statement = first_statement)

1431 elif s.sy == 'yield':

1432 node = p_yield_expression(s)

1433 elif s.sy == 'assert':

1434 node = p_assert_statement(s)

1435 elif s.sy == 'pass':

1436 node = p_pass_statement(s)

1437 else:

1438 node = p_expression_or_assignment(s)

1439 return node

1440

1441 def p_simple_statement_list(s, ctx, first_statement = 0):

1442 # Parse a series of simple statements on one line

1443 # separated by semicolons.

1444 stat = p_simple_statement(s, first_statement = first_statement)

1445 if s.sy == ';':

1446 stats = [stat]

1447 while s.sy == ';':

1448 #print "p_simple_statement_list: maybe more to follow" ###

1449 s.next()

1450 if s.sy in ('NEWLINE', 'EOF'):

1451 break

1452 stats.append(p_simple_statement(s))

1453 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1454 s.expect_newline("Syntax error in simple statement list")

1455 return stat

1456

1457 def p_compile_time_expr(s):

1458 old = s.compile_time_expr

1459 s.compile_time_expr = 1

1460 expr = p_expr(s)

1461 s.compile_time_expr = old

1462 return expr

1463

1464 def p_DEF_statement(s):

1465 pos = s.position()

1466 denv = s.compile_time_env

1467 s.next() # 'DEF'

1468 name = p_ident(s)

1469 s.expect('=')

1470 expr = p_compile_time_expr(s)

1471 value = expr.compile_time_value(denv)

1472 #print "p_DEF_statement: %s = %r" % (name, value) ###

1473 denv.declare(name, value)

1474 s.expect_newline()

1475 return Nodes.PassStatNode(pos)

1476

1477 def p_IF_statement(s, ctx):

1478 pos = s.position()

1479 saved_eval = s.compile_time_eval

1480 current_eval = saved_eval

1481 denv = s.compile_time_env

1482 result = None

1483 while 1:

1484 s.next() # 'IF' or 'ELIF'

1485 expr = p_compile_time_expr(s)

1486 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1487 body = p_suite(s, ctx)

1488 if s.compile_time_eval:

1489 result = body

1490 current_eval = 0

1491 if s.sy != 'ELIF':

1492 break

1493 if s.sy == 'ELSE':

1494 s.next()

1495 s.compile_time_eval = current_eval

1496 body = p_suite(s, ctx)

1497 if current_eval:

1498 result = body

1499 if not result:

1500 result = Nodes.PassStatNode(pos)

1501 s.compile_time_eval = saved_eval

1502 return result

1503

1504 def p_statement(s, ctx, first_statement = 0):

1505 cdef_flag = ctx.cdef_flag

1506 decorators = None

1507 if s.sy == 'ctypedef':

1508 if ctx.level not in ('module', 'module_pxd'):

1509 s.error("ctypedef statement not allowed here")

1510 #if ctx.api:

1511 # error(s.position(), "'api' not allowed with 'ctypedef'")

1512 return p_ctypedef_statement(s, ctx)

1513 elif s.sy == 'DEF':

1514 return p_DEF_statement(s)

1515 elif s.sy == 'IF':

1516 return p_IF_statement(s, ctx)

1517 elif s.sy == 'DECORATOR':

1518 if ctx.level not in ('module', 'class', 'c_class', 'property', 'module_pxd', 'c_class_pxd'):

1519 print ctx.level

1520 s.error('decorator not allowed here')

1521 s.level = ctx.level

1522 decorators = p_decorators(s)

1523 if s.sy not in ('def', 'cdef', 'cpdef', 'class'):

1524 s.error("Decorators can only be followed by functions or classes")

1525 elif s.sy == 'pass' and cdef_flag:

1526 # empty cdef block

1527 return p_pass_statement(s, with_newline = 1)

1528

1529 overridable = 0

1530 if s.sy == 'cdef':

1531 cdef_flag = 1

1532 s.next()

1533 elif s.sy == 'cpdef':

1534 cdef_flag = 1

1535 overridable = 1

1536 s.next()

1537 if cdef_flag:

1538 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1539 s.error('cdef statement not allowed here')

1540 s.level = ctx.level

1541 node = p_cdef_statement(s, ctx(overridable = overridable))

1542 if decorators is not None:

1543 if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):

1544 s.error("Decorators can only be followed by functions or Python classes")

1545 node.decorators = decorators

1546 return node

1547 else:

1548 if ctx.api:

1549 error(s.pos, "'api' not allowed with this statement")

1550 elif s.sy == 'def':

1551 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1552 s.error('def statement not allowed here')

1553 s.level = ctx.level

1554 return p_def_statement(s, decorators)

1555 elif s.sy == 'class':

1556 if ctx.level != 'module':

1557 s.error("class definition not allowed here")

1558 return p_class_statement(s, decorators)

1559 elif s.sy == 'include':

1560 if ctx.level not in ('module', 'module_pxd'):

1561 s.error("include statement not allowed here")

1562 return p_include_statement(s, ctx)

1563 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1564 return p_property_decl(s)

1565 elif s.sy == 'pass' and ctx.level != 'property':

1566 return p_pass_statement(s, with_newline = 1)

1567 else:

1568 if ctx.level in ('c_class_pxd', 'property'):

1569 s.error("Executable statement not allowed here")

1570 if s.sy == 'if':

1571 return p_if_statement(s)

1572 elif s.sy == 'while':

1573 return p_while_statement(s)

1574 elif s.sy == 'for':

1575 return p_for_statement(s)

1576 elif s.sy == 'try':

1577 return p_try_statement(s)

1578 elif s.sy == 'with':

1579 return p_with_statement(s)

1580 else:

1581 return p_simple_statement_list(

1582 s, ctx, first_statement = first_statement)

1583

1584 def p_statement_list(s, ctx, first_statement = 0):

1585 # Parse a series of statements separated by newlines.

1586 pos = s.position()

1587 stats = []

1588 while s.sy not in ('DEDENT', 'EOF'):

1589 stats.append(p_statement(s, ctx, first_statement = first_statement))

1590 first_statement = 0

1591 if len(stats) == 1:

1592 return stats[0]

1593 else:

1594 return Nodes.StatListNode(pos, stats = stats)

1595

1596 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1597 pos = s.position()

1598 s.expect(':')

1599 doc = None

1600 stmts = []

1601 if s.sy == 'NEWLINE':

1602 s.next()

1603 s.expect_indent()

1604 if with_doc or with_pseudo_doc:

1605 doc = p_doc_string(s)

1606 body = p_statement_list(s, ctx)

1607 s.expect_dedent()

1608 else:

1609 if ctx.api:

1610 error(s.pos, "'api' not allowed with this statement")

1611 if ctx.level in ('module', 'class', 'function', 'other'):

1612 body = p_simple_statement_list(s, ctx)

1613 else:

1614 body = p_pass_statement(s)

1615 s.expect_newline("Syntax error in declarations")

1616 if with_doc:

1617 return doc, body

1618 else:

1619 return body

1620

1621 def p_positional_and_keyword_args(s, end_sy_set, templates = None):

1622 """

1623 Parses positional and keyword arguments. end_sy_set

1624 should contain any s.sy that terminate the argument list.

1625 Argument expansion (* and **) are not allowed.

1626

1627 Returns: (positional_args, keyword_args)

1628 """

1629 positional_args = []

1630 keyword_args = []

1631 pos_idx = 0

1632

1633 while s.sy not in end_sy_set:

1634 if s.sy == '*' or s.sy == '**':

1635 s.error('Argument expansion not allowed here.')

1636

1637 parsed_type = False

1638 if s.sy == 'IDENT' and s.peek()[0] == '=':

1639 ident = s.systring

1640 s.next() # s.sy is '='

1641 s.next()

1642 if looking_at_expr(s):

1643 arg = p_simple_expr(s)

1644 else:

1645 base_type = p_c_base_type(s, templates = templates)

1646 declarator = p_c_declarator(s, empty = 1)

1647 arg = Nodes.CComplexBaseTypeNode(base_type.pos,

1648 base_type = base_type, declarator = declarator)

1649 parsed_type = True

1650 keyword_node = ExprNodes.IdentifierStringNode(

1651 arg.pos, value = EncodedString(ident))

1652 keyword_args.append((keyword_node, arg))

1653 was_keyword = True

1654

1655 else:

1656 if looking_at_expr(s):

1657 arg = p_simple_expr(s)

1658 else:

1659 base_type = p_c_base_type(s, templates = templates)

1660 declarator = p_c_declarator(s, empty = 1)

1661 arg = Nodes.CComplexBaseTypeNode(base_type.pos,

1662 base_type = base_type, declarator = declarator)

1663 parsed_type = True

1664 positional_args.append(arg)

1665 pos_idx += 1

1666 if len(keyword_args) > 0:

1667 s.error("Non-keyword arg following keyword arg",

1668 pos = arg.pos)

1669

1670 if s.sy != ',':

1671 if s.sy not in end_sy_set:

1672 if parsed_type:

1673 s.error("Unmatched %s" % " or ".join(end_sy_set))

1674 break

1675 s.next()

1676 return positional_args, keyword_args

1677

1678 def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):

1679 # If self_flag is true, this is the base type for the

1680 # self argument of a C method of an extension type.

1681 if s.sy == '(':

1682 return p_c_complex_base_type(s)

1683 else:

1684 return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)

1685

1686 def p_calling_convention(s):

1687 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1688 result = s.systring

1689 s.next()

1690 return result

1691 else:

1692 return ""

1693

1694 calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

1695

1696 def p_c_complex_base_type(s):

1697 # s.sy == '('

1698 pos = s.position()

1699 s.next()

1700 base_type = p_c_base_type(s)

1701 declarator = p_c_declarator(s, empty = 1)

1702 s.expect(')')

1703 return Nodes.CComplexBaseTypeNode(pos,

1704 base_type = base_type, declarator = declarator)

1705

1706 def p_c_simple_base_type(s, self_flag, nonempty, templates = None):

1707 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1708 is_basic = 0

1709 signed = 1

1710 longness = 0

1711 complex = 0

1712 module_path = []

1713 pos = s.position()

1714 if not s.sy == 'IDENT':

1715 error(pos, "Expected an identifier, found '%s'" % s.sy)

1716 if looking_at_base_type(s):

1717 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1718 is_basic = 1

1719 if s.sy == 'IDENT' and s.systring in special_basic_c_types:

1720 signed, longness = special_basic_c_types[s.systring]

1721 name = s.systring

1722 s.next()

1723 else:

1724 signed, longness = p_sign_and_longness(s)

1725 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1726 name = s.systring

1727 s.next()

1728 else:

1729 name = 'int'

1730 if s.sy == 'IDENT' and s.systring == 'complex':

1731 complex = 1

1732 s.next()

1733 elif looking_at_dotted_name(s):

1734 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1735 name = s.systring

1736 s.next()

1737 while s.sy == '.':

1738 module_path.append(name)

1739 s.next()

1740 name = p_ident(s)

1741 else:

1742 name = s.systring

1743 s.next()

1744 if nonempty and s.sy != 'IDENT':

1745 # Make sure this is not a declaration of a variable or function.

1746 if s.sy == '(':

1747 s.next()

1748 if s.sy == '*' or s.sy == '**':

1749 s.put_back('(', '(')

1750 else:

1751 s.put_back('(', '(')

1752 s.put_back('IDENT', name)

1753 name = None

1754 elif s.sy not in ('*', '**', '['):

1755 s.put_back('IDENT', name)

1756 name = None

1757

1758 type_node = Nodes.CSimpleBaseTypeNode(pos,

1759 name = name, module_path = module_path,

1760 is_basic_c_type = is_basic, signed = signed,

1761 complex = complex, longness = longness,

1762 is_self_arg = self_flag, templates = templates)

1763

1764 if s.sy == '[':

1765 type_node = p_buffer_or_template(s, type_node, templates)

1766

1767 if s.sy == '.':

1768 s.next()

1769 name = p_ident(s)

1770 type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)

1771

1772 return type_node

1773

1774 def p_buffer_or_template(s, base_type_node, templates):

1775 # s.sy == '['

1776 pos = s.position()

1777 s.next()

1778 # Note that buffer_positional_options_count=1, so the only positional argument is dtype.

1779 # For templated types, all parameters are types.

1780 positional_args, keyword_args = (

1781 p_positional_and_keyword_args(s, (']',), templates)

1782 )

1783 s.expect(']')

1784

1785 keyword_dict = ExprNodes.DictNode(pos,

1786 key_value_pairs = [

1787 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1788 for key, value in keyword_args

1789 ])

1790 result = Nodes.TemplatedTypeNode(pos,

1791 positional_args = positional_args,

1792 keyword_args = keyword_dict,

1793 base_type_node = base_type_node)

1794 return result

1795

1796

1797 def looking_at_name(s):

1798 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1799

1800 def looking_at_expr(s):

1801 if s.systring in base_type_start_words:

1802 return False

1803 elif s.sy == 'IDENT':

1804 is_type = False

1805 name = s.systring

1806 dotted_path = []

1807 s.next()

1808 while s.sy == '.':

1809 s.next()

1810 dotted_path.append(s.systring)

1811 s.expect('IDENT')

1812 saved = s.sy, s.systring

1813 if s.sy == 'IDENT':

1814 is_type = True

1815 elif s.sy == '*' or s.sy == '**':

1816 s.next()

1817 is_type = s.sy == ')'

1818 s.put_back(*saved)

1819 elif s.sy == '(':

1820 s.next()

1821 is_type = s.sy == '*'

1822 s.put_back(*saved)

1823 elif s.sy == '[':

1824 s.next()

1825 is_type = s.sy == ']'

1826 s.put_back(*saved)

1827 dotted_path.reverse()

1828 for p in dotted_path:

1829 s.put_back('IDENT', p)

1830 s.put_back('.', '.')

1831 s.put_back('IDENT', name)

1832 return not is_type

1833 else:

1834 return True

1835

1836 def looking_at_base_type(s):

1837 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1838 return s.sy == 'IDENT' and s.systring in base_type_start_words

1839

1840 def looking_at_dotted_name(s):

1841 if s.sy == 'IDENT':

1842 name = s.systring

1843 s.next()

1844 result = s.sy == '.'

1845 s.put_back('IDENT', name)

1846 return result

1847 else:

1848 return 0

1849

1850 basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

1851

1852 special_basic_c_types = {

1853 # name : (signed, longness)

1854 "Py_ssize_t" : (2, 0),

1855 "size_t" : (0, 0),

1856 }

1857

1858 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1859

1860 base_type_start_words = \

1861 basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)

1862

1863 def p_sign_and_longness(s):

1864 signed = 1

1865 longness = 0

1866 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1867 if s.systring == 'unsigned':

1868 signed = 0

1869 elif s.systring == 'signed':

1870 signed = 2

1871 elif s.systring == 'short':

1872 longness = -1

1873 elif s.systring == 'long':

1874 longness += 1

1875 s.next()

1876 return signed, longness

1877

1878 def p_opt_cname(s):

1879 literal = p_opt_string_literal(s)

1880 if literal:

1881 _, cname = literal

1882 cname = EncodedString(cname)

1883 cname.encoding = s.source_encoding

1884 else:

1885 cname = None

1886 return cname

1887

1888 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1889 assignable = 0, nonempty = 0,

1890 calling_convention_allowed = 0):

1891 # If empty is true, the declarator must be empty. If nonempty is true,

1892 # the declarator must be nonempty. Otherwise we don't care.

1893 # If cmethod_flag is true, then if this declarator declares

1894 # a function, it's a C method of an extension type.

1895 pos = s.position()

1896 if s.sy == '(':

1897 s.next()

1898 if s.sy == ')' or looking_at_name(s):

1899 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1900 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1901 else:

1902 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1903 cmethod_flag = cmethod_flag,

1904 nonempty = nonempty,

1905 calling_convention_allowed = 1)

1906 s.expect(')')

1907 else:

1908 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1909 assignable, nonempty)

1910 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1911 error(s.position(), "%s on something that is not a function"

1912 % result.calling_convention)

1913 while s.sy in ('[', '('):

1914 pos = s.position()

1915 if s.sy == '[':

1916 result = p_c_array_declarator(s, result)

1917 else: # sy == '('

1918 s.next()

1919 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1920 cmethod_flag = 0

1921 return result

1922

1923 def p_c_array_declarator(s, base):

1924 pos = s.position()

1925 s.next() # '['

1926 if s.sy != ']':

1927 dim = p_expr(s)

1928 else:

1929 dim = None

1930 s.expect(']')

1931 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1932

1933 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

1934 # Opening paren has already been skipped

1935 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

1936 nonempty_declarators = 0)

1937 ellipsis = p_optional_ellipsis(s)

1938 s.expect(')')

1939 nogil = p_nogil(s)

1940 exc_val, exc_check = p_exception_value_clause(s)

1941 with_gil = p_with_gil(s)

1942 return Nodes.CFuncDeclaratorNode(pos,

1943 base = base, args = args, has_varargs = ellipsis,

1944 exception_value = exc_val, exception_check = exc_check,

1945 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

1946

1947 supported_overloaded_operators = set([

1948 '+', '-', '*', '/', '%',

1949 '++', '--', '~', '|', '&', '^', '<<', '>>',

1950 '==', '!=', '>=', '>', '<=', '<',

1951 '[]', '()',

1952 ])

1953

1954 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1955 assignable, nonempty):

1956 pos = s.position()

1957 calling_convention = p_calling_convention(s)

1958 if s.sy == '*':

1959 s.next()

1960 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1961 cmethod_flag = cmethod_flag,

1962 assignable = assignable, nonempty = nonempty)

1963 result = Nodes.CPtrDeclaratorNode(pos,

1964 base = base)

1965 elif s.sy == '**': # scanner returns this as a single token

1966 s.next()

1967 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1968 cmethod_flag = cmethod_flag,

1969 assignable = assignable, nonempty = nonempty)

1970 result = Nodes.CPtrDeclaratorNode(pos,

1971 base = Nodes.CPtrDeclaratorNode(pos,

1972 base = base))

1973 elif s.sy == '&':

1974 s.next()

1975 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1976 cmethod_flag = cmethod_flag,

1977 assignable = assignable, nonempty = nonempty)

1978 result = Nodes.CReferenceDeclaratorNode(pos, base = base)

1979 else:

1980 rhs = None

1981 if s.sy == 'IDENT':

1982 name = EncodedString(s.systring)

1983 if empty:

1984 error(s.position(), "Declarator should be empty")

1985 s.next()

1986 cname = p_opt_cname(s)

1987 if s.sy == '=' and assignable:

1988 s.next()

1989 rhs = p_simple_expr(s)

1990 else:

1991 if nonempty:

1992 error(s.position(), "Empty declarator")

1993 name = ""

1994 cname = None

1995 if cname is None and ctx.namespace is not None:

1996 cname = ctx.namespace + "::" + name

1997 if name == 'operator' and ctx.visibility == 'extern':

1998 op = s.sy

1999 s.next()

2000 # Handle diphthong operators.

2001 if op == '(':

2002 s.expect(')')

2003 op = '()'

2004 elif op == '[':

2005 s.expect(']')

2006 op = '[]'

2007 if op in ['-', '+', '|', '&'] and s.sy == op:

2008 op = op*2

2009 s.next()

2010 if s.sy == '=':

2011 op += s.sy

2012 s.next()

2013 if op not in supported_overloaded_operators:

2014 s.error("Overloading operator '%s' not yet supported." % op)

2015 name = name+op

2016 result = Nodes.CNameDeclaratorNode(pos,

2017 name = name, cname = cname, default = rhs)

2018 result.calling_convention = calling_convention

2019 return result

2020

2021 def p_nogil(s):

2022 if s.sy == 'IDENT' and s.systring == 'nogil':

2023 s.next()

2024 return 1

2025 else:

2026 return 0

2027

2028 def p_with_gil(s):

2029 if s.sy == 'with':

2030 s.next()

2031 s.expect_keyword('gil')

2032 return 1

2033 else:

2034 return 0

2035

2036 def p_exception_value_clause(s):

2037 exc_val = None

2038 exc_check = 0

2039 if s.sy == 'except':

2040 s.next()

2041 if s.sy == '*':

2042 exc_check = 1

2043 s.next()

2044 elif s.sy == '+':

2045 exc_check = '+'

2046 s.next()

2047 if s.sy == 'IDENT':

2048 name = s.systring

2049 s.next()

2050 exc_val = p_name(s, name)

2051 else:

2052 if s.sy == '?':

2053 exc_check = 1

2054 s.next()

2055 exc_val = p_simple_expr(s)

2056 return exc_val, exc_check

2057

2058 c_arg_list_terminators = ('*', '**', '.', ')')

2059

2060 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

2061 nonempty_declarators = 0, kw_only = 0):

2062 # Comma-separated list of C argument declarations, possibly empty.

2063 # May have a trailing comma.

2064 args = []

2065 is_self_arg = cmethod_flag

2066 while s.sy not in c_arg_list_terminators:

2067 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

2068 nonempty = nonempty_declarators, kw_only = kw_only))

2069 if s.sy != ',':

2070 break

2071 s.next()

2072 is_self_arg = 0

2073 return args

2074

2075 def p_optional_ellipsis(s):

2076 if s.sy == '.':

2077 expect_ellipsis(s)

2078 return 1

2079 else:

2080 return 0

2081

2082 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

2083 pos = s.position()

2084 not_none = 0

2085 default = None

2086 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2087 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2088 if s.sy == 'not':

2089 s.next()

2090 if s.sy == 'IDENT' and s.systring == 'None':

2091 s.next()

2092 else:

2093 s.error("Expected 'None'")

2094 if not in_pyfunc:

2095 error(pos, "'not None' only allowed in Python functions")

2096 not_none = 1

2097 if s.sy == '=':

2098 s.next()

2099 if 'pxd' in s.level:

2100 if s.sy not in ['*', '?']:

2101 error(pos, "default values cannot be specified in pxd files, use ? or *")

2102 default = ExprNodes.BoolNode(1)

2103 s.next()

2104 else:

2105 default = p_simple_expr(s)

2106 return Nodes.CArgDeclNode(pos,

2107 base_type = base_type,

2108 declarator = declarator,

2109 not_none = not_none,

2110 default = default,

2111 kw_only = kw_only)

2112

2113 def p_api(s):

2114 if s.sy == 'IDENT' and s.systring == 'api':

2115 s.next()

2116 return 1

2117 else:

2118 return 0

2119

2120 def p_cdef_statement(s, ctx):

2121 pos = s.position()

2122 ctx.visibility = p_visibility(s, ctx.visibility)

2123 ctx.api = ctx.api or p_api(s)

2124 if ctx.api:

2125 if ctx.visibility not in ('private', 'public'):

2126 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2127 if (ctx.visibility == 'extern') and s.sy == 'from':

2128 return p_cdef_extern_block(s, pos, ctx)

2129 elif s.sy == 'import':

2130 s.next()

2131 return p_cdef_extern_block(s, pos, ctx)

2132 elif p_nogil(s):

2133 ctx.nogil = 1

2134 if ctx.overridable:

2135 error(pos, "cdef blocks cannot be declared cpdef")

2136 return p_cdef_block(s, ctx)

2137 elif s.sy == ':':

2138 if ctx.overridable:

2139 error(pos, "cdef blocks cannot be declared cpdef")

2140 return p_cdef_block(s, ctx)

2141 elif s.sy == 'class':

2142 if ctx.level not in ('module', 'module_pxd'):

2143 error(pos, "Extension type definition not allowed here")

2144 if ctx.overridable:

2145 error(pos, "Extension types cannot be declared cpdef")

2146 return p_c_class_definition(s, pos, ctx)

2147 elif s.sy == 'IDENT' and s.systring == 'cppclass':

2148 if ctx.visibility != 'extern':

2149 error(pos, "C++ classes need to be declared extern")

2150 return p_cpp_class_definition(s, pos, ctx)

2151 elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):

2152 if ctx.level not in ('module', 'module_pxd'):

2153 error(pos, "C struct/union/enum definition not allowed here")

2154 if ctx.overridable:

2155 error(pos, "C struct/union/enum cannot be declared cpdef")

2156 if s.systring == "enum":

2157 return p_c_enum_definition(s, pos, ctx)

2158 else:

2159 return p_c_struct_or_union_definition(s, pos, ctx)

2160 else:

2161 return p_c_func_or_var_declaration(s, pos, ctx)

2162

2163 def p_cdef_block(s, ctx):

2164 return p_suite(s, ctx(cdef_flag = 1))

2165

2166 def p_cdef_extern_block(s, pos, ctx):

2167 if ctx.overridable:

2168 error(pos, "cdef extern blocks cannot be declared cpdef")

2169 include_file = None

2170 s.expect('from')

2171 if s.sy == '*':

2172 s.next()

2173 else:

2174 _, include_file = p_string_literal(s)

2175 if s.systring == "namespace":

2176 s.next()

2177 ctx.namespace = p_string_literal(s)[1]

2178 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2179 if p_nogil(s):

2180 ctx.nogil = 1

2181 body = p_suite(s, ctx)

2182 return Nodes.CDefExternNode(pos,

2183 include_file = include_file,

2184 body = body,

2185 namespace = ctx.namespace)

2186

2187 def p_c_enum_definition(s, pos, ctx):

2188 # s.sy == ident 'enum'

2189 s.next()

2190 if s.sy == 'IDENT':

2191 name = s.systring

2192 s.next()

2193 cname = p_opt_cname(s)

2194 if cname is None and ctx.namespace is not None:

2195 cname = ctx.namespace + "::" + name

2196 else:

2197 name = None

2198 cname = None

2199 items = None

2200 s.expect(':')

2201 items = []

2202 if s.sy != 'NEWLINE':

2203 p_c_enum_line(s, items)

2204 else:

2205 s.next() # 'NEWLINE'

2206 s.expect_indent()

2207 while s.sy not in ('DEDENT', 'EOF'):

2208 p_c_enum_line(s, items)

2209 s.expect_dedent()

2210 return Nodes.CEnumDefNode(

2211 pos, name = name, cname = cname, items = items,

2212 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2213 in_pxd = ctx.level == 'module_pxd')

2214

2215 def p_c_enum_line(s, items):

2216 if s.sy != 'pass':

2217 p_c_enum_item(s, items)

2218 while s.sy == ',':

2219 s.next()

2220 if s.sy in ('NEWLINE', 'EOF'):

2221 break

2222 p_c_enum_item(s, items)

2223 else:

2224 s.next()

2225 s.expect_newline("Syntax error in enum item list")

2226

2227 def p_c_enum_item(s, items):

2228 pos = s.position()

2229 name = p_ident(s)

2230 cname = p_opt_cname(s)

2231 value = None

2232 if s.sy == '=':

2233 s.next()

2234 value = p_simple_expr(s)

2235 items.append(Nodes.CEnumDefItemNode(pos,

2236 name = name, cname = cname, value = value))

2237

2238 def p_c_struct_or_union_definition(s, pos, ctx):

2239 packed = False

2240 if s.systring == 'packed':

2241 packed = True

2242 s.next()

2243 if s.sy != 'IDENT' or s.systring != 'struct':

2244 s.expected('struct')

2245 # s.sy == ident 'struct' or 'union'

2246 kind = s.systring

2247 s.next()

2248 name = p_ident(s)

2249 cname = p_opt_cname(s)

2250 if cname is None and ctx.namespace is not None:

2251 cname = ctx.namespace + "::" + name

2252 attributes = None

2253 if s.sy == ':':

2254 s.next()

2255 s.expect('NEWLINE')

2256 s.expect_indent()

2257 attributes = []

2258 body_ctx = Ctx()

2259 while s.sy != 'DEDENT':

2260 if s.sy != 'pass':

2261 attributes.append(

2262 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2263 else:

2264 s.next()

2265 s.expect_newline("Expected a newline")

2266 s.expect_dedent()

2267 else:

2268 s.expect_newline("Syntax error in struct or union definition")

2269 return Nodes.CStructOrUnionDefNode(pos,

2270 name = name, cname = cname, kind = kind, attributes = attributes,

2271 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2272 in_pxd = ctx.level == 'module_pxd', packed = packed)

2273

2274 def p_visibility(s, prev_visibility):

2275 pos = s.position()

2276 visibility = prev_visibility

2277 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2278 visibility = s.systring

2279 if prev_visibility != 'private' and visibility != prev_visibility:

2280 s.error("Conflicting visibility options '%s' and '%s'"

2281 % (prev_visibility, visibility))

2282 s.next()

2283 return visibility

2284

2285 def p_c_modifiers(s):

2286 if s.sy == 'IDENT' and s.systring in ('inline',):

2287 modifier = s.systring

2288 s.next()

2289 return [modifier] + p_c_modifiers(s)

2290 return []

2291

2292 def p_c_func_or_var_declaration(s, pos, ctx):

2293 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2294 modifiers = p_c_modifiers(s)

2295 base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)

2296 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2297 assignable = 1, nonempty = 1)

2298 declarator.overridable = ctx.overridable

2299 if s.sy == ':':

2300 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd') and not ctx.templates:

2301 s.error("C function definition not allowed here")

2302 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2303 result = Nodes.CFuncDefNode(pos,

2304 visibility = ctx.visibility,

2305 base_type = base_type,

2306 declarator = declarator,

2307 body = suite,

2308 doc = doc,

2309 modifiers = modifiers,

2310 api = ctx.api,

2311 overridable = ctx.overridable)

2312 else:

2313 #if api:

2314 # error(s.pos, "'api' not allowed with variable declaration")

2315 declarators = [declarator]

2316 while s.sy == ',':

2317 s.next()

2318 if s.sy == 'NEWLINE':

2319 break

2320 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2321 assignable = 1, nonempty = 1)

2322 declarators.append(declarator)

2323 s.expect_newline("Syntax error in C variable declaration")

2324 result = Nodes.CVarDefNode(pos,

2325 visibility = ctx.visibility,

2326 base_type = base_type,

2327 declarators = declarators,

2328 in_pxd = ctx.level == 'module_pxd',

2329 api = ctx.api,

2330 overridable = ctx.overridable)

2331 return result

2332

2333 def p_ctypedef_statement(s, ctx):

2334 # s.sy == 'ctypedef'

2335 pos = s.position()

2336 s.next()

2337 visibility = p_visibility(s, ctx.visibility)

2338 api = p_api(s)

2339 ctx = ctx(typedef_flag = 1, visibility = visibility)

2340 if api:

2341 ctx.api = 1

2342 if s.sy == 'class':

2343 return p_c_class_definition(s, pos, ctx)

2344 elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):

2345 if s.systring == 'enum':

2346 return p_c_enum_definition(s, pos, ctx)

2347 else:

2348 return p_c_struct_or_union_definition(s, pos, ctx)

2349 else:

2350 base_type = p_c_base_type(s, nonempty = 1)

2351 if base_type.name is None:

2352 s.error("Syntax error in ctypedef statement")

2353 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2354 s.expect_newline("Syntax error in ctypedef statement")

2355 return Nodes.CTypeDefNode(

2356 pos, base_type = base_type,

2357 declarator = declarator, visibility = visibility,

2358 in_pxd = ctx.level == 'module_pxd')

2359

2360 def p_decorators(s):

2361 decorators = []

2362 while s.sy == 'DECORATOR':

2363 pos = s.position()

2364 s.next()

2365 decstring = p_dotted_name(s, as_allowed=0)[2]

2366 names = decstring.split('.')

2367 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2368 for name in names[1:]:

2369 decorator = ExprNodes.AttributeNode(pos,

2370 attribute=EncodedString(name),

2371 obj=decorator)

2372 if s.sy == '(':

2373 decorator = p_call(s, decorator)

2374 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2375 s.expect_newline("Expected a newline after decorator")

2376 return decorators

2377

2378 def p_def_statement(s, decorators=None):

2379 # s.sy == 'def'

2380 pos = s.position()

2381 s.next()

2382 name = EncodedString( p_ident(s) )

2383 #args = []

2384 s.expect('(');

2385 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2386 star_arg = None

2387 starstar_arg = None

2388 if s.sy == '*':

2389 s.next()

2390 if s.sy == 'IDENT':

2391 star_arg = p_py_arg_decl(s)

2392 if s.sy == ',':

2393 s.next()

2394 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2395 nonempty_declarators = 1, kw_only = 1))

2396 elif s.sy != ')':

2397 s.error("Syntax error in Python function argument list")

2398 if s.sy == '**':

2399 s.next()

2400 starstar_arg = p_py_arg_decl(s)

2401 s.expect(')')

2402 if p_nogil(s):

2403 error(s.pos, "Python function cannot be declared nogil")

2404 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2405 return Nodes.DefNode(pos, name = name, args = args,

2406 star_arg = star_arg, starstar_arg = starstar_arg,

2407 doc = doc, body = body, decorators = decorators)

2408

2409 def p_py_arg_decl(s):

2410 pos = s.position()

2411 name = p_ident(s)

2412 return Nodes.PyArgDeclNode(pos, name = name)

2413

2414 def p_class_statement(s, decorators):

2415 # s.sy == 'class'

2416 pos = s.position()

2417 s.next()

2418 class_name = EncodedString( p_ident(s) )

2419 class_name.encoding = s.source_encoding

2420 if s.sy == '(':

2421 s.next()

2422 base_list = p_simple_expr_list(s)

2423 s.expect(')')

2424 else:

2425 base_list = []

2426 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2427 return Nodes.PyClassDefNode(pos,

2428 name = class_name,

2429 bases = ExprNodes.TupleNode(pos, args = base_list),

2430 doc = doc, body = body, decorators = decorators)

2431

2432 def p_c_class_definition(s, pos, ctx):

2433 # s.sy == 'class'

2434 s.next()

2435 module_path = []

2436 class_name = p_ident(s)

2437 while s.sy == '.':

2438 s.next()

2439 module_path.append(class_name)

2440 class_name = p_ident(s)

2441 if module_path and ctx.visibility != 'extern':

2442 error(pos, "Qualified class name only allowed for 'extern' C class")

2443 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2444 s.next()

2445 as_name = p_ident(s)

2446 else:

2447 as_name = class_name

2448 objstruct_name = None

2449 typeobj_name = None

2450 base_class_module = None

2451 base_class_name = None

2452 if s.sy == '(':

2453 s.next()

2454 base_class_path = [p_ident(s)]

2455 while s.sy == '.':

2456 s.next()

2457 base_class_path.append(p_ident(s))

2458 if s.sy == ',':

2459 s.error("C class may only have one base class")

2460 s.expect(')')

2461 base_class_module = ".".join(base_class_path[:-1])

2462 base_class_name = base_class_path[-1]

2463 if s.sy == '[':

2464 if ctx.visibility not in ('public', 'extern'):

2465 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2466 objstruct_name, typeobj_name = p_c_class_options(s)

2467 if s.sy == ':':

2468 if ctx.level == 'module_pxd':

2469 body_level = 'c_class_pxd'

2470 else:

2471 body_level = 'c_class'

2472 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2473 else:

2474 s.expect_newline("Syntax error in C class definition")

2475 doc = None

2476 body = None

2477 if ctx.visibility == 'extern':

2478 if not module_path:

2479 error(pos, "Module name required for 'extern' C class")

2480 if typeobj_name:

2481 error(pos, "Type object name specification not allowed for 'extern' C class")

2482 elif ctx.visibility == 'public':

2483 if not objstruct_name:

2484 error(pos, "Object struct name specification required for 'public' C class")

2485 if not typeobj_name:

2486 error(pos, "Type object name specification required for 'public' C class")

2487 elif ctx.visibility == 'private':

2488 if ctx.api:

2489 error(pos, "Only 'public' C class can be declared 'api'")

2490 else:

2491 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2492 return Nodes.CClassDefNode(pos,

2493 visibility = ctx.visibility,

2494 typedef_flag = ctx.typedef_flag,

2495 api = ctx.api,

2496 module_name = ".".join(module_path),

2497 class_name = class_name,

2498 as_name = as_name,

2499 base_class_module = base_class_module,

2500 base_class_name = base_class_name,

2501 objstruct_name = objstruct_name,

2502 typeobj_name = typeobj_name,

2503 in_pxd = ctx.level == 'module_pxd',

2504 doc = doc,

2505 body = body)

2506

2507 def p_c_class_options(s):

2508 objstruct_name = None

2509 typeobj_name = None

2510 s.expect('[')

2511 while 1:

2512 if s.sy != 'IDENT':

2513 break

2514 if s.systring == 'object':

2515 s.next()

2516 objstruct_name = p_ident(s)

2517 elif s.systring == 'type':

2518 s.next()

2519 typeobj_name = p_ident(s)

2520 if s.sy != ',':

2521 break

2522 s.next()

2523 s.expect(']', "Expected 'object' or 'type'")

2524 return objstruct_name, typeobj_name

2525

2526 def p_property_decl(s):

2527 pos = s.position()

2528 s.next() # 'property'

2529 name = p_ident(s)

2530 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2531 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2532

2533 def p_doc_string(s):

2534 if s.sy == 'BEGIN_STRING':

2535 pos = s.position()

2536 kind, result = p_cat_string_literal(s)

2537 if s.sy != 'EOF':

2538 s.expect_newline("Syntax error in doc string")

2539 if kind != 'u':

2540 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2541 if kind == 'b':

2542 result.encoding = None # force a unicode string

2543 return result

2544 else:

2545 return None

2546

2547 def p_code(s, level=None):

2548 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2549 if s.sy != 'EOF':

2550 s.error("Syntax error in statement [%s,%s]" % (

2551 repr(s.sy), repr(s.systring)))

2552 return body

2553

2554 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*((\w|[.])+\s*=.*)$")

2555

2556 def p_compiler_directive_comments(s):

2557 result = {}

2558 while s.sy == 'commentline':

2559 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2560 if m:

2561 directives = m.group(1).strip()

2562 try:

2563 result.update( Options.parse_directive_list(

2564 directives, ignore_unknown=True) )

2565 except ValueError, e:

2566 s.error(e.args[0], fatal=False)

2567 s.next()

2568 return result

2569

2570 def p_module(s, pxd, full_module_name):

2571 pos = s.position()

2572

2573 directive_comments = p_compiler_directive_comments(s)

2574 s.parse_comments = False

2575

2576 doc = p_doc_string(s)

2577 if pxd:

2578 level = 'module_pxd'

2579 else:

2580 level = 'module'

2581

2582 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2583 if s.sy != 'EOF':

2584 s.error("Syntax error in statement [%s,%s]" % (

2585 repr(s.sy), repr(s.systring)))

2586 return ModuleNode(pos, doc = doc, body = body,

2587 full_module_name = full_module_name,

2588 directive_comments = directive_comments)

2589

2590 def p_cpp_class_definition(s, pos, ctx):

2591 # s.sy == 'cppclass'

2592 s.next()

2593 module_path = []

2594 class_name = p_ident(s)

2595 cname = p_opt_cname(s)

2596 if cname is None and ctx.namespace is not None:

2597 cname = ctx.namespace + "::" + class_name

2598 if s.sy == '.':

2599 error(pos, "Qualified class name not allowed C++ class")

2600 if s.sy == '[':

2601 s.next()

2602 templates = [p_ident(s)]

2603 while s.sy == ',':

2604 s.next()

2605 templates.append(p_ident(s))

2606 s.expect(']')

2607 else:

2608 templates = None

2609 if s.sy == '(':

2610 s.next()

2611 base_classes = [p_dotted_name(s, False)[2]]

2612 while s.sy == ',':

2613 s.next()

2614 base_classes.append(p_dotted_name(s, False)[2])

2615 s.expect(')')

2616 else:

2617 base_classes = []

2618 if s.sy == '[':

2619 error(s.position(), "Name options not allowed for C++ class")

2620 if s.sy == ':':

2621 s.next()

2622 s.expect('NEWLINE')

2623 s.expect_indent()

2624 attributes = []

2625 body_ctx = Ctx(visibility = ctx.visibility)

2626 body_ctx.templates = templates

2627 while s.sy != 'DEDENT':

2628 if s.systring == 'cppclass':

2629 attributes.append(

2630 p_cpp_class_definition(s, s.position(), body_ctx))

2631 elif s.sy != 'pass':

2632 attributes.append(

2633 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2634 else:

2635 s.next()

2636 s.expect_newline("Expected a newline")

2637 s.expect_dedent()

2638 else:

2639 s.expect_newline("Syntax error in C++ class definition")

2640 return Nodes.CppClassNode(pos,

2641 name = class_name,

2642 cname = cname,

2643 base_classes = base_classes,

2644 visibility = ctx.visibility,

2645 in_pxd = ctx.level == 'module_pxd',

2646 attributes = attributes,

2647 templates = templates)

2648

2649

2650

2651 #----------------------------------------------

2652 #

2653 # Debugging

2654 #

2655 #----------------------------------------------

2656

2657 def print_parse_tree(f, node, level, key = None):

2658 from Nodes import Node

2659 ind = " " * level

2660 if node:

2661 f.write(ind)

2662 if key:

2663 f.write("%s: " % key)

2664 t = type(node)

2665 if t is tuple:

2666 f.write("(%s @ %s\n" % (node[0], node[1]))

2667 for i in xrange(2, len(node)):

2668 print_parse_tree(f, node[i], level+1)

2669 f.write("%s)\n" % ind)

2670 return

2671 elif isinstance(node, Node):

2672 try:

2673 tag = node.tag

2674 except AttributeError:

2675 tag = node.__class__.__name__

2676 f.write("%s @ %s\n" % (tag, node.pos))

2677 for name, value in node.__dict__.items():

2678 if name != 'tag' and name != 'pos':

2679 print_parse_tree(f, value, level+1, name)

2680 return

2681 elif t is list:

2682 f.write("[\n")

2683 for i in xrange(len(node)):

2684 print_parse_tree(f, node[i], level+1)

2685 f.write("%s]\n" % ind)

2686 return

2687 f.write("%s%s\n" % (ind, node))

2688