1 # cython: auto_cpdef=True, infer_types=True

2 #

3 # Pyrex Parser

4 #

5

6 # This should be done automatically

7 import cython

8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object)

9

10 import os

11 import re

12 import sys

13

14 try:

15 from __builtin__ import set

16 except (ImportError, AttributeError):

17 try:

18 from builtins import set

19 except (ImportError, AttributeError):

20 from sets import Set as set

21

22 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor

23 import Nodes

24 import ExprNodes

25 import StringEncoding

26 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes

27 from ModuleNode import ModuleNode

28 from Errors import error, warning, InternalError

29 from Cython import Utils

30 import Future

31 import Options

32

33 class Ctx(object):

34 # Parsing context

35 level = 'other'

36 visibility = 'private'

37 cdef_flag = 0

38 typedef_flag = 0

39 api = 0

40 overridable = 0

41 nogil = 0

42 namespace = None

43 templates = None

44

45 def __init__(self, **kwds):

46 self.__dict__.update(kwds)

47

48 def __call__(self, **kwds):

49 ctx = Ctx()

50 d = ctx.__dict__

51 d.update(self.__dict__)

52 d.update(kwds)

53 return ctx

54

55 def p_ident(s, message = "Expected an identifier"):

56 if s.sy == 'IDENT':

57 name = s.systring

58 s.next()

59 return name

60 else:

61 s.error(message)

62

63 def p_ident_list(s):

64 names = []

65 while s.sy == 'IDENT':

66 names.append(s.systring)

67 s.next()

68 if s.sy != ',':

69 break

70 s.next()

71 return names

72

73 #------------------------------------------

74 #

75 # Expressions

76 #

77 #------------------------------------------

78

79 def p_binop_operator(s):

80 pos = s.position()

81 op = s.sy

82 s.next()

83 return op, pos

84

85 def p_binop_expr(s, ops, p_sub_expr):

86 n1 = p_sub_expr(s)

87 while s.sy in ops:

88 op, pos = p_binop_operator(s)

89 n2 = p_sub_expr(s)

90 n1 = ExprNodes.binop_node(pos, op, n1, n2)

91 if op == '/':

92 if Future.division in s.context.future_directives:

93 n1.truedivision = True

94 else:

95 n1.truedivision = None # unknown

96 return n1

97

98 #expression: or_test [if or_test else test] | lambda_form

99

100 def p_simple_expr(s):

101 pos = s.position()

102 expr = p_or_test(s)

103 if s.sy == 'if':

104 s.next()

105 test = p_or_test(s)

106 s.expect('else')

107 other = p_test(s)

108 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

109 else:

110 return expr

111

112 #test: or_test | lambda_form

113

114 def p_test(s):

115 return p_or_test(s)

116

117 #or_test: and_test ('or' and_test)*

118

119 def p_or_test(s):

120 return p_rassoc_binop_expr(s, ('or',), p_and_test)

121

122 def p_rassoc_binop_expr(s, ops, p_subexpr):

123 n1 = p_subexpr(s)

124 if s.sy in ops:

125 pos = s.position()

126 op = s.sy

127 s.next()

128 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

129 n1 = ExprNodes.binop_node(pos, op, n1, n2)

130 return n1

131

132 #and_test: not_test ('and' not_test)*

133

134 def p_and_test(s):

135 #return p_binop_expr(s, ('and',), p_not_test)

136 return p_rassoc_binop_expr(s, ('and',), p_not_test)

137

138 #not_test: 'not' not_test | comparison

139

140 def p_not_test(s):

141 if s.sy == 'not':

142 pos = s.position()

143 s.next()

144 return ExprNodes.NotNode(pos, operand = p_not_test(s))

145 else:

146 return p_comparison(s)

147

148 #comparison: expr (comp_op expr)*

149 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

150

151 def p_comparison(s):

152 n1 = p_starred_expr(s)

153 if s.sy in comparison_ops:

154 pos = s.position()

155 op = p_cmp_op(s)

156 n2 = p_starred_expr(s)

157 n1 = ExprNodes.PrimaryCmpNode(pos,

158 operator = op, operand1 = n1, operand2 = n2)

159 if s.sy in comparison_ops:

160 n1.cascade = p_cascaded_cmp(s)

161 return n1

162

163 def p_starred_expr(s):

164 pos = s.position()

165 if s.sy == '*':

166 starred = True

167 s.next()

168 else:

169 starred = False

170 expr = p_bit_expr(s)

171 if starred:

172 expr = ExprNodes.StarredTargetNode(pos, expr)

173 return expr

174

175 def p_cascaded_cmp(s):

176 pos = s.position()

177 op = p_cmp_op(s)

178 n2 = p_starred_expr(s)

179 result = ExprNodes.CascadedCmpNode(pos,

180 operator = op, operand2 = n2)

181 if s.sy in comparison_ops:

182 result.cascade = p_cascaded_cmp(s)

183 return result

184

185 def p_cmp_op(s):

186 if s.sy == 'not':

187 s.next()

188 s.expect('in')

189 op = 'not_in'

190 elif s.sy == 'is':

191 s.next()

192 if s.sy == 'not':

193 s.next()

194 op = 'is_not'

195 else:

196 op = 'is'

197 else:

198 op = s.sy

199 s.next()

200 if op == '<>':

201 op = '!='

202 return op

203

204 comparison_ops = (

205 '<', '>', '==', '>=', '<=', '<>', '!=',

206 'in', 'is', 'not'

207 )

208

209 #expr: xor_expr ('|' xor_expr)*

210

211 def p_bit_expr(s):

212 return p_binop_expr(s, ('|',), p_xor_expr)

213

214 #xor_expr: and_expr ('^' and_expr)*

215

216 def p_xor_expr(s):

217 return p_binop_expr(s, ('^',), p_and_expr)

218

219 #and_expr: shift_expr ('&' shift_expr)*

220

221 def p_and_expr(s):

222 return p_binop_expr(s, ('&',), p_shift_expr)

223

224 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

225

226 def p_shift_expr(s):

227 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

228

229 #arith_expr: term (('+'|'-') term)*

230

231 def p_arith_expr(s):

232 return p_binop_expr(s, ('+', '-'), p_term)

233

234 #term: factor (('*'|'/'|'%') factor)*

235

236 def p_term(s):

237 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

238

239 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

240

241 def p_factor(s):

242 sy = s.sy

243 if sy in ('+', '-', '~'):

244 op = s.sy

245 pos = s.position()

246 s.next()

247 return ExprNodes.unop_node(pos, op, p_factor(s))

248 elif sy == '&':

249 pos = s.position()

250 s.next()

251 arg = p_factor(s)

252 return ExprNodes.AmpersandNode(pos, operand = arg)

253 elif sy == "<":

254 return p_typecast(s)

255 elif sy == 'IDENT' and s.systring == "sizeof":

256 return p_sizeof(s)

257 else:

258 return p_power(s)

259

260 def p_typecast(s):

261 # s.sy == "<"

262 pos = s.position()

263 s.next()

264 base_type = p_c_base_type(s)

265 if base_type.name is None:

266 s.error("Unknown type")

267 declarator = p_c_declarator(s, empty = 1)

268 if s.sy == '?':

269 s.next()

270 typecheck = 1

271 else:

272 typecheck = 0

273 s.expect(">")

274 operand = p_factor(s)

275 return ExprNodes.TypecastNode(pos,

276 base_type = base_type,

277 declarator = declarator,

278 operand = operand,

279 typecheck = typecheck)

280

281 def p_sizeof(s):

282 # s.sy == ident "sizeof"

283 pos = s.position()

284 s.next()

285 s.expect('(')

286 # Here we decide if we are looking at an expression or type

287 # If it is actually a type, but parsable as an expression,

288 # we treat it as an expression here.

289 if looking_at_expr(s):

290 operand = p_simple_expr(s)

291 node = ExprNodes.SizeofVarNode(pos, operand = operand)

292 else:

293 base_type = p_c_base_type(s)

294 declarator = p_c_declarator(s, empty = 1)

295 node = ExprNodes.SizeofTypeNode(pos,

296 base_type = base_type, declarator = declarator)

297 s.expect(')')

298 return node

299

300 def p_yield_expression(s):

301 # s.sy == "yield"

302 pos = s.position()

303 s.next()

304 if s.sy not in ('EOF', 'NEWLINE', ')'):

305 expr = p_expr(s)

306 s.error("generators ('yield') are not currently supported")

307 return Nodes.PassStatNode(pos)

308

309 #power: atom trailer* ('**' factor)*

310

311 def p_power(s):

312 if s.systring == 'new' and s.peek()[0] == 'IDENT':

313 return p_new_expr(s)

314 n1 = p_atom(s)

315 while s.sy in ('(', '[', '.'):

316 n1 = p_trailer(s, n1)

317 if s.sy == '**':

318 pos = s.position()

319 s.next()

320 n2 = p_factor(s)

321 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

322 return n1

323

324 def p_new_expr(s):

325 # s.systring == 'new'.

326 pos = s.position()

327 s.next()

328 cppclass = p_c_base_type(s)

329 return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))

330

331 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

332

333 def p_trailer(s, node1):

334 pos = s.position()

335 if s.sy == '(':

336 return p_call(s, node1)

337 elif s.sy == '[':

338 return p_index(s, node1)

339 else: # s.sy == '.'

340 s.next()

341 name = EncodedString( p_ident(s) )

342 return ExprNodes.AttributeNode(pos,

343 obj = node1, attribute = name)

344

345 # arglist: argument (',' argument)* [',']

346 # argument: [test '='] test # Really [keyword '='] test

347

348 def p_call(s, function):

349 # s.sy == '('

350 pos = s.position()

351 s.next()

352 positional_args = []

353 keyword_args = []

354 star_arg = None

355 starstar_arg = None

356 while s.sy not in ('**', ')'):

357 if s.sy == '*':

358 if star_arg:

359 s.error("only one star-arg parameter allowed",

360 pos = s.position())

361 s.next()

362 star_arg = p_simple_expr(s)

363 else:

364 arg = p_simple_expr(s)

365 if s.sy == '=':

366 s.next()

367 if not arg.is_name:

368 s.error("Expected an identifier before '='",

369 pos = arg.pos)

370 encoded_name = EncodedString(arg.name)

371 keyword = ExprNodes.IdentifierStringNode(arg.pos, value = encoded_name)

372 arg = p_simple_expr(s)

373 keyword_args.append((keyword, arg))

374 else:

375 if keyword_args:

376 s.error("Non-keyword arg following keyword arg",

377 pos = arg.pos)

378 if star_arg:

379 s.error("Non-keyword arg following star-arg",

380 pos = arg.pos)

381 positional_args.append(arg)

382 if s.sy != ',':

383 break

384 s.next()

385

386 if s.sy == '**':

387 s.next()

388 starstar_arg = p_simple_expr(s)

389 if s.sy == ',':

390 s.next()

391 s.expect(')')

392 if not (keyword_args or star_arg or starstar_arg):

393 return ExprNodes.SimpleCallNode(pos,

394 function = function,

395 args = positional_args)

396 else:

397 arg_tuple = None

398 keyword_dict = None

399 if positional_args or not star_arg:

400 arg_tuple = ExprNodes.TupleNode(pos,

401 args = positional_args)

402 if star_arg:

403 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

404 if arg_tuple:

405 arg_tuple = ExprNodes.binop_node(pos,

406 operator = '+', operand1 = arg_tuple,

407 operand2 = star_arg_tuple)

408 else:

409 arg_tuple = star_arg_tuple

410 if keyword_args:

411 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

412 for key, value in keyword_args]

413 keyword_dict = ExprNodes.DictNode(pos,

414 key_value_pairs = keyword_args)

415 return ExprNodes.GeneralCallNode(pos,

416 function = function,

417 positional_args = arg_tuple,

418 keyword_args = keyword_dict,

419 starstar_arg = starstar_arg)

420

421 #lambdef: 'lambda' [varargslist] ':' test

422

423 #subscriptlist: subscript (',' subscript)* [',']

424

425 def p_index(s, base):

426 # s.sy == '['

427 pos = s.position()

428 s.next()

429 subscripts = p_subscript_list(s)

430 if len(subscripts) == 1 and len(subscripts[0]) == 2:

431 start, stop = subscripts[0]

432 result = ExprNodes.SliceIndexNode(pos,

433 base = base, start = start, stop = stop)

434 else:

435 indexes = make_slice_nodes(pos, subscripts)

436 if len(indexes) == 1:

437 index = indexes[0]

438 else:

439 index = ExprNodes.TupleNode(pos, args = indexes)

440 result = ExprNodes.IndexNode(pos,

441 base = base, index = index)

442 s.expect(']')

443 return result

444

445 def p_subscript_list(s):

446 items = [p_subscript(s)]

447 while s.sy == ',':

448 s.next()

449 if s.sy == ']':

450 break

451 items.append(p_subscript(s))

452 return items

453

454 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

455

456 def p_subscript(s):

457 # Parse a subscript and return a list of

458 # 1, 2 or 3 ExprNodes, depending on how

459 # many slice elements were encountered.

460 pos = s.position()

461 if s.sy == '.':

462 expect_ellipsis(s)

463 return [ExprNodes.EllipsisNode(pos)]

464 else:

465 start = p_slice_element(s, (':',))

466 if s.sy != ':':

467 return [start]

468 s.next()

469 stop = p_slice_element(s, (':', ',', ']'))

470 if s.sy != ':':

471 return [start, stop]

472 s.next()

473 step = p_slice_element(s, (':', ',', ']'))

474 return [start, stop, step]

475

476 def p_slice_element(s, follow_set):

477 # Simple expression which may be missing iff

478 # it is followed by something in follow_set.

479 if s.sy not in follow_set:

480 return p_simple_expr(s)

481 else:

482 return None

483

484 def expect_ellipsis(s):

485 s.expect('.')

486 s.expect('.')

487 s.expect('.')

488

489 def make_slice_nodes(pos, subscripts):

490 # Convert a list of subscripts as returned

491 # by p_subscript_list into a list of ExprNodes,

492 # creating SliceNodes for elements with 2 or

493 # more components.

494 result = []

495 for subscript in subscripts:

496 if len(subscript) == 1:

497 result.append(subscript[0])

498 else:

499 result.append(make_slice_node(pos, *subscript))

500 return result

501

502 def make_slice_node(pos, start, stop = None, step = None):

503 if not start:

504 start = ExprNodes.NoneNode(pos)

505 if not stop:

506 stop = ExprNodes.NoneNode(pos)

507 if not step:

508 step = ExprNodes.NoneNode(pos)

509 return ExprNodes.SliceNode(pos,

510 start = start, stop = stop, step = step)

511

512 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

513

514 def p_atom(s):

515 pos = s.position()

516 sy = s.sy

517 if sy == '(':

518 s.next()

519 if s.sy == ')':

520 result = ExprNodes.TupleNode(pos, args = [])

521 elif s.sy == 'yield':

522 result = p_yield_expression(s)

523 else:

524 result = p_expr(s)

525 s.expect(')')

526 return result

527 elif sy == '[':

528 return p_list_maker(s)

529 elif sy == '{':

530 return p_dict_or_set_maker(s)

531 elif sy == '`':

532 return p_backquote_expr(s)

533 elif sy == 'INT':

534 value = s.systring

535 s.next()

536 unsigned = ""

537 longness = ""

538 while value[-1] in "UuLl":

539 if value[-1] in "Ll":

540 longness += "L"

541 else:

542 unsigned += "U"

543 value = value[:-1]

544 return ExprNodes.IntNode(pos,

545 value = value,

546 unsigned = unsigned,

547 longness = longness)

548 elif sy == 'FLOAT':

549 value = s.systring

550 s.next()

551 return ExprNodes.FloatNode(pos, value = value)

552 elif sy == 'IMAG':

553 value = s.systring[:-1]

554 s.next()

555 return ExprNodes.ImagNode(pos, value = value)

556 elif sy == 'BEGIN_STRING':

557 kind, value = p_cat_string_literal(s)

558 if kind == 'c':

559 return ExprNodes.CharNode(pos, value = value)

560 elif kind == 'u':

561 return ExprNodes.UnicodeNode(pos, value = value)

562 elif kind == 'b':

563 return ExprNodes.BytesNode(pos, value = value)

564 else:

565 return ExprNodes.StringNode(pos, value = value)

566 elif sy == 'IDENT':

567 name = EncodedString( s.systring )

568 s.next()

569 if name == "None":

570 return ExprNodes.NoneNode(pos)

571 elif name == "True":

572 return ExprNodes.BoolNode(pos, value=True)

573 elif name == "False":

574 return ExprNodes.BoolNode(pos, value=False)

575 elif name == "NULL":

576 return ExprNodes.NullNode(pos)

577 else:

578 return p_name(s, name)

579 else:

580 s.error("Expected an identifier or literal")

581

582 def p_name(s, name):

583 pos = s.position()

584 if not s.compile_time_expr and name in s.compile_time_env:

585 value = s.compile_time_env.lookup_here(name)

586 rep = repr(value)

587 if isinstance(value, bool):

588 return ExprNodes.BoolNode(pos, value = value)

589 elif isinstance(value, int):

590 return ExprNodes.IntNode(pos, value = rep)

591 elif isinstance(value, long):

592 return ExprNodes.IntNode(pos, value = rep, longness = "L")

593 elif isinstance(value, float):

594 return ExprNodes.FloatNode(pos, value = rep)

595 elif isinstance(value, _unicode):

596 return ExprNodes.UnicodeNode(pos, value = value)

597 elif isinstance(value, _bytes):

598 return ExprNodes.BytesNode(pos, value = value)

599 else:

600 error(pos, "Invalid type for compile-time constant: %s"

601 % value.__class__.__name__)

602 return ExprNodes.NameNode(pos, name = name)

603

604 def p_cat_string_literal(s):

605 # A sequence of one or more adjacent string literals.

606 # Returns (kind, value) where kind in ('b', 'c', 'u', '')

607 kind, value = p_string_literal(s)

608 if s.sy != 'BEGIN_STRING':

609 return kind, value

610 if kind != 'c':

611 strings = [value]

612 while s.sy == 'BEGIN_STRING':

613 pos = s.position()

614 next_kind, next_value = p_string_literal(s)

615 if next_kind == 'c':

616 error(pos, "Cannot concatenate char literal with another string or char literal")

617 elif next_kind != kind:

618 error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %

619 (kind, next_kind))

620 else:

621 strings.append(next_value)

622 if kind == 'u':

623 value = EncodedString( u''.join(strings) )

624 else:

625 value = BytesLiteral( StringEncoding.join_bytes(strings) )

626 value.encoding = s.source_encoding

627 return kind, value

628

629 def p_opt_string_literal(s):

630 if s.sy == 'BEGIN_STRING':

631 return p_string_literal(s)

632 else:

633 return None

634

635 def p_string_literal(s):

636 # A single string or char literal.

637 # Returns (kind, value) where kind in ('b', 'c', 'u')

638 # s.sy == 'BEGIN_STRING'

639 pos = s.position()

640 is_raw = 0

641 kind = s.systring[:1].lower()

642 if kind == 'r':

643 kind = ''

644 is_raw = 1

645 elif kind in 'ub':

646 is_raw = s.systring[1:2].lower() == 'r'

647 elif kind != 'c':

648 kind = ''

649 if Future.unicode_literals in s.context.future_directives:

650 if kind == '':

651 kind = 'u'

652 if kind == 'u':

653 chars = StringEncoding.UnicodeLiteralBuilder()

654 else:

655 chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)

656 while 1:

657 s.next()

658 sy = s.sy

659 #print "p_string_literal: sy =", sy, repr(s.systring) ###

660 if sy == 'CHARS':

661 chars.append(s.systring)

662 elif sy == 'ESCAPE':

663 has_escape = True

664 systr = s.systring

665 if is_raw:

666 if systr == u'\\\n':

667 chars.append(u'\\\n')

668 elif systr == u'\\\"':

669 chars.append(u'"')

670 elif systr == u'\\\'':

671 chars.append(u"'")

672 else:

673 chars.append(systr)

674 else:

675 c = systr[1]

676 if c in u"01234567":

677 chars.append_charval( int(systr[1:], 8) )

678 elif c in u"'\"\\":

679 chars.append(c)

680 elif c in u"abfnrtv":

681 chars.append(

682 StringEncoding.char_from_escape_sequence(systr))

683 elif c == u'\n':

684 pass

685 elif c in u'Uux':

686 if kind == 'u' or c == 'x':

687 chrval = int(systr[2:], 16)

688 if chrval > 1114111: # sys.maxunicode:

689 s.error("Invalid unicode escape '%s'" % systr,

690 pos = pos)

691 elif chrval > 65535:

692 warning(s.position(),

693 "Unicode characters above 65535 are not "

694 "necessarily portable across Python installations", 1)

695 chars.append_charval(chrval)

696 else:

697 # unicode escapes in plain byte strings are not unescaped

698 chars.append(systr)

699 else:

700 chars.append(u'\\' + systr[1:])

701 elif sy == 'NEWLINE':

702 chars.append(u'\n')

703 elif sy == 'END_STRING':

704 break

705 elif sy == 'EOF':

706 s.error("Unclosed string literal", pos = pos)

707 else:

708 s.error(

709 "Unexpected token %r:%r in string literal" %

710 (sy, s.systring))

711 if kind == 'c':

712 value = chars.getchar()

713 if len(value) != 1:

714 error(pos, u"invalid character literal: %r" % value)

715 else:

716 value = chars.getstring()

717 s.next()

718 #print "p_string_literal: value =", repr(value) ###

719 return kind, value

720

721 # list_display ::= "[" [listmaker] "]"

722 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

723 # list_iter ::= list_for | list_if

724 # list_for ::= "for" expression_list "in" testlist [list_iter]

725 # list_if ::= "if" test [list_iter]

726

727 def p_list_maker(s):

728 # s.sy == '['

729 pos = s.position()

730 s.next()

731 if s.sy == ']':

732 s.expect(']')

733 return ExprNodes.ListNode(pos, args = [])

734 expr = p_simple_expr(s)

735 if s.sy == 'for':

736 target = ExprNodes.ListNode(pos, args = [])

737 append = ExprNodes.ComprehensionAppendNode(

738 pos, expr=expr, target=ExprNodes.CloneNode(target))

739 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

740 s.expect(']')

741 return ExprNodes.ComprehensionNode(

742 pos, loop=loop, append=append, target=target)

743 else:

744 exprs = [expr]

745 if s.sy == ',':

746 s.next()

747 exprs += p_simple_expr_list(s)

748 s.expect(']')

749 return ExprNodes.ListNode(pos, args = exprs)

750

751 def p_list_iter(s, body):

752 if s.sy == 'for':

753 return p_list_for(s, body)

754 elif s.sy == 'if':

755 return p_list_if(s, body)

756 else:

757 # insert the 'append' operation into the loop

758 return body

759

760 def p_list_for(s, body):

761 # s.sy == 'for'

762 pos = s.position()

763 s.next()

764 kw = p_for_bounds(s)

765 kw['else_clause'] = None

766 kw['body'] = p_list_iter(s, body)

767 return Nodes.ForStatNode(pos, **kw)

768

769 def p_list_if(s, body):

770 # s.sy == 'if'

771 pos = s.position()

772 s.next()

773 test = p_test(s)

774 return Nodes.IfStatNode(pos,

775 if_clauses = [Nodes.IfClauseNode(pos, condition = test,

776 body = p_list_iter(s, body))],

777 else_clause = None )

778

779 #dictmaker: test ':' test (',' test ':' test)* [',']

780

781 def p_dict_or_set_maker(s):

782 # s.sy == '{'

783 pos = s.position()

784 s.next()

785 if s.sy == '}':

786 s.next()

787 return ExprNodes.DictNode(pos, key_value_pairs = [])

788 item = p_simple_expr(s)

789 if s.sy == ',' or s.sy == '}':

790 # set literal

791 values = [item]

792 while s.sy == ',':

793 s.next()

794 if s.sy == '}':

795 break

796 values.append( p_simple_expr(s) )

797 s.expect('}')

798 return ExprNodes.SetNode(pos, args=values)

799 elif s.sy == 'for':

800 # set comprehension

801 target = ExprNodes.SetNode(pos, args=[])

802 append = ExprNodes.ComprehensionAppendNode(

803 item.pos, expr=item, target=ExprNodes.CloneNode(target))

804 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

805 s.expect('}')

806 return ExprNodes.ComprehensionNode(

807 pos, loop=loop, append=append, target=target)

808 elif s.sy == ':':

809 # dict literal or comprehension

810 key = item

811 s.next()

812 value = p_simple_expr(s)

813 if s.sy == 'for':

814 # dict comprehension

815 target = ExprNodes.DictNode(pos, key_value_pairs = [])

816 append = ExprNodes.DictComprehensionAppendNode(

817 item.pos, key_expr=key, value_expr=value,

818 target=ExprNodes.CloneNode(target))

819 loop = p_list_for(s, Nodes.ExprStatNode(append.pos, expr=append))

820 s.expect('}')

821 return ExprNodes.ComprehensionNode(

822 pos, loop=loop, append=append, target=target)

823 else:

824 # dict literal

825 items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]

826 while s.sy == ',':

827 s.next()

828 if s.sy == '}':

829 break

830 key = p_simple_expr(s)

831 s.expect(':')

832 value = p_simple_expr(s)

833 items.append(

834 ExprNodes.DictItemNode(key.pos, key=key, value=value))

835 s.expect('}')

836 return ExprNodes.DictNode(pos, key_value_pairs=items)

837 else:

838 # raise an error

839 s.expect('}')

840 return ExprNodes.DictNode(pos, key_value_pairs = [])

841

842 def p_backquote_expr(s):

843 # s.sy == '`'

844 pos = s.position()

845 s.next()

846 arg = p_expr(s)

847 s.expect('`')

848 return ExprNodes.BackquoteNode(pos, arg = arg)

849

850 def p_simple_expr_list(s):

851 exprs = []

852 while s.sy not in expr_terminators:

853 expr = p_simple_expr(s)

854 exprs.append(expr)

855 if s.sy != ',':

856 break

857 s.next()

858 return exprs

859

860 def p_expr(s):

861 pos = s.position()

862 expr = p_simple_expr(s)

863 if s.sy == ',':

864 s.next()

865 exprs = [expr] + p_simple_expr_list(s)

866 return ExprNodes.TupleNode(pos, args = exprs)

867 else:

868 return expr

869

870

871 #testlist: test (',' test)* [',']

872 # differs from p_expr only in the fact that it cannot contain conditional expressions

873

874 def p_testlist(s):

875 pos = s.position()

876 expr = p_test(s)

877 if s.sy == ',':

878 exprs = [expr]

879 while s.sy == ',':

880 s.next()

881 exprs.append(p_test(s))

882 return ExprNodes.TupleNode(pos, args = exprs)

883 else:

884 return expr

885

886 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

887

888 #-------------------------------------------------------

889 #

890 # Statements

891 #

892 #-------------------------------------------------------

893

894 def p_global_statement(s):

895 # assume s.sy == 'global'

896 pos = s.position()

897 s.next()

898 names = p_ident_list(s)

899 return Nodes.GlobalNode(pos, names = names)

900

901 def p_expression_or_assignment(s):

902 expr_list = [p_expr(s)]

903 while s.sy == '=':

904 s.next()

905 expr_list.append(p_expr(s))

906 if len(expr_list) == 1:

907 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

908 lhs = expr_list[0]

909 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

910 error(lhs.pos, "Illegal operand for inplace operation.")

911 operator = s.sy[:-1]

912 s.next()

913 rhs = p_expr(s)

914 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

915 expr = expr_list[0]

916 if isinstance(expr, (ExprNodes.UnicodeNode, ExprNodes.StringNode, ExprNodes.BytesNode)):

917 return Nodes.PassStatNode(expr.pos)

918 else:

919 return Nodes.ExprStatNode(expr.pos, expr = expr)

920

921 rhs = expr_list[-1]

922 if len(expr_list) == 2:

923 return Nodes.SingleAssignmentNode(rhs.pos,

924 lhs = expr_list[0], rhs = rhs)

925 else:

926 return Nodes.CascadedAssignmentNode(rhs.pos,

927 lhs_list = expr_list[:-1], rhs = rhs)

928

929 def p_print_statement(s):

930 # s.sy == 'print'

931 pos = s.position()

932 ends_with_comma = 0

933 s.next()

934 if s.sy == '>>':

935 s.next()

936 stream = p_simple_expr(s)

937 if s.sy == ',':

938 s.next()

939 ends_with_comma = s.sy in ('NEWLINE', 'EOF')

940 else:

941 stream = None

942 args = []

943 if s.sy not in ('NEWLINE', 'EOF'):

944 args.append(p_simple_expr(s))

945 while s.sy == ',':

946 s.next()

947 if s.sy in ('NEWLINE', 'EOF'):

948 ends_with_comma = 1

949 break

950 args.append(p_simple_expr(s))

951 arg_tuple = ExprNodes.TupleNode(pos, args = args)

952 return Nodes.PrintStatNode(pos,

953 arg_tuple = arg_tuple, stream = stream,

954 append_newline = not ends_with_comma)

955

956 def p_exec_statement(s):

957 # s.sy == 'exec'

958 pos = s.position()

959 s.next()

960 args = [ p_bit_expr(s) ]

961 if s.sy == 'in':

962 s.next()

963 args.append(p_simple_expr(s))

964 if s.sy == ',':

965 s.next()

966 args.append(p_simple_expr(s))

967 else:

968 error(pos, "'exec' currently requires a target mapping (globals/locals)")

969 return Nodes.ExecStatNode(pos, args = args)

970

971 def p_del_statement(s):

972 # s.sy == 'del'

973 pos = s.position()

974 s.next()

975 args = p_simple_expr_list(s)

976 return Nodes.DelStatNode(pos, args = args)

977

978 def p_pass_statement(s, with_newline = 0):

979 pos = s.position()

980 s.expect('pass')

981 if with_newline:

982 s.expect_newline("Expected a newline")

983 return Nodes.PassStatNode(pos)

984

985 def p_break_statement(s):

986 # s.sy == 'break'

987 pos = s.position()

988 s.next()

989 return Nodes.BreakStatNode(pos)

990

991 def p_continue_statement(s):

992 # s.sy == 'continue'

993 pos = s.position()

994 s.next()

995 return Nodes.ContinueStatNode(pos)

996

997 def p_return_statement(s):

998 # s.sy == 'return'

999 pos = s.position()

1000 s.next()

1001 if s.sy not in statement_terminators:

1002 value = p_expr(s)

1003 else:

1004 value = None

1005 return Nodes.ReturnStatNode(pos, value = value)

1006

1007 def p_raise_statement(s):

1008 # s.sy == 'raise'

1009 pos = s.position()

1010 s.next()

1011 exc_type = None

1012 exc_value = None

1013 exc_tb = None

1014 if s.sy not in statement_terminators:

1015 exc_type = p_simple_expr(s)

1016 if s.sy == ',':

1017 s.next()

1018 exc_value = p_simple_expr(s)

1019 if s.sy == ',':

1020 s.next()

1021 exc_tb = p_simple_expr(s)

1022 if exc_type or exc_value or exc_tb:

1023 return Nodes.RaiseStatNode(pos,

1024 exc_type = exc_type,

1025 exc_value = exc_value,

1026 exc_tb = exc_tb)

1027 else:

1028 return Nodes.ReraiseStatNode(pos)

1029

1030 def p_import_statement(s):

1031 # s.sy in ('import', 'cimport')

1032 pos = s.position()

1033 kind = s.sy

1034 s.next()

1035 items = [p_dotted_name(s, as_allowed = 1)]

1036 while s.sy == ',':

1037 s.next()

1038 items.append(p_dotted_name(s, as_allowed = 1))

1039 stats = []

1040 for pos, target_name, dotted_name, as_name in items:

1041 dotted_name = EncodedString(dotted_name)

1042 if kind == 'cimport':

1043 stat = Nodes.CImportStatNode(pos,

1044 module_name = dotted_name,

1045 as_name = as_name)

1046 else:

1047 if as_name and "." in dotted_name:

1048 name_list = ExprNodes.ListNode(pos, args = [

1049 ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])

1050 else:

1051 name_list = None

1052 stat = Nodes.SingleAssignmentNode(pos,

1053 lhs = ExprNodes.NameNode(pos,

1054 name = as_name or target_name),

1055 rhs = ExprNodes.ImportNode(pos,

1056 module_name = ExprNodes.IdentifierStringNode(

1057 pos, value = dotted_name),

1058 name_list = name_list))

1059 stats.append(stat)

1060 return Nodes.StatListNode(pos, stats = stats)

1061

1062 def p_from_import_statement(s, first_statement = 0):

1063 # s.sy == 'from'

1064 pos = s.position()

1065 s.next()

1066 (dotted_name_pos, _, dotted_name, _) = \

1067 p_dotted_name(s, as_allowed = 0)

1068 if s.sy in ('import', 'cimport'):

1069 kind = s.sy

1070 s.next()

1071 else:

1072 s.error("Expected 'import' or 'cimport'")

1073 is_cimport = kind == 'cimport'

1074 is_parenthesized = False

1075 if s.sy == '*':

1076 imported_names = [(s.position(), "*", None, None)]

1077 s.next()

1078 else:

1079 if s.sy == '(':

1080 is_parenthesized = True

1081 s.next()

1082 imported_names = [p_imported_name(s, is_cimport)]

1083 while s.sy == ',':

1084 s.next()

1085 imported_names.append(p_imported_name(s, is_cimport))

1086 if is_parenthesized:

1087 s.expect(')')

1088 dotted_name = EncodedString(dotted_name)

1089 if dotted_name == '__future__':

1090 if not first_statement:

1091 s.error("from __future__ imports must occur at the beginning of the file")

1092 else:

1093 for (name_pos, name, as_name, kind) in imported_names:

1094 if name == "braces":

1095 s.error("not a chance", name_pos)

1096 break

1097 try:

1098 directive = getattr(Future, name)

1099 except AttributeError:

1100 s.error("future feature %s is not defined" % name, name_pos)

1101 break

1102 s.context.future_directives.add(directive)

1103 return Nodes.PassStatNode(pos)

1104 elif kind == 'cimport':

1105 return Nodes.FromCImportStatNode(pos,

1106 module_name = dotted_name,

1107 imported_names = imported_names)

1108 else:

1109 imported_name_strings = []

1110 items = []

1111 for (name_pos, name, as_name, kind) in imported_names:

1112 encoded_name = EncodedString(name)

1113 imported_name_strings.append(

1114 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1115 items.append(

1116 (name,

1117 ExprNodes.NameNode(name_pos,

1118 name = as_name or name)))

1119 import_list = ExprNodes.ListNode(

1120 imported_names[0][0], args = imported_name_strings)

1121 dotted_name = EncodedString(dotted_name)

1122 return Nodes.FromImportStatNode(pos,

1123 module = ExprNodes.ImportNode(dotted_name_pos,

1124 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1125 name_list = import_list),

1126 items = items)

1127

1128 imported_name_kinds = ('class', 'struct', 'union')

1129

1130 def p_imported_name(s, is_cimport):

1131 pos = s.position()

1132 kind = None

1133 if is_cimport and s.systring in imported_name_kinds:

1134 kind = s.systring

1135 s.next()

1136 name = p_ident(s)

1137 as_name = p_as_name(s)

1138 return (pos, name, as_name, kind)

1139

1140 def p_dotted_name(s, as_allowed):

1141 pos = s.position()

1142 target_name = p_ident(s)

1143 as_name = None

1144 names = [target_name]

1145 while s.sy == '.':

1146 s.next()

1147 names.append(p_ident(s))

1148 if as_allowed:

1149 as_name = p_as_name(s)

1150 return (pos, target_name, u'.'.join(names), as_name)

1151

1152 def p_as_name(s):

1153 if s.sy == 'IDENT' and s.systring == 'as':

1154 s.next()

1155 return p_ident(s)

1156 else:

1157 return None

1158

1159 def p_assert_statement(s):

1160 # s.sy == 'assert'

1161 pos = s.position()

1162 s.next()

1163 cond = p_simple_expr(s)

1164 if s.sy == ',':

1165 s.next()

1166 value = p_simple_expr(s)

1167 else:

1168 value = None

1169 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1170

1171 statement_terminators = (';', 'NEWLINE', 'EOF')

1172

1173 def p_if_statement(s):

1174 # s.sy == 'if'

1175 pos = s.position()

1176 s.next()

1177 if_clauses = [p_if_clause(s)]

1178 while s.sy == 'elif':

1179 s.next()

1180 if_clauses.append(p_if_clause(s))

1181 else_clause = p_else_clause(s)

1182 return Nodes.IfStatNode(pos,

1183 if_clauses = if_clauses, else_clause = else_clause)

1184

1185 def p_if_clause(s):

1186 pos = s.position()

1187 test = p_simple_expr(s)

1188 body = p_suite(s)

1189 return Nodes.IfClauseNode(pos,

1190 condition = test, body = body)

1191

1192 def p_else_clause(s):

1193 if s.sy == 'else':

1194 s.next()

1195 return p_suite(s)

1196 else:

1197 return None

1198

1199 def p_while_statement(s):

1200 # s.sy == 'while'

1201 pos = s.position()

1202 s.next()

1203 test = p_simple_expr(s)

1204 body = p_suite(s)

1205 else_clause = p_else_clause(s)

1206 return Nodes.WhileStatNode(pos,

1207 condition = test, body = body,

1208 else_clause = else_clause)

1209

1210 def p_for_statement(s):

1211 # s.sy == 'for'

1212 pos = s.position()

1213 s.next()

1214 kw = p_for_bounds(s)

1215 kw['body'] = p_suite(s)

1216 kw['else_clause'] = p_else_clause(s)

1217 return Nodes.ForStatNode(pos, **kw)

1218

1219 def p_for_bounds(s):

1220 target = p_for_target(s)

1221 if s.sy == 'in':

1222 s.next()

1223 iterator = p_for_iterator(s)

1224 return { 'target': target, 'iterator': iterator }

1225 else:

1226 if s.sy == 'from':

1227 s.next()

1228 bound1 = p_bit_expr(s)

1229 else:

1230 # Support shorter "for a <= x < b" syntax

1231 bound1, target = target, None

1232 rel1 = p_for_from_relation(s)

1233 name2_pos = s.position()

1234 name2 = p_ident(s)

1235 rel2_pos = s.position()

1236 rel2 = p_for_from_relation(s)

1237 bound2 = p_bit_expr(s)

1238 step = p_for_from_step(s)

1239 if target is None:

1240 target = ExprNodes.NameNode(name2_pos, name = name2)

1241 else:

1242 if not target.is_name:

1243 error(target.pos,

1244 "Target of for-from statement must be a variable name")

1245 elif name2 != target.name:

1246 error(name2_pos,

1247 "Variable name in for-from range does not match target")

1248 if rel1[0] != rel2[0]:

1249 error(rel2_pos,

1250 "Relation directions in for-from do not match")

1251 return {'target': target,

1252 'bound1': bound1,

1253 'relation1': rel1,

1254 'relation2': rel2,

1255 'bound2': bound2,

1256 'step': step }

1257

1258 def p_for_from_relation(s):

1259 if s.sy in inequality_relations:

1260 op = s.sy

1261 s.next()

1262 return op

1263 else:

1264 s.error("Expected one of '<', '<=', '>' '>='")

1265

1266 def p_for_from_step(s):

1267 if s.sy == 'by':

1268 s.next()

1269 step = p_bit_expr(s)

1270 return step

1271 else:

1272 return None

1273

1274 inequality_relations = ('<', '<=', '>', '>=')

1275

1276 def p_target(s, terminator):

1277 pos = s.position()

1278 expr = p_starred_expr(s)

1279 if s.sy == ',':

1280 s.next()

1281 exprs = [expr]

1282 while s.sy != terminator:

1283 exprs.append(p_starred_expr(s))

1284 if s.sy != ',':

1285 break

1286 s.next()

1287 return ExprNodes.TupleNode(pos, args = exprs)

1288 else:

1289 return expr

1290

1291 def p_for_target(s):

1292 return p_target(s, 'in')

1293

1294 def p_for_iterator(s):

1295 pos = s.position()

1296 expr = p_testlist(s)

1297 return ExprNodes.IteratorNode(pos, sequence = expr)

1298

1299 def p_try_statement(s):

1300 # s.sy == 'try'

1301 pos = s.position()

1302 s.next()

1303 body = p_suite(s)

1304 except_clauses = []

1305 else_clause = None

1306 if s.sy in ('except', 'else'):

1307 while s.sy == 'except':

1308 except_clauses.append(p_except_clause(s))

1309 if s.sy == 'else':

1310 s.next()

1311 else_clause = p_suite(s)

1312 body = Nodes.TryExceptStatNode(pos,

1313 body = body, except_clauses = except_clauses,

1314 else_clause = else_clause)

1315 if s.sy != 'finally':

1316 return body

1317 # try-except-finally is equivalent to nested try-except/try-finally

1318 if s.sy == 'finally':

1319 s.next()

1320 finally_clause = p_suite(s)

1321 return Nodes.TryFinallyStatNode(pos,

1322 body = body, finally_clause = finally_clause)

1323 else:

1324 s.error("Expected 'except' or 'finally'")

1325

1326 def p_except_clause(s):

1327 # s.sy == 'except'

1328 pos = s.position()

1329 s.next()

1330 exc_type = None

1331 exc_value = None

1332 if s.sy != ':':

1333 exc_type = p_simple_expr(s)

1334 if s.sy == ',':

1335 s.next()

1336 exc_value = p_simple_expr(s)

1337 elif s.sy == 'IDENT' and s.systring == 'as':

1338 ## XXX In Python 3, it should be:

1339 ## s.next()

1340 ## pos2 = s.position()

1341 ## name = p_ident(s)

1342 ## exc_value = ExprNodes.NameNode(pos2, name = name)

1343 s.next()

1344 exc_value = p_simple_expr(s)

1345 body = p_suite(s)

1346 return Nodes.ExceptClauseNode(pos,

1347 pattern = exc_type, target = exc_value, body = body)

1348

1349 def p_include_statement(s, ctx):

1350 pos = s.position()

1351 s.next() # 'include'

1352 _, include_file_name = p_string_literal(s)

1353 s.expect_newline("Syntax error in include statement")

1354 if s.compile_time_eval:

1355 include_file_name = include_file_name.decode(s.source_encoding)

1356 include_file_path = s.context.find_include_file(include_file_name, pos)

1357 if include_file_path:

1358 s.included_files.append(include_file_name)

1359 f = Utils.open_source_file(include_file_path, mode="rU")

1360 source_desc = FileSourceDescriptor(include_file_path)

1361 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)

1362 try:

1363 tree = p_statement_list(s2, ctx)

1364 finally:

1365 f.close()

1366 return tree

1367 else:

1368 return None

1369 else:

1370 return Nodes.PassStatNode(pos)

1371

1372 def p_with_statement(s):

1373 pos = s.position()

1374 s.next() # 'with'

1375 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1376 if s.sy == 'IDENT' and s.systring == 'nogil':

1377 state = s.systring

1378 s.next()

1379 body = p_suite(s)

1380 return Nodes.GILStatNode(pos, state = state, body = body)

1381 elif s.systring == 'template':

1382 templates = []

1383 s.next()

1384 s.expect('[')

1385 #s.next()

1386 templates.append(s.systring)

1387 s.next()

1388 while s.systring == ',':

1389 s.next()

1390 templates.append(s.systring)

1391 s.next()

1392 s.expect(']')

1393 if s.sy == ':':

1394 s.next()

1395 s.expect_newline("Syntax error in template function declaration")

1396 s.expect_indent()

1397 body_ctx = Ctx()

1398 body_ctx.templates = templates

1399 func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)

1400 s.expect_dedent()

1401 return func_or_var

1402 else:

1403 error(pos, "Syntax error in template function declaration")

1404 else:

1405 manager = p_expr(s)

1406 target = None

1407 if s.sy == 'IDENT' and s.systring == 'as':

1408 s.next()

1409 allow_multi = (s.sy == '(')

1410 target = p_target(s, ':')

1411 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1412 s.error("Multiple with statement target values not allowed without paranthesis")

1413 body = p_suite(s)

1414 return Nodes.WithStatNode(pos, manager = manager,

1415 target = target, body = body)

1416

1417 def p_simple_statement(s, first_statement = 0):

1418 #print "p_simple_statement:", s.sy, s.systring ###

1419 if s.sy == 'global':

1420 node = p_global_statement(s)

1421 elif s.sy == 'print':

1422 node = p_print_statement(s)

1423 elif s.sy == 'exec':

1424 node = p_exec_statement(s)

1425 elif s.sy == 'del':

1426 node = p_del_statement(s)

1427 elif s.sy == 'break':

1428 node = p_break_statement(s)

1429 elif s.sy == 'continue':

1430 node = p_continue_statement(s)

1431 elif s.sy == 'return':

1432 node = p_return_statement(s)

1433 elif s.sy == 'raise':

1434 node = p_raise_statement(s)

1435 elif s.sy in ('import', 'cimport'):

1436 node = p_import_statement(s)

1437 elif s.sy == 'from':

1438 node = p_from_import_statement(s, first_statement = first_statement)

1439 elif s.sy == 'yield':

1440 node = p_yield_expression(s)

1441 elif s.sy == 'assert':

1442 node = p_assert_statement(s)

1443 elif s.sy == 'pass':

1444 node = p_pass_statement(s)

1445 else:

1446 node = p_expression_or_assignment(s)

1447 return node

1448

1449 def p_simple_statement_list(s, ctx, first_statement = 0):

1450 # Parse a series of simple statements on one line

1451 # separated by semicolons.

1452 stat = p_simple_statement(s, first_statement = first_statement)

1453 if s.sy == ';':

1454 stats = [stat]

1455 while s.sy == ';':

1456 #print "p_simple_statement_list: maybe more to follow" ###

1457 s.next()

1458 if s.sy in ('NEWLINE', 'EOF'):

1459 break

1460 stats.append(p_simple_statement(s))

1461 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1462 s.expect_newline("Syntax error in simple statement list")

1463 return stat

1464

1465 def p_compile_time_expr(s):

1466 old = s.compile_time_expr

1467 s.compile_time_expr = 1

1468 expr = p_expr(s)

1469 s.compile_time_expr = old

1470 return expr

1471

1472 def p_DEF_statement(s):

1473 pos = s.position()

1474 denv = s.compile_time_env

1475 s.next() # 'DEF'

1476 name = p_ident(s)

1477 s.expect('=')

1478 expr = p_compile_time_expr(s)

1479 value = expr.compile_time_value(denv)

1480 #print "p_DEF_statement: %s = %r" % (name, value) ###

1481 denv.declare(name, value)

1482 s.expect_newline()

1483 return Nodes.PassStatNode(pos)

1484

1485 def p_IF_statement(s, ctx):

1486 pos = s.position()

1487 saved_eval = s.compile_time_eval

1488 current_eval = saved_eval

1489 denv = s.compile_time_env

1490 result = None

1491 while 1:

1492 s.next() # 'IF' or 'ELIF'

1493 expr = p_compile_time_expr(s)

1494 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1495 body = p_suite(s, ctx)

1496 if s.compile_time_eval:

1497 result = body

1498 current_eval = 0

1499 if s.sy != 'ELIF':

1500 break

1501 if s.sy == 'ELSE':

1502 s.next()

1503 s.compile_time_eval = current_eval

1504 body = p_suite(s, ctx)

1505 if current_eval:

1506 result = body

1507 if not result:

1508 result = Nodes.PassStatNode(pos)

1509 s.compile_time_eval = saved_eval

1510 return result

1511

1512 def p_statement(s, ctx, first_statement = 0):

1513 cdef_flag = ctx.cdef_flag

1514 decorators = None

1515 if s.sy == 'ctypedef':

1516 if ctx.level not in ('module', 'module_pxd'):

1517 s.error("ctypedef statement not allowed here")

1518 #if ctx.api:

1519 # error(s.position(), "'api' not allowed with 'ctypedef'")

1520 return p_ctypedef_statement(s, ctx)

1521 elif s.sy == 'DEF':

1522 return p_DEF_statement(s)

1523 elif s.sy == 'IF':

1524 return p_IF_statement(s, ctx)

1525 elif s.sy == 'DECORATOR':

1526 if ctx.level not in ('module', 'class', 'c_class', 'property', 'module_pxd', 'c_class_pxd'):

1527 print ctx.level

1528 s.error('decorator not allowed here')

1529 s.level = ctx.level

1530 decorators = p_decorators(s)

1531 if s.sy not in ('def', 'cdef', 'cpdef', 'class'):

1532 s.error("Decorators can only be followed by functions or classes")

1533 elif s.sy == 'pass' and cdef_flag:

1534 # empty cdef block

1535 return p_pass_statement(s, with_newline = 1)

1536

1537 overridable = 0

1538 if s.sy == 'cdef':

1539 cdef_flag = 1

1540 s.next()

1541 elif s.sy == 'cpdef':

1542 cdef_flag = 1

1543 overridable = 1

1544 s.next()

1545 if cdef_flag:

1546 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1547 s.error('cdef statement not allowed here')

1548 s.level = ctx.level

1549 node = p_cdef_statement(s, ctx(overridable = overridable))

1550 if decorators is not None:

1551 if not isinstance(node, (Nodes.CFuncDefNode, Nodes.CVarDefNode)):

1552 s.error("Decorators can only be followed by functions or Python classes")

1553 node.decorators = decorators

1554 return node

1555 else:

1556 if ctx.api:

1557 error(s.pos, "'api' not allowed with this statement")

1558 elif s.sy == 'def':

1559 if ctx.level not in ('module', 'class', 'c_class', 'c_class_pxd', 'property'):

1560 s.error('def statement not allowed here')

1561 s.level = ctx.level

1562 return p_def_statement(s, decorators)

1563 elif s.sy == 'class':

1564 if ctx.level != 'module':

1565 s.error("class definition not allowed here")

1566 return p_class_statement(s, decorators)

1567 elif s.sy == 'include':

1568 if ctx.level not in ('module', 'module_pxd'):

1569 s.error("include statement not allowed here")

1570 return p_include_statement(s, ctx)

1571 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1572 return p_property_decl(s)

1573 elif s.sy == 'pass' and ctx.level != 'property':

1574 return p_pass_statement(s, with_newline = 1)

1575 else:

1576 if ctx.level in ('c_class_pxd', 'property'):

1577 s.error("Executable statement not allowed here")

1578 if s.sy == 'if':

1579 return p_if_statement(s)

1580 elif s.sy == 'while':

1581 return p_while_statement(s)

1582 elif s.sy == 'for':

1583 return p_for_statement(s)

1584 elif s.sy == 'try':

1585 return p_try_statement(s)

1586 elif s.sy == 'with':

1587 return p_with_statement(s)

1588 else:

1589 return p_simple_statement_list(

1590 s, ctx, first_statement = first_statement)

1591

1592 def p_statement_list(s, ctx, first_statement = 0):

1593 # Parse a series of statements separated by newlines.

1594 pos = s.position()

1595 stats = []

1596 while s.sy not in ('DEDENT', 'EOF'):

1597 stats.append(p_statement(s, ctx, first_statement = first_statement))

1598 first_statement = 0

1599 if len(stats) == 1:

1600 return stats[0]

1601 else:

1602 return Nodes.StatListNode(pos, stats = stats)

1603

1604 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1605 pos = s.position()

1606 s.expect(':')

1607 doc = None

1608 stmts = []

1609 if s.sy == 'NEWLINE':

1610 s.next()

1611 s.expect_indent()

1612 if with_doc or with_pseudo_doc:

1613 doc = p_doc_string(s)

1614 body = p_statement_list(s, ctx)

1615 s.expect_dedent()

1616 else:

1617 if ctx.api:

1618 error(s.pos, "'api' not allowed with this statement")

1619 if ctx.level in ('module', 'class', 'function', 'other'):

1620 body = p_simple_statement_list(s, ctx)

1621 else:

1622 body = p_pass_statement(s)

1623 s.expect_newline("Syntax error in declarations")

1624 if with_doc:

1625 return doc, body

1626 else:

1627 return body

1628

1629 def p_positional_and_keyword_args(s, end_sy_set, templates = None):

1630 """

1631 Parses positional and keyword arguments. end_sy_set

1632 should contain any s.sy that terminate the argument list.

1633 Argument expansion (* and **) are not allowed.

1634

1635 Returns: (positional_args, keyword_args)

1636 """

1637 positional_args = []

1638 keyword_args = []

1639 pos_idx = 0

1640

1641 while s.sy not in end_sy_set:

1642 if s.sy == '*' or s.sy == '**':

1643 s.error('Argument expansion not allowed here.')

1644

1645 parsed_type = False

1646 if s.sy == 'IDENT' and s.peek()[0] == '=':

1647 ident = s.systring

1648 s.next() # s.sy is '='

1649 s.next()

1650 if looking_at_expr(s):

1651 arg = p_simple_expr(s)

1652 else:

1653 base_type = p_c_base_type(s, templates = templates)

1654 declarator = p_c_declarator(s, empty = 1)

1655 arg = Nodes.CComplexBaseTypeNode(base_type.pos,

1656 base_type = base_type, declarator = declarator)

1657 parsed_type = True

1658 keyword_node = ExprNodes.IdentifierStringNode(

1659 arg.pos, value = EncodedString(ident))

1660 keyword_args.append((keyword_node, arg))

1661 was_keyword = True

1662

1663 else:

1664 if looking_at_expr(s):

1665 arg = p_simple_expr(s)

1666 else:

1667 base_type = p_c_base_type(s, templates = templates)

1668 declarator = p_c_declarator(s, empty = 1)

1669 arg = Nodes.CComplexBaseTypeNode(base_type.pos,

1670 base_type = base_type, declarator = declarator)

1671 parsed_type = True

1672 positional_args.append(arg)

1673 pos_idx += 1

1674 if len(keyword_args) > 0:

1675 s.error("Non-keyword arg following keyword arg",

1676 pos = arg.pos)

1677

1678 if s.sy != ',':

1679 if s.sy not in end_sy_set:

1680 if parsed_type:

1681 s.error("Unmatched %s" % " or ".join(end_sy_set))

1682 break

1683 s.next()

1684 return positional_args, keyword_args

1685

1686 def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):

1687 # If self_flag is true, this is the base type for the

1688 # self argument of a C method of an extension type.

1689 if s.sy == '(':

1690 return p_c_complex_base_type(s)

1691 else:

1692 return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)

1693

1694 def p_calling_convention(s):

1695 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1696 result = s.systring

1697 s.next()

1698 return result

1699 else:

1700 return ""

1701

1702 calling_convention_words = ("__stdcall", "__cdecl", "__fastcall")

1703

1704 def p_c_complex_base_type(s):

1705 # s.sy == '('

1706 pos = s.position()

1707 s.next()

1708 base_type = p_c_base_type(s)

1709 declarator = p_c_declarator(s, empty = 1)

1710 s.expect(')')

1711 return Nodes.CComplexBaseTypeNode(pos,

1712 base_type = base_type, declarator = declarator)

1713

1714 def p_c_simple_base_type(s, self_flag, nonempty, templates = None):

1715 #print "p_c_simple_base_type: self_flag =", self_flag, nonempty

1716 is_basic = 0

1717 signed = 1

1718 longness = 0

1719 complex = 0

1720 module_path = []

1721 pos = s.position()

1722 if not s.sy == 'IDENT':

1723 error(pos, "Expected an identifier, found '%s'" % s.sy)

1724 if looking_at_base_type(s):

1725 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1726 is_basic = 1

1727 if s.sy == 'IDENT' and s.systring in special_basic_c_types:

1728 signed, longness = special_basic_c_types[s.systring]

1729 name = s.systring

1730 s.next()

1731 else:

1732 signed, longness = p_sign_and_longness(s)

1733 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1734 name = s.systring

1735 s.next()

1736 else:

1737 name = 'int'

1738 if s.sy == 'IDENT' and s.systring == 'complex':

1739 complex = 1

1740 s.next()

1741 elif looking_at_dotted_name(s):

1742 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1743 name = s.systring

1744 s.next()

1745 while s.sy == '.':

1746 module_path.append(name)

1747 s.next()

1748 name = p_ident(s)

1749 else:

1750 name = s.systring

1751 s.next()

1752 if nonempty and s.sy != 'IDENT':

1753 # Make sure this is not a declaration of a variable or function.

1754 if s.sy == '(':

1755 s.next()

1756 if s.sy == '*' or s.sy == '**' or s.sy == '&':

1757 s.put_back('(', '(')

1758 else:

1759 s.put_back('(', '(')

1760 s.put_back('IDENT', name)

1761 name = None

1762 elif s.sy not in ('*', '**', '[', '&'):

1763 s.put_back('IDENT', name)

1764 name = None

1765

1766 type_node = Nodes.CSimpleBaseTypeNode(pos,

1767 name = name, module_path = module_path,

1768 is_basic_c_type = is_basic, signed = signed,

1769 complex = complex, longness = longness,

1770 is_self_arg = self_flag, templates = templates)

1771

1772 if s.sy == '[':

1773 type_node = p_buffer_or_template(s, type_node, templates)

1774

1775 if s.sy == '.':

1776 s.next()

1777 name = p_ident(s)

1778 type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)

1779

1780 return type_node

1781

1782 def p_buffer_or_template(s, base_type_node, templates):

1783 # s.sy == '['

1784 pos = s.position()

1785 s.next()

1786 # Note that buffer_positional_options_count=1, so the only positional argument is dtype.

1787 # For templated types, all parameters are types.

1788 positional_args, keyword_args = (

1789 p_positional_and_keyword_args(s, (']',), templates)

1790 )

1791 s.expect(']')

1792

1793 keyword_dict = ExprNodes.DictNode(pos,

1794 key_value_pairs = [

1795 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1796 for key, value in keyword_args

1797 ])

1798 result = Nodes.TemplatedTypeNode(pos,

1799 positional_args = positional_args,

1800 keyword_args = keyword_dict,

1801 base_type_node = base_type_node)

1802 return result

1803

1804

1805 def looking_at_name(s):

1806 return s.sy == 'IDENT' and not s.systring in calling_convention_words

1807

1808 def looking_at_expr(s):

1809 if s.systring in base_type_start_words:

1810 return False

1811 elif s.sy == 'IDENT':

1812 is_type = False

1813 name = s.systring

1814 dotted_path = []

1815 s.next()

1816 while s.sy == '.':

1817 s.next()

1818 dotted_path.append(s.systring)

1819 s.expect('IDENT')

1820 saved = s.sy, s.systring

1821 if s.sy == 'IDENT':

1822 is_type = True

1823 elif s.sy == '*' or s.sy == '**':

1824 s.next()

1825 is_type = s.sy == ')'

1826 s.put_back(*saved)

1827 elif s.sy == '(':

1828 s.next()

1829 is_type = s.sy == '*'

1830 s.put_back(*saved)

1831 elif s.sy == '[':

1832 s.next()

1833 is_type = s.sy == ']'

1834 s.put_back(*saved)

1835 dotted_path.reverse()

1836 for p in dotted_path:

1837 s.put_back('IDENT', p)

1838 s.put_back('.', '.')

1839 s.put_back('IDENT', name)

1840 return not is_type

1841 else:

1842 return True

1843

1844 def looking_at_base_type(s):

1845 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1846 return s.sy == 'IDENT' and s.systring in base_type_start_words

1847

1848 def looking_at_dotted_name(s):

1849 if s.sy == 'IDENT':

1850 name = s.systring

1851 s.next()

1852 result = s.sy == '.'

1853 s.put_back('IDENT', name)

1854 return result

1855 else:

1856 return 0

1857

1858 basic_c_type_names = ("void", "char", "int", "float", "double", "bint")

1859

1860 special_basic_c_types = {

1861 # name : (signed, longness)

1862 "Py_ssize_t" : (2, 0),

1863 "size_t" : (0, 0),

1864 }

1865

1866 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1867

1868 base_type_start_words = \

1869 basic_c_type_names + sign_and_longness_words + tuple(special_basic_c_types)

1870

1871 def p_sign_and_longness(s):

1872 signed = 1

1873 longness = 0

1874 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1875 if s.systring == 'unsigned':

1876 signed = 0

1877 elif s.systring == 'signed':

1878 signed = 2

1879 elif s.systring == 'short':

1880 longness = -1

1881 elif s.systring == 'long':

1882 longness += 1

1883 s.next()

1884 return signed, longness

1885

1886 def p_opt_cname(s):

1887 literal = p_opt_string_literal(s)

1888 if literal:

1889 _, cname = literal

1890 cname = EncodedString(cname)

1891 cname.encoding = s.source_encoding

1892 else:

1893 cname = None

1894 return cname

1895

1896 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1897 assignable = 0, nonempty = 0,

1898 calling_convention_allowed = 0):

1899 # If empty is true, the declarator must be empty. If nonempty is true,

1900 # the declarator must be nonempty. Otherwise we don't care.

1901 # If cmethod_flag is true, then if this declarator declares

1902 # a function, it's a C method of an extension type.

1903 pos = s.position()

1904 if s.sy == '(':

1905 s.next()

1906 if s.sy == ')' or looking_at_name(s):

1907 base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)

1908 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1909 else:

1910 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1911 cmethod_flag = cmethod_flag,

1912 nonempty = nonempty,

1913 calling_convention_allowed = 1)

1914 s.expect(')')

1915 else:

1916 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1917 assignable, nonempty)

1918 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1919 error(s.position(), "%s on something that is not a function"

1920 % result.calling_convention)

1921 while s.sy in ('[', '('):

1922 pos = s.position()

1923 if s.sy == '[':

1924 result = p_c_array_declarator(s, result)

1925 else: # sy == '('

1926 s.next()

1927 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1928 cmethod_flag = 0

1929 return result

1930

1931 def p_c_array_declarator(s, base):

1932 pos = s.position()

1933 s.next() # '['

1934 if s.sy != ']':

1935 dim = p_expr(s)

1936 else:

1937 dim = None

1938 s.expect(']')

1939 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1940

1941 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

1942 # Opening paren has already been skipped

1943 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

1944 nonempty_declarators = 0)

1945 ellipsis = p_optional_ellipsis(s)

1946 s.expect(')')

1947 nogil = p_nogil(s)

1948 exc_val, exc_check = p_exception_value_clause(s)

1949 with_gil = p_with_gil(s)

1950 return Nodes.CFuncDeclaratorNode(pos,

1951 base = base, args = args, has_varargs = ellipsis,

1952 exception_value = exc_val, exception_check = exc_check,

1953 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

1954

1955 supported_overloaded_operators = set([

1956 '+', '-', '*', '/', '%',

1957 '++', '--', '~', '|', '&', '^', '<<', '>>',

1958 '==', '!=', '>=', '>', '<=', '<',

1959 '[]', '()',

1960 ])

1961

1962 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1963 assignable, nonempty):

1964 pos = s.position()

1965 calling_convention = p_calling_convention(s)

1966 if s.sy == '*':

1967 s.next()

1968 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1969 cmethod_flag = cmethod_flag,

1970 assignable = assignable, nonempty = nonempty)

1971 result = Nodes.CPtrDeclaratorNode(pos,

1972 base = base)

1973 elif s.sy == '**': # scanner returns this as a single token

1974 s.next()

1975 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1976 cmethod_flag = cmethod_flag,

1977 assignable = assignable, nonempty = nonempty)

1978 result = Nodes.CPtrDeclaratorNode(pos,

1979 base = Nodes.CPtrDeclaratorNode(pos,

1980 base = base))

1981 elif s.sy == '&':

1982 s.next()

1983 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1984 cmethod_flag = cmethod_flag,

1985 assignable = assignable, nonempty = nonempty)

1986 result = Nodes.CReferenceDeclaratorNode(pos, base = base)

1987 else:

1988 rhs = None

1989 if s.sy == 'IDENT':

1990 name = EncodedString(s.systring)

1991 if empty:

1992 error(s.position(), "Declarator should be empty")

1993 s.next()

1994 cname = p_opt_cname(s)

1995 if name != "operator" and s.sy == '=' and assignable:

1996 s.next()

1997 rhs = p_simple_expr(s)

1998 else:

1999 if nonempty:

2000 error(s.position(), "Empty declarator")

2001 name = ""

2002 cname = None

2003 if cname is None and ctx.namespace is not None:

2004 cname = ctx.namespace + "::" + name

2005 if name == 'operator' and ctx.visibility == 'extern':

2006 op = s.sy

2007 s.next()

2008 # Handle diphthong operators.

2009 if op == '(':

2010 s.expect(')')

2011 op = '()'

2012 elif op == '[':

2013 s.expect(']')

2014 op = '[]'

2015 if op in ['-', '+', '|', '&'] and s.sy == op:

2016 op = op*2

2017 s.next()

2018 if s.sy == '=':

2019 op += s.sy

2020 s.next()

2021 if op not in supported_overloaded_operators:

2022 s.error("Overloading operator '%s' not yet supported." % op)

2023 name = name+op

2024 result = Nodes.CNameDeclaratorNode(pos,

2025 name = name, cname = cname, default = rhs)

2026 result.calling_convention = calling_convention

2027 return result

2028

2029 def p_nogil(s):

2030 if s.sy == 'IDENT' and s.systring == 'nogil':

2031 s.next()

2032 return 1

2033 else:

2034 return 0

2035

2036 def p_with_gil(s):

2037 if s.sy == 'with':

2038 s.next()

2039 s.expect_keyword('gil')

2040 return 1

2041 else:

2042 return 0

2043

2044 def p_exception_value_clause(s):

2045 exc_val = None

2046 exc_check = 0

2047 if s.sy == 'except':

2048 s.next()

2049 if s.sy == '*':

2050 exc_check = 1

2051 s.next()

2052 elif s.sy == '+':

2053 exc_check = '+'

2054 s.next()

2055 if s.sy == 'IDENT':

2056 name = s.systring

2057 s.next()

2058 exc_val = p_name(s, name)

2059 else:

2060 if s.sy == '?':

2061 exc_check = 1

2062 s.next()

2063 exc_val = p_simple_expr(s)

2064 return exc_val, exc_check

2065

2066 c_arg_list_terminators = ('*', '**', '.', ')')

2067

2068 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

2069 nonempty_declarators = 0, kw_only = 0):

2070 # Comma-separated list of C argument declarations, possibly empty.

2071 # May have a trailing comma.

2072 args = []

2073 is_self_arg = cmethod_flag

2074 while s.sy not in c_arg_list_terminators:

2075 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

2076 nonempty = nonempty_declarators, kw_only = kw_only))

2077 if s.sy != ',':

2078 break

2079 s.next()

2080 is_self_arg = 0

2081 return args

2082

2083 def p_optional_ellipsis(s):

2084 if s.sy == '.':

2085 expect_ellipsis(s)

2086 return 1

2087 else:

2088 return 0

2089

2090 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

2091 pos = s.position()

2092 not_none = 0

2093 default = None

2094 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

2095 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

2096 if s.sy == 'not':

2097 s.next()

2098 if s.sy == 'IDENT' and s.systring == 'None':

2099 s.next()

2100 else:

2101 s.error("Expected 'None'")

2102 if not in_pyfunc:

2103 error(pos, "'not None' only allowed in Python functions")

2104 not_none = 1

2105 if s.sy == '=':

2106 s.next()

2107 if 'pxd' in s.level:

2108 if s.sy not in ['*', '?']:

2109 error(pos, "default values cannot be specified in pxd files, use ? or *")

2110 default = ExprNodes.BoolNode(1)

2111 s.next()

2112 else:

2113 default = p_simple_expr(s)

2114 return Nodes.CArgDeclNode(pos,

2115 base_type = base_type,

2116 declarator = declarator,

2117 not_none = not_none,

2118 default = default,

2119 kw_only = kw_only)

2120

2121 def p_api(s):

2122 if s.sy == 'IDENT' and s.systring == 'api':

2123 s.next()

2124 return 1

2125 else:

2126 return 0

2127

2128 def p_cdef_statement(s, ctx):

2129 pos = s.position()

2130 ctx.visibility = p_visibility(s, ctx.visibility)

2131 ctx.api = ctx.api or p_api(s)

2132 if ctx.api:

2133 if ctx.visibility not in ('private', 'public'):

2134 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

2135 if (ctx.visibility == 'extern') and s.sy == 'from':

2136 return p_cdef_extern_block(s, pos, ctx)

2137 elif s.sy == 'import':

2138 s.next()

2139 return p_cdef_extern_block(s, pos, ctx)

2140 elif p_nogil(s):

2141 ctx.nogil = 1

2142 if ctx.overridable:

2143 error(pos, "cdef blocks cannot be declared cpdef")

2144 return p_cdef_block(s, ctx)

2145 elif s.sy == ':':

2146 if ctx.overridable:

2147 error(pos, "cdef blocks cannot be declared cpdef")

2148 return p_cdef_block(s, ctx)

2149 elif s.sy == 'class':

2150 if ctx.level not in ('module', 'module_pxd'):

2151 error(pos, "Extension type definition not allowed here")

2152 if ctx.overridable:

2153 error(pos, "Extension types cannot be declared cpdef")

2154 return p_c_class_definition(s, pos, ctx)

2155 elif s.sy == 'IDENT' and s.systring == 'cppclass':

2156 if ctx.visibility != 'extern':

2157 error(pos, "C++ classes need to be declared extern")

2158 return p_cpp_class_definition(s, pos, ctx)

2159 elif s.sy == 'IDENT' and s.systring in ("struct", "union", "enum", "packed"):

2160 if ctx.level not in ('module', 'module_pxd'):

2161 error(pos, "C struct/union/enum definition not allowed here")

2162 if ctx.overridable:

2163 error(pos, "C struct/union/enum cannot be declared cpdef")

2164 if s.systring == "enum":

2165 return p_c_enum_definition(s, pos, ctx)

2166 else:

2167 return p_c_struct_or_union_definition(s, pos, ctx)

2168 else:

2169 return p_c_func_or_var_declaration(s, pos, ctx)

2170

2171 def p_cdef_block(s, ctx):

2172 return p_suite(s, ctx(cdef_flag = 1))

2173

2174 def p_cdef_extern_block(s, pos, ctx):

2175 if ctx.overridable:

2176 error(pos, "cdef extern blocks cannot be declared cpdef")

2177 include_file = None

2178 s.expect('from')

2179 if s.sy == '*':

2180 s.next()

2181 else:

2182 _, include_file = p_string_literal(s)

2183 if s.systring == "namespace":

2184 s.next()

2185 ctx.namespace = p_string_literal(s)[1]

2186 ctx = ctx(cdef_flag = 1, visibility = 'extern')

2187 if p_nogil(s):

2188 ctx.nogil = 1

2189 body = p_suite(s, ctx)

2190 return Nodes.CDefExternNode(pos,

2191 include_file = include_file,

2192 body = body,

2193 namespace = ctx.namespace)

2194

2195 def p_c_enum_definition(s, pos, ctx):

2196 # s.sy == ident 'enum'

2197 s.next()

2198 if s.sy == 'IDENT':

2199 name = s.systring

2200 s.next()

2201 cname = p_opt_cname(s)

2202 if cname is None and ctx.namespace is not None:

2203 cname = ctx.namespace + "::" + name

2204 else:

2205 name = None

2206 cname = None

2207 items = None

2208 s.expect(':')

2209 items = []

2210 if s.sy != 'NEWLINE':

2211 p_c_enum_line(s, items)

2212 else:

2213 s.next() # 'NEWLINE'

2214 s.expect_indent()

2215 while s.sy not in ('DEDENT', 'EOF'):

2216 p_c_enum_line(s, items)

2217 s.expect_dedent()

2218 return Nodes.CEnumDefNode(

2219 pos, name = name, cname = cname, items = items,

2220 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2221 in_pxd = ctx.level == 'module_pxd')

2222

2223 def p_c_enum_line(s, items):

2224 if s.sy != 'pass':

2225 p_c_enum_item(s, items)

2226 while s.sy == ',':

2227 s.next()

2228 if s.sy in ('NEWLINE', 'EOF'):

2229 break

2230 p_c_enum_item(s, items)

2231 else:

2232 s.next()

2233 s.expect_newline("Syntax error in enum item list")

2234

2235 def p_c_enum_item(s, items):

2236 pos = s.position()

2237 name = p_ident(s)

2238 cname = p_opt_cname(s)

2239 value = None

2240 if s.sy == '=':

2241 s.next()

2242 value = p_simple_expr(s)

2243 items.append(Nodes.CEnumDefItemNode(pos,

2244 name = name, cname = cname, value = value))

2245

2246 def p_c_struct_or_union_definition(s, pos, ctx):

2247 packed = False

2248 if s.systring == 'packed':

2249 packed = True

2250 s.next()

2251 if s.sy != 'IDENT' or s.systring != 'struct':

2252 s.expected('struct')

2253 # s.sy == ident 'struct' or 'union'

2254 kind = s.systring

2255 s.next()

2256 name = p_ident(s)

2257 cname = p_opt_cname(s)

2258 if cname is None and ctx.namespace is not None:

2259 cname = ctx.namespace + "::" + name

2260 attributes = None

2261 if s.sy == ':':

2262 s.next()

2263 s.expect('NEWLINE')

2264 s.expect_indent()

2265 attributes = []

2266 body_ctx = Ctx()

2267 while s.sy != 'DEDENT':

2268 if s.sy != 'pass':

2269 attributes.append(

2270 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2271 else:

2272 s.next()

2273 s.expect_newline("Expected a newline")

2274 s.expect_dedent()

2275 else:

2276 s.expect_newline("Syntax error in struct or union definition")

2277 return Nodes.CStructOrUnionDefNode(pos,

2278 name = name, cname = cname, kind = kind, attributes = attributes,

2279 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2280 in_pxd = ctx.level == 'module_pxd', packed = packed)

2281

2282 def p_visibility(s, prev_visibility):

2283 pos = s.position()

2284 visibility = prev_visibility

2285 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2286 visibility = s.systring

2287 if prev_visibility != 'private' and visibility != prev_visibility:

2288 s.error("Conflicting visibility options '%s' and '%s'"

2289 % (prev_visibility, visibility))

2290 s.next()

2291 return visibility

2292

2293 def p_c_modifiers(s):

2294 if s.sy == 'IDENT' and s.systring in ('inline',):

2295 modifier = s.systring

2296 s.next()

2297 return [modifier] + p_c_modifiers(s)

2298 return []

2299

2300 def p_c_func_or_var_declaration(s, pos, ctx):

2301 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2302 modifiers = p_c_modifiers(s)

2303 base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)

2304 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2305 assignable = 1, nonempty = 1)

2306 declarator.overridable = ctx.overridable

2307 if s.sy == ':':

2308 if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd') and not ctx.templates:

2309 s.error("C function definition not allowed here")

2310 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2311 result = Nodes.CFuncDefNode(pos,

2312 visibility = ctx.visibility,

2313 base_type = base_type,

2314 declarator = declarator,

2315 body = suite,

2316 doc = doc,

2317 modifiers = modifiers,

2318 api = ctx.api,

2319 overridable = ctx.overridable)

2320 else:

2321 #if api:

2322 # error(s.pos, "'api' not allowed with variable declaration")

2323 declarators = [declarator]

2324 while s.sy == ',':

2325 s.next()

2326 if s.sy == 'NEWLINE':

2327 break

2328 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2329 assignable = 1, nonempty = 1)

2330 declarators.append(declarator)

2331 s.expect_newline("Syntax error in C variable declaration")

2332 result = Nodes.CVarDefNode(pos,

2333 visibility = ctx.visibility,

2334 base_type = base_type,

2335 declarators = declarators,

2336 in_pxd = ctx.level == 'module_pxd',

2337 api = ctx.api,

2338 overridable = ctx.overridable)

2339 return result

2340

2341 def p_ctypedef_statement(s, ctx):

2342 # s.sy == 'ctypedef'

2343 pos = s.position()

2344 s.next()

2345 visibility = p_visibility(s, ctx.visibility)

2346 api = p_api(s)

2347 ctx = ctx(typedef_flag = 1, visibility = visibility)

2348 if api:

2349 ctx.api = 1

2350 if s.sy == 'class':

2351 return p_c_class_definition(s, pos, ctx)

2352 elif s.sy == 'IDENT' and s.systring in ('packed', 'struct', 'union', 'enum'):

2353 if s.systring == 'enum':

2354 return p_c_enum_definition(s, pos, ctx)

2355 else:

2356 return p_c_struct_or_union_definition(s, pos, ctx)

2357 else:

2358 base_type = p_c_base_type(s, nonempty = 1)

2359 if base_type.name is None:

2360 s.error("Syntax error in ctypedef statement")

2361 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2362 s.expect_newline("Syntax error in ctypedef statement")

2363 return Nodes.CTypeDefNode(

2364 pos, base_type = base_type,

2365 declarator = declarator, visibility = visibility,

2366 in_pxd = ctx.level == 'module_pxd')

2367

2368 def p_decorators(s):

2369 decorators = []

2370 while s.sy == 'DECORATOR':

2371 pos = s.position()

2372 s.next()

2373 decstring = p_dotted_name(s, as_allowed=0)[2]

2374 names = decstring.split('.')

2375 decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))

2376 for name in names[1:]:

2377 decorator = ExprNodes.AttributeNode(pos,

2378 attribute=EncodedString(name),

2379 obj=decorator)

2380 if s.sy == '(':

2381 decorator = p_call(s, decorator)

2382 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2383 s.expect_newline("Expected a newline after decorator")

2384 return decorators

2385

2386 def p_def_statement(s, decorators=None):

2387 # s.sy == 'def'

2388 pos = s.position()

2389 s.next()

2390 name = EncodedString( p_ident(s) )

2391 #args = []

2392 s.expect('(');

2393 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2394 star_arg = None

2395 starstar_arg = None

2396 if s.sy == '*':

2397 s.next()

2398 if s.sy == 'IDENT':

2399 star_arg = p_py_arg_decl(s)

2400 if s.sy == ',':

2401 s.next()

2402 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2403 nonempty_declarators = 1, kw_only = 1))

2404 elif s.sy != ')':

2405 s.error("Syntax error in Python function argument list")

2406 if s.sy == '**':

2407 s.next()

2408 starstar_arg = p_py_arg_decl(s)

2409 s.expect(')')

2410 if p_nogil(s):

2411 error(s.pos, "Python function cannot be declared nogil")

2412 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2413 return Nodes.DefNode(pos, name = name, args = args,

2414 star_arg = star_arg, starstar_arg = starstar_arg,

2415 doc = doc, body = body, decorators = decorators)

2416

2417 def p_py_arg_decl(s):

2418 pos = s.position()

2419 name = p_ident(s)

2420 return Nodes.PyArgDeclNode(pos, name = name)

2421

2422 def p_class_statement(s, decorators):

2423 # s.sy == 'class'

2424 pos = s.position()

2425 s.next()

2426 class_name = EncodedString( p_ident(s) )

2427 class_name.encoding = s.source_encoding

2428 if s.sy == '(':

2429 s.next()

2430 base_list = p_simple_expr_list(s)

2431 s.expect(')')

2432 else:

2433 base_list = []

2434 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2435 return Nodes.PyClassDefNode(pos,

2436 name = class_name,

2437 bases = ExprNodes.TupleNode(pos, args = base_list),

2438 doc = doc, body = body, decorators = decorators)

2439

2440 def p_c_class_definition(s, pos, ctx):

2441 # s.sy == 'class'

2442 s.next()

2443 module_path = []

2444 class_name = p_ident(s)

2445 while s.sy == '.':

2446 s.next()

2447 module_path.append(class_name)

2448 class_name = p_ident(s)

2449 if module_path and ctx.visibility != 'extern':

2450 error(pos, "Qualified class name only allowed for 'extern' C class")

2451 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2452 s.next()

2453 as_name = p_ident(s)

2454 else:

2455 as_name = class_name

2456 objstruct_name = None

2457 typeobj_name = None

2458 base_class_module = None

2459 base_class_name = None

2460 if s.sy == '(':

2461 s.next()

2462 base_class_path = [p_ident(s)]

2463 while s.sy == '.':

2464 s.next()

2465 base_class_path.append(p_ident(s))

2466 if s.sy == ',':

2467 s.error("C class may only have one base class")

2468 s.expect(')')

2469 base_class_module = ".".join(base_class_path[:-1])

2470 base_class_name = base_class_path[-1]

2471 if s.sy == '[':

2472 if ctx.visibility not in ('public', 'extern'):

2473 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2474 objstruct_name, typeobj_name = p_c_class_options(s)

2475 if s.sy == ':':

2476 if ctx.level == 'module_pxd':

2477 body_level = 'c_class_pxd'

2478 else:

2479 body_level = 'c_class'

2480 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2481 else:

2482 s.expect_newline("Syntax error in C class definition")

2483 doc = None

2484 body = None

2485 if ctx.visibility == 'extern':

2486 if not module_path:

2487 error(pos, "Module name required for 'extern' C class")

2488 if typeobj_name:

2489 error(pos, "Type object name specification not allowed for 'extern' C class")

2490 elif ctx.visibility == 'public':

2491 if not objstruct_name:

2492 error(pos, "Object struct name specification required for 'public' C class")

2493 if not typeobj_name:

2494 error(pos, "Type object name specification required for 'public' C class")

2495 elif ctx.visibility == 'private':

2496 if ctx.api:

2497 error(pos, "Only 'public' C class can be declared 'api'")

2498 else:

2499 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2500 return Nodes.CClassDefNode(pos,

2501 visibility = ctx.visibility,

2502 typedef_flag = ctx.typedef_flag,

2503 api = ctx.api,

2504 module_name = ".".join(module_path),

2505 class_name = class_name,

2506 as_name = as_name,

2507 base_class_module = base_class_module,

2508 base_class_name = base_class_name,

2509 objstruct_name = objstruct_name,

2510 typeobj_name = typeobj_name,

2511 in_pxd = ctx.level == 'module_pxd',

2512 doc = doc,

2513 body = body)

2514

2515 def p_c_class_options(s):

2516 objstruct_name = None

2517 typeobj_name = None

2518 s.expect('[')

2519 while 1:

2520 if s.sy != 'IDENT':

2521 break

2522 if s.systring == 'object':

2523 s.next()

2524 objstruct_name = p_ident(s)

2525 elif s.systring == 'type':

2526 s.next()

2527 typeobj_name = p_ident(s)

2528 if s.sy != ',':

2529 break

2530 s.next()

2531 s.expect(']', "Expected 'object' or 'type'")

2532 return objstruct_name, typeobj_name

2533

2534 def p_property_decl(s):

2535 pos = s.position()

2536 s.next() # 'property'

2537 name = p_ident(s)

2538 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2539 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2540

2541 def p_doc_string(s):

2542 if s.sy == 'BEGIN_STRING':

2543 pos = s.position()

2544 kind, result = p_cat_string_literal(s)

2545 if s.sy != 'EOF':

2546 s.expect_newline("Syntax error in doc string")

2547 if kind != 'u':

2548 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2549 if kind == 'b':

2550 result.encoding = None # force a unicode string

2551 return result

2552 else:

2553 return None

2554

2555 def p_code(s, level=None):

2556 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2557 if s.sy != 'EOF':

2558 s.error("Syntax error in statement [%s,%s]" % (

2559 repr(s.sy), repr(s.systring)))

2560 return body

2561

2562 COMPILER_DIRECTIVE_COMMENT_RE = re.compile(r"^#\s*cython:\s*((\w|[.])+\s*=.*)$")

2563

2564 def p_compiler_directive_comments(s):

2565 result = {}

2566 while s.sy == 'commentline':

2567 m = COMPILER_DIRECTIVE_COMMENT_RE.match(s.systring)

2568 if m:

2569 directives = m.group(1).strip()

2570 try:

2571 result.update( Options.parse_directive_list(

2572 directives, ignore_unknown=True) )

2573 except ValueError, e:

2574 s.error(e.args[0], fatal=False)

2575 s.next()

2576 return result

2577

2578 def p_module(s, pxd, full_module_name):

2579 pos = s.position()

2580

2581 directive_comments = p_compiler_directive_comments(s)

2582 s.parse_comments = False

2583

2584 doc = p_doc_string(s)

2585 if pxd:

2586 level = 'module_pxd'

2587 else:

2588 level = 'module'

2589

2590 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2591 if s.sy != 'EOF':

2592 s.error("Syntax error in statement [%s,%s]" % (

2593 repr(s.sy), repr(s.systring)))

2594 return ModuleNode(pos, doc = doc, body = body,

2595 full_module_name = full_module_name,

2596 directive_comments = directive_comments)

2597

2598 def p_cpp_class_definition(s, pos, ctx):

2599 # s.sy == 'cppclass'

2600 s.next()

2601 module_path = []

2602 class_name = p_ident(s)

2603 cname = p_opt_cname(s)

2604 if cname is None and ctx.namespace is not None:

2605 cname = ctx.namespace + "::" + class_name

2606 if s.sy == '.':

2607 error(pos, "Qualified class name not allowed C++ class")

2608 if s.sy == '[':

2609 s.next()

2610 templates = [p_ident(s)]

2611 while s.sy == ',':

2612 s.next()

2613 templates.append(p_ident(s))

2614 s.expect(']')

2615 else:

2616 templates = None

2617 if s.sy == '(':

2618 s.next()

2619 base_classes = [p_dotted_name(s, False)[2]]

2620 while s.sy == ',':

2621 s.next()

2622 base_classes.append(p_dotted_name(s, False)[2])

2623 s.expect(')')

2624 else:

2625 base_classes = []

2626 if s.sy == '[':

2627 error(s.position(), "Name options not allowed for C++ class")

2628 if s.sy == ':':

2629 s.next()

2630 s.expect('NEWLINE')

2631 s.expect_indent()

2632 attributes = []

2633 body_ctx = Ctx(visibility = ctx.visibility)

2634 body_ctx.templates = templates

2635 while s.sy != 'DEDENT':

2636 if s.systring == 'cppclass':

2637 attributes.append(

2638 p_cpp_class_definition(s, s.position(), body_ctx))

2639 elif s.sy != 'pass':

2640 attributes.append(

2641 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2642 else:

2643 s.next()

2644 s.expect_newline("Expected a newline")

2645 s.expect_dedent()

2646 else:

2647 s.expect_newline("Syntax error in C++ class definition")

2648 return Nodes.CppClassNode(pos,

2649 name = class_name,

2650 cname = cname,

2651 base_classes = base_classes,

2652 visibility = ctx.visibility,

2653 in_pxd = ctx.level == 'module_pxd',

2654 attributes = attributes,

2655 templates = templates)

2656

2657

2658

2659 #----------------------------------------------

2660 #

2661 # Debugging

2662 #

2663 #----------------------------------------------

2664

2665 def print_parse_tree(f, node, level, key = None):

2666 from Nodes import Node

2667 ind = " " * level

2668 if node:

2669 f.write(ind)

2670 if key:

2671 f.write("%s: " % key)

2672 t = type(node)

2673 if t is tuple:

2674 f.write("(%s @ %s\n" % (node[0], node[1]))

2675 for i in xrange(2, len(node)):

2676 print_parse_tree(f, node[i], level+1)

2677 f.write("%s)\n" % ind)

2678 return

2679 elif isinstance(node, Node):

2680 try:

2681 tag = node.tag

2682 except AttributeError:

2683 tag = node.__class__.__name__

2684 f.write("%s @ %s\n" % (tag, node.pos))

2685 for name, value in node.__dict__.items():

2686 if name != 'tag' and name != 'pos':

2687 print_parse_tree(f, value, level+1, name)

2688 return

2689 elif t is list:

2690 f.write("[\n")

2691 for i in xrange(len(node)):

2692 print_parse_tree(f, node[i], level+1)

2693 f.write("%s]\n" % ind)

2694 return

2695 f.write("%s%s\n" % (ind, node))

2696