1 #

2 # Pyrex Parser

3 #

4

5 import os, re

6 from types import ListType, TupleType

7 from Scanning import PyrexScanner, FileSourceDescriptor

8 import Nodes

9 import ExprNodes

10 from ModuleNode import ModuleNode

11 from Errors import error, warning, InternalError

12 from Cython import Utils

13 import Future

14

15 class Ctx(object):

16 # Parsing context

17 level = 'other'

18 visibility = 'private'

19 cdef_flag = 0

20 typedef_flag = 0

21 api = 0

22 overridable = 0

23 nogil = 0

24

25 def __init__(self, **kwds):

26 self.__dict__.update(kwds)

27

28 def __call__(self, **kwds):

29 ctx = Ctx()

30 d = ctx.__dict__

31 d.update(self.__dict__)

32 d.update(kwds)

33 return ctx

34

35 def p_ident(s, message = "Expected an identifier"):

36 if s.sy == 'IDENT':

37 name = s.systring

38 s.next()

39 return name

40 else:

41 s.error(message)

42

43 def p_ident_list(s):

44 names = []

45 while s.sy == 'IDENT':

46 names.append(s.systring)

47 s.next()

48 if s.sy != ',':

49 break

50 s.next()

51 return names

52

53 #------------------------------------------

54 #

55 # Expressions

56 #

57 #------------------------------------------

58

59 def p_binop_expr(s, ops, p_sub_expr):

60 n1 = p_sub_expr(s)

61 while s.sy in ops:

62 op = s.sy

63 pos = s.position()

64 s.next()

65 n2 = p_sub_expr(s)

66 n1 = ExprNodes.binop_node(pos, op, n1, n2)

67 return n1

68

69 #expression: or_test [if or_test else test] | lambda_form

70

71 def p_simple_expr(s):

72 pos = s.position()

73 expr = p_or_test(s)

74 if s.sy == 'if':

75 s.next()

76 test = p_or_test(s)

77 if s.sy == 'else':

78 s.next()

79 other = p_test(s)

80 return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)

81 else:

82 s.error("Expected 'else'")

83 else:

84 return expr

85

86 #test: or_test | lambda_form

87

88 def p_test(s):

89 return p_or_test(s)

90

91 #or_test: and_test ('or' and_test)*

92

93 def p_or_test(s):

94 return p_rassoc_binop_expr(s, ('or',), p_and_test)

95

96 def p_rassoc_binop_expr(s, ops, p_subexpr):

97 n1 = p_subexpr(s)

98 if s.sy in ops:

99 pos = s.position()

100 op = s.sy

101 s.next()

102 n2 = p_rassoc_binop_expr(s, ops, p_subexpr)

103 n1 = ExprNodes.binop_node(pos, op, n1, n2)

104 return n1

105

106 #and_test: not_test ('and' not_test)*

107

108 def p_and_test(s):

109 #return p_binop_expr(s, ('and',), p_not_test)

110 return p_rassoc_binop_expr(s, ('and',), p_not_test)

111

112 #not_test: 'not' not_test | comparison

113

114 def p_not_test(s):

115 if s.sy == 'not':

116 pos = s.position()

117 s.next()

118 return ExprNodes.NotNode(pos, operand = p_not_test(s))

119 else:

120 return p_comparison(s)

121

122 #comparison: expr (comp_op expr)*

123 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'

124

125 def p_comparison(s):

126 n1 = p_bit_expr(s)

127 if s.sy in comparison_ops:

128 pos = s.position()

129 op = p_cmp_op(s)

130 n2 = p_bit_expr(s)

131 n1 = ExprNodes.PrimaryCmpNode(pos,

132 operator = op, operand1 = n1, operand2 = n2)

133 if s.sy in comparison_ops:

134 n1.cascade = p_cascaded_cmp(s)

135 return n1

136

137 def p_cascaded_cmp(s):

138 pos = s.position()

139 op = p_cmp_op(s)

140 n2 = p_bit_expr(s)

141 result = ExprNodes.CascadedCmpNode(pos,

142 operator = op, operand2 = n2)

143 if s.sy in comparison_ops:

144 result.cascade = p_cascaded_cmp(s)

145 return result

146

147 def p_cmp_op(s):

148 if s.sy == 'not':

149 s.next()

150 s.expect('in')

151 op = 'not_in'

152 elif s.sy == 'is':

153 s.next()

154 if s.sy == 'not':

155 s.next()

156 op = 'is_not'

157 else:

158 op = 'is'

159 else:

160 op = s.sy

161 s.next()

162 if op == '<>':

163 op = '!='

164 return op

165

166 comparison_ops = (

167 '<', '>', '==', '>=', '<=', '<>', '!=',

168 'in', 'is', 'not'

169 )

170

171 #expr: xor_expr ('|' xor_expr)*

172

173 def p_bit_expr(s):

174 return p_binop_expr(s, ('|',), p_xor_expr)

175

176 #xor_expr: and_expr ('^' and_expr)*

177

178 def p_xor_expr(s):

179 return p_binop_expr(s, ('^',), p_and_expr)

180

181 #and_expr: shift_expr ('&' shift_expr)*

182

183 def p_and_expr(s):

184 return p_binop_expr(s, ('&',), p_shift_expr)

185

186 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*

187

188 def p_shift_expr(s):

189 return p_binop_expr(s, ('<<', '>>'), p_arith_expr)

190

191 #arith_expr: term (('+'|'-') term)*

192

193 def p_arith_expr(s):

194 return p_binop_expr(s, ('+', '-'), p_term)

195

196 #term: factor (('*'|'/'|'%') factor)*

197

198 def p_term(s):

199 return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)

200

201 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power

202

203 def p_factor(s):

204 sy = s.sy

205 if sy in ('+', '-', '~'):

206 op = s.sy

207 pos = s.position()

208 s.next()

209 return ExprNodes.unop_node(pos, op, p_factor(s))

210 elif sy == '&':

211 pos = s.position()

212 s.next()

213 arg = p_factor(s)

214 return ExprNodes.AmpersandNode(pos, operand = arg)

215 elif sy == "<":

216 return p_typecast(s)

217 elif sy == 'IDENT' and s.systring == "sizeof":

218 return p_sizeof(s)

219 else:

220 return p_power(s)

221

222 def p_typecast(s):

223 # s.sy == "<"

224 pos = s.position()

225 s.next()

226 base_type = p_c_base_type(s)

227 declarator = p_c_declarator(s, empty = 1)

228 if s.sy == '?':

229 s.next()

230 typecheck = 1

231 else:

232 typecheck = 0

233 s.expect(">")

234 operand = p_factor(s)

235 return ExprNodes.TypecastNode(pos,

236 base_type = base_type,

237 declarator = declarator,

238 operand = operand,

239 typecheck = typecheck)

240

241 def p_sizeof(s):

242 # s.sy == ident "sizeof"

243 pos = s.position()

244 s.next()

245 s.expect('(')

246 if looking_at_type(s) or looking_at_dotted_name(s):

247 base_type = p_c_base_type(s)

248 declarator = p_c_declarator(s, empty = 1)

249 node = ExprNodes.SizeofTypeNode(pos,

250 base_type = base_type, declarator = declarator)

251 else:

252 operand = p_simple_expr(s)

253 node = ExprNodes.SizeofVarNode(pos, operand = operand)

254 s.expect(')')

255 return node

256

257 #power: atom trailer* ('**' factor)*

258

259 def p_power(s):

260 n1 = p_atom(s)

261 while s.sy in ('(', '[', '.'):

262 n1 = p_trailer(s, n1)

263 if s.sy == '**':

264 pos = s.position()

265 s.next()

266 n2 = p_factor(s)

267 n1 = ExprNodes.binop_node(pos, '**', n1, n2)

268 return n1

269

270 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME

271

272 def p_trailer(s, node1):

273 pos = s.position()

274 if s.sy == '(':

275 return p_call(s, node1)

276 elif s.sy == '[':

277 return p_index(s, node1)

278 else: # s.sy == '.'

279 s.next()

280 name = Utils.EncodedString( p_ident(s) )

281 return ExprNodes.AttributeNode(pos,

282 obj = node1, attribute = name)

283

284 # arglist: argument (',' argument)* [',']

285 # argument: [test '='] test # Really [keyword '='] test

286

287 def p_call(s, function):

288 # s.sy == '('

289 pos = s.position()

290 s.next()

291 positional_args = []

292 keyword_args = []

293 star_arg = None

294 starstar_arg = None

295 while s.sy not in ('*', '**', ')'):

296 arg = p_simple_expr(s)

297 if s.sy == '=':

298 s.next()

299 if not arg.is_name:

300 s.error("Expected an identifier before '='",

301 pos = arg.pos)

302 encoded_name = Utils.EncodedString(arg.name)

303 keyword = ExprNodes.IdentifierStringNode(arg.pos,

304 value = encoded_name)

305 arg = p_simple_expr(s)

306 keyword_args.append((keyword, arg))

307 else:

308 if keyword_args:

309 s.error("Non-keyword arg following keyword arg",

310 pos = arg.pos)

311 positional_args.append(arg)

312 if s.sy != ',':

313 break

314 s.next()

315

316 if s.sy == '*':

317 s.next()

318 star_arg = p_simple_expr(s)

319 if s.sy == ',':

320 s.next()

321 if s.sy == '**':

322 s.next()

323 starstar_arg = p_simple_expr(s)

324 if s.sy == ',':

325 s.next()

326 s.expect(')')

327 if not (keyword_args or star_arg or starstar_arg):

328 return ExprNodes.SimpleCallNode(pos,

329 function = function,

330 args = positional_args)

331 else:

332 arg_tuple = None

333 keyword_dict = None

334 if positional_args or not star_arg:

335 arg_tuple = ExprNodes.TupleNode(pos,

336 args = positional_args)

337 if star_arg:

338 star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)

339 if arg_tuple:

340 arg_tuple = ExprNodes.binop_node(pos,

341 operator = '+', operand1 = arg_tuple,

342 operand2 = star_arg_tuple)

343 else:

344 arg_tuple = star_arg_tuple

345 if keyword_args:

346 keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

347 for key, value in keyword_args]

348 keyword_dict = ExprNodes.DictNode(pos,

349 key_value_pairs = keyword_args)

350 return ExprNodes.GeneralCallNode(pos,

351 function = function,

352 positional_args = arg_tuple,

353 keyword_args = keyword_dict,

354 starstar_arg = starstar_arg)

355

356 #lambdef: 'lambda' [varargslist] ':' test

357

358 #subscriptlist: subscript (',' subscript)* [',']

359

360 def p_index(s, base):

361 # s.sy == '['

362 pos = s.position()

363 s.next()

364 subscripts = p_subscript_list(s)

365 if len(subscripts) == 1 and len(subscripts[0]) == 2:

366 start, stop = subscripts[0]

367 result = ExprNodes.SliceIndexNode(pos,

368 base = base, start = start, stop = stop)

369 else:

370 indexes = make_slice_nodes(pos, subscripts)

371 if len(indexes) == 1:

372 index = indexes[0]

373 else:

374 index = ExprNodes.TupleNode(pos, args = indexes)

375 result = ExprNodes.IndexNode(pos,

376 base = base, index = index)

377 s.expect(']')

378 return result

379

380 def p_subscript_list(s):

381 items = [p_subscript(s)]

382 while s.sy == ',':

383 s.next()

384 if s.sy == ']':

385 break

386 items.append(p_subscript(s))

387 return items

388

389 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]

390

391 def p_subscript(s):

392 # Parse a subscript and return a list of

393 # 1, 2 or 3 ExprNodes, depending on how

394 # many slice elements were encountered.

395 pos = s.position()

396 if s.sy == '.':

397 expect_ellipsis(s)

398 return [ExprNodes.EllipsisNode(pos)]

399 else:

400 start = p_slice_element(s, (':',))

401 if s.sy != ':':

402 return [start]

403 s.next()

404 stop = p_slice_element(s, (':', ',', ']'))

405 if s.sy != ':':

406 return [start, stop]

407 s.next()

408 step = p_slice_element(s, (':', ',', ']'))

409 return [start, stop, step]

410

411 def p_slice_element(s, follow_set):

412 # Simple expression which may be missing iff

413 # it is followed by something in follow_set.

414 if s.sy not in follow_set:

415 return p_simple_expr(s)

416 else:

417 return None

418

419 def expect_ellipsis(s):

420 s.expect('.')

421 s.expect('.')

422 s.expect('.')

423

424 def make_slice_nodes(pos, subscripts):

425 # Convert a list of subscripts as returned

426 # by p_subscript_list into a list of ExprNodes,

427 # creating SliceNodes for elements with 2 or

428 # more components.

429 result = []

430 for subscript in subscripts:

431 if len(subscript) == 1:

432 result.append(subscript[0])

433 else:

434 result.append(make_slice_node(pos, *subscript))

435 return result

436

437 def make_slice_node(pos, start, stop = None, step = None):

438 if not start:

439 start = ExprNodes.NoneNode(pos)

440 if not stop:

441 stop = ExprNodes.NoneNode(pos)

442 if not step:

443 step = ExprNodes.NoneNode(pos)

444 return ExprNodes.SliceNode(pos,

445 start = start, stop = stop, step = step)

446

447 #atom: '(' [testlist] ')' | '[' [listmaker] ']' | '{' [dictmaker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+

448

449 def p_atom(s):

450 pos = s.position()

451 sy = s.sy

452 if sy == '(':

453 s.next()

454 if s.sy == ')':

455 result = ExprNodes.TupleNode(pos, args = [])

456 else:

457 result = p_expr(s)

458 s.expect(')')

459 return result

460 elif sy == '[':

461 return p_list_maker(s)

462 elif sy == '{':

463 return p_dict_maker(s)

464 elif sy == '`':

465 return p_backquote_expr(s)

466 elif sy == 'INT':

467 value = s.systring

468 s.next()

469 unsigned = ""

470 longness = ""

471 while value[-1] in "UuLl":

472 if value[-1] in "Ll":

473 longness += "L"

474 else:

475 unsigned += "U"

476 value = value[:-1]

477 return ExprNodes.IntNode(pos,

478 value = value,

479 unsigned = unsigned,

480 longness = longness)

481 elif sy == 'FLOAT':

482 value = s.systring

483 s.next()

484 return ExprNodes.FloatNode(pos, value = value)

485 elif sy == 'IMAG':

486 value = s.systring[:-1]

487 s.next()

488 return ExprNodes.ImagNode(pos, value = value)

489 elif sy == 'BEGIN_STRING':

490 kind, value = p_cat_string_literal(s)

491 if kind == 'c':

492 return ExprNodes.CharNode(pos, value = value)

493 else:

494 return ExprNodes.StringNode(pos, value = value)

495 elif sy == 'IDENT':

496 name = Utils.EncodedString( s.systring )

497 s.next()

498 if name == "None":

499 return ExprNodes.NoneNode(pos)

500 elif name == "True":

501 return ExprNodes.BoolNode(pos, value=True)

502 elif name == "False":

503 return ExprNodes.BoolNode(pos, value=False)

504 else:

505 return p_name(s, name)

506 elif sy == 'NULL':

507 s.next()

508 return ExprNodes.NullNode(pos)

509 else:

510 s.error("Expected an identifier or literal")

511

512 def p_name(s, name):

513 pos = s.position()

514 if not s.compile_time_expr:

515 try:

516 value = s.compile_time_env.lookup_here(name)

517 except KeyError:

518 pass

519 else:

520 rep = repr(value)

521 if isinstance(value, bool):

522 return ExprNodes.BoolNode(pos, value = value)

523 elif isinstance(value, int):

524 return ExprNodes.IntNode(pos, value = rep)

525 elif isinstance(value, long):

526 return ExprNodes.IntNode(pos, value = rep, longness = "L")

527 elif isinstance(value, float):

528 return ExprNodes.FloatNode(pos, value = rep)

529 elif isinstance(value, unicode):

530 return ExprNodes.StringNode(pos, value = value)

531 else:

532 error(pos, "Invalid type for compile-time constant: %s"

533 % value.__class__.__name__)

534 return ExprNodes.NameNode(pos, name = name)

535

536 def p_cat_string_literal(s):

537 # A sequence of one or more adjacent string literals.

538 # Returns (kind, value) where kind in ('b', 'c', 'u')

539 kind, value = p_string_literal(s)

540 if kind != 'c':

541 strings = [value]

542 while s.sy == 'BEGIN_STRING':

543 next_kind, next_value = p_string_literal(s)

544 if next_kind == 'c':

545 error(s.position(),

546 "Cannot concatenate char literal with another string or char literal")

547 elif next_kind == 'u':

548 kind = 'u'

549 strings.append(next_value)

550 value = Utils.EncodedString( u''.join(strings) )

551 if kind != 'u':

552 value.encoding = s.source_encoding

553 return kind, value

554

555 def p_opt_string_literal(s):

556 if s.sy == 'BEGIN_STRING':

557 return p_string_literal(s)

558 else:

559 return None

560

561 def p_string_literal(s):

562 # A single string or char literal.

563 # Returns (kind, value) where kind in ('b', 'c', 'u')

564 # s.sy == 'BEGIN_STRING'

565 pos = s.position()

566 is_raw = 0

567 kind = s.systring[:1].lower()

568 if kind == 'r':

569 kind = ''

570 is_raw = 1

571 elif kind in 'ub':

572 is_raw = s.systring[1:2].lower() == 'r'

573 elif kind != 'c':

574 kind = ''

575 if Future.unicode_literals in s.context.future_directives:

576 if kind == '':

577 kind = 'u'

578 elif kind == '':

579 kind = 'b'

580 chars = []

581 while 1:

582 s.next()

583 sy = s.sy

584 #print "p_string_literal: sy =", sy, repr(s.systring) ###

585 if sy == 'CHARS':

586 systr = s.systring

587 if len(systr) == 1 and systr in "'\"\n":

588 chars.append('\\')

589 chars.append(systr)

590 elif sy == 'ESCAPE':

591 systr = s.systring

592 if is_raw:

593 if systr == '\\\n':

594 chars.append(r'\\\n')

595 elif systr == r'\"':

596 chars.append(r'\\\"')

597 elif systr == r'\\':

598 chars.append(r'\\\\')

599 else:

600 chars.append('\\' + systr)

601 else:

602 c = systr[1]

603 if c in "'\"\\abfnrtv01234567":

604 chars.append(systr)

605 elif c == '\n':

606 pass

607 elif c in 'ux':

608 if kind == 'u':

609 try:

610 chars.append(

611 systr.encode("ASCII").decode('unicode_escape'))

612 except UnicodeDecodeError:

613 s.error("Invalid unicode escape '%s'" % systr,

614 pos = pos)

615 elif c == 'x':

616 chars.append('\\x0' + systr[2:])

617 else:

618 chars.append(systr)

619 else:

620 chars.append(r'\\' + systr[1:])

621 elif sy == 'NEWLINE':

622 chars.append(r'\n')

623 elif sy == 'END_STRING':

624 break

625 elif sy == 'EOF':

626 s.error("Unclosed string literal", pos = pos)

627 else:

628 s.error(

629 "Unexpected token %r:%r in string literal" %

630 (sy, s.systring))

631 s.next()

632 value = Utils.EncodedString( u''.join(chars) )

633 if kind != 'u':

634 value.encoding = s.source_encoding

635 #print "p_string_literal: value =", repr(value) ###

636 return kind, value

637

638 # list_display ::= "[" [listmaker] "]"

639 # listmaker ::= expression ( list_for | ( "," expression )* [","] )

640 # list_iter ::= list_for | list_if

641 # list_for ::= "for" expression_list "in" testlist [list_iter]

642 # list_if ::= "if" test [list_iter]

643

644 def p_list_maker(s):

645 # s.sy == '['

646 pos = s.position()

647 s.next()

648 if s.sy == ']':

649 s.expect(']')

650 return ExprNodes.ListNode(pos, args = [])

651 expr = p_simple_expr(s)

652 if s.sy == 'for':

653 loop = p_list_for(s)

654 s.expect(']')

655 inner_loop = loop

656 while not isinstance(inner_loop.body, Nodes.PassStatNode):

657 inner_loop = inner_loop.body

658 if isinstance(inner_loop, Nodes.IfStatNode):

659 inner_loop = inner_loop.if_clauses[0]

660 append = ExprNodes.ListComprehensionAppendNode( pos, expr = expr )

661 inner_loop.body = Nodes.ExprStatNode(pos, expr = append)

662 return ExprNodes.ListComprehensionNode(pos, loop = loop, append = append)

663 else:

664 exprs = [expr]

665 if s.sy == ',':

666 s.next()

667 exprs += p_simple_expr_list(s)

668 s.expect(']')

669 return ExprNodes.ListNode(pos, args = exprs)

670

671 def p_list_iter(s):

672 if s.sy == 'for':

673 return p_list_for(s)

674 elif s.sy == 'if':

675 return p_list_if(s)

676 else:

677 return Nodes.PassStatNode(s.position())

678

679 def p_list_for(s):

680 # s.sy == 'for'

681 pos = s.position()

682 s.next()

683 kw = p_for_bounds(s)

684 kw['else_clause'] = None

685 kw['body'] = p_list_iter(s)

686 return Nodes.ForStatNode(pos, **kw)

687

688 def p_list_if(s):

689 # s.sy == 'if'

690 pos = s.position()

691 s.next()

692 test = p_test(s)

693 return Nodes.IfStatNode(pos,

694 if_clauses = [Nodes.IfClauseNode(pos, condition = test, body = p_list_iter(s))],

695 else_clause = None )

696

697 #dictmaker: test ':' test (',' test ':' test)* [',']

698

699 def p_dict_maker(s):

700 # s.sy == '{'

701 pos = s.position()

702 s.next()

703 items = []

704 while s.sy != '}':

705 items.append(p_dict_item(s))

706 if s.sy != ',':

707 break

708 s.next()

709 s.expect('}')

710 return ExprNodes.DictNode(pos, key_value_pairs = items)

711

712 def p_dict_item(s):

713 key = p_simple_expr(s)

714 s.expect(':')

715 value = p_simple_expr(s)

716 return ExprNodes.DictItemNode(key.pos, key=key, value=value)

717

718 def p_backquote_expr(s):

719 # s.sy == '`'

720 pos = s.position()

721 s.next()

722 arg = p_expr(s)

723 s.expect('`')

724 return ExprNodes.BackquoteNode(pos, arg = arg)

725

726 def p_simple_expr_list(s):

727 exprs = []

728 while s.sy not in expr_terminators:

729 exprs.append(p_simple_expr(s))

730 if s.sy != ',':

731 break

732 s.next()

733 return exprs

734

735 def p_expr(s):

736 pos = s.position()

737 expr = p_simple_expr(s)

738 if s.sy == ',':

739 s.next()

740 exprs = [expr] + p_simple_expr_list(s)

741 return ExprNodes.TupleNode(pos, args = exprs)

742 else:

743 return expr

744

745

746 #testlist: test (',' test)* [',']

747 # differs from p_expr only in the fact that it cannot contain conditional expressions

748

749 def p_testlist(s):

750 pos = s.position()

751 expr = p_test(s)

752 if s.sy == ',':

753 exprs = [expr]

754 while s.sy == ',':

755 s.next()

756 exprs.append(p_test(s))

757 return ExprNodes.TupleNode(pos, args = exprs)

758 else:

759 return expr

760

761 expr_terminators = (')', ']', '}', ':', '=', 'NEWLINE')

762

763 #-------------------------------------------------------

764 #

765 # Statements

766 #

767 #-------------------------------------------------------

768

769 def p_global_statement(s):

770 # assume s.sy == 'global'

771 pos = s.position()

772 s.next()

773 names = p_ident_list(s)

774 return Nodes.GlobalNode(pos, names = names)

775

776 def p_expression_or_assignment(s):

777 expr_list = [p_expr(s)]

778 while s.sy == '=':

779 s.next()

780 expr_list.append(p_expr(s))

781 if len(expr_list) == 1:

782 if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):

783 lhs = expr_list[0]

784 if not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):

785 error(lhs.pos, "Illegal operand for inplace operation.")

786 operator = s.sy[:-1]

787 s.next()

788 rhs = p_expr(s)

789 return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)

790 expr = expr_list[0]

791 if isinstance(expr, ExprNodes.StringNode):

792 return Nodes.PassStatNode(expr.pos)

793 else:

794 return Nodes.ExprStatNode(expr.pos, expr = expr)

795 else:

796 expr_list_list = []

797 flatten_parallel_assignments(expr_list, expr_list_list)

798 nodes = []

799 for expr_list in expr_list_list:

800 lhs_list = expr_list[:-1]

801 rhs = expr_list[-1]

802 if len(lhs_list) == 1:

803 node = Nodes.SingleAssignmentNode(rhs.pos,

804 lhs = lhs_list[0], rhs = rhs)

805 else:

806 node = Nodes.CascadedAssignmentNode(rhs.pos,

807 lhs_list = lhs_list, rhs = rhs)

808 nodes.append(node)

809 if len(nodes) == 1:

810 return nodes[0]

811 else:

812 return Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

813

814 def flatten_parallel_assignments(input, output):

815 # The input is a list of expression nodes, representing

816 # the LHSs and RHS of one (possibly cascaded) assignment

817 # statement. If they are all sequence constructors with

818 # the same number of arguments, rearranges them into a

819 # list of equivalent assignments between the individual

820 # elements. This transformation is applied recursively.

821 size = find_parallel_assignment_size(input)

822 if size >= 0:

823 for i in range(size):

824 new_exprs = [expr.args[i] for expr in input]

825 flatten_parallel_assignments(new_exprs, output)

826 else:

827 output.append(input)

828

829 def find_parallel_assignment_size(input):

830 # The input is a list of expression nodes. If

831 # they are all sequence constructors with the same number

832 # of arguments, return that number, else return -1.

833 # Produces an error message if they are all sequence

834 # constructors but not all the same size.

835 for expr in input:

836 if not expr.is_sequence_constructor:

837 return -1

838 rhs = input[-1]

839 rhs_size = len(rhs.args)

840 for lhs in input[:-1]:

841 lhs_size = len(lhs.args)

842 if lhs_size != rhs_size:

843 error(lhs.pos, "Unpacking sequence of wrong size (expected %d, got %d)"

844 % (lhs_size, rhs_size))

845 return -1

846 return rhs_size

847

848 def p_print_statement(s):

849 # s.sy == 'print'

850 pos = s.position()

851 s.next()

852 if s.sy == '>>':

853 s.error("'print >>' not yet implemented")

854 args = []

855 ends_with_comma = 0

856 if s.sy not in ('NEWLINE', 'EOF'):

857 args.append(p_simple_expr(s))

858 while s.sy == ',':

859 s.next()

860 if s.sy in ('NEWLINE', 'EOF'):

861 ends_with_comma = 1

862 break

863 args.append(p_simple_expr(s))

864 arg_tuple = ExprNodes.TupleNode(pos, args = args)

865 return Nodes.PrintStatNode(pos,

866 arg_tuple = arg_tuple, append_newline = not ends_with_comma)

867

868 def p_del_statement(s):

869 # s.sy == 'del'

870 pos = s.position()

871 s.next()

872 args = p_simple_expr_list(s)

873 return Nodes.DelStatNode(pos, args = args)

874

875 def p_pass_statement(s, with_newline = 0):

876 pos = s.position()

877 s.expect('pass')

878 if with_newline:

879 s.expect_newline("Expected a newline")

880 return Nodes.PassStatNode(pos)

881

882 def p_break_statement(s):

883 # s.sy == 'break'

884 pos = s.position()

885 s.next()

886 return Nodes.BreakStatNode(pos)

887

888 def p_continue_statement(s):

889 # s.sy == 'continue'

890 pos = s.position()

891 s.next()

892 return Nodes.ContinueStatNode(pos)

893

894 def p_return_statement(s):

895 # s.sy == 'return'

896 pos = s.position()

897 s.next()

898 if s.sy not in statement_terminators:

899 value = p_expr(s)

900 else:

901 value = None

902 return Nodes.ReturnStatNode(pos, value = value)

903

904 def p_raise_statement(s):

905 # s.sy == 'raise'

906 pos = s.position()

907 s.next()

908 exc_type = None

909 exc_value = None

910 exc_tb = None

911 if s.sy not in statement_terminators:

912 exc_type = p_simple_expr(s)

913 if s.sy == ',':

914 s.next()

915 exc_value = p_simple_expr(s)

916 if s.sy == ',':

917 s.next()

918 exc_tb = p_simple_expr(s)

919 if exc_type or exc_value or exc_tb:

920 return Nodes.RaiseStatNode(pos,

921 exc_type = exc_type,

922 exc_value = exc_value,

923 exc_tb = exc_tb)

924 else:

925 return Nodes.ReraiseStatNode(pos)

926

927 def p_import_statement(s):

928 # s.sy in ('import', 'cimport')

929 pos = s.position()

930 kind = s.sy

931 s.next()

932 items = [p_dotted_name(s, as_allowed = 1)]

933 while s.sy == ',':

934 s.next()

935 items.append(p_dotted_name(s, as_allowed = 1))

936 stats = []

937 for pos, target_name, dotted_name, as_name in items:

938 dotted_name = Utils.EncodedString(dotted_name)

939 if kind == 'cimport':

940 stat = Nodes.CImportStatNode(pos,

941 module_name = dotted_name,

942 as_name = as_name)

943 else:

944 if as_name and "." in dotted_name:

945 name_list = ExprNodes.ListNode(pos, args = [

946 ExprNodes.StringNode(pos, value = Utils.EncodedString("*"))])

947 else:

948 name_list = None

949 stat = Nodes.SingleAssignmentNode(pos,

950 lhs = ExprNodes.NameNode(pos,

951 name = as_name or target_name),

952 rhs = ExprNodes.ImportNode(pos,

953 module_name = ExprNodes.IdentifierStringNode(

954 pos, value = dotted_name),

955 name_list = name_list))

956 stats.append(stat)

957 return Nodes.StatListNode(pos, stats = stats)

958

959 def p_from_import_statement(s, first_statement = 0):

960 # s.sy == 'from'

961 pos = s.position()

962 s.next()

963 (dotted_name_pos, _, dotted_name, _) = \

964 p_dotted_name(s, as_allowed = 0)

965 if s.sy in ('import', 'cimport'):

966 kind = s.sy

967 s.next()

968 else:

969 s.error("Expected 'import' or 'cimport'")

970 is_cimport = kind == 'cimport'

971 if s.sy == '*':

972 imported_names = [(s.position(), "*", None, None)]

973 s.next()

974 else:

975 imported_names = [p_imported_name(s, is_cimport)]

976 while s.sy == ',':

977 s.next()

978 imported_names.append(p_imported_name(s, is_cimport))

979 dotted_name = Utils.EncodedString(dotted_name)

980 if dotted_name == '__future__':

981 if not first_statement:

982 s.error("from __future__ imports must occur at the beginning of the file")

983 else:

984 for (name_pos, name, as_name, kind) in imported_names:

985 if name == "braces":

986 s.error("not a chance", name_pos)

987 break

988 try:

989 directive = getattr(Future, name)

990 except AttributeError:

991 s.error("future feature %s is not defined" % name, name_pos)

992 break

993 s.context.future_directives.add(directive)

994 return Nodes.PassStatNode(pos)

995 elif kind == 'cimport':

996 for (name_pos, name, as_name, kind) in imported_names:

997 local_name = as_name or name

998 s.add_type_name(local_name)

999 return Nodes.FromCImportStatNode(pos,

1000 module_name = dotted_name,

1001 imported_names = imported_names)

1002 else:

1003 imported_name_strings = []

1004 items = []

1005 for (name_pos, name, as_name, kind) in imported_names:

1006 encoded_name = Utils.EncodedString(name)

1007 imported_name_strings.append(

1008 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))

1009 items.append(

1010 (name,

1011 ExprNodes.NameNode(name_pos,

1012 name = as_name or name)))

1013 import_list = ExprNodes.ListNode(

1014 imported_names[0][0], args = imported_name_strings)

1015 dotted_name = Utils.EncodedString(dotted_name)

1016 return Nodes.FromImportStatNode(pos,

1017 module = ExprNodes.ImportNode(dotted_name_pos,

1018 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),

1019 name_list = import_list),

1020 items = items)

1021

1022 imported_name_kinds = ('class', 'struct', 'union')

1023

1024 def p_imported_name(s, is_cimport):

1025 pos = s.position()

1026 kind = None

1027 if is_cimport and s.systring in imported_name_kinds:

1028 kind = s.systring

1029 s.next()

1030 name = p_ident(s)

1031 as_name = p_as_name(s)

1032 return (pos, name, as_name, kind)

1033

1034 def p_dotted_name(s, as_allowed):

1035 pos = s.position()

1036 target_name = p_ident(s)

1037 as_name = None

1038 names = [target_name]

1039 while s.sy == '.':

1040 s.next()

1041 names.append(p_ident(s))

1042 if as_allowed:

1043 as_name = p_as_name(s)

1044 return (pos, target_name, u'.'.join(names), as_name)

1045

1046 def p_as_name(s):

1047 if s.sy == 'IDENT' and s.systring == 'as':

1048 s.next()

1049 return p_ident(s)

1050 else:

1051 return None

1052

1053 def p_assert_statement(s):

1054 # s.sy == 'assert'

1055 pos = s.position()

1056 s.next()

1057 cond = p_simple_expr(s)

1058 if s.sy == ',':

1059 s.next()

1060 value = p_simple_expr(s)

1061 else:

1062 value = None

1063 return Nodes.AssertStatNode(pos, cond = cond, value = value)

1064

1065 statement_terminators = (';', 'NEWLINE', 'EOF')

1066

1067 def p_if_statement(s):

1068 # s.sy == 'if'

1069 pos = s.position()

1070 s.next()

1071 if_clauses = [p_if_clause(s)]

1072 while s.sy == 'elif':

1073 s.next()

1074 if_clauses.append(p_if_clause(s))

1075 else_clause = p_else_clause(s)

1076 return Nodes.IfStatNode(pos,

1077 if_clauses = if_clauses, else_clause = else_clause)

1078

1079 def p_if_clause(s):

1080 pos = s.position()

1081 test = p_simple_expr(s)

1082 body = p_suite(s)

1083 return Nodes.IfClauseNode(pos,

1084 condition = test, body = body)

1085

1086 def p_else_clause(s):

1087 if s.sy == 'else':

1088 s.next()

1089 return p_suite(s)

1090 else:

1091 return None

1092

1093 def p_while_statement(s):

1094 # s.sy == 'while'

1095 pos = s.position()

1096 s.next()

1097 test = p_simple_expr(s)

1098 body = p_suite(s)

1099 else_clause = p_else_clause(s)

1100 return Nodes.WhileStatNode(pos,

1101 condition = test, body = body,

1102 else_clause = else_clause)

1103

1104 def p_for_statement(s):

1105 # s.sy == 'for'

1106 pos = s.position()

1107 s.next()

1108 kw = p_for_bounds(s)

1109 kw['body'] = p_suite(s)

1110 kw['else_clause'] = p_else_clause(s)

1111 return Nodes.ForStatNode(pos, **kw)

1112

1113 def p_for_bounds(s):

1114 target = p_for_target(s)

1115 if s.sy == 'in':

1116 s.next()

1117 iterator = p_for_iterator(s)

1118 return { 'target': target, 'iterator': iterator }

1119 else:

1120 if s.sy == 'from':

1121 s.next()

1122 bound1 = p_bit_expr(s)

1123 else:

1124 # Support shorter "for a <= x < b" syntax

1125 bound1, target = target, None

1126 rel1 = p_for_from_relation(s)

1127 name2_pos = s.position()

1128 name2 = p_ident(s)

1129 rel2_pos = s.position()

1130 rel2 = p_for_from_relation(s)

1131 bound2 = p_bit_expr(s)

1132 step = p_for_from_step(s)

1133 if target is None:

1134 target = ExprNodes.NameNode(name2_pos, name = name2)

1135 else:

1136 if not target.is_name:

1137 error(target.pos,

1138 "Target of for-from statement must be a variable name")

1139 elif name2 != target.name:

1140 error(name2_pos,

1141 "Variable name in for-from range does not match target")

1142 if rel1[0] != rel2[0]:

1143 error(rel2_pos,

1144 "Relation directions in for-from do not match")

1145 return {'target': target,

1146 'bound1': bound1,

1147 'relation1': rel1,

1148 'relation2': rel2,

1149 'bound2': bound2,

1150 'step': step }

1151

1152 def p_for_from_relation(s):

1153 if s.sy in inequality_relations:

1154 op = s.sy

1155 s.next()

1156 return op

1157 else:

1158 s.error("Expected one of '<', '<=', '>' '>='")

1159

1160 def p_for_from_step(s):

1161 if s.sy == 'by':

1162 s.next()

1163 step = p_bit_expr(s)

1164 return step

1165 else:

1166 return None

1167

1168 inequality_relations = ('<', '<=', '>', '>=')

1169

1170 def p_target(s, terminator):

1171 pos = s.position()

1172 expr = p_bit_expr(s)

1173 if s.sy == ',':

1174 s.next()

1175 exprs = [expr]

1176 while s.sy != terminator:

1177 exprs.append(p_bit_expr(s))

1178 if s.sy != ',':

1179 break

1180 s.next()

1181 return ExprNodes.TupleNode(pos, args = exprs)

1182 else:

1183 return expr

1184

1185 def p_for_target(s):

1186 return p_target(s, 'in')

1187

1188 def p_for_iterator(s):

1189 pos = s.position()

1190 expr = p_testlist(s)

1191 return ExprNodes.IteratorNode(pos, sequence = expr)

1192

1193 def p_try_statement(s):

1194 # s.sy == 'try'

1195 pos = s.position()

1196 s.next()

1197 body = p_suite(s)

1198 except_clauses = []

1199 else_clause = None

1200 if s.sy in ('except', 'else'):

1201 while s.sy == 'except':

1202 except_clauses.append(p_except_clause(s))

1203 if s.sy == 'else':

1204 s.next()

1205 else_clause = p_suite(s)

1206 body = Nodes.TryExceptStatNode(pos,

1207 body = body, except_clauses = except_clauses,

1208 else_clause = else_clause)

1209 if s.sy != 'finally':

1210 return body

1211 # try-except-finally is equivalent to nested try-except/try-finally

1212 if s.sy == 'finally':

1213 s.next()

1214 finally_clause = p_suite(s)

1215 return Nodes.TryFinallyStatNode(pos,

1216 body = body, finally_clause = finally_clause)

1217 else:

1218 s.error("Expected 'except' or 'finally'")

1219

1220 def p_except_clause(s):

1221 # s.sy == 'except'

1222 pos = s.position()

1223 s.next()

1224 exc_type = None

1225 exc_value = None

1226 if s.sy != ':':

1227 exc_type = p_simple_expr(s)

1228 if s.sy == ',':

1229 s.next()

1230 exc_value = p_simple_expr(s)

1231 body = p_suite(s)

1232 return Nodes.ExceptClauseNode(pos,

1233 pattern = exc_type, target = exc_value, body = body)

1234

1235 def p_include_statement(s, ctx):

1236 pos = s.position()

1237 s.next() # 'include'

1238 _, include_file_name = p_string_literal(s)

1239 s.expect_newline("Syntax error in include statement")

1240 if s.compile_time_eval:

1241 include_file_path = s.context.find_include_file(include_file_name, pos)

1242 if include_file_path:

1243 s.included_files.append(include_file_name)

1244 f = Utils.open_source_file(include_file_path, mode="rU")

1245 source_desc = FileSourceDescriptor(include_file_path)

1246 s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding)

1247 try:

1248 tree = p_statement_list(s2, ctx)

1249 finally:

1250 f.close()

1251 return tree

1252 else:

1253 return None

1254 else:

1255 return Nodes.PassStatNode(pos)

1256

1257 def p_with_statement(s):

1258 pos = s.position()

1259 s.next() # 'with'

1260 # if s.sy == 'IDENT' and s.systring in ('gil', 'nogil'):

1261 if s.sy == 'IDENT' and s.systring == 'nogil':

1262 state = s.systring

1263 s.next()

1264 body = p_suite(s)

1265 return Nodes.GILStatNode(pos, state = state, body = body)

1266 else:

1267 manager = p_expr(s)

1268 target = None

1269 if s.sy == 'IDENT' and s.systring == 'as':

1270 s.next()

1271 allow_multi = (s.sy == '(')

1272 target = p_target(s, ':')

1273 if not allow_multi and isinstance(target, ExprNodes.TupleNode):

1274 s.error("Multiple with statement target values not allowed without paranthesis")

1275 body = p_suite(s)

1276 return Nodes.WithStatNode(pos, manager = manager,

1277 target = target, body = body)

1278

1279 def p_simple_statement(s, first_statement = 0):

1280 #print "p_simple_statement:", s.sy, s.systring ###

1281 if s.sy == 'global':

1282 node = p_global_statement(s)

1283 elif s.sy == 'print':

1284 node = p_print_statement(s)

1285 elif s.sy == 'del':

1286 node = p_del_statement(s)

1287 elif s.sy == 'break':

1288 node = p_break_statement(s)

1289 elif s.sy == 'continue':

1290 node = p_continue_statement(s)

1291 elif s.sy == 'return':

1292 node = p_return_statement(s)

1293 elif s.sy == 'raise':

1294 node = p_raise_statement(s)

1295 elif s.sy in ('import', 'cimport'):

1296 node = p_import_statement(s)

1297 elif s.sy == 'from':

1298 node = p_from_import_statement(s, first_statement = first_statement)

1299 elif s.sy == 'assert':

1300 node = p_assert_statement(s)

1301 elif s.sy == 'pass':

1302 node = p_pass_statement(s)

1303 else:

1304 node = p_expression_or_assignment(s)

1305 return node

1306

1307 def p_simple_statement_list(s, ctx, first_statement = 0):

1308 # Parse a series of simple statements on one line

1309 # separated by semicolons.

1310 stat = p_simple_statement(s, first_statement = first_statement)

1311 if s.sy == ';':

1312 stats = [stat]

1313 while s.sy == ';':

1314 #print "p_simple_statement_list: maybe more to follow" ###

1315 s.next()

1316 if s.sy in ('NEWLINE', 'EOF'):

1317 break

1318 stats.append(p_simple_statement(s))

1319 stat = Nodes.StatListNode(stats[0].pos, stats = stats)

1320 s.expect_newline("Syntax error in simple statement list")

1321 return stat

1322

1323 def p_compile_time_expr(s):

1324 old = s.compile_time_expr

1325 s.compile_time_expr = 1

1326 expr = p_expr(s)

1327 s.compile_time_expr = old

1328 return expr

1329

1330 def p_DEF_statement(s):

1331 pos = s.position()

1332 denv = s.compile_time_env

1333 s.next() # 'DEF'

1334 name = p_ident(s)

1335 s.expect('=')

1336 expr = p_compile_time_expr(s)

1337 value = expr.compile_time_value(denv)

1338 #print "p_DEF_statement: %s = %r" % (name, value) ###

1339 denv.declare(name, value)

1340 s.expect_newline()

1341 return Nodes.PassStatNode(pos)

1342

1343 def p_IF_statement(s, ctx):

1344 pos = s.position()

1345 saved_eval = s.compile_time_eval

1346 current_eval = saved_eval

1347 denv = s.compile_time_env

1348 result = None

1349 while 1:

1350 s.next() # 'IF' or 'ELIF'

1351 expr = p_compile_time_expr(s)

1352 s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))

1353 body = p_suite(s, ctx)

1354 if s.compile_time_eval:

1355 result = body

1356 current_eval = 0

1357 if s.sy != 'ELIF':

1358 break

1359 if s.sy == 'ELSE':

1360 s.next()

1361 s.compile_time_eval = current_eval

1362 body = p_suite(s, ctx)

1363 if current_eval:

1364 result = body

1365 if not result:

1366 result = Nodes.PassStatNode(pos)

1367 s.compile_time_eval = saved_eval

1368 return result

1369

1370 def p_statement(s, ctx, first_statement = 0):

1371 cdef_flag = ctx.cdef_flag

1372 if s.sy == 'ctypedef':

1373 if ctx.level not in ('module', 'module_pxd'):

1374 s.error("ctypedef statement not allowed here")

1375 if ctx.api:

1376 error(s.position(), "'api' not allowed with 'ctypedef'")

1377 return p_ctypedef_statement(s, ctx)

1378 elif s.sy == 'DEF':

1379 return p_DEF_statement(s)

1380 elif s.sy == 'IF':

1381 return p_IF_statement(s, ctx)

1382 elif s.sy == 'DECORATOR':

1383 if ctx.level not in ('module', 'class', 'c_class', 'property'):

1384 s.error('decorator not allowed here')

1385 s.level = ctx.level

1386 decorators = p_decorators(s)

1387 if s.sy != 'def':

1388 s.error("Decorators can only be followed by functions ")

1389 return p_def_statement(s, decorators)

1390 else:

1391 overridable = 0

1392 if s.sy == 'cdef':

1393 cdef_flag = 1

1394 s.next()

1395 if s.sy == 'cpdef':

1396 cdef_flag = 1

1397 overridable = 1

1398 s.next()

1399 if cdef_flag:

1400 if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):

1401 s.error('cdef statement not allowed here')

1402 s.level = ctx.level

1403 return p_cdef_statement(s, ctx(overridable = overridable))

1404 else:

1405 if ctx.api:

1406 error(s.pos, "'api' not allowed with this statement")

1407 elif s.sy == 'def':

1408 if ctx.level not in ('module', 'class', 'c_class', 'property'):

1409 s.error('def statement not allowed here')

1410 s.level = ctx.level

1411 return p_def_statement(s)

1412 elif s.sy == 'class':

1413 if ctx.level != 'module':

1414 s.error("class definition not allowed here")

1415 return p_class_statement(s)

1416 elif s.sy == 'include':

1417 if ctx.level not in ('module', 'module_pxd'):

1418 s.error("include statement not allowed here")

1419 return p_include_statement(s, ctx)

1420 elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':

1421 return p_property_decl(s)

1422 elif s.sy == 'pass' and ctx.level != 'property':

1423 return p_pass_statement(s, with_newline = 1)

1424 else:

1425 if ctx.level in ('c_class_pxd', 'property'):

1426 s.error("Executable statement not allowed here")

1427 if s.sy == 'if':

1428 return p_if_statement(s)

1429 elif s.sy == 'while':

1430 return p_while_statement(s)

1431 elif s.sy == 'for':

1432 return p_for_statement(s)

1433 elif s.sy == 'try':

1434 return p_try_statement(s)

1435 elif s.sy == 'with':

1436 return p_with_statement(s)

1437 else:

1438 return p_simple_statement_list(

1439 s, ctx, first_statement = first_statement)

1440

1441 def p_statement_list(s, ctx, first_statement = 0):

1442 # Parse a series of statements separated by newlines.

1443 pos = s.position()

1444 stats = []

1445 while s.sy not in ('DEDENT', 'EOF'):

1446 stats.append(p_statement(s, ctx, first_statement = first_statement))

1447 first_statement = 0

1448 if len(stats) == 1:

1449 return stats[0]

1450 else:

1451 return Nodes.StatListNode(pos, stats = stats)

1452

1453 def p_suite(s, ctx = Ctx(), with_doc = 0, with_pseudo_doc = 0):

1454 pos = s.position()

1455 s.expect(':')

1456 doc = None

1457 stmts = []

1458 if s.sy == 'NEWLINE':

1459 s.next()

1460 s.expect_indent()

1461 if with_doc or with_pseudo_doc:

1462 doc = p_doc_string(s)

1463 body = p_statement_list(s, ctx)

1464 s.expect_dedent()

1465 else:

1466 if ctx.api:

1467 error(s.pos, "'api' not allowed with this statement")

1468 if ctx.level in ('module', 'class', 'function', 'other'):

1469 body = p_simple_statement_list(s, ctx)

1470 else:

1471 body = p_pass_statement(s)

1472 s.expect_newline("Syntax error in declarations")

1473 if with_doc:

1474 return doc, body

1475 else:

1476 return body

1477

1478 def p_positional_and_keyword_args(s, end_sy_set, type_positions=(), type_keywords=()):

1479 """

1480 Parses positional and keyword arguments. end_sy_set

1481 should contain any s.sy that terminate the argument list.

1482 Argument expansion (* and **) are not allowed.

1483

1484 type_positions and type_keywords specifies which argument

1485 positions and/or names which should be interpreted as

1486 types. Other arguments will be treated as expressions.

1487

1488 Returns: (positional_args, keyword_args)

1489 """

1490 positional_args = []

1491 keyword_args = []

1492 pos_idx = 0

1493

1494 while s.sy not in end_sy_set:

1495 if s.sy == '*' or s.sy == '**':

1496 s.error('Argument expansion not allowed here.')

1497

1498 was_keyword = False

1499 parsed_type = False

1500 if s.sy == 'IDENT':

1501 # Since we can have either types or expressions as positional args,

1502 # we use a strategy of looking an extra step forward for a '=' and

1503 # if it is a positional arg we backtrack.

1504 ident = s.systring

1505 s.next()

1506 if s.sy == '=':

1507 s.next()

1508 # Is keyword arg

1509 if ident in type_keywords:

1510 arg = p_c_base_type(s)

1511 parsed_type = True

1512 else:

1513 arg = p_simple_expr(s)

1514 keyword_node = ExprNodes.IdentifierStringNode(arg.pos,

1515 value = Utils.EncodedString(ident))

1516 keyword_args.append((keyword_node, arg))

1517 was_keyword = True

1518 else:

1519 s.put_back('IDENT', ident)

1520

1521 if not was_keyword:

1522 if pos_idx in type_positions:

1523 arg = p_c_base_type(s)

1524 parsed_type = True

1525 else:

1526 arg = p_simple_expr(s)

1527 positional_args.append(arg)

1528 pos_idx += 1

1529 if len(keyword_args) > 0:

1530 s.error("Non-keyword arg following keyword arg",

1531 pos = arg.pos)

1532

1533 if s.sy != ',':

1534 if s.sy not in end_sy_set:

1535 if parsed_type:

1536 s.error("Expected: type")

1537 else:

1538 s.error("Expected: expression")

1539 break

1540 s.next()

1541 return positional_args, keyword_args

1542

1543 def p_c_base_type(s, self_flag = 0, nonempty = 0):

1544 # If self_flag is true, this is the base type for the

1545 # self argument of a C method of an extension type.

1546 if s.sy == '(':

1547 return p_c_complex_base_type(s)

1548 else:

1549 return p_c_simple_base_type(s, self_flag, nonempty = nonempty)

1550

1551 def p_calling_convention(s):

1552 if s.sy == 'IDENT' and s.systring in calling_convention_words:

1553 result = s.systring

1554 s.next()

1555 return result

1556 else:

1557 return ""

1558

1559 calling_convention_words = ("__stdcall", "__cdecl")

1560

1561 def p_c_complex_base_type(s):

1562 # s.sy == '('

1563 pos = s.position()

1564 s.next()

1565 base_type = p_c_base_type(s)

1566 declarator = p_c_declarator(s, empty = 1)

1567 s.expect(')')

1568 return Nodes.CComplexBaseTypeNode(pos,

1569 base_type = base_type, declarator = declarator)

1570

1571 def p_c_simple_base_type(s, self_flag, nonempty):

1572 #print "p_c_simple_base_type: self_flag =", self_flag

1573 is_basic = 0

1574 signed = 1

1575 longness = 0

1576 module_path = []

1577 pos = s.position()

1578 if looking_at_base_type(s):

1579 #print "p_c_simple_base_type: looking_at_base_type at", s.position()

1580 is_basic = 1

1581 signed, longness = p_sign_and_longness(s)

1582 if s.sy == 'IDENT' and s.systring in basic_c_type_names:

1583 name = s.systring

1584 s.next()

1585 else:

1586 name = 'int'

1587 elif s.looking_at_type_name():

1588 name = s.systring

1589 s.next()

1590 if nonempty and s.sy != 'IDENT':

1591 # Make sure this is not a declaration of a variable or

1592 # function with the same name as a type.

1593 if s.sy == '(':

1594 s.next()

1595 if s.sy == '*' or s.sy == '**':

1596 s.put_back('(', '(')

1597 else:

1598 s.put_back('(', '(')

1599 s.put_back('IDENT', name)

1600 name = None

1601 elif s.sy not in ('*', '**', '['):

1602 s.put_back('IDENT', name)

1603 name = None

1604 elif looking_at_dotted_name(s):

1605 #print "p_c_simple_base_type: looking_at_type_name at", s.position()

1606 name = s.systring

1607 s.next()

1608 while s.sy == '.':

1609 module_path.append(name)

1610 s.next()

1611 name = p_ident(s)

1612 else:

1613 #print "p_c_simple_base_type: not looking at type at", s.position()

1614 name = None

1615

1616 type_node = Nodes.CSimpleBaseTypeNode(pos,

1617 name = name, module_path = module_path,

1618 is_basic_c_type = is_basic, signed = signed,

1619 longness = longness, is_self_arg = self_flag)

1620

1621

1622 # Treat trailing [] on type as buffer access

1623 if not is_basic and s.sy == '[':

1624 return p_buffer_access(s, type_node)

1625 else:

1626 return type_node

1627

1628 def p_buffer_access(s, type_node):

1629 # s.sy == '['

1630 pos = s.position()

1631 s.next()

1632 positional_args, keyword_args = (

1633 p_positional_and_keyword_args(s, (']',), (0,), ('dtype',))

1634 )

1635 s.expect(']')

1636

1637 keyword_dict = ExprNodes.DictNode(pos,

1638 key_value_pairs = [

1639 ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)

1640 for key, value in keyword_args

1641 ])

1642

1643 result = Nodes.CBufferAccessTypeNode(pos,

1644 positional_args = positional_args,

1645 keyword_args = keyword_dict,

1646 base_type_node = type_node)

1647 return result

1648

1649

1650 def looking_at_type(s):

1651 return looking_at_base_type(s) or s.looking_at_type_name()

1652

1653 def looking_at_base_type(s):

1654 #print "looking_at_base_type?", s.sy, s.systring, s.position()

1655 return s.sy == 'IDENT' and s.systring in base_type_start_words

1656

1657 def looking_at_dotted_name(s):

1658 if s.sy == 'IDENT':

1659 name = s.systring

1660 s.next()

1661 result = s.sy == '.'

1662 s.put_back('IDENT', name)

1663 return result

1664 else:

1665 return 0

1666

1667 basic_c_type_names = ("void", "char", "int", "float", "double", "Py_ssize_t", "bint")

1668

1669 sign_and_longness_words = ("short", "long", "signed", "unsigned")

1670

1671 base_type_start_words = \

1672 basic_c_type_names + sign_and_longness_words

1673

1674 def p_sign_and_longness(s):

1675 signed = 1

1676 longness = 0

1677 while s.sy == 'IDENT' and s.systring in sign_and_longness_words:

1678 if s.systring == 'unsigned':

1679 signed = 0

1680 elif s.systring == 'signed':

1681 signed = 2

1682 elif s.systring == 'short':

1683 longness = -1

1684 elif s.systring == 'long':

1685 longness += 1

1686 s.next()

1687 return signed, longness

1688

1689 def p_opt_cname(s):

1690 literal = p_opt_string_literal(s)

1691 if literal:

1692 _, cname = literal

1693 else:

1694 cname = None

1695 return cname

1696

1697 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,

1698 assignable = 0, nonempty = 0,

1699 calling_convention_allowed = 0):

1700 # If empty is true, the declarator must be empty. If nonempty is true,

1701 # the declarator must be nonempty. Otherwise we don't care.

1702 # If cmethod_flag is true, then if this declarator declares

1703 # a function, it's a C method of an extension type.

1704 pos = s.position()

1705 if s.sy == '(':

1706 s.next()

1707 if s.sy == ')' or looking_at_type(s):

1708 base = Nodes.CNameDeclaratorNode(pos, name = "", cname = None)

1709 result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)

1710 else:

1711 result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1712 cmethod_flag = cmethod_flag,

1713 nonempty = nonempty,

1714 calling_convention_allowed = 1)

1715 s.expect(')')

1716 else:

1717 result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1718 assignable, nonempty)

1719 if not calling_convention_allowed and result.calling_convention and s.sy != '(':

1720 error(s.position(), "%s on something that is not a function"

1721 % result.calling_convention)

1722 while s.sy in ('[', '('):

1723 pos = s.position()

1724 if s.sy == '[':

1725 result = p_c_array_declarator(s, result)

1726 else: # sy == '('

1727 s.next()

1728 result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)

1729 cmethod_flag = 0

1730 return result

1731

1732 def p_c_array_declarator(s, base):

1733 pos = s.position()

1734 s.next() # '['

1735 if s.sy != ']':

1736 dim = p_expr(s)

1737 else:

1738 dim = None

1739 s.expect(']')

1740 return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)

1741

1742 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):

1743 # Opening paren has already been skipped

1744 args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,

1745 nonempty_declarators = 0)

1746 ellipsis = p_optional_ellipsis(s)

1747 s.expect(')')

1748 nogil = p_nogil(s)

1749 exc_val, exc_check = p_exception_value_clause(s)

1750 with_gil = p_with_gil(s)

1751 return Nodes.CFuncDeclaratorNode(pos,

1752 base = base, args = args, has_varargs = ellipsis,

1753 exception_value = exc_val, exception_check = exc_check,

1754 nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)

1755

1756 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,

1757 assignable, nonempty):

1758 pos = s.position()

1759 calling_convention = p_calling_convention(s)

1760 if s.sy == '*':

1761 s.next()

1762 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1763 cmethod_flag = cmethod_flag,

1764 assignable = assignable, nonempty = nonempty)

1765 result = Nodes.CPtrDeclaratorNode(pos,

1766 base = base)

1767 elif s.sy == '**': # scanner returns this as a single token

1768 s.next()

1769 base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,

1770 cmethod_flag = cmethod_flag,

1771 assignable = assignable, nonempty = nonempty)

1772 result = Nodes.CPtrDeclaratorNode(pos,

1773 base = Nodes.CPtrDeclaratorNode(pos,

1774 base = base))

1775 else:

1776 rhs = None

1777 if s.sy == 'IDENT':

1778 name = s.systring

1779 if is_type:

1780 s.add_type_name(name)

1781 if empty:

1782 error(s.position(), "Declarator should be empty")

1783 s.next()

1784 cname = p_opt_cname(s)

1785 if s.sy == '=' and assignable:

1786 s.next()

1787 rhs = p_simple_expr(s)

1788 else:

1789 if nonempty:

1790 error(s.position(), "Empty declarator")

1791 name = ""

1792 cname = None

1793 result = Nodes.CNameDeclaratorNode(pos,

1794 name = name, cname = cname, default = rhs)

1795 result.calling_convention = calling_convention

1796 return result

1797

1798 def p_nogil(s):

1799 if s.sy == 'IDENT' and s.systring == 'nogil':

1800 s.next()

1801 return 1

1802 else:

1803 return 0

1804

1805 def p_with_gil(s):

1806 if s.sy == 'with':

1807 s.next()

1808 s.expect_keyword('gil')

1809 return 1

1810 else:

1811 return 0

1812

1813 def p_exception_value_clause(s):

1814 exc_val = None

1815 exc_check = 0

1816 if s.sy == 'except':

1817 s.next()

1818 if s.sy == '*':

1819 exc_check = 1

1820 s.next()

1821 elif s.sy == '+':

1822 exc_check = '+'

1823 s.next()

1824 if s.sy == 'IDENT':

1825 name = s.systring

1826 s.next()

1827 exc_val = p_name(s, name)

1828 else:

1829 if s.sy == '?':

1830 exc_check = 1

1831 s.next()

1832 exc_val = p_simple_expr(s)

1833 return exc_val, exc_check

1834

1835 c_arg_list_terminators = ('*', '**', '.', ')')

1836

1837 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,

1838 nonempty_declarators = 0, kw_only = 0):

1839 # Comma-separated list of C argument declarations, possibly empty.

1840 # May have a trailing comma.

1841 args = []

1842 is_self_arg = cmethod_flag

1843 while s.sy not in c_arg_list_terminators:

1844 args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,

1845 nonempty = nonempty_declarators, kw_only = kw_only))

1846 if s.sy != ',':

1847 break

1848 s.next()

1849 is_self_arg = 0

1850 return args

1851

1852 def p_optional_ellipsis(s):

1853 if s.sy == '.':

1854 expect_ellipsis(s)

1855 return 1

1856 else:

1857 return 0

1858

1859 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0, kw_only = 0):

1860 pos = s.position()

1861 not_none = 0

1862 default = None

1863 base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)

1864 declarator = p_c_declarator(s, ctx, nonempty = nonempty)

1865 if s.sy == 'not':

1866 s.next()

1867 if s.sy == 'IDENT' and s.systring == 'None':

1868 s.next()

1869 else:

1870 s.error("Expected 'None'")

1871 if not in_pyfunc:

1872 error(pos, "'not None' only allowed in Python functions")

1873 not_none = 1

1874 if s.sy == '=':

1875 s.next()

1876 if 'pxd' in s.level:

1877 if s.sy not in ['*', '?']:

1878 error(pos, "default values cannot be specified in pxd files, use ? or *")

1879 default = 1

1880 s.next()

1881 else:

1882 default = p_simple_expr(s)

1883 return Nodes.CArgDeclNode(pos,

1884 base_type = base_type,

1885 declarator = declarator,

1886 not_none = not_none,

1887 default = default,

1888 kw_only = kw_only)

1889

1890 def p_api(s):

1891 if s.sy == 'IDENT' and s.systring == 'api':

1892 s.next()

1893 return 1

1894 else:

1895 return 0

1896

1897 def p_cdef_statement(s, ctx):

1898 pos = s.position()

1899 ctx.visibility = p_visibility(s, ctx.visibility)

1900 ctx.api = ctx.api or p_api(s)

1901 if ctx.api:

1902 if ctx.visibility not in ('private', 'public'):

1903 error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)

1904 if (ctx.visibility == 'extern') and s.sy == 'from':

1905 return p_cdef_extern_block(s, pos, ctx)

1906 elif s.sy == 'import':

1907 s.next()

1908 return p_cdef_extern_block(s, pos, ctx)

1909 if p_nogil(s):

1910 ctx.nogil = 1

1911 if s.sy == ':':

1912 return p_cdef_block(s, ctx)

1913 elif s.sy == 'class':

1914 if ctx.level not in ('module', 'module_pxd'):

1915 error(pos, "Extension type definition not allowed here")

1916 #if ctx.api:

1917 # error(pos, "'api' not allowed with extension class")

1918 return p_c_class_definition(s, pos, ctx)

1919 elif s.sy == 'IDENT' and s.systring in struct_union_or_enum:

1920 if ctx.level not in ('module', 'module_pxd'):

1921 error(pos, "C struct/union/enum definition not allowed here")

1922 #if ctx.visibility == 'public':

1923 # error(pos, "Public struct/union/enum definition not implemented")

1924 #if ctx.api:

1925 # error(pos, "'api' not allowed with '%s'" % s.systring)

1926 if s.systring == "enum":

1927 return p_c_enum_definition(s, pos, ctx)

1928 else:

1929 return p_c_struct_or_union_definition(s, pos, ctx)

1930 elif s.sy == 'pass':

1931 node = p_pass_statement(s)

1932 s.expect_newline('Expected a newline')

1933 return node

1934 else:

1935 return p_c_func_or_var_declaration(s, pos, ctx)

1936

1937 def p_cdef_block(s, ctx):

1938 return p_suite(s, ctx(cdef_flag = 1))

1939

1940 def p_cdef_extern_block(s, pos, ctx):

1941 include_file = None

1942 s.expect('from')

1943 if s.sy == '*':

1944 s.next()

1945 else:

1946 _, include_file = p_string_literal(s)

1947 ctx = ctx(cdef_flag = 1, visibility = 'extern')

1948 if p_nogil(s):

1949 ctx.nogil = 1

1950 body = p_suite(s, ctx)

1951 return Nodes.CDefExternNode(pos,

1952 include_file = include_file,

1953 body = body)

1954

1955 struct_union_or_enum = (

1956 "struct", "union", "enum"

1957 )

1958

1959 def p_c_enum_definition(s, pos, ctx):

1960 # s.sy == ident 'enum'

1961 s.next()

1962 if s.sy == 'IDENT':

1963 name = s.systring

1964 s.next()

1965 s.add_type_name(name)

1966 cname = p_opt_cname(s)

1967 else:

1968 name = None

1969 cname = None

1970 items = None

1971 s.expect(':')

1972 items = []

1973 if s.sy != 'NEWLINE':

1974 p_c_enum_line(s, items)

1975 else:

1976 s.next() # 'NEWLINE'

1977 s.expect_indent()

1978 while s.sy not in ('DEDENT', 'EOF'):

1979 p_c_enum_line(s, items)

1980 s.expect_dedent()

1981 return Nodes.CEnumDefNode(

1982 pos, name = name, cname = cname, items = items,

1983 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

1984 in_pxd = ctx.level == 'module_pxd')

1985

1986 def p_c_enum_line(s, items):

1987 if s.sy != 'pass':

1988 p_c_enum_item(s, items)

1989 while s.sy == ',':

1990 s.next()

1991 if s.sy in ('NEWLINE', 'EOF'):

1992 break

1993 p_c_enum_item(s, items)

1994 else:

1995 s.next()

1996 s.expect_newline("Syntax error in enum item list")

1997

1998 def p_c_enum_item(s, items):

1999 pos = s.position()

2000 name = p_ident(s)

2001 cname = p_opt_cname(s)

2002 value = None

2003 if s.sy == '=':

2004 s.next()

2005 value = p_simple_expr(s)

2006 items.append(Nodes.CEnumDefItemNode(pos,

2007 name = name, cname = cname, value = value))

2008

2009 def p_c_struct_or_union_definition(s, pos, ctx):

2010 # s.sy == ident 'struct' or 'union'

2011 kind = s.systring

2012 s.next()

2013 name = p_ident(s)

2014 cname = p_opt_cname(s)

2015 s.add_type_name(name)

2016 attributes = None

2017 if s.sy == ':':

2018 s.next()

2019 s.expect('NEWLINE')

2020 s.expect_indent()

2021 attributes = []

2022 body_ctx = Ctx()

2023 while s.sy != 'DEDENT':

2024 if s.sy != 'pass':

2025 attributes.append(

2026 p_c_func_or_var_declaration(s, s.position(), body_ctx))

2027 else:

2028 s.next()

2029 s.expect_newline("Expected a newline")

2030 s.expect_dedent()

2031 else:

2032 s.expect_newline("Syntax error in struct or union definition")

2033 return Nodes.CStructOrUnionDefNode(pos,

2034 name = name, cname = cname, kind = kind, attributes = attributes,

2035 typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,

2036 in_pxd = ctx.level == 'module_pxd')

2037

2038 def p_visibility(s, prev_visibility):

2039 pos = s.position()

2040 visibility = prev_visibility

2041 if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):

2042 visibility = s.systring

2043 if prev_visibility != 'private' and visibility != prev_visibility:

2044 s.error("Conflicting visibility options '%s' and '%s'"

2045 % (prev_visibility, visibility))

2046 s.next()

2047 return visibility

2048

2049 def p_c_modifiers(s):

2050 if s.sy == 'IDENT' and s.systring in ('inline',):

2051 modifier = s.systring

2052 s.next()

2053 return [modifier] + p_c_modifiers(s)

2054 return []

2055

2056 def p_c_func_or_var_declaration(s, pos, ctx):

2057 cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')

2058 modifiers = p_c_modifiers(s)

2059 base_type = p_c_base_type(s, nonempty = 1)

2060 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2061 assignable = 1, nonempty = 1)

2062 declarator.overridable = ctx.overridable

2063 if s.sy == ':':

2064 if ctx.level not in ('module', 'c_class'):

2065 s.error("C function definition not allowed here")

2066 doc, suite = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2067 result = Nodes.CFuncDefNode(pos,

2068 visibility = ctx.visibility,

2069 base_type = base_type,

2070 declarator = declarator,

2071 body = suite,

2072 doc = doc,

2073 modifiers = modifiers,

2074 api = ctx.api,

2075 overridable = ctx.overridable)

2076 else:

2077 #if api:

2078 # error(s.pos, "'api' not allowed with variable declaration")

2079 declarators = [declarator]

2080 while s.sy == ',':

2081 s.next()

2082 if s.sy == 'NEWLINE':

2083 break

2084 declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,

2085 assignable = 1, nonempty = 1)

2086 declarators.append(declarator)

2087 s.expect_newline("Syntax error in C variable declaration")

2088 result = Nodes.CVarDefNode(pos,

2089 visibility = ctx.visibility,

2090 base_type = base_type,

2091 declarators = declarators,

2092 in_pxd = ctx.level == 'module_pxd',

2093 api = ctx.api,

2094 overridable = ctx.overridable)

2095 return result

2096

2097 def p_ctypedef_statement(s, ctx):

2098 # s.sy == 'ctypedef'

2099 pos = s.position()

2100 s.next()

2101 visibility = p_visibility(s, ctx.visibility)

2102 ctx = ctx(typedef_flag = 1, visibility = visibility)

2103 if s.sy == 'class':

2104 return p_c_class_definition(s, pos, ctx)

2105 elif s.sy == 'IDENT' and s.systring in ('struct', 'union', 'enum'):

2106 if s.systring == 'enum':

2107 return p_c_enum_definition(s, pos, ctx)

2108 else:

2109 return p_c_struct_or_union_definition(s, pos, ctx)

2110 else:

2111 base_type = p_c_base_type(s, nonempty = 1)

2112 declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)

2113 s.expect_newline("Syntax error in ctypedef statement")

2114 return Nodes.CTypeDefNode(

2115 pos, base_type = base_type,

2116 declarator = declarator, visibility = visibility,

2117 in_pxd = ctx.level == 'module_pxd')

2118

2119 def p_decorators(s):

2120 decorators = []

2121 while s.sy == 'DECORATOR':

2122 pos = s.position()

2123 s.next()

2124 decorator = ExprNodes.NameNode(

2125 pos, name = Utils.EncodedString(

2126 p_dotted_name(s, as_allowed=0)[2] ))

2127 if s.sy == '(':

2128 decorator = p_call(s, decorator)

2129 decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))

2130 s.expect_newline("Expected a newline after decorator")

2131 return decorators

2132

2133 def p_def_statement(s, decorators=None):

2134 # s.sy == 'def'

2135 pos = s.position()

2136 s.next()

2137 name = p_ident(s)

2138 #args = []

2139 s.expect('(');

2140 args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1)

2141 star_arg = None

2142 starstar_arg = None

2143 if s.sy == '*':

2144 s.next()

2145 if s.sy == 'IDENT':

2146 star_arg = p_py_arg_decl(s)

2147 if s.sy == ',':

2148 s.next()

2149 args.extend(p_c_arg_list(s, in_pyfunc = 1,

2150 nonempty_declarators = 1, kw_only = 1))

2151 elif s.sy != ')':

2152 s.error("Syntax error in Python function argument list")

2153 if s.sy == '**':

2154 s.next()

2155 starstar_arg = p_py_arg_decl(s)

2156 s.expect(')')

2157 if p_nogil(s):

2158 error(s.pos, "Python function cannot be declared nogil")

2159 doc, body = p_suite(s, Ctx(level = 'function'), with_doc = 1)

2160 return Nodes.DefNode(pos, name = name, args = args,

2161 star_arg = star_arg, starstar_arg = starstar_arg,

2162 doc = doc, body = body, decorators = decorators)

2163

2164 def p_py_arg_decl(s):

2165 pos = s.position()

2166 name = p_ident(s)

2167 return Nodes.PyArgDeclNode(pos, name = name)

2168

2169 def p_class_statement(s):

2170 # s.sy == 'class'

2171 pos = s.position()

2172 s.next()

2173 class_name = Utils.EncodedString( p_ident(s) )

2174 class_name.encoding = s.source_encoding

2175 if s.sy == '(':

2176 s.next()

2177 base_list = p_simple_expr_list(s)

2178 s.expect(')')

2179 else:

2180 base_list = []

2181 doc, body = p_suite(s, Ctx(level = 'class'), with_doc = 1)

2182 return Nodes.PyClassDefNode(pos,

2183 name = class_name,

2184 bases = ExprNodes.TupleNode(pos, args = base_list),

2185 doc = doc, body = body)

2186

2187 def p_c_class_definition(s, pos, ctx):

2188 # s.sy == 'class'

2189 s.next()

2190 module_path = []

2191 class_name = p_ident(s)

2192 while s.sy == '.':

2193 s.next()

2194 module_path.append(class_name)

2195 class_name = p_ident(s)

2196 if module_path and ctx.visibility != 'extern':

2197 error(pos, "Qualified class name only allowed for 'extern' C class")

2198 if module_path and s.sy == 'IDENT' and s.systring == 'as':

2199 s.next()

2200 as_name = p_ident(s)

2201 else:

2202 as_name = class_name

2203 s.add_type_name(as_name)

2204 objstruct_name = None

2205 typeobj_name = None

2206 base_class_module = None

2207 base_class_name = None

2208 if s.sy == '(':

2209 s.next()

2210 base_class_path = [p_ident(s)]

2211 while s.sy == '.':

2212 s.next()

2213 base_class_path.append(p_ident(s))

2214 if s.sy == ',':

2215 s.error("C class may only have one base class")

2216 s.expect(')')

2217 base_class_module = ".".join(base_class_path[:-1])

2218 base_class_name = base_class_path[-1]

2219 if s.sy == '[':

2220 if ctx.visibility not in ('public', 'extern'):

2221 error(s.position(), "Name options only allowed for 'public' or 'extern' C class")

2222 objstruct_name, typeobj_name = p_c_class_options(s)

2223 if s.sy == ':':

2224 if ctx.level == 'module_pxd':

2225 body_level = 'c_class_pxd'

2226 else:

2227 body_level = 'c_class'

2228 doc, body = p_suite(s, Ctx(level = body_level), with_doc = 1)

2229 else:

2230 s.expect_newline("Syntax error in C class definition")

2231 doc = None

2232 body = None

2233 if ctx.visibility == 'extern':

2234 if not module_path:

2235 error(pos, "Module name required for 'extern' C class")

2236 if typeobj_name:

2237 error(pos, "Type object name specification not allowed for 'extern' C class")

2238 elif ctx.visibility == 'public':

2239 if not objstruct_name:

2240 error(pos, "Object struct name specification required for 'public' C class")

2241 if not typeobj_name:

2242 error(pos, "Type object name specification required for 'public' C class")

2243 elif ctx.visibility == 'private':

2244 if ctx.api:

2245 error(pos, "Only 'public' C class can be declared 'api'")

2246 else:

2247 error(pos, "Invalid class visibility '%s'" % ctx.visibility)

2248 return Nodes.CClassDefNode(pos,

2249 visibility = ctx.visibility,

2250 typedef_flag = ctx.typedef_flag,

2251 api = ctx.api,

2252 module_name = ".".join(module_path),

2253 class_name = class_name,

2254 as_name = as_name,

2255 base_class_module = base_class_module,

2256 base_class_name = base_class_name,

2257 objstruct_name = objstruct_name,

2258 typeobj_name = typeobj_name,

2259 in_pxd = ctx.level == 'module_pxd',

2260 doc = doc,

2261 body = body)

2262

2263 def p_c_class_options(s):

2264 objstruct_name = None

2265 typeobj_name = None

2266 s.expect('[')

2267 while 1:

2268 if s.sy != 'IDENT':

2269 break

2270 if s.systring == 'object':

2271 s.next()

2272 objstruct_name = p_ident(s)

2273 elif s.systring == 'type':

2274 s.next()

2275 typeobj_name = p_ident(s)

2276 if s.sy != ',':

2277 break

2278 s.next()

2279 s.expect(']', "Expected 'object' or 'type'")

2280 return objstruct_name, typeobj_name

2281

2282 def p_property_decl(s):

2283 pos = s.position()

2284 s.next() # 'property'

2285 name = p_ident(s)

2286 doc, body = p_suite(s, Ctx(level = 'property'), with_doc = 1)

2287 return Nodes.PropertyNode(pos, name = name, doc = doc, body = body)

2288

2289 def p_doc_string(s):

2290 if s.sy == 'BEGIN_STRING':

2291 pos = s.position()

2292 kind, result = p_cat_string_literal(s)

2293 if s.sy != 'EOF':

2294 s.expect_newline("Syntax error in doc string")

2295 if kind != 'u':

2296 # warning(pos, "Python 3 requires docstrings to be unicode strings")

2297 if kind == 'b':

2298 result.encoding = None # force a unicode string

2299 return result

2300 else:

2301 return None

2302

2303 def p_code(s, level=None):

2304 s.add_type_name("object")

2305 s.add_type_name("Py_buffer")

2306 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2307 if s.sy != 'EOF':

2308 s.error("Syntax error in statement [%s,%s]" % (

2309 repr(s.sy), repr(s.systring)))

2310 return body

2311

2312 def p_module(s, pxd, full_module_name):

2313 s.add_type_name("object")

2314 s.add_type_name("Py_buffer")

2315 pos = s.position()

2316 doc = p_doc_string(s)

2317 if pxd:

2318 level = 'module_pxd'

2319 else:

2320 level = 'module'

2321 body = p_statement_list(s, Ctx(level = level), first_statement = 1)

2322 if s.sy != 'EOF':

2323 s.error("Syntax error in statement [%s,%s]" % (

2324 repr(s.sy), repr(s.systring)))

2325 return ModuleNode(pos, doc = doc, body = body, full_module_name = full_module_name)

2326

2327 #----------------------------------------------

2328 #

2329 # Debugging

2330 #

2331 #----------------------------------------------

2332

2333 def print_parse_tree(f, node, level, key = None):

2334 from Nodes import Node

2335 ind = " " * level

2336 if node:

2337 f.write(ind)

2338 if key:

2339 f.write("%s: " % key)

2340 t = type(node)

2341 if t == TupleType:

2342 f.write("(%s @ %s\n" % (node[0], node[1]))

2343 for i in xrange(2, len(node)):

2344 print_parse_tree(f, node[i], level+1)

2345 f.write("%s)\n" % ind)

2346 return

2347 elif isinstance(node, Node):

2348 try:

2349 tag = node.tag

2350 except AttributeError:

2351 tag = node.__class__.__name__

2352 f.write("%s @ %s\n" % (tag, node.pos))

2353 for name, value in node.__dict__.items():

2354 if name != 'tag' and name != 'pos':

2355 print_parse_tree(f, value, level+1, name)

2356 return

2357 elif t == ListType:

2358 f.write("[\n")

2359 for i in xrange(len(node)):

2360 print_parse_tree(f, node[i], level+1)

2361 f.write("%s]\n" % ind)

2362 return

2363 f.write("%s%s\n" % (ind, node))

2364