1 from Cython.Compiler.Visitor import VisitorTransform, temp_name_handle, CythonTransform

2 from Cython.Compiler.ModuleNode import ModuleNode

3 from Cython.Compiler.Nodes import *

4 from Cython.Compiler.ExprNodes import *

5 from Cython.Compiler.TreeFragment import TreeFragment

6 from Cython.Utils import EncodedString

7 from Cython.Compiler.Errors import CompileError

8 from sets import Set as set

9

10 class NormalizeTree(CythonTransform):

11 """

12 This transform fixes up a few things after parsing

13 in order to make the parse tree more suitable for

14 transforms.

15

16 a) After parsing, blocks with only one statement will

17 be represented by that statement, not by a StatListNode.

18 When doing transforms this is annoying and inconsistent,

19 as one cannot in general remove a statement in a consistent

20 way and so on. This transform wraps any single statements

21 in a StatListNode containing a single statement.

22

23 b) The PassStatNode is a noop and serves no purpose beyond

24 plugging such one-statement blocks; i.e., once parsed a

25 ` "pass" can just as well be represented using an empty

26 StatListNode. This means less special cases to worry about

27 in subsequent transforms (one always checks to see if a

28 StatListNode has no children to see if the block is empty).

29 """

30

31 def __init__(self, context):

32 super(NormalizeTree, self).__init__(context)

33 self.is_in_statlist = False

34 self.is_in_expr = False

35

36 def visit_ExprNode(self, node):

37 stacktmp = self.is_in_expr

38 self.is_in_expr = True

39 self.visitchildren(node)

40 self.is_in_expr = stacktmp

41 return node

42

43 def visit_StatNode(self, node, is_listcontainer=False):

44 stacktmp = self.is_in_statlist

45 self.is_in_statlist = is_listcontainer

46 self.visitchildren(node)

47 self.is_in_statlist = stacktmp

48 if not self.is_in_statlist and not self.is_in_expr:

49 return StatListNode(pos=node.pos, stats=[node])

50 else:

51 return node

52

53 def visit_StatListNode(self, node):

54 self.is_in_statlist = True

55 self.visitchildren(node)

56 self.is_in_statlist = False

57 return node

58

59 def visit_ParallelAssignmentNode(self, node):

60 return self.visit_StatNode(node, True)

61

62 def visit_CEnumDefNode(self, node):

63 return self.visit_StatNode(node, True)

64

65 def visit_CStructOrUnionDefNode(self, node):

66 return self.visit_StatNode(node, True)

67

68 # Eliminate PassStatNode

69 def visit_PassStatNode(self, node):

70 if not self.is_in_statlist:

71 return StatListNode(pos=node.pos, stats=[])

72 else:

73 return []

74

75

76 class PostParseError(CompileError): pass

77

78 # error strings checked by unit tests, so define them

79 ERR_BUF_OPTION_UNKNOWN = '"%s" is not a buffer option'

80 ERR_BUF_TOO_MANY = 'Too many buffer options'

81 ERR_BUF_DUP = '"%s" buffer option already supplied'

82 ERR_BUF_MISSING = '"%s" missing'

83 ERR_BUF_INT = '"%s" must be an integer'

84 ERR_BUF_NONNEG = '"%s" must be non-negative'

85 ERR_CDEF_INCLASS = 'Cannot assign default value to cdef class attributes'

86 ERR_BUF_LOCALONLY = 'Buffer types only allowed as function local variables'

87 ERR_BUF_MODEHELP = 'Only allowed buffer modes are "full" or "strided" (as a compile-time string)'

88 class PostParse(CythonTransform):

89 """

90 Basic interpretation of the parse tree, as well as validity

91 checking that can be done on a very basic level on the parse

92 tree (while still not being a problem with the basic syntax,

93 as such).

94

95 Specifically:

96 - Default values to cdef assignments are turned into single

97 assignments following the declaration (everywhere but in class

98 bodies, where they raise a compile error)

99 - CBufferAccessTypeNode has its options interpreted:

100 Any first positional argument goes into the "dtype" attribute,

101 any "ndim" keyword argument goes into the "ndim" attribute and

102 so on. Also it is checked that the option combination is valid.

103

104 Note: Currently Parsing.py does a lot of interpretation and

105 reorganization that can be refactored into this transform

106 if a more pure Abstract Syntax Tree is wanted.

107 """

108

109 # Track our context.

110 scope_type = None # can be either of 'module', 'function', 'class'

111

112 def visit_ModuleNode(self, node):

113 self.scope_type = 'module'

114 self.visitchildren(node)

115 return node

116

117 def visit_ClassDefNode(self, node):

118 prev = self.scope_type

119 self.scope_type = 'class'

120 self.visitchildren(node)

121 self.scope_type = prev

122 return node

123

124 def visit_FuncDefNode(self, node):

125 prev = self.scope_type

126 self.scope_type = 'function'

127 self.visitchildren(node)

128 self.scope_type = prev

129 return node

130

131 # cdef variables

132 def visit_CVarDefNode(self, node):

133 # This assumes only plain names and pointers are assignable on

134 # declaration. Also, it makes use of the fact that a cdef decl

135 # must appear before the first use, so we don't have to deal with

136 # "i = 3; cdef int i = i" and can simply move the nodes around.

137 try:

138 self.visitchildren(node)

139 except PostParseError, e:

140 # An error in a cdef clause is ok, simply remove the declaration

141 # and try to move on to report more errors

142 self.context.nonfatal_error(e)

143 return None

144 stats = [node]

145 for decl in node.declarators:

146 while isinstance(decl, CPtrDeclaratorNode):

147 decl = decl.base

148 if isinstance(decl, CNameDeclaratorNode):

149 if decl.default is not None:

150 if self.scope_type == 'class':

151 raise PostParseError(decl.pos, ERR_CDEF_INCLASS)

152 stats.append(SingleAssignmentNode(node.pos,

153 lhs=NameNode(node.pos, name=decl.name),

154 rhs=decl.default, first=True))

155 decl.default = None

156 return stats

157

158 # buffer access

159 buffer_options = ("dtype", "ndim", "mode") # ordered!

160 def visit_CBufferAccessTypeNode(self, node):

161 if not self.scope_type == 'function':

162 raise PostParseError(node.pos, ERR_BUF_LOCALONLY)

163

164 options = {}

165 # Fetch positional arguments

166 if len(node.positional_args) > len(self.buffer_options):

167 raise PostParseError(node.pos, ERR_BUF_TOO_MANY)

168 for arg, unicode_name in zip(node.positional_args, self.buffer_options):

169 name = str(unicode_name)

170 options[name] = arg

171 # Fetch named arguments

172 for item in node.keyword_args.key_value_pairs:

173 name = str(item.key.value)

174 if not name in self.buffer_options:

175 raise PostParseError(item.key.pos, ERR_BUF_OPTION_UNKNOWN % name)

176 if name in options.keys():

177 raise PostParseError(item.key.pos, ERR_BUF_DUP % key)

178 options[name] = item.value

179

180 # get dtype

181 dtype = options.get("dtype")

182 if dtype is None:

183 raise PostParseError(node.pos, ERR_BUF_MISSING % 'dtype')

184 node.dtype_node = dtype

185

186 # get ndim

187 if "ndim" in options:

188 ndimnode = options["ndim"]

189 if not isinstance(ndimnode, IntNode):

190 # Compile-time values (DEF) are currently resolved by the parser,

191 # so nothing more to do here

192 raise PostParseError(ndimnode.pos, ERR_BUF_INT % 'ndim')

193 ndim_value = int(ndimnode.value)

194 if ndim_value < 0:

195 raise PostParseError(ndimnode.pos, ERR_BUF_NONNEG % 'ndim')

196 node.ndim = int(ndimnode.value)

197 else:

198 node.ndim = 1

199

200 if "mode" in options:

201 modenode = options["mode"]

202 if not isinstance(modenode, StringNode):

203 raise PostParseError(modenode.pos, ERR_BUF_MODEHELP)

204 mode = modenode.value

205 if not mode in ('full', 'strided'):

206 raise PostParseError(modenode.pos, ERR_BUF_MODEHELP)

207 node.mode = mode

208 else:

209 node.mode = 'full'

210

211 # We're done with the parse tree args

212 node.positional_args = None

213 node.keyword_args = None

214 return node

215

216 class WithTransform(CythonTransform):

217

218 # EXCINFO is manually set to a variable that contains

219 # the exc_info() tuple that can be generated by the enclosing except

220 # statement.

221 template_without_target = TreeFragment(u"""

222 MGR = EXPR

223 EXIT = MGR.__exit__

224 MGR.__enter__()

225 EXC = True

226 try:

227 try:

228 BODY

229 except:

230 EXC = False

231 if not EXIT(*EXCINFO):

232 raise

233 finally:

234 if EXC:

235 EXIT(None, None, None)

236 """, temps=[u'MGR', u'EXC', u"EXIT", u"SYS)"],

237 pipeline=[NormalizeTree(None)])

238

239 template_with_target = TreeFragment(u"""

240 MGR = EXPR

241 EXIT = MGR.__exit__

242 VALUE = MGR.__enter__()

243 EXC = True

244 try:

245 try:

246 TARGET = VALUE

247 BODY

248 except:

249 EXC = False

250 if not EXIT(*EXCINFO):

251 raise

252 finally:

253 if EXC:

254 EXIT(None, None, None)

255 """, temps=[u'MGR', u'EXC', u"EXIT", u"VALUE", u"SYS"],

256 pipeline=[NormalizeTree(None)])

257

258 def visit_WithStatNode(self, node):

259 excinfo_name = temp_name_handle('EXCINFO')

260 excinfo_namenode = NameNode(pos=node.pos, name=excinfo_name)

261 excinfo_target = NameNode(pos=node.pos, name=excinfo_name)

262 if node.target is not None:

263 result = self.template_with_target.substitute({

264 u'EXPR' : node.manager,

265 u'BODY' : node.body,

266 u'TARGET' : node.target,

267 u'EXCINFO' : excinfo_namenode

268 }, pos = node.pos)

269 # Set except excinfo target to EXCINFO

270 result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target

271 else:

272 result = self.template_without_target.substitute({

273 u'EXPR' : node.manager,

274 u'BODY' : node.body,

275 u'EXCINFO' : excinfo_namenode

276 }, pos = node.pos)

277 # Set except excinfo target to EXCINFO

278 result.stats[4].body.stats[0].except_clauses[0].excinfo_target = excinfo_target

279

280 return result.stats

281

282 class DecoratorTransform(CythonTransform):

283

284 def visit_DefNode(self, func_node):

285 if not func_node.decorators:

286 return func_node

287

288 decorator_result = NameNode(func_node.pos, name = func_node.name)

289 for decorator in func_node.decorators[::-1]:

290 decorator_result = SimpleCallNode(

291 decorator.pos,

292 function = decorator.decorator,

293 args = [decorator_result])

294

295 func_name_node = NameNode(func_node.pos, name = func_node.name)

296 reassignment = SingleAssignmentNode(

297 func_node.pos,

298 lhs = func_name_node,

299 rhs = decorator_result)

300 return [func_node, reassignment]

301

302 class AnalyseDeclarationsTransform(CythonTransform):

303

304 basic_property = TreeFragment(u"""

305 property NAME:

306 def __get__(self):

307 return ATTR

308 def __set__(self, value):

309 ATTR = value

310 """, level='c_class')

311

312 def __call__(self, root):

313 self.env_stack = [root.scope]

314 return super(AnalyseDeclarationsTransform, self).__call__(root)

315

316 def visit_ModuleNode(self, node):

317 node.analyse_declarations(self.env_stack[-1])

318 self.visitchildren(node)

319 return node

320

321 def visit_FuncDefNode(self, node):

322 lenv = node.create_local_scope(self.env_stack[-1])

323 node.body.analyse_control_flow(lenv) # this will be totally refactored

324 node.declare_arguments(lenv)

325 node.body.analyse_declarations(lenv)

326 self.env_stack.append(lenv)

327 self.visitchildren(node)

328 self.env_stack.pop()

329 return node

330

331 # Some nodes are no longer needed after declaration

332 # analysis and can be dropped. The analysis was performed

333 # on these nodes in a seperate recursive process from the

334 # enclosing function or module, so we can simply drop them.

335 def visit_CVarDefNode(self, node):

336 if node.need_properties:

337 # cdef public attributes may need type testing on

338 # assignment, so we create a property accesss

339 # mechanism for them.

340 stats = []

341 for entry in node.need_properties:

342 property = self.basic_property.substitute({

343 u"ATTR": AttributeNode(pos=entry.pos, obj=NameNode(pos=entry.pos, name="self"), attribute=entry.name),

344 }, pos=entry.pos)

345 property.stats[0].name = entry.name

346 property.analyse_declarations(node.dest_scope)

347 self.visit(property)

348 stats.append(property)

349 return StatListNode(pos=node.pos, stats=stats)

350 else:

351 return None

352

353 class AnalyseExpressionsTransform(CythonTransform):

354 def visit_ModuleNode(self, node):

355 node.body.analyse_expressions(node.scope)

356 self.visitchildren(node)

357 return node

358

359 def visit_FuncDefNode(self, node):

360 node.body.analyse_expressions(node.local_scope)

361 self.visitchildren(node)

362 return node

363

364 class MarkClosureVisitor(CythonTransform):

365

366 needs_closure = False

367

368 def visit_FuncDefNode(self, node):

369 self.needs_closure = False

370 self.visitchildren(node)

371 node.needs_closure = self.needs_closure

372 self.needs_closure = True

373 return node

374

375 def visit_ClassDefNode(self, node):

376 self.visitchildren(node)

377 self.needs_closure = True

378 return node

379

380 def visit_YieldNode(self, node):

381 self.needs_closure = True

382

383 class CreateClosureClasses(CythonTransform):

384 # Output closure classes in module scope for all functions

385 # that need it.

386

387 def visit_ModuleNode(self, node):

388 self.module_scope = node.scope

389 self.visitchildren(node)

390 return node

391

392 def create_class_from_scope(self, node, target_module_scope):

393 as_name = temp_name_handle("closure")

394 func_scope = node.local_scope

395

396 entry = target_module_scope.declare_c_class(name = as_name,

397 pos = node.pos, defining = True, implementing = True)

398 class_scope = entry.type.scope

399 for entry in func_scope.entries.values():

400 class_scope.declare_var(pos=node.pos,

401 name=entry.name,

402 cname=entry.cname,

403 type=entry.type,

404 is_cdef=True)

405

406 def visit_FuncDefNode(self, node):

407 self.create_class_from_scope(node, self.module_scope)

408 return node

409

410