Cython has moved to github.
cython-devel
view Cython/Compiler/Visitor.py @ 1600:e279baa0c002
dump complete syntax tree path on compiler crashes, including nodes traversed in recursive calls that are not controlled by a transform
| author | Stefan Behnel <scoder@users.berlios.de> |
|---|---|
| date | Sat Jan 10 16:50:43 2009 +0100 (3 years ago) |
| parents | 7110f62bc826 |
| children | 55ccb1795a87 |
line source
1 #
2 # Tree visitor and transform framework
3 #
4 import cython
5 import inspect
6 import Nodes
7 import ExprNodes
8 import Naming
9 import Errors
10 from StringEncoding import EncodedString
12 class BasicVisitor(object):
13 """A generic visitor base class which can be used for visiting any kind of object."""
14 # Note: If needed, this can be replaced with a more efficient metaclass
15 # approach, resolving the jump table at module load time rather than per visitor
16 # instance.
17 def __init__(self):
18 self.dispatch_table = {}
20 def visit(self, obj):
21 cls = type(obj)
22 try:
23 handler_method = self.dispatch_table[cls]
24 except KeyError:
25 #print "Cache miss for class %s in visitor %s" % (
26 # cls.__name__, type(self).__name__)
27 # Must resolve, try entire hierarchy
28 pattern = "visit_%s"
29 mro = inspect.getmro(cls)
30 handler_method = None
31 for mro_cls in mro:
32 if hasattr(self, pattern % mro_cls.__name__):
33 handler_method = getattr(self, pattern % mro_cls.__name__)
34 break
35 if handler_method is None:
36 print type(self), type(obj)
37 if hasattr(self, 'access_path') and self.access_path:
38 print self.access_path
39 if self.access_path:
40 print self.access_path[-1][0].pos
41 print self.access_path[-1][0].__dict__
42 raise RuntimeError("Visitor does not accept object: %s" % obj)
43 #print "Caching " + cls.__name__
44 self.dispatch_table[cls] = handler_method
45 return handler_method(obj)
47 class TreeVisitor(BasicVisitor):
48 """
49 Base class for writing visitors for a Cython tree, contains utilities for
50 recursing such trees using visitors. Each node is
51 expected to have a child_attrs iterable containing the names of attributes
52 containing child nodes or lists of child nodes. Lists are not considered
53 part of the tree structure (i.e. contained nodes are considered direct
54 children of the parent node).
56 visit_children visits each of the children of a given node (see the visit_children
57 documentation). When recursing the tree using visit_children, an attribute
58 access_path is maintained which gives information about the current location
59 in the tree as a stack of tuples: (parent_node, attrname, index), representing
60 the node, attribute and optional list index that was taken in each step in the path to
61 the current node.
63 Example:
65 >>> class SampleNode(object):
66 ... child_attrs = ["head", "body"]
67 ... def __init__(self, value, head=None, body=None):
68 ... self.value = value
69 ... self.head = head
70 ... self.body = body
71 ... def __repr__(self): return "SampleNode(%s)" % self.value
72 ...
73 >>> tree = SampleNode(0, SampleNode(1), [SampleNode(2), SampleNode(3)])
74 >>> class MyVisitor(TreeVisitor):
75 ... def visit_SampleNode(self, node):
76 ... print "in", node.value, self.access_path
77 ... self.visitchildren(node)
78 ... print "out", node.value
79 ...
80 >>> MyVisitor().visit(tree)
81 in 0 []
82 in 1 [(SampleNode(0), 'head', None)]
83 out 1
84 in 2 [(SampleNode(0), 'body', 0)]
85 out 2
86 in 3 [(SampleNode(0), 'body', 1)]
87 out 3
88 out 0
89 """
91 def __init__(self):
92 super(TreeVisitor, self).__init__()
93 self.access_path = []
95 def dump_node(self, node, indent=0):
96 ignored = list(node.child_attrs) + [u'child_attrs', u'pos',
97 u'gil_message', u'subexprs']
98 values = []
99 pos = node.pos
100 if pos:
101 source = pos[0]
102 if source:
103 import os.path
104 source = os.path.basename(source.get_description())
105 values.append(u'%s:%s:%s' % (source, pos[1], pos[2]))
106 attribute_names = dir(node)
107 attribute_names.sort()
108 for attr in attribute_names:
109 if attr in ignored:
110 continue
111 if attr.startswith(u'_') or attr.endswith(u'_'):
112 continue
113 try:
114 value = getattr(node, attr)
115 except AttributeError:
116 continue
117 if value is None:
118 continue
119 elif isinstance(value, list):
120 value = u'[...]'
121 elif not isinstance(value, (str, unicode, long, int, float)):
122 continue
123 else:
124 value = repr(value)
125 values.append(u'%s = %s' % (attr, value))
126 return u'%s(%s)' % (node.__class__.__name__,
127 u',\n '.join(values))
129 def _find_node_path(self, stacktrace):
130 import os.path
131 last_traceback = stacktrace
132 nodes = []
133 while hasattr(stacktrace, 'tb_frame'):
134 frame = stacktrace.tb_frame
135 node = frame.f_locals.get(u'self')
136 if isinstance(node, Nodes.Node):
137 code = frame.f_code
138 method_name = code.co_name
139 pos = (os.path.basename(code.co_filename),
140 code.co_firstlineno)
141 nodes.append((node, method_name, pos))
142 last_traceback = stacktrace
143 stacktrace = stacktrace.tb_next
144 return (last_traceback, nodes)
146 def visitchild(self, child, parent, attrname, idx):
147 self.access_path.append((parent, attrname, idx))
148 try:
149 result = self.visit(child)
150 except Errors.CompilerCrash:
151 raise
152 except Exception, e:
153 import sys
154 trace = ['']
155 for parent, attribute, index in self.access_path:
156 node = getattr(parent, attribute)
157 if index is None:
158 index = ''
159 else:
160 node = node[index]
161 index = u'[%d]' % index
162 trace.append(u'%s.%s%s = %s' % (
163 parent.__class__.__name__, attribute, index,
164 self.dump_node(node)))
165 stacktrace, called_nodes = self._find_node_path(sys.exc_info()[2])
166 last_node = child
167 for node, method_name, pos in called_nodes:
168 last_node = node
169 trace.append(u"File '%s', line %d, in %s: %s" % (
170 pos[0], pos[1], method_name, self.dump_node(node)))
171 raise Errors.CompilerCrash(
172 last_node.pos, self.__class__.__name__,
173 u'\n'.join(trace), e, stacktrace)
174 self.access_path.pop()
175 return result
177 def visitchildren(self, parent, attrs=None):
178 """
179 Visits the children of the given parent. If parent is None, returns
180 immediately (returning None).
182 The return value is a dictionary giving the results for each
183 child (mapping the attribute name to either the return value
184 or a list of return values (in the case of multiple children
185 in an attribute)).
186 """
188 if parent is None: return None
189 result = {}
190 for attr in parent.child_attrs:
191 if attrs is not None and attr not in attrs: continue
192 child = getattr(parent, attr)
193 if child is not None:
194 if isinstance(child, list):
195 childretval = [self.visitchild(x, parent, attr, idx) for idx, x in enumerate(child)]
196 else:
197 childretval = self.visitchild(child, parent, attr, None)
198 assert not isinstance(childretval, list), 'Cannot insert list here: %s in %r' % (attr, parent)
199 result[attr] = childretval
200 return result
203 class VisitorTransform(TreeVisitor):
204 """
205 A tree transform is a base class for visitors that wants to do stream
206 processing of the structure (rather than attributes etc.) of a tree.
208 It implements __call__ to simply visit the argument node.
210 It requires the visitor methods to return the nodes which should take
211 the place of the visited node in the result tree (which can be the same
212 or one or more replacement). Specifically, if the return value from
213 a visitor method is:
215 - [] or None; the visited node will be removed (set to None if an attribute and
216 removed if in a list)
217 - A single node; the visited node will be replaced by the returned node.
218 - A list of nodes; the visited nodes will be replaced by all the nodes in the
219 list. This will only work if the node was already a member of a list; if it
220 was not, an exception will be raised. (Typically you want to ensure that you
221 are within a StatListNode or similar before doing this.)
222 """
223 def __init__(self):
224 super(VisitorTransform, self).__init__()
225 self._super_visitchildren = super(VisitorTransform, self).visitchildren
227 def visitchildren(self, parent, attrs=None):
228 result = cython.declare(dict)
229 result = self._super_visitchildren(parent, attrs)
230 for attr, newnode in result.iteritems():
231 if not isinstance(newnode, list):
232 setattr(parent, attr, newnode)
233 else:
234 # Flatten the list one level and remove any None
235 newlist = []
236 for x in newnode:
237 if x is not None:
238 if isinstance(x, list):
239 newlist += x
240 else:
241 newlist.append(x)
242 setattr(parent, attr, newlist)
243 return result
245 def recurse_to_children(self, node):
246 self.visitchildren(node)
247 return node
249 def __call__(self, root):
250 return self.visit(root)
252 class CythonTransform(VisitorTransform):
253 """
254 Certain common conventions and utilitues for Cython transforms.
255 """
256 def __init__(self, context):
257 super(CythonTransform, self).__init__()
258 self.context = context
260 def __call__(self, node):
261 import ModuleNode
262 if isinstance(node, ModuleNode.ModuleNode):
263 self.current_directives = node.directives
264 return super(CythonTransform, self).__call__(node)
266 def visit_CompilerDirectivesNode(self, node):
267 old = self.current_directives
268 self.current_directives = node.directives
269 self.visitchildren(node)
270 self.current_directives = old
271 return node
273 def visit_Node(self, node):
274 self.visitchildren(node)
275 return node
280 # Utils
281 def ensure_statlist(node):
282 if not isinstance(node, Nodes.StatListNode):
283 node = Nodes.StatListNode(pos=node.pos, stats=[node])
284 return node
286 def replace_node(ptr, value):
287 """Replaces a node. ptr is of the form used on the access path stack
288 (parent, attrname, listidx|None)
289 """
290 parent, attrname, listidx = ptr
291 if listidx is None:
292 setattr(parent, attrname, value)
293 else:
294 getattr(parent, attrname)[listidx] = value
296 class PrintTree(TreeVisitor):
297 """Prints a representation of the tree to standard output.
298 Subclass and override repr_of to provide more information
299 about nodes. """
300 def __init__(self):
301 TreeVisitor.__init__(self)
302 self._indent = ""
304 def indent(self):
305 self._indent += " "
306 def unindent(self):
307 self._indent = self._indent[:-2]
309 def __call__(self, tree, phase=None):
310 print("Parse tree dump at phase '%s'" % phase)
311 self.visit(tree)
312 return tree
314 # Don't do anything about process_list, the defaults gives
315 # nice-looking name[idx] nodes which will visually appear
316 # under the parent-node, not displaying the list itself in
317 # the hierarchy.
318 def visit_Node(self, node):
319 if len(self.access_path) == 0:
320 name = "(root)"
321 else:
322 parent, attr, idx = self.access_path[-1]
323 if idx is not None:
324 name = "%s[%d]" % (attr, idx)
325 else:
326 name = attr
327 print("%s- %s: %s" % (self._indent, name, self.repr_of(node)))
328 self.indent()
329 self.visitchildren(node)
330 self.unindent()
331 return node
333 def repr_of(self, node):
334 if node is None:
335 return "(none)"
336 else:
337 result = node.__class__.__name__
338 if isinstance(node, ExprNodes.NameNode):
339 result += "(type=%s, name=\"%s\")" % (repr(node.type), node.name)
340 elif isinstance(node, Nodes.DefNode):
341 result += "(name=\"%s\")" % node.name
342 elif isinstance(node, ExprNodes.ExprNode):
343 t = node.type
344 result += "(type=%s)" % repr(t)
346 return result
348 if __name__ == "__main__":
349 import doctest
350 doctest.testmod()