1 #

2 # TreeFragments - parsing of strings to trees

3 #

4

5 import re

6 from cStringIO import StringIO

7 from Scanning import PyrexScanner, StringSourceDescriptor

8 from Symtab import BuiltinScope, ModuleScope

9 import Symtab

10 import PyrexTypes

11 from Visitor import VisitorTransform

12 from Nodes import Node, StatListNode

13 from ExprNodes import NameNode

14 import Parsing

15 import Main

16 import UtilNodes

17

18 """

19 Support for parsing strings into code trees.

20 """

21

22 class StringParseContext(Main.Context):

23 def __init__(self, include_directories, name):

24 Main.Context.__init__(self, include_directories, {})

25 self.module_name = name

26

27 def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):

28 if module_name != self.module_name:

29 raise AssertionError("Not yet supporting any cimports/includes from string code snippets")

30 return ModuleScope(module_name, parent_module = None, context = self)

31

32 def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):

33 """

34 Utility method to parse a (unicode) string of code. This is mostly

35 used for internal Cython compiler purposes (creating code snippets

36 that transforms should emit, as well as unit testing).

37

38 code - a unicode string containing Cython (module-level) code

39 name - a descriptive name for the code source (to use in error messages etc.)

40 """

41

42 # Since source files carry an encoding, it makes sense in this context

43 # to use a unicode string so that code fragments don't have to bother

44 # with encoding. This means that test code passed in should not have an

45 # encoding header.

46 assert isinstance(code, unicode), "unicode code snippets only please"

47 encoding = "UTF-8"

48

49 module_name = name

50 if initial_pos is None:

51 initial_pos = (name, 1, 0)

52 code_source = StringSourceDescriptor(name, code)

53

54 context = StringParseContext([], name)

55 scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)

56

57 buf = StringIO(code.encode(encoding))

58

59 scanner = PyrexScanner(buf, code_source, source_encoding = encoding,

60 scope = scope, context = context, initial_pos = initial_pos)

61 if level is None:

62 tree = Parsing.p_module(scanner, 0, module_name)

63 else:

64 tree = Parsing.p_code(scanner, level=level)

65 return tree

66

67 class TreeCopier(VisitorTransform):

68 def visit_Node(self, node):

69 if node is None:

70 return node

71 else:

72 c = node.clone_node()

73 self.visitchildren(c)

74 return c

75

76 class ApplyPositionAndCopy(TreeCopier):

77 def __init__(self, pos):

78 super(ApplyPositionAndCopy, self).__init__()

79 self.pos = pos

80

81 def visit_Node(self, node):

82 copy = super(ApplyPositionAndCopy, self).visit_Node(node)

83 copy.pos = self.pos

84 return copy

85

86 class TemplateTransform(VisitorTransform):

87 """

88 Makes a copy of a template tree while doing substitutions.

89

90 A dictionary "substitutions" should be passed in when calling

91 the transform; mapping names to replacement nodes. Then replacement

92 happens like this:

93 - If an ExprStatNode contains a single NameNode, whose name is

94 a key in the substitutions dictionary, the ExprStatNode is

95 replaced with a copy of the tree given in the dictionary.

96 It is the responsibility of the caller that the replacement

97 node is a valid statement.

98 - If a single NameNode is otherwise encountered, it is replaced

99 if its name is listed in the substitutions dictionary in the

100 same way. It is the responsibility of the caller to make sure

101 that the replacement nodes is a valid expression.

102

103 Also a list "temps" should be passed. Any names listed will

104 be transformed into anonymous, temporary names.

105

106 Currently supported for tempnames is:

107 NameNode

108 (various function and class definition nodes etc. should be added to this)

109

110 Each replacement node gets the position of the substituted node

111 recursively applied to every member node.

112 """

113

114 temp_name_counter = 0

115

116 def __call__(self, node, substitutions, temps, pos):

117 self.substitutions = substitutions

118 self.pos = pos

119 tempmap = {}

120 temphandles = []

121 for temp in temps:

122 TemplateTransform.temp_name_counter += 1

123 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter

124 # handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)

125 tempmap[temp] = handle

126 # temphandles.append(handle)

127 self.tempmap = tempmap

128 result = super(TemplateTransform, self).__call__(node)

129 # if temps:

130 # result = UtilNodes.TempsBlockNode(self.get_pos(node),

131 # temps=temphandles,

132 # body=result)

133 return result

134

135 def get_pos(self, node):

136 if self.pos:

137 return self.pos

138 else:

139 return node.pos

140

141 def visit_Node(self, node):

142 if node is None:

143 return None

144 else:

145 c = node.clone_node()

146 if self.pos is not None:

147 c.pos = self.pos

148 self.visitchildren(c)

149 return c

150

151 def try_substitution(self, node, key):

152 sub = self.substitutions.get(key)

153 if sub is not None:

154 pos = self.pos

155 if pos is None: pos = node.pos

156 return ApplyPositionAndCopy(pos)(sub)

157 else:

158 return self.visit_Node(node) # make copy as usual

159

160 def visit_NameNode(self, node):

161 temphandle = self.tempmap.get(node.name)

162 if temphandle:

163 return NameNode(pos=node.pos, name=temphandle)

164 # Replace name with temporary

165 #return temphandle.ref(self.get_pos(node))

166 else:

167 return self.try_substitution(node, node.name)

168

169 def visit_ExprStatNode(self, node):

170 # If an expression-as-statement consists of only a replaceable

171 # NameNode, we replace the entire statement, not only the NameNode

172 if isinstance(node.expr, NameNode):

173 return self.try_substitution(node, node.expr.name)

174 else:

175 return self.visit_Node(node)

176

177 def copy_code_tree(node):

178 return TreeCopier()(node)

179

180 INDENT_RE = re.compile(ur"^ *")

181 def strip_common_indent(lines):

182 "Strips empty lines and common indentation from the list of strings given in lines"

183 # TODO: Facilitate textwrap.indent instead

184 lines = [x for x in lines if x.strip() != u""]

185 minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])

186 lines = [x[minindent:] for x in lines]

187 return lines

188

189 class TreeFragment(object):

190 def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[], level=None, initial_pos=None):

191 if isinstance(code, unicode):

192 def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))

193

194 fmt_code = fmt(code)

195 fmt_pxds = {}

196 for key, value in pxds.iteritems():

197 fmt_pxds[key] = fmt(value)

198 mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos)

199 if level is None:

200 t = t.body # Make sure a StatListNode is at the top

201 if not isinstance(t, StatListNode):

202 t = StatListNode(pos=mod.pos, stats=[t])

203 for transform in pipeline:

204 t = transform(t)

205 self.root = t

206 elif isinstance(code, Node):

207 if pxds != {}: raise NotImplementedError()

208 self.root = code

209 else:

210 raise ValueError("Unrecognized code format (accepts unicode and Node)")

211 self.temps = temps

212

213 def copy(self):

214 return copy_code_tree(self.root)

215

216 def substitute(self, nodes={}, temps=[], pos = None):

217 return TemplateTransform()(self.root,

218 substitutions = nodes,

219 temps = self.temps + temps, pos = pos)

220

221

222

223