1 #

2 # TreeFragments - parsing of strings to trees

3 #

4

5 import re

6 from cStringIO import StringIO

7 from Scanning import PyrexScanner, StringSourceDescriptor

8 from Symtab import BuiltinScope, ModuleScope

9 import Symtab

10 import PyrexTypes

11 from Visitor import VisitorTransform

12 from Nodes import Node, StatListNode

13 from ExprNodes import NameNode

14 import Parsing

15 import Main

16 import UtilNodes

17

18 """

19 Support for parsing strings into code trees.

20 """

21

22 class StringParseContext(Main.Context):

23 def __init__(self, include_directories, name):

24 Main.Context.__init__(self, include_directories, {})

25 self.module_name = name

26

27 def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):

28 if module_name != self.module_name:

29 raise AssertionError("Not yet supporting any cimports/includes from string code snippets")

30 return ModuleScope(module_name, parent_module = None, context = self)

31

32 def parse_from_strings(name, code, pxds={}, level=None, initial_pos=None):

33 """

34 Utility method to parse a (unicode) string of code. This is mostly

35 used for internal Cython compiler purposes (creating code snippets

36 that transforms should emit, as well as unit testing).

37

38 code - a unicode string containing Cython (module-level) code

39 name - a descriptive name for the code source (to use in error messages etc.)

40 """

41

42 # Since source files carry an encoding, it makes sense in this context

43 # to use a unicode string so that code fragments don't have to bother

44 # with encoding. This means that test code passed in should not have an

45 # encoding header.

46 assert isinstance(code, unicode), "unicode code snippets only please"

47 encoding = "UTF-8"

48

49 module_name = name

50 if initial_pos is None:

51 initial_pos = (name, 1, 0)

52 code_source = StringSourceDescriptor(name, code)

53

54 context = StringParseContext([], name)

55 scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)

56

57 buf = StringIO(code.encode(encoding))

58

59 scanner = PyrexScanner(buf, code_source, source_encoding = encoding,

60 scope = scope, context = context, initial_pos = initial_pos)

61 if level is None:

62 tree = Parsing.p_module(scanner, 0, module_name)

63 else:

64 tree = Parsing.p_code(scanner, level=level)

65 return tree

66

67 class TreeCopier(VisitorTransform):

68 def visit_Node(self, node):

69 if node is None:

70 return node

71 else:

72 c = node.clone_node()

73 self.visitchildren(c)

74 return c

75

76 class ApplyPositionAndCopy(TreeCopier):

77 def __init__(self, pos):

78 super(ApplyPositionAndCopy, self).__init__()

79 self.pos = pos

80

81 def visit_Node(self, node):

82 copy = super(ApplyPositionAndCopy, self).visit_Node(node)

83 copy.pos = self.pos

84 return copy

85

86 class TemplateTransform(VisitorTransform):

87 """

88 Makes a copy of a template tree while doing substitutions.

89

90 A dictionary "substitutions" should be passed in when calling

91 the transform; mapping names to replacement nodes. Then replacement

92 happens like this:

93 - If an ExprStatNode contains a single NameNode, whose name is

94 a key in the substitutions dictionary, the ExprStatNode is

95 replaced with a copy of the tree given in the dictionary.

96 It is the responsibility of the caller that the replacement

97 node is a valid statement.

98 - If a single NameNode is otherwise encountered, it is replaced

99 if its name is listed in the substitutions dictionary in the

100 same way. It is the responsibility of the caller to make sure

101 that the replacement nodes is a valid expression.

102

103 Also a list "temps" should be passed. Any names listed will

104 be transformed into anonymous, temporary names.

105

106 Currently supported for tempnames is:

107 NameNode

108 (various function and class definition nodes etc. should be added to this)

109

110 Each replacement node gets the position of the substituted node

111 recursively applied to every member node.

112 """

113

114 temp_name_counter = 0

115

116 def __call__(self, node, substitutions, temps, pos):

117 self.substitutions = substitutions

118 self.pos = pos

119 tempmap = {}

120 temphandles = []

121 for temp in temps:

122 TemplateTransform.temp_name_counter += 1

123 handle = "__tmpvar_%d" % TemplateTransform.temp_name_counter

124 # handle = UtilNodes.TempHandle(PyrexTypes.py_object_type)

125 tempmap[temp] = handle

126 # temphandles.append(handle)

127 self.tempmap = tempmap

128 result = super(TemplateTransform, self).__call__(node)

129 # if temps:

130 # result = UtilNodes.TempsBlockNode(self.get_pos(node),

131 # temps=temphandles,

132 # body=result)

133 return result

134

135 def get_pos(self, node):

136 if self.pos:

137 return self.pos

138 else:

139 return node.pos

140

141 def visit_Node(self, node):

142 if node is None:

143 return None

144 else:

145 c = node.clone_node()

146 if self.pos is not None:

147 c.pos = self.pos

148 self.visitchildren(c)

149 return c

150

151 def try_substitution(self, node, key):

152 sub = self.substitutions.get(key)

153 if sub is not None:

154 pos = self.pos

155 if pos is None: pos = node.pos

156 return ApplyPositionAndCopy(pos)(sub)

157 else:

158 return self.visit_Node(node) # make copy as usual

159

160 def visit_NameNode(self, node):

161 temphandle = self.tempmap.get(node.name)

162 if temphandle:

163 node.name = temphandle

164 return node

165 # Replace name with temporary

166 #return temphandle.ref(self.get_pos(node))

167 else:

168 return self.try_substitution(node, node.name)

169

170 def visit_ExprStatNode(self, node):

171 # If an expression-as-statement consists of only a replaceable

172 # NameNode, we replace the entire statement, not only the NameNode

173 if isinstance(node.expr, NameNode):

174 return self.try_substitution(node, node.expr.name)

175 else:

176 return self.visit_Node(node)

177

178 def copy_code_tree(node):

179 return TreeCopier()(node)

180

181 INDENT_RE = re.compile(ur"^ *")

182 def strip_common_indent(lines):

183 "Strips empty lines and common indentation from the list of strings given in lines"

184 # TODO: Facilitate textwrap.indent instead

185 lines = [x for x in lines if x.strip() != u""]

186 minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])

187 lines = [x[minindent:] for x in lines]

188 return lines

189

190 class TreeFragment(object):

191 def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[], level=None, initial_pos=None):

192 if isinstance(code, unicode):

193 def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))

194

195 fmt_code = fmt(code)

196 fmt_pxds = {}

197 for key, value in pxds.iteritems():

198 fmt_pxds[key] = fmt(value)

199 mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level, initial_pos=initial_pos)

200 if level is None:

201 t = t.body # Make sure a StatListNode is at the top

202 if not isinstance(t, StatListNode):

203 t = StatListNode(pos=mod.pos, stats=[t])

204 for transform in pipeline:

205 t = transform(t)

206 self.root = t

207 elif isinstance(code, Node):

208 if pxds != {}: raise NotImplementedError()

209 self.root = code

210 else:

211 raise ValueError("Unrecognized code format (accepts unicode and Node)")

212 self.temps = temps

213

214 def copy(self):

215 return copy_code_tree(self.root)

216

217 def substitute(self, nodes={}, temps=[], pos = None):

218 return TemplateTransform()(self.root,

219 substitutions = nodes,

220 temps = self.temps + temps, pos = pos)

221

222

223

224