1 #

2 # TreeFragments - parsing of strings to trees

3 #

4

5 import re

6 from cStringIO import StringIO

7 from Scanning import PyrexScanner, StringSourceDescriptor

8 from Symtab import BuiltinScope, ModuleScope

9 import Symtab

10 import PyrexTypes

11 from Visitor import VisitorTransform, temp_name_handle

12 from Nodes import Node, StatListNode

13 from ExprNodes import NameNode

14 import Parsing

15 import Main

16

17 """

18 Support for parsing strings into code trees.

19 """

20

21 class StringParseContext(Main.Context):

22 def __init__(self, include_directories, name):

23 Main.Context.__init__(self, include_directories)

24 self.module_name = name

25

26 def find_module(self, module_name, relative_to = None, pos = None, need_pxd = 1):

27 if module_name != self.module_name:

28 raise AssertionError("Not yet supporting any cimports/includes from string code snippets")

29 return ModuleScope(module_name, parent_module = None, context = self)

30

31 def parse_from_strings(name, code, pxds={}, level=None):

32 """

33 Utility method to parse a (unicode) string of code. This is mostly

34 used for internal Cython compiler purposes (creating code snippets

35 that transforms should emit, as well as unit testing).

36

37 code - a unicode string containing Cython (module-level) code

38 name - a descriptive name for the code source (to use in error messages etc.)

39 """

40

41 # Since source files carry an encoding, it makes sense in this context

42 # to use a unicode string so that code fragments don't have to bother

43 # with encoding. This means that test code passed in should not have an

44 # encoding header.

45 assert isinstance(code, unicode), "unicode code snippets only please"

46 encoding = "UTF-8"

47

48 module_name = name

49 initial_pos = (name, 1, 0)

50 code_source = StringSourceDescriptor(name, code)

51

52 context = StringParseContext([], name)

53 scope = context.find_module(module_name, pos = initial_pos, need_pxd = 0)

54

55 buf = StringIO(code.encode(encoding))

56

57 scanner = PyrexScanner(buf, code_source, source_encoding = encoding,

58 scope = scope, context = context)

59 if level is None:

60 tree = Parsing.p_module(scanner, 0, module_name)

61 else:

62 tree = Parsing.p_code(scanner, level=level)

63 return tree

64

65 class TreeCopier(VisitorTransform):

66 def visit_Node(self, node):

67 if node is None:

68 return node

69 else:

70 c = node.clone_node()

71 self.visitchildren(c)

72 return c

73

74 class ApplyPositionAndCopy(TreeCopier):

75 def __init__(self, pos):

76 super(ApplyPositionAndCopy, self).__init__()

77 self.pos = pos

78

79 def visit_Node(self, node):

80 copy = super(ApplyPositionAndCopy, self).visit_Node(node)

81 copy.pos = self.pos

82 return copy

83

84 class TemplateTransform(VisitorTransform):

85 """

86 Makes a copy of a template tree while doing substitutions.

87

88 A dictionary "substitutions" should be passed in when calling

89 the transform; mapping names to replacement nodes. Then replacement

90 happens like this:

91 - If an ExprStatNode contains a single NameNode, whose name is

92 a key in the substitutions dictionary, the ExprStatNode is

93 replaced with a copy of the tree given in the dictionary.

94 It is the responsibility of the caller that the replacement

95 node is a valid statement.

96 - If a single NameNode is otherwise encountered, it is replaced

97 if its name is listed in the substitutions dictionary in the

98 same way. It is the responsibility of the caller to make sure

99 that the replacement nodes is a valid expression.

100

101 Also a list "temps" should be passed. Any names listed will

102 be transformed into anonymous, temporary names.

103

104 Currently supported for tempnames is:

105 NameNode

106 (various function and class definition nodes etc. should be added to this)

107

108 Each replacement node gets the position of the substituted node

109 recursively applied to every member node.

110 """

111

112 def __call__(self, node, substitutions, temps, pos):

113 self.substitutions = substitutions

114 tempdict = {}

115 for key in temps:

116 tempdict[key] = temp_name_handle(key) # pending result_code refactor: Symtab.new_temp(PyrexTypes.py_object_type, key)

117 self.temp_key_to_entries = tempdict

118 self.pos = pos

119 return super(TemplateTransform, self).__call__(node)

120

121 def get_pos(self, node):

122 if self.pos:

123 return self.pos

124 else:

125 return node.pos

126

127 def visit_Node(self, node):

128 if node is None:

129 return None

130 else:

131 c = node.clone_node()

132 if self.pos is not None:

133 c.pos = self.pos

134 self.visitchildren(c)

135 return c

136

137 def try_substitution(self, node, key):

138 sub = self.substitutions.get(key)

139 if sub is not None:

140 pos = self.pos

141 if pos is None: pos = node.pos

142 return ApplyPositionAndCopy(pos)(sub)

143 else:

144 return self.visit_Node(node) # make copy as usual

145

146

147 def visit_NameNode(self, node):

148 tempentry = self.temp_key_to_entries.get(node.name)

149 if tempentry is not None:

150 # Replace name with temporary

151 return NameNode(self.get_pos(node), name=tempentry)

152 # Pending result_code refactor: return NameNode(self.get_pos(node), entry=tempentry)

153 else:

154 return self.try_substitution(node, node.name)

155

156 def visit_ExprStatNode(self, node):

157 # If an expression-as-statement consists of only a replaceable

158 # NameNode, we replace the entire statement, not only the NameNode

159 if isinstance(node.expr, NameNode):

160 return self.try_substitution(node, node.expr.name)

161 else:

162 return self.visit_Node(node)

163

164 def copy_code_tree(node):

165 return TreeCopier()(node)

166

167 INDENT_RE = re.compile(ur"^ *")

168 def strip_common_indent(lines):

169 "Strips empty lines and common indentation from the list of strings given in lines"

170 # TODO: Facilitate textwrap.indent instead

171 lines = [x for x in lines if x.strip() != u""]

172 minindent = min([len(INDENT_RE.match(x).group(0)) for x in lines])

173 lines = [x[minindent:] for x in lines]

174 return lines

175

176 class TreeFragment(object):

177 def __init__(self, code, name="(tree fragment)", pxds={}, temps=[], pipeline=[], level=None):

178 if isinstance(code, unicode):

179 def fmt(x): return u"\n".join(strip_common_indent(x.split(u"\n")))

180

181 fmt_code = fmt(code)

182 fmt_pxds = {}

183 for key, value in pxds.iteritems():

184 fmt_pxds[key] = fmt(value)

185

186 mod = t = parse_from_strings(name, fmt_code, fmt_pxds, level=level)

187 if level is None:

188 t = t.body # Make sure a StatListNode is at the top

189 if not isinstance(t, StatListNode):

190 t = StatListNode(pos=mod.pos, stats=[t])

191 for transform in pipeline:

192 t = transform(t)

193 self.root = t

194 elif isinstance(code, Node):

195 if pxds != {}: raise NotImplementedError()

196 self.root = code

197 else:

198 raise ValueError("Unrecognized code format (accepts unicode and Node)")

199 self.temps = temps

200

201 def copy(self):

202 return copy_code_tree(self.root)

203

204 def substitute(self, nodes={}, temps=[], pos = None):

205 return TemplateTransform()(self.root,

206 substitutions = nodes,

207 temps = self.temps + temps, pos = pos)

208

209

210

211