1 #

2 # Pyrex - Code output module

3 #

4

5 import re

6 import Naming

7 import Options

8 import StringEncoding

9 from Cython import Utils

10 from Scanning import SourceDescriptor

11 from Cython.StringIOTree import StringIOTree

12 try:

13 set

14 except NameError:

15 from sets import Set as set

16 import DebugFlags

17

18 from Cython.Utils import none_or_sub

19

20 class UtilityCode(object):

21 # Stores utility code to add during code generation.

22 #

23 # See GlobalState.put_utility_code.

24 #

25 # hashes/equals by instance

26

27 def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,

28 proto_block='utility_code_proto'):

29 # proto_block: Which code block to dump prototype in. See GlobalState.

30 self.proto = proto

31 self.impl = impl

32 self.init = init

33 self.cleanup = cleanup

34 self.requires = requires

35 self._cache = {}

36 self.specialize_list = []

37 self.proto_block = proto_block

38

39 def specialize(self, pyrex_type=None, **data):

40 # Dicts aren't hashable...

41 if pyrex_type is not None:

42 data['type'] = pyrex_type.declaration_code('')

43 data['type_name'] = pyrex_type.specalization_name()

44 key = data.items(); key.sort(); key = tuple(key)

45 try:

46 return self._cache[key]

47 except KeyError:

48 if self.requires is None:

49 requires = None

50 else:

51 requires = [r.specialize(data) for r in self.requires]

52 s = self._cache[key] = UtilityCode(

53 none_or_sub(self.proto, data),

54 none_or_sub(self.impl, data),

55 none_or_sub(self.init, data),

56 none_or_sub(self.cleanup, data),

57 requires, self.proto_block)

58 self.specialize_list.append(s)

59 return s

60

61 def put_code(self, output):

62 if self.requires:

63 for dependency in self.requires:

64 output.use_utility_code(dependency)

65 if self.proto:

66 output[self.proto_block].put(self.proto)

67 if self.impl:

68 output['utility_code_def'].put(self.impl)

69 if self.init:

70 writer = output['init_globals']

71 if isinstance(self.init, basestring):

72 writer.put(self.init)

73 else:

74 self.init(writer, output.module_pos)

75 if self.cleanup and Options.generate_cleanup_code:

76 writer = output['cleanup_globals']

77 if isinstance(self.cleanup, basestring):

78 writer.put(self.cleanup)

79 else:

80 self.cleanup(writer, output.module_pos)

81

82

83

84 class FunctionState(object):

85 # return_label string function return point label

86 # error_label string error catch point label

87 # continue_label string loop continue point label

88 # break_label string loop break point label

89 # return_from_error_cleanup_label string

90 # label_counter integer counter for naming labels

91 # in_try_finally boolean inside try of try...finally

92 # exc_vars (string * 3) exception variables for reraise, or None

93

94 # Not used for now, perhaps later

95 def __init__(self, owner, names_taken=set()):

96 self.names_taken = names_taken

97 self.owner = owner

98

99 self.error_label = None

100 self.label_counter = 0

101 self.labels_used = {}

102 self.return_label = self.new_label()

103 self.new_error_label()

104 self.continue_label = None

105 self.break_label = None

106

107 self.in_try_finally = 0

108 self.exc_vars = None

109

110 self.temps_allocated = [] # of (name, type, manage_ref)

111 self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status

112 self.temps_used_type = {} # name -> (type, manage_ref)

113 self.temp_counter = 0

114

115 # labels

116

117 def new_label(self, name=None):

118 n = self.label_counter

119 self.label_counter = n + 1

120 label = "%s%d" % (Naming.label_prefix, n)

121 if name is not None:

122 label += '_' + name

123 return label

124

125 def new_error_label(self):

126 old_err_lbl = self.error_label

127 self.error_label = self.new_label('error')

128 return old_err_lbl

129

130 def get_loop_labels(self):

131 return (

132 self.continue_label,

133 self.break_label)

134

135 def set_loop_labels(self, labels):

136 (self.continue_label,

137 self.break_label) = labels

138

139 def new_loop_labels(self):

140 old_labels = self.get_loop_labels()

141 self.set_loop_labels(

142 (self.new_label("continue"),

143 self.new_label("break")))

144 return old_labels

145

146 def get_all_labels(self):

147 return (

148 self.continue_label,

149 self.break_label,

150 self.return_label,

151 self.error_label)

152

153 def set_all_labels(self, labels):

154 (self.continue_label,

155 self.break_label,

156 self.return_label,

157 self.error_label) = labels

158

159 def all_new_labels(self):

160 old_labels = self.get_all_labels()

161 new_labels = []

162 for old_label in old_labels:

163 if old_label:

164 new_labels.append(self.new_label())

165 else:

166 new_labels.append(old_label)

167 self.set_all_labels(new_labels)

168 return old_labels

169

170 def use_label(self, lbl):

171 self.labels_used[lbl] = 1

172

173 def label_used(self, lbl):

174 return lbl in self.labels_used

175

176 # temp handling

177

178 def allocate_temp(self, type, manage_ref):

179 """

180 Allocates a temporary (which may create a new one or get a previously

181 allocated and released one of the same type). Type is simply registered

182 and handed back, but will usually be a PyrexType.

183

184 If type.is_pyobject, manage_ref comes into play. If manage_ref is set to

185 True, the temp will be decref-ed on return statements and in exception

186 handling clauses. Otherwise the caller has to deal with any reference

187 counting of the variable.

188

189 If not type.is_pyobject, then manage_ref will be ignored, but it

190 still has to be passed. It is recommended to pass False by convention

191 if it is known that type will never be a Python object.

192

193 A C string referring to the variable is returned.

194 """

195 if not type.is_pyobject:

196 # Make manage_ref canonical, so that manage_ref will always mean

197 # a decref is needed.

198 manage_ref = False

199 freelist = self.temps_free.get((type, manage_ref))

200 if freelist is not None and len(freelist) > 0:

201 result = freelist.pop()

202 else:

203 while True:

204 self.temp_counter += 1

205 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)

206 if not result in self.names_taken: break

207 self.temps_allocated.append((result, type, manage_ref))

208 self.temps_used_type[result] = (type, manage_ref)

209 if DebugFlags.debug_temp_code_comments:

210 self.owner.putln("/* %s allocated */" % result)

211 return result

212

213 def release_temp(self, name):

214 """

215 Releases a temporary so that it can be reused by other code needing

216 a temp of the same type.

217 """

218 type, manage_ref = self.temps_used_type[name]

219 freelist = self.temps_free.get((type, manage_ref))

220 if freelist is None:

221 freelist = []

222 self.temps_free[(type, manage_ref)] = freelist

223 if name in freelist:

224 raise RuntimeError("Temp %s freed twice!" % name)

225 freelist.append(name)

226 if DebugFlags.debug_temp_code_comments:

227 self.owner.putln("/* %s released */" % name)

228

229 def temps_in_use(self):

230 """Return a list of (cname,type,manage_ref) tuples of temp names and their type

231 that are currently in use.

232 """

233 used = []

234 for name, type, manage_ref in self.temps_allocated:

235 freelist = self.temps_free.get((type, manage_ref))

236 if freelist is None or name not in freelist:

237 used.append((name, type, manage_ref))

238 return used

239

240 def temps_holding_reference(self):

241 """Return a list of (cname,type) tuples of temp names and their type

242 that are currently in use. This includes only temps of a

243 Python object type which owns its reference.

244 """

245 return [(name, type)

246 for name, type, manage_ref in self.temps_in_use()

247 if manage_ref]

248

249 def all_managed_temps(self):

250 """Return a list of (cname, type) tuples of refcount-managed Python objects.

251 """

252 return [(cname, type)

253 for cname, type, manage_ref in self.temps_allocated

254 if manage_ref]

255

256 def all_free_managed_temps(self):

257 """Return a list of (cname, type) tuples of refcount-managed Python

258 objects that are not currently in use. This is used by

259 try-except and try-finally blocks to clean up temps in the

260 error case.

261 """

262 return [(cname, type)

263 for (type, manage_ref), freelist in self.temps_free.iteritems()

264 if manage_ref

265 for cname in freelist]

266

267

268 class IntConst(object):

269 """Global info about a Python integer constant held by GlobalState.

270 """

271 # cname string

272 # value int

273 # is_long boolean

274

275 def __init__(self, cname, value, is_long):

276 self.cname = cname

277 self.value = value

278 self.is_long = is_long

279

280 class PyObjectConst(object):

281 """Global info about a generic constant held by GlobalState.

282 """

283 # cname string

284 # type PyrexType

285

286 def __init__(self, cname, type):

287 self.cname = cname

288 self.type = type

289

290 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match

291 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match

292 nice_identifier = re.compile('^[a-zA-Z0-9_]+$').match

293 find_alphanums = re.compile('([a-zA-Z0-9]+)').findall

294

295 class StringConst(object):

296 """Global info about a C string constant held by GlobalState.

297 """

298 # cname string

299 # text EncodedString or BytesLiteral

300 # py_strings {(identifier, encoding) : PyStringConst}

301

302 def __init__(self, cname, text, byte_string):

303 self.cname = cname

304 self.text = text

305 self.escaped_value = StringEncoding.escape_byte_string(byte_string)

306 self.py_strings = None

307

308 def get_py_string_const(self, encoding, identifier=None, is_str=False):

309 py_strings = self.py_strings

310 text = self.text

311

312 is_str = bool(identifier or is_str)

313 is_unicode = encoding is None and not is_str

314

315 if encoding is None:

316 # unicode string

317 encoding_key = None

318 else:

319 # bytes or str

320 encoding = encoding.lower()

321 if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):

322 encoding = None

323 encoding_key = None

324 else:

325 encoding_key = ''.join(find_alphanums(encoding))

326

327 key = (is_str, is_unicode, encoding_key)

328 if py_strings is not None and key in py_strings:

329 py_string = py_strings[key]

330 else:

331 if py_strings is None:

332 self.py_strings = {}

333 if identifier:

334 intern = True

335 elif identifier is None:

336 if isinstance(text, unicode):

337 intern = bool(possible_unicode_identifier(text))

338 else:

339 intern = bool(possible_bytes_identifier(text))

340 else:

341 intern = False

342 if intern:

343 prefix = Naming.interned_str_prefix

344 else:

345 prefix = Naming.py_const_prefix

346 pystring_cname = "%s%s_%s" % (

347 prefix,

348 (is_str and 's') or (is_unicode and 'u') or 'b',

349 self.cname[len(Naming.const_prefix):])

350

351 py_string = PyStringConst(

352 pystring_cname, encoding, is_unicode, is_str, intern)

353 self.py_strings[key] = py_string

354

355 return py_string

356

357 class PyStringConst(object):

358 """Global info about a Python string constant held by GlobalState.

359 """

360 # cname string

361 # encoding string

362 # intern boolean

363 # is_unicode boolean

364 # is_str boolean

365

366 def __init__(self, cname, encoding, is_unicode, is_str=False, intern=False):

367 self.cname = cname

368 self.encoding = encoding

369 self.is_str = is_str

370 self.is_unicode = is_unicode

371 self.intern = intern

372

373 def __lt__(self, other):

374 return self.cname < other.cname

375

376

377 class GlobalState(object):

378 # filename_table {string : int} for finding filename table indexes

379 # filename_list [string] filenames in filename table order

380 # input_file_contents dict contents (=list of lines) of any file that was used as input

381 # to create this output C code. This is

382 # used to annotate the comments.

383 #

384 # utility_codes set IDs of used utility code (to avoid reinsertion)

385 #

386 # declared_cnames {string:Entry} used in a transition phase to merge pxd-declared

387 # constants etc. into the pyx-declared ones (i.e,

388 # check if constants are already added).

389 # In time, hopefully the literals etc. will be

390 # supplied directly instead.

391 #

392 # const_cname_counter int global counter for constant identifiers

393 #

394

395 # parts {string:CCodeWriter}

396

397

398 # interned_strings

399 # consts

400 # interned_nums

401

402 # directives set Temporary variable used to track

403 # the current set of directives in the code generation

404 # process.

405

406 directives = {}

407

408 code_layout = [

409 'h_code',

410 'utility_code_proto_before_types',

411 'numeric_typedefs', # Let these detailed individual parts stay!,

412 'complex_type_declarations', # as the proper solution is to make a full DAG...

413 'type_declarations', # More coarse-grained blocks would simply hide

414 'utility_code_proto', # the ugliness, not fix it

415 'module_declarations',

416 'typeinfo',

417 'before_global_var',

418 'global_var',

419 'decls',

420 'all_the_rest',

421 'pystring_table',

422 'cached_builtins',

423 'init_globals',

424 'init_module',

425 'cleanup_globals',

426 'cleanup_module',

427 'main_method',

428 'filename_table',

429 'utility_code_def',

430 'end'

431 ]

432

433

434 def __init__(self, writer, emit_linenums=False):

435 self.filename_table = {}

436 self.filename_list = []

437 self.input_file_contents = {}

438 self.utility_codes = set()

439 self.declared_cnames = {}

440 self.in_utility_code_generation = False

441 self.emit_linenums = emit_linenums

442 self.parts = {}

443

444 self.const_cname_counter = 1

445 self.string_const_index = {}

446 self.int_const_index = {}

447 self.py_constants = []

448

449 assert writer.globalstate is None

450 writer.globalstate = self

451 self.rootwriter = writer

452

453 def initialize_main_c_code(self):

454 rootwriter = self.rootwriter

455 for part in self.code_layout:

456 self.parts[part] = rootwriter.insertion_point()

457

458 if not Options.cache_builtins:

459 del self.parts['cached_builtins']

460 else:

461 w = self.parts['cached_builtins']

462 w.enter_cfunc_scope()

463 w.putln("static int __Pyx_InitCachedBuiltins(void) {")

464

465

466 w = self.parts['init_globals']

467 w.enter_cfunc_scope()

468 w.putln("")

469 w.putln("static int __Pyx_InitGlobals(void) {")

470

471 if not Options.generate_cleanup_code:

472 del self.parts['cleanup_globals']

473 else:

474 w = self.parts['cleanup_globals']

475 w.enter_cfunc_scope()

476 w.putln("")

477 w.putln("static void __Pyx_CleanupGlobals(void) {")

478

479 #

480 # utility_code_def

481 #

482 code = self.parts['utility_code_def']

483 if self.emit_linenums:

484 code.write('\n#line 1 "cython_utility"\n')

485 code.putln("")

486 code.putln("/* Runtime support code */")

487 code.putln("")

488 code.putln("static void %s(void) {" % Naming.fileinit_cname)

489 code.putln("%s = %s;" %

490 (Naming.filetable_cname, Naming.filenames_cname))

491 code.putln("}")

492

493 def finalize_main_c_code(self):

494 self.close_global_decls()

495

496 #

497 # utility_code_def

498 #

499 code = self.parts['utility_code_def']

500 import PyrexTypes

501 code.put(PyrexTypes.type_conversion_functions)

502 code.putln("")

503

504 def __getitem__(self, key):

505 return self.parts[key]

506

507 #

508 # Global constants, interned objects, etc.

509 #

510 def close_global_decls(self):

511 # This is called when it is known that no more global declarations will

512 # declared.

513 self.generate_const_declarations()

514 if Options.cache_builtins:

515 w = self.parts['cached_builtins']

516 w.putln("return 0;")

517 w.put_label(w.error_label)

518 w.putln("return -1;")

519 w.putln("}")

520 w.exit_cfunc_scope()

521

522 w = self.parts['init_globals']

523 w.putln("return 0;")

524 w.put_label(w.error_label)

525 w.putln("return -1;")

526 w.putln("}")

527 w.exit_cfunc_scope()

528

529 if Options.generate_cleanup_code:

530 w = self.parts['cleanup_globals']

531 w.putln("}")

532 w.exit_cfunc_scope()

533

534 if Options.generate_cleanup_code:

535 w = self.parts['cleanup_module']

536 w.putln("}")

537 w.exit_cfunc_scope()

538

539 def put_pyobject_decl(self, entry):

540 self['global_var'].putln("static PyObject *%s;" % entry.cname)

541

542 # constant handling at code generation time

543

544 def get_int_const(self, str_value, longness=False):

545 longness = bool(longness or Utils.long_literal(str_value))

546 try:

547 c = self.int_const_index[(str_value, longness)]

548 except KeyError:

549 c = self.new_int_const(str_value, longness)

550 return c

551

552 def get_py_const(self, type):

553 # create a new Python object constant

554 return self.new_py_const(type)

555

556 def get_string_const(self, text):

557 # return a C string constant, creating a new one if necessary

558 if text.is_unicode:

559 byte_string = text.utf8encode()

560 else:

561 byte_string = text.byteencode()

562 try:

563 c = self.string_const_index[byte_string]

564 except KeyError:

565 c = self.new_string_const(text, byte_string)

566 return c

567

568 def get_py_string_const(self, text, identifier=None, is_str=False):

569 # return a Python string constant, creating a new one if necessary

570 c_string = self.get_string_const(text)

571 py_string = c_string.get_py_string_const(text.encoding, identifier, is_str)

572 return py_string

573

574 def get_interned_identifier(self, text):

575 return self.get_py_string_const(text, identifier=True)

576

577 def new_string_const(self, text, byte_string):

578 cname = self.new_string_const_cname(byte_string)

579 c = StringConst(cname, text, byte_string)

580 self.string_const_index[byte_string] = c

581 return c

582

583 def new_int_const(self, value, longness):

584 cname = self.new_int_const_cname(value, longness)

585 c = IntConst(cname, value, longness)

586 self.int_const_index[(value, longness)] = c

587 return c

588

589 def new_py_const(self, type):

590 cname = self.new_const_cname()

591 c = PyObjectConst(cname, type)

592 self.py_constants.append(c)

593 return c

594

595 def new_string_const_cname(self, bytes_value, intern=None):

596 # Create a new globally-unique nice name for a C string constant.

597 try:

598 value = bytes_value.decode('ASCII')

599 except UnicodeError:

600 return self.new_const_cname()

601

602 if len(value) < 20 and nice_identifier(value):

603 return "%s_%s" % (Naming.const_prefix, value)

604 else:

605 return self.new_const_cname()

606

607 def new_int_const_cname(self, value, longness):

608 if longness:

609 value += 'L'

610 cname = "%s%s" % (Naming.interned_num_prefix, value)

611 cname = cname.replace('-', 'neg_').replace('.','_')

612 return cname

613

614 def new_const_cname(self, prefix=''):

615 n = self.const_cname_counter

616 self.const_cname_counter += 1

617 return "%s%s%d" % (Naming.const_prefix, prefix, n)

618

619 def add_cached_builtin_decl(self, entry):

620 if Options.cache_builtins:

621 if self.should_declare(entry.cname, entry):

622 interned_cname = self.get_interned_identifier(entry.name).cname

623 self.put_pyobject_decl(entry)

624 w = self.parts['cached_builtins']

625 w.putln('%s = __Pyx_GetName(%s, %s); if (!%s) %s' % (

626 entry.cname,

627 Naming.builtins_cname,

628 interned_cname,

629 entry.cname,

630 w.error_goto(entry.pos)))

631

632 def generate_const_declarations(self):

633 self.generate_string_constants()

634 self.generate_int_constants()

635 self.generate_object_constant_decls()

636

637 def generate_object_constant_decls(self):

638 consts = [ (len(c.cname), c.cname, c)

639 for c in self.py_constants ]

640 consts.sort()

641 decls_writer = self.parts['decls']

642 for _, cname, c in consts:

643 decls_writer.putln(

644 "static %s;" % c.type.declaration_code(cname))

645

646 def generate_string_constants(self):

647 c_consts = [ (len(c.cname), c.cname, c)

648 for c in self.string_const_index.itervalues() ]

649 c_consts.sort()

650 py_strings = []

651

652 decls_writer = self.parts['decls']

653 for _, cname, c in c_consts:

654 decls_writer.putln('static char %s[] = "%s";' % (

655 cname, c.escaped_value))

656 if c.py_strings is not None:

657 for py_string in c.py_strings.itervalues():

658 py_strings.append((c.cname, len(py_string.cname), py_string))

659

660 if py_strings:

661 import Nodes

662 self.use_utility_code(Nodes.init_string_tab_utility_code)

663

664 py_strings.sort()

665 w = self.parts['pystring_table']

666 w.putln("")

667 w.putln("static __Pyx_StringTabEntry %s[] = {" %

668 Naming.stringtab_cname)

669 for c_cname, _, py_string in py_strings:

670 if not py_string.is_str or not py_string.encoding or \

671 py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',

672 'UTF8', 'UTF-8'):

673 encoding = '0'

674 else:

675 encoding = '"%s"' % py_string.encoding.lower()

676

677 decls_writer.putln(

678 "static PyObject *%s;" % py_string.cname)

679 w.putln(

680 "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (

681 py_string.cname,

682 c_cname,

683 c_cname,

684 encoding,

685 py_string.is_unicode,

686 py_string.is_str,

687 py_string.intern

688 ))

689 w.putln("{0, 0, 0, 0, 0, 0, 0}")

690 w.putln("};")

691

692 init_globals = self.parts['init_globals']

693 init_globals.putln(

694 "if (__Pyx_InitStrings(%s) < 0) %s;" % (

695 Naming.stringtab_cname,

696 init_globals.error_goto(self.module_pos)))

697

698 def generate_int_constants(self):

699 consts = [ (len(c.value), c.value, c.is_long, c)

700 for c in self.int_const_index.itervalues() ]

701 consts.sort()

702 decls_writer = self.parts['decls']

703 for _, value, longness, c in consts:

704 cname = c.cname

705 decls_writer.putln("static PyObject *%s;" % cname)

706 if longness:

707 function = '%s = PyLong_FromString((char *)"%s", 0, 0); %s;'

708 else:

709 function = "%s = PyInt_FromLong(%s); %s;"

710 init_globals = self.parts['init_globals']

711 init_globals.putln(function % (

712 cname,

713 value,

714 init_globals.error_goto_if_null(cname, self.module_pos)))

715

716 # The functions below are there in a transition phase only

717 # and will be deprecated. They are called from Nodes.BlockNode.

718 # The copy&paste duplication is intentional in order to be able

719 # to see quickly how BlockNode worked, until this is replaced.

720

721 def should_declare(self, cname, entry):

722 if cname in self.declared_cnames:

723 other = self.declared_cnames[cname]

724 assert str(entry.type) == str(other.type)

725 assert entry.init == other.init

726 return False

727 else:

728 self.declared_cnames[cname] = entry

729 return True

730

731 #

732 # File name state

733 #

734

735 def lookup_filename(self, filename):

736 try:

737 index = self.filename_table[filename]

738 except KeyError:

739 index = len(self.filename_list)

740 self.filename_list.append(filename)

741 self.filename_table[filename] = index

742 return index

743

744 def commented_file_contents(self, source_desc):

745 try:

746 return self.input_file_contents[source_desc]

747 except KeyError:

748 F = [u' * ' + line.rstrip().replace(

749 u'*/', u'*[inserted by cython to avoid comment closer]/'

750 ).replace(

751 u'/*', u'/[inserted by cython to avoid comment start]*'

752 )

753 for line in source_desc.get_lines(encoding='ASCII',

754 error_handling='ignore')]

755 if len(F) == 0: F.append(u'')

756 self.input_file_contents[source_desc] = F

757 return F

758

759 #

760 # Utility code state

761 #

762

763 def use_utility_code(self, utility_code):

764 """

765 Adds code to the C file. utility_code should

766 a) implement __eq__/__hash__ for the purpose of knowing whether the same

767 code has already been included

768 b) implement put_code, which takes a globalstate instance

769

770 See UtilityCode.

771 """

772 if utility_code not in self.utility_codes:

773 self.utility_codes.add(utility_code)

774 utility_code.put_code(self)

775

776

777 def funccontext_property(name):

778 try:

779 import operator

780 attribute_of = operator.attrgetter(name)

781 except:

782 def attribute_of(o):

783 return getattr(o, name)

784

785 def get(self):

786 return attribute_of(self.funcstate)

787 def set(self, value):

788 setattr(self.funcstate, name, value)

789 return property(get, set)

790

791

792 class CCodeWriter(object):

793 """

794 Utility class to output C code.

795

796 When creating an insertion point one must care about the state that is

797 kept:

798 - formatting state (level, bol) is cloned and used in insertion points

799 as well

800 - labels, temps, exc_vars: One must construct a scope in which these can

801 exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for

802 sanity checking and forward compatabilty). Created insertion points

803 looses this scope and cannot access it.

804 - marker: Not copied to insertion point

805 - filename_table, filename_list, input_file_contents: All codewriters

806 coming from the same root share the same instances simultaneously.

807 """

808

809 # f file output file

810 # buffer StringIOTree

811

812 # level int indentation level

813 # bol bool beginning of line?

814 # marker string comment to emit before next line

815 # funcstate FunctionState contains state local to a C function used for code

816 # generation (labels and temps state etc.)

817 # globalstate GlobalState contains state global for a C file (input file info,

818 # utility code, declared constants etc.)

819 # emit_linenums boolean whether or not to write #line pragmas

820 #

821 # pyclass_stack list used during recursive code generation to pass information

822 # about the current class one is in

823

824 globalstate = None

825

826 def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None):

827 if buffer is None: buffer = StringIOTree()

828 self.buffer = buffer

829 self.marker = None

830 self.last_marker_line = 0

831 self.source_desc = ""

832 self.pyclass_stack = []

833

834 self.funcstate = None

835 self.level = 0

836 self.call_level = 0

837 self.bol = 1

838

839 if create_from is not None:

840 # Use same global state

841 self.globalstate = create_from.globalstate

842 # Clone formatting state

843 if copy_formatting:

844 self.level = create_from.level

845 self.bol = create_from.bol

846 self.call_level = create_from.call_level

847 if emit_linenums is None and self.globalstate:

848 self.emit_linenums = self.globalstate.emit_linenums

849 else:

850 self.emit_linenums = emit_linenums

851

852 def create_new(self, create_from, buffer, copy_formatting):

853 # polymorphic constructor -- very slightly more versatile

854 # than using __class__

855 result = CCodeWriter(create_from, buffer, copy_formatting)

856 return result

857

858 def copyto(self, f):

859 self.buffer.copyto(f)

860

861 def getvalue(self):

862 return self.buffer.getvalue()

863

864 def write(self, s):

865 self.buffer.write(s)

866

867 def insertion_point(self):

868 other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)

869 return other

870

871 def new_writer(self):

872 """

873 Creates a new CCodeWriter connected to the same global state, which

874 can later be inserted using insert.

875 """

876 return CCodeWriter(create_from=self)

877

878 def insert(self, writer):

879 """

880 Inserts the contents of another code writer (created with

881 the same global state) in the current location.

882

883 It is ok to write to the inserted writer also after insertion.

884 """

885 assert writer.globalstate is self.globalstate

886 self.buffer.insert(writer.buffer)

887

888 # Properties delegated to function scope

889 label_counter = funccontext_property("label_counter")

890 return_label = funccontext_property("return_label")

891 error_label = funccontext_property("error_label")

892 labels_used = funccontext_property("labels_used")

893 continue_label = funccontext_property("continue_label")

894 break_label = funccontext_property("break_label")

895 return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")

896

897 # Functions delegated to function scope

898 def new_label(self, name=None): return self.funcstate.new_label(name)

899 def new_error_label(self): return self.funcstate.new_error_label()

900 def get_loop_labels(self): return self.funcstate.get_loop_labels()

901 def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)

902 def new_loop_labels(self): return self.funcstate.new_loop_labels()

903 def get_all_labels(self): return self.funcstate.get_all_labels()

904 def set_all_labels(self, labels): return self.funcstate.set_all_labels(labels)

905 def all_new_labels(self): return self.funcstate.all_new_labels()

906 def use_label(self, lbl): return self.funcstate.use_label(lbl)

907 def label_used(self, lbl): return self.funcstate.label_used(lbl)

908

909

910 def enter_cfunc_scope(self):

911 self.funcstate = FunctionState(self)

912

913 def exit_cfunc_scope(self):

914 self.funcstate = None

915

916 # constant handling

917

918 def get_py_num(self, str_value, longness):

919 return self.globalstate.get_int_const(str_value, longness).cname

920

921 def get_string_const(self, text):

922 return self.globalstate.get_string_const(text).cname

923

924 def get_py_string_const(self, text, identifier=None, is_str=False):

925 return self.globalstate.get_py_string_const(text, identifier, is_str).cname

926

927 def get_argument_default_const(self, type):

928 return self.globalstate.get_py_const(type).cname

929

930 def intern(self, text):

931 return self.get_py_string_const(text)

932

933 def intern_identifier(self, text):

934 return self.get_py_string_const(text, identifier=True)

935

936 # code generation

937

938 def putln(self, code = "", safe=False):

939 if self.marker and self.bol:

940 self.emit_marker()

941 if self.emit_linenums and self.last_marker_line != 0:

942 self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))

943 if code:

944 if safe:

945 self.put_safe(code)

946 else:

947 self.put(code)

948 self.write("\n");

949 self.bol = 1

950

951 def emit_marker(self):

952 self.write("\n");

953 self.indent()

954 self.write("/* %s */\n" % self.marker[1])

955 self.last_marker_line = self.marker[0]

956 self.marker = None

957

958 def put_safe(self, code):

959 # put code, but ignore {}

960 self.write(code)

961 self.bol = 0

962

963 def put(self, code):

964 fix_indent = False

965 if "{" in code:

966 dl = code.count("{")

967 else:

968 dl = 0

969 if "}" in code:

970 dl -= code.count("}")

971 if dl < 0:

972 self.level += dl

973 elif dl == 0 and code[0] == "}":

974 # special cases like "} else {" need a temporary dedent

975 fix_indent = True

976 self.level -= 1

977 if self.bol:

978 self.indent()

979 self.write(code)

980 self.bol = 0

981 if dl > 0:

982 self.level += dl

983 elif fix_indent:

984 self.level += 1

985

986 def increase_indent(self):

987 self.level = self.level + 1

988

989 def decrease_indent(self):

990 self.level = self.level - 1

991

992 def begin_block(self):

993 self.putln("{")

994 self.increase_indent()

995

996 def end_block(self):

997 self.decrease_indent()

998 self.putln("}")

999

1000 def indent(self):

1001 self.write(" " * self.level)

1002

1003 def get_py_version_hex(self, pyversion):

1004 return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

1005

1006 def mark_pos(self, pos):

1007 if pos is None:

1008 return

1009 source_desc, line, col = pos

1010 if self.last_marker_line == line:

1011 return

1012 assert isinstance(source_desc, SourceDescriptor)

1013 contents = self.globalstate.commented_file_contents(source_desc)

1014 lines = contents[max(0,line-3):line] # line numbers start at 1

1015 lines[-1] += u' # <<<<<<<<<<<<<<'

1016 lines += contents[line:line+2]

1017

1018 marker = u'"%s":%d\n%s\n' % (

1019 source_desc.get_escaped_description(), line, u'\n'.join(lines))

1020 self.marker = (line, marker)

1021 if self.emit_linenums:

1022 self.source_desc = source_desc.get_escaped_description()

1023

1024 def put_label(self, lbl):

1025 if lbl in self.funcstate.labels_used:

1026 self.putln("%s:;" % lbl)

1027

1028 def put_goto(self, lbl):

1029 self.funcstate.use_label(lbl)

1030 self.putln("goto %s;" % lbl)

1031

1032 def put_var_declarations(self, entries, static = 0, dll_linkage = None,

1033 definition = True):

1034 for entry in entries:

1035 if not entry.in_cinclude:

1036 self.put_var_declaration(entry, static, dll_linkage, definition)

1037

1038 def put_var_declaration(self, entry, static = 0, dll_linkage = None,

1039 definition = True):

1040 #print "Code.put_var_declaration:", entry.name, "definition =", definition ###

1041 if entry.in_closure:

1042 return

1043 visibility = entry.visibility

1044 if visibility == 'private' and not definition:

1045 #print "...private and not definition, skipping" ###

1046 return

1047 if not entry.used and visibility == "private":

1048 #print "not used and private, skipping", entry.cname ###

1049 return

1050 storage_class = ""

1051 if visibility == 'extern':

1052 storage_class = Naming.extern_c_macro

1053 elif visibility == 'public':

1054 if not definition:

1055 storage_class = Naming.extern_c_macro

1056 elif visibility == 'private':

1057 if static:

1058 storage_class = "static"

1059 if storage_class:

1060 self.put("%s " % storage_class)

1061 if visibility != 'public':

1062 dll_linkage = None

1063 self.put(entry.type.declaration_code(entry.cname,

1064 dll_linkage = dll_linkage))

1065 if entry.init is not None:

1066 self.put_safe(" = %s" % entry.type.literal_code(entry.init))

1067 self.putln(";")

1068

1069 def put_temp_declarations(self, func_context):

1070 for name, type, manage_ref in func_context.temps_allocated:

1071 decl = type.declaration_code(name)

1072 if type.is_pyobject:

1073 self.putln("%s = NULL;" % decl)

1074 else:

1075 self.putln("%s;" % decl)

1076

1077 def put_h_guard(self, guard):

1078 self.putln("#ifndef %s" % guard)

1079 self.putln("#define %s" % guard)

1080

1081 def unlikely(self, cond):

1082 if Options.gcc_branch_hints:

1083 return 'unlikely(%s)' % cond

1084 else:

1085 return cond

1086

1087 # Python objects and reference counting

1088

1089 def entry_as_pyobject(self, entry):

1090 type = entry.type

1091 if (not entry.is_self_arg and not entry.type.is_complete()

1092 or entry.type.is_extension_type):

1093 return "(PyObject *)" + entry.cname

1094 else:

1095 return entry.cname

1096

1097 def as_pyobject(self, cname, type):

1098 from PyrexTypes import py_object_type, typecast

1099 return typecast(py_object_type, type, cname)

1100

1101 def put_gotref(self, cname):

1102 self.putln("__Pyx_GOTREF(%s);" % cname)

1103

1104 def put_giveref(self, cname):

1105 self.putln("__Pyx_GIVEREF(%s);" % cname)

1106

1107 def put_xgiveref(self, cname):

1108 self.putln("__Pyx_XGIVEREF(%s);" % cname)

1109

1110 def put_xgotref(self, cname):

1111 self.putln("__Pyx_XGOTREF(%s);" % cname)

1112

1113 def put_incref(self, cname, type, nanny=True):

1114 if nanny:

1115 self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))

1116 else:

1117 self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))

1118

1119 def put_decref(self, cname, type, nanny=True):

1120 if nanny:

1121 self.putln("__Pyx_DECREF(%s);" % self.as_pyobject(cname, type))

1122 else:

1123 self.putln("Py_DECREF(%s);" % self.as_pyobject(cname, type))

1124

1125 def put_var_gotref(self, entry):

1126 if entry.type.is_pyobject:

1127 self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))

1128

1129 def put_var_giveref(self, entry):

1130 if entry.type.is_pyobject:

1131 self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))

1132

1133 def put_var_xgotref(self, entry):

1134 if entry.type.is_pyobject:

1135 self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))

1136

1137 def put_var_xgiveref(self, entry):

1138 if entry.type.is_pyobject:

1139 self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))

1140

1141 def put_var_incref(self, entry):

1142 if entry.type.is_pyobject:

1143 self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))

1144

1145 def put_decref_clear(self, cname, type, nanny=True):

1146 from PyrexTypes import py_object_type, typecast

1147 if nanny:

1148 self.putln("__Pyx_DECREF(%s); %s = 0;" % (

1149 typecast(py_object_type, type, cname), cname))

1150 else:

1151 self.putln("Py_DECREF(%s); %s = 0;" % (

1152 typecast(py_object_type, type, cname), cname))

1153

1154 def put_xdecref(self, cname, type, nanny=True):

1155 if nanny:

1156 self.putln("__Pyx_XDECREF(%s);" % self.as_pyobject(cname, type))

1157 else:

1158 self.putln("Py_XDECREF(%s);" % self.as_pyobject(cname, type))

1159

1160 def put_xdecref_clear(self, cname, type, nanny=True):

1161 if nanny:

1162 self.putln("__Pyx_XDECREF(%s); %s = 0;" % (

1163 self.as_pyobject(cname, type), cname))

1164 else:

1165 self.putln("Py_XDECREF(%s); %s = 0;" % (

1166 self.as_pyobject(cname, type), cname))

1167

1168 def put_var_decref(self, entry):

1169 if entry.type.is_pyobject:

1170 if entry.init_to_none is False:

1171 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))

1172 else:

1173 self.putln("__Pyx_DECREF(%s);" % self.entry_as_pyobject(entry))

1174

1175 def put_var_decref_clear(self, entry):

1176 if entry.type.is_pyobject:

1177 self.putln("__Pyx_DECREF(%s); %s = 0;" % (

1178 self.entry_as_pyobject(entry), entry.cname))

1179

1180 def put_var_xdecref(self, entry):

1181 if entry.type.is_pyobject:

1182 self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))

1183

1184 def put_var_xdecref_clear(self, entry):

1185 if entry.type.is_pyobject:

1186 self.putln("__Pyx_XDECREF(%s); %s = 0;" % (

1187 self.entry_as_pyobject(entry), entry.cname))

1188

1189 def put_var_decrefs(self, entries, used_only = 0):

1190 for entry in entries:

1191 if not used_only or entry.used:

1192 if entry.xdecref_cleanup:

1193 self.put_var_xdecref(entry)

1194 else:

1195 self.put_var_decref(entry)

1196

1197 def put_var_xdecrefs(self, entries):

1198 for entry in entries:

1199 self.put_var_xdecref(entry)

1200

1201 def put_var_xdecrefs_clear(self, entries):

1202 for entry in entries:

1203 self.put_var_xdecref_clear(entry)

1204

1205 def put_init_to_py_none(self, cname, type, nanny=True):

1206 from PyrexTypes import py_object_type, typecast

1207 py_none = typecast(type, py_object_type, "Py_None")

1208 if nanny:

1209 self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))

1210 else:

1211 self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))

1212

1213 def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):

1214 code = template % entry.cname

1215 #if entry.type.is_extension_type:

1216 # code = "((PyObject*)%s)" % code

1217 self.put_init_to_py_none(code, entry.type, nanny)

1218

1219 def put_pymethoddef(self, entry, term):

1220 from TypeSlots import method_coexist

1221 if entry.doc:

1222 doc_code = entry.doc_cname

1223 else:

1224 doc_code = 0

1225 method_flags = entry.signature.method_flags()

1226 if method_flags:

1227 if entry.is_special:

1228 method_flags += [method_coexist]

1229 self.putln(

1230 '{__Pyx_NAMESTR("%s"), (PyCFunction)%s, %s, __Pyx_DOCSTR(%s)}%s' % (

1231 entry.name,

1232 entry.func_cname,

1233 "|".join(method_flags),

1234 doc_code,

1235 term))

1236

1237 # error handling

1238

1239 def put_error_if_neg(self, pos, value):

1240 # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos))) # TODO this path is almost _never_ taken, yet this macro makes is slower!

1241 return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))

1242

1243 def set_error_info(self, pos):

1244 if Options.c_line_in_traceback:

1245 cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)

1246 else:

1247 cinfo = ""

1248 return "%s = %s[%s]; %s = %s;%s" % (

1249 Naming.filename_cname,

1250 Naming.filetable_cname,

1251 self.lookup_filename(pos[0]),

1252 Naming.lineno_cname,

1253 pos[1],

1254 cinfo)

1255

1256 def error_goto(self, pos):

1257 lbl = self.funcstate.error_label

1258 self.funcstate.use_label(lbl)

1259 return "{%s goto %s;}" % (

1260 self.set_error_info(pos),

1261 lbl)

1262

1263 def error_goto_if(self, cond, pos):

1264 return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))

1265

1266 def error_goto_if_null(self, cname, pos):

1267 return self.error_goto_if("!%s" % cname, pos)

1268

1269 def error_goto_if_neg(self, cname, pos):

1270 return self.error_goto_if("%s < 0" % cname, pos)

1271

1272 def error_goto_if_PyErr(self, pos):

1273 return self.error_goto_if("PyErr_Occurred()", pos)

1274

1275 def lookup_filename(self, filename):

1276 return self.globalstate.lookup_filename(filename)

1277

1278 def put_setup_refcount_context(self, name):

1279 self.putln('__Pyx_RefNannySetupContext("%s");' % name)

1280

1281 def put_finish_refcount_context(self):

1282 self.putln("__Pyx_RefNannyFinishContext();")

1283

1284 def put_trace_call(self, name, pos):

1285 self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]));

1286

1287 def put_trace_exception(self):

1288 self.putln("__Pyx_TraceException();")

1289

1290 def put_trace_return(self, retvalue_cname):

1291 self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)

1292

1293

1294 class PyrexCodeWriter(object):

1295 # f file output file

1296 # level int indentation level

1297

1298 def __init__(self, outfile_name):

1299 self.f = Utils.open_new_file(outfile_name)

1300 self.level = 0

1301

1302 def putln(self, code):

1303 self.f.write("%s%s\n" % (" " * self.level, code))

1304

1305 def indent(self):

1306 self.level += 1

1307

1308 def dedent(self):

1309 self.level -= 1

1310