1 #

2 # Code for automatically running tests

3 #

4

5 import glob, os, sys, re, traceback

6 from os import path

7 from string import replace, strip

8

9 from Pyrex.Compiler import Main

10 from Pyrex.Utils import replace_suffix

11 from Pyrex.Compiler.Main import CCompilerError

12

13 from Pyrex.Compiler import ModuleNode

14 ModuleNode.testing_mode = 1

15

16 platform = sys.platform

17 if platform == "mac":

18 from Pyrex.Testing.MacTesting import run_python_file

19 from Pyrex.Testing.Mac.FileMarking import mark_item, item_mark

20 from Pyrex.Mac.MacSystem import c_compile

21 elif platform == "darwin":

22 from Pyrex.Testing.UnixTesting import run_python_file

23 from Pyrex.Testing.Mac.FileMarking import mark_item, item_mark

24 from Pyrex.Mac.DarwinSystem import c_compile, c_link

25 elif "linux" in platform:

26 from Pyrex.Unix.UnixTesting import \

27 run_python_file, mark_item, item_mark

28 from Pyrex.Unix.LinuxSystem import c_compile

29 else:

30 raise Exception(

31 "Testing not supported on platform '%s'" % sys.platform)

32

33

34 class FailureError(Exception):

35 pass

36

37 class AbortTesting(Exception):

38 pass

39

40 failure_flag = 0

41 stop_at_first_failure = 0

42

43 def run_tests():

44 "Run tests given as command line arguments."

45 global stop_at_first_failure, new_results

46 stop_at_first_failure = 0

47 new_results = []

48 args = sys.argv[1:]

49 if args and args[0] == "-1":

50 print "Setting stop_at_first_failure flag" ###

51 stop_at_first_failure = 1

52 del args[0]

53 try:

54 if args:

55 for arg in args:

56 run_test(arg)

57 else:

58 run_all_tests()

59 except AbortTesting:

60 pass

61 for f in new_results:

62 print "NEW RESULT:", f

63 if not failure_flag:

64 print "All tests passed."

65 else:

66 if stop_at_first_failure:

67 print "Testing aborted."

68 else:

69 print "All tests complete."

70 print "FAILURES OCCURRED"

71

72 def run_all_tests():

73 "Run all tests in the Tests folder of the current dir."

74 #test_dir = path.join(os.pardir, "Tests")

75 test_dir = "Tests"

76 return run_tests_in_dir(test_dir)

77

78 def run_test(item_path):

79 "Run a single test or directory of tests."

80 item_name = path.basename(item_path)

81 if item_name.startswith("("):

82 return "passed"

83 elif item_marked_tested(item_path):

84 print "Already tested", item_path

85 return "passed"

86 else:

87 print "Running test", item_path

88 if path.isdir(item_path):

89 if item_name.startswith("R_"):

90 return run_functional_test(item_path)

91 else:

92 return run_tests_in_dir(item_path)

93 elif item_name.startswith("r_"):

94 return run_functional_test(item_path)

95 elif item_name.startswith("b_"):

96 return "passed"

97 elif item_name.startswith("l_"):

98 return run_compile_test(item_path, link = 1)

99 else:

100 return run_compile_test(item_path)

101

102 def run_tests_in_dir(dir):

103 "Run all tests in given directory."

104 #print "*** run_tests_in_dir:", dir ###

105 print "Running tests in", dir

106 items = glob.glob(path.join(dir, "*.pyx"))

107 result = "passed"

108 for item in items:

109 #print "*** run_tests_in_dir: doing file", item ###

110 #item = path.join(dir, name)

111 if run_test(item) <> "passed":

112 result = "failed"

113 names = os.listdir(dir)

114 for name in names:

115 if name not in ignore_dir_names:

116 #print "*** run_tests_in_dir: checking name", name ###

117 item = path.join(dir, name)

118 if path.isdir(item):

119 if run_test(item) <> "passed":

120 result = "failed"

121 mark_item(dir, result)

122 return result

123

124 ignore_dir_names = (

125 "Reference", "CantTestYet"

126 )

127

128 def run_compile_test(item, link = 0):

129 """Run a single compile-only or compile-and-link test.

130 If linking, the linked extension module is kept for use by later tests.

131 """

132 try:

133 mark_item(item, "failed")

134 dir = path.dirname(item)

135 name = path.basename(item)

136 global mangled_module_name

137 module_name, _ = os.path.splitext(name)

138 mangled_module_name = "%d%s_" % (len(module_name), module_name)

139 produces_include_files = name.startswith("i_") or name.startswith("ia_")

140 produces_api_file = name.startswith("a_") or name.startswith("ia_")

141 is_error_test = (

142 name[:2] == "e_" or

143 name[:3] == "se_")

144 options = Main.CompilationOptions(Main.default_options)

145 if is_error_test:

146 options.use_listing_file = 1

147 options.errors_to_stderr = 0

148 try:

149 result = Main.compile(item, options)

150 except CCompilerError:

151 fail("C compilation error.")

152 except:

153 fail_with_exception("Exception raised in Pyrex compiler.")

154 #print "result =", result.__dict__ ###

155 if is_error_test:

156 if result.num_errors == 0:

157 fail("No errors produced, expected some")

158 if result.listing_file is None:

159 fail("No listing file produced")

160 compare_with_reference(result.listing_file, show_diffs = 0,

161 line_munger = munge_error_line)

162 remove_file(replace_suffix(item, ".c"))

163 else:

164 if result.num_errors <> 0:

165 #display_files(replace_suffix(item, ".lis"))

166 fail("%s errors reported, expected none" %

167 result.num_errors)

168 if result.c_file is None:

169 fail("No C file produced")

170 compare_with_reference(result.c_file, show_diffs = 1,

171 line_munger = munge_c_line)

172 if produces_include_files:

173 if result.h_file is None:

174 fail("No header file produced")

175 compare_with_reference(result.h_file, show_diffs = 1,

176 line_munger = munge_c_line)

177 if result.i_file is None:

178 pass

179 # .pxi files no longer produced by default

180 #fail("No include file produced")

181 else:

182 compare_with_reference(result.i_file, show_diffs = 1,

183 line_munger = None)

184 if produces_api_file:

185 if result.api_file is None:

186 fail("No api header file produced")

187 compare_with_reference(result.api_file, show_diffs = 1,

188 line_munger = munge_c_line)

189 try:

190 result.object_file = c_compile(result.c_file)

191 except CCompilerError:

192 fail("C compilation error.")

193 except:

194 fail_with_exception("C compiler failed.")

195 try:

196 cplus_object_file = c_compile(result.c_file, cplus = 1, obj_suffix = ".cplus.o")

197 except CCompilerError:

198 fail("C++ compilation error.")

199 except:

200 fail_with_exception("C++ compiler failed.")

201 if link:

202 try:

203 c_link(result.object_file)

204 except CCompilerError:

205 fail("C linking error.")

206 remove_file(result.listing_file)

207 remove_file(result.object_file)

208 remove_file(cplus_object_file)

209 mark_item(item, "passed")

210 return "passed"

211 except FailureError:

212 return "failed"

213

214 def run_functional_test_dir(dir, keep_files = 0):

215 pyx_files = glob.glob(path.join(dir, "*.pyx"))

216 if not pyx_files:

217 fail("No .pyx file")

218 if len(pyx_files) > 1:

219 fail("Too many .pyx files")

220 pyx_file = pyx_files[0]

221 return run_functional_test(pyx_file, keep_files)

222

223 def compile_and_link(pyx_file):

224 try:

225 result = Main.compile(pyx_file, c_compile = 1, c_link = 1)

226 except CCompilerError:

227 fail("C compilation failed.")

228 except:

229 fail_with_exception("Pyrex compiler failed.")

230 if result.num_errors <> 0:

231 fail("%d Pyrex errors reported" % result.num_errors)

232 return result

233

234 def functional_test_aux_file(pyx_file):

235 dir, name = os.path.split(pyx_file)

236 aux_name = "b_" + name[2:]

237 return os.path.join(dir, aux_name)

238

239 def run_functional_test(pyx_file, keep_files = 0):

240 "Run a compile, link and execute test."

241 try:

242 mark_item(pyx_file, "failed")

243 result = compile_and_link(pyx_file)

244 aux_pyx_file = functional_test_aux_file(pyx_file)

245 if os.path.exists(aux_pyx_file):

246 aux_result = compile_and_link(aux_pyx_file)

247 else:

248 aux_result = None

249 #new_c = compare_with_reference(result.c_file, show_diffs = 1)

250 new_c = 0

251 py_file = replace_suffix(pyx_file, "_t.py")

252 out_file = replace_suffix(pyx_file, ".out")

253 err_file = replace_suffix(pyx_file, ".err")

254 try:

255 stat = run_python_file(py_file, out_file, err_file)

256 except:

257 print_file(err_file)

258 fail_with_exception("Python script execution failed.")

259 if stat:

260 fail("Exit status %s" % stat)

261 new_output = compare_with_reference(out_file, show_diffs = 0,

262 line_munger = munge_runnable_test_output_line)

263 if not keep_files:

264 remove_file(replace_suffix(pyx_file, ".lis"))

265 if not new_c:

266 remove_file(result.c_file)

267 remove_file(result.object_file)

268 remove_file(result.extension_file)

269 remove_file(err_file)

270 if aux_result:

271 remove_file(replace_suffix(aux_pyx_file, ".lis"))

272 remove_file(aux_result.c_file)

273 remove_file(aux_result.object_file)

274 remove_file(aux_result.extension_file)

275 mark_item(pyx_file, "passed")

276 return "passed"

277 except FailureError:

278 return "failed"

279

280 def print_file(path):

281 text = open(path).read()

282 sys.stdout.write(text)

283 if not text.endswith("\n"):

284 sys.stdout.write("\n")

285

286 def compare_with_reference(file_in_question, show_diffs,

287 line_munger):

288 dir = path.dirname(file_in_question)

289 name = path.basename(file_in_question)

290 refdir = path.join(dir, "Reference")

291 reference_file = path.join(refdir, name)

292 if not path.exists(reference_file):

293 #print "NEW RESULT:", file_in_question

294 new_results.append(file_in_question)

295 #display_files(file_in_question)

296 return 1

297 lines1 = get_lines(file_in_question, line_munger)

298 lines2 = get_lines(reference_file, line_munger)

299 if not munged_lines_equal(lines1, lines2):

300 print "%s differs from reference." % name

301 show_munged_lines_difference(lines1, lines2)

302 fail("%s differs from reference" % name)

303 return 0

304

305 def munged_lines_equal(lines1, lines2):

306 if len(lines1) <> len(lines2):

307 #print "Different numbers of munged lines:", \

308 # len(lines1), len(lines2) ###

309 return 0

310 for i in xrange(min(len(lines1), len(lines2))):

311 #print "%4d: '%r'" % lines1[i] ###

312 #print "%4d: '%r'" % lines2[i] ###

313 if lines1[i][1] <> lines2[i][1]:

314 return 0

315 return 1

316

317 class StopComparison(Exception):

318 pass

319

320 def get_lines(filename, line_munger):

321 try:

322 f = open(filename)

323 lines = f.readlines()

324 f.close()

325 except IOError, e:

326 fail(str(e))

327 lines2 = []

328 i = 0

329 for line in lines:

330 i += 1

331 line = strip(line)

332 if line_munger:

333 try:

334 line = line_munger(line)

335 except StopComparison:

336 break

337 if line:

338 lines2.append((i, line))

339 return lines2

340

341 def munge_error_line(line):

342 line = line.replace('"', '')

343 file, mess = line.split(None, 1)

344 i = line.rfind(":", 3)

345 #print "Testing.munge_error_line:" ###

346 #print "...file =", repr(file) ###

347 #print "...mess =", repr(mess) ###

348 #print "...i =", i ###

349 line = "%s %s" % (file[i:], mess)

350 #print "...new line =", repr(line)

351 return line

352

353 def munge_runnable_test_output_line(line):

354 # Get rid of the leading components of anything

355 # that looks like a double-quoted pathname.

356 pat = r'\"([^:/]*[:/])*'

357 line = re.sub(pat, '"', line)

358 line = line.replace("in <module>", "in ?")

359 return line

360

361 #mangled_module_name = None

362

363 def munge_c_line(line):

364 #

365 # Try to compensate for changes in code generation

366 # strategy.

367 #

368 # MINOR HACKs are relatively harmless since any

369 # problems they mask will be caught due to the

370 # C code failing to compile.

371 #

372 # HACKs, on the other hand, could mask real problems.

373 # The reference files should be updated as soon as

374 # possible to make them unnecessary, and they

375 # should be removed.

376 #

377 # MINOR HACK: Ignore runtime support code

378 if line == "/* Runtime support code */":

379 raise StopComparison

380 # Ignore comments and other easily recognisable junk

381 if line[:2] == "/*" and line[-2:] == "*/":

382 line = ""

383 elif line in ignore_lines:

384 line = ""

385 line = replace(line, " ", "")

386

387 # MINOR HACK: ignore prototypes

388 if line[-9:] == "/*proto*/":

389 line = ""

390

391 # HACK: treat all temp vars as equivalent

392 line = re.sub("__pyx_[0-9]+", "__pyx_x", line)

393

394 # HACK: ignore differences in source line numbers

395 line = re.sub("__pyx_lineno=[0-9]+", "__pyx_lineno=x", line)

396

397 # MINOR HACK: ignore temp var declarations

398 if line == "PyObject*__pyx_x=0;":

399 line = ""

400 if line == "int__pyx_x;":

401 line = ""

402 if line == "Py_ssize_t__pyx_x;":

403 line = ""

404

405 ## MINOR HACK: ignore gcc3.3 bug workaround lines

406 #if "__pyx_gcc33_" in line:

407 # line = ""

408

409 # ------ End of standing hacks -----

410

411 # HACKS for string const changes

412

413 if line.startswith("staticchar__pyx_k") \

414 or line.startswith("staticPyObject*__pyx_k") \

415 or line.startswith("staticPyObject*__pyx_n"):

416 line = ""

417 line = re.sub("__pyx_k[0-9]+", "__pyx_kx", line)

418 line = re.sub("__pyx_k[0-9]+p", "__pyx_kxp", line)

419

420

421 # ---------- END HACKS ----------

422

423 return line

424

425 # The following are matched after removing leading and trailing

426 # whitespace but before removing embedded whitespace.

427

428 ignore_lines = (

429 "#ifndef __stdcall"

430 "#define __stdcall"

431 "#endif"

432 "#ifndef __cdecl"

433 "#define __cdecl"

434 )

435

436 def show_munged_lines_difference(newlines, reflines):

437 print "Flexidiff:"

438 for i in range(min(len(newlines), len(reflines))):

439 n1, line1 = newlines[i]

440 n2, line2 = reflines[i]

441 if line1 <> line2:

442 print "New %4d: %s" % (n1, repr(line1))

443 print "Ref %4d: %s" % (n2, repr(line2))

444 return

445 if len(newlines) > len(reflines):

446 n1, line1 = newlines[len(reflines)]

447 print "New %4d: %s" % (n1, repr(line1))

448 elif len(reflines) > len(newlines):

449 n2, line2 = reflines[len(newlines)]

450 print "Ref %4d: %s" % (n2, repr(line2))

451

452 def fail(mess):

453 global failure_flag

454 failure_flag = 1

455 print "TEST FAILED:", mess

456 if 0:

457 ans = raw_input("Continue testing [y/n]? ")

458 if ans[:1] <> "y":

459 print "Testing aborted."

460 sys.exit(1)

461 if stop_at_first_failure:

462 raise AbortTesting

463 else:

464 raise FailureError

465

466 def fail_with_exception(mess):

467 traceback.print_exc()

468 fail(mess)

469

470 #def display_files(*filenames):

471 # if sys.platform == "mac":

472 # for filename in filenames:

473 # bbedit_open(filename)

474

475 def remove_file(file):

476 #print "Removing:", file ###

477 if file:

478 try:

479 os.unlink(file)

480 except (IOError, OSError):

481 pass

482

483 def item_marked_tested(path):

484 return item_mark(path) == "passed"