You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
348 lines
9.6 KiB
348 lines
9.6 KiB
#!/usr/bin/env python
|
|
|
|
import os, difflib, time, gc, codecs, platform, sys
|
|
from pprint import pprint
|
|
import textwrap
|
|
|
|
# Setup a logger manually for compatibility with Python 2.3
|
|
import logging
|
|
logging.getLogger('MARKDOWN').addHandler(logging.StreamHandler())
|
|
import markdown
|
|
|
|
TEST_DIR = "tests"
|
|
TMP_DIR = "./tmp/"
|
|
WRITE_BENCHMARK = True
|
|
WRITE_BENCHMARK = False
|
|
ACTUALLY_MEASURE_MEMORY = True
|
|
|
|
######################################################################
|
|
|
|
if platform.system().lower() == "darwin": # Darwin
|
|
_proc_status = '/proc/%d/stat' % os.getpid()
|
|
else: # Linux
|
|
_proc_status = '/proc/%d/status' % os.getpid()
|
|
|
|
_scale = {'kB': 1024.0, 'mB': 1024.0*1024.0,
|
|
'KB': 1024.0, 'MB': 1024.0*1024.0}
|
|
|
|
def _VmB(VmKey):
|
|
'''Private.
|
|
'''
|
|
global _proc_status, _scale
|
|
# get pseudo file /proc/<pid>/status
|
|
try:
|
|
t = open(_proc_status)
|
|
v = t.read()
|
|
t.close()
|
|
except:
|
|
return 0.0 # non-Linux?
|
|
# get VmKey line e.g. 'VmRSS: 9999 kB\n ...'
|
|
i = v.index(VmKey)
|
|
v = v[i:].split(None, 3) # whitespace
|
|
if len(v) < 3:
|
|
return 0.0 # invalid format?
|
|
# convert Vm value to bytes
|
|
return float(v[1]) * _scale[v[2]]
|
|
|
|
|
|
def memory(since=0.0):
|
|
'''Return memory usage in bytes.
|
|
'''
|
|
if ACTUALLY_MEASURE_MEMORY :
|
|
return _VmB('VmSize:') - since
|
|
|
|
|
|
def resident(since=0.0):
|
|
'''Return resident memory usage in bytes.
|
|
'''
|
|
return _VmB('VmRSS:') - since
|
|
|
|
|
|
def stacksize(since=0.0):
|
|
'''Return stack size in bytes.
|
|
'''
|
|
return _VmB('VmStk:') - since
|
|
|
|
|
|
############################################################
|
|
|
|
DIFF_FILE_TEMPLATE = """
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<style>
|
|
td {
|
|
padding-left: 10px;
|
|
padding-right: 10px;
|
|
}
|
|
colgroup {
|
|
margin: 10px;
|
|
}
|
|
.diff_header {
|
|
color: gray;
|
|
}
|
|
.ok {
|
|
color: green;
|
|
}
|
|
.gray {
|
|
color: gray;
|
|
}
|
|
.failed a {
|
|
color: red;
|
|
}
|
|
.failed {
|
|
color: red;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<h1>Results Summary</h1>
|
|
<table rules="groups" >
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<th>
|
|
<td></td>
|
|
<td>Seconds</td>
|
|
<td></td>
|
|
<td>Memory</td>
|
|
</th>
|
|
<tbody>
|
|
"""
|
|
|
|
FOOTER = """
|
|
</body>
|
|
</html>
|
|
"""
|
|
|
|
DIFF_TABLE_TEMPLATE = """
|
|
<table class="diff" rules="groups" >
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<colgroup></colgroup>
|
|
<th>
|
|
<td></td>
|
|
<td>Expected</td>
|
|
<td></td>
|
|
<td></td>
|
|
<td>Actual</td>
|
|
</th>
|
|
<tbody>
|
|
%s
|
|
</tbody>
|
|
</table>
|
|
"""
|
|
|
|
|
|
def smart_split(text) :
|
|
result = []
|
|
for x in text.splitlines() :
|
|
for y in textwrap.wrap(textwrap.dedent(x), 40):
|
|
result.append(y)
|
|
return result
|
|
|
|
|
|
differ = difflib.Differ()
|
|
try :
|
|
htmldiff = difflib.HtmlDiff()
|
|
except:
|
|
htmldiff = None
|
|
|
|
class TestRunner :
|
|
|
|
def __init__ (self) :
|
|
self.failedTests = []
|
|
if not os.path.exists(TMP_DIR):
|
|
os.mkdir(TMP_DIR)
|
|
|
|
def test_directory(self, dir, measure_time=False, safe_mode=False, encoding="utf-8", output_format='xhtml1') :
|
|
self.encoding = encoding
|
|
benchmark_file_name = os.path.join(dir, "benchmark.dat")
|
|
self.saved_benchmarks = {}
|
|
|
|
if measure_time :
|
|
if os.path.exists(benchmark_file_name) :
|
|
file = open(benchmark_file_name)
|
|
for line in file.readlines() :
|
|
test, str_time, str_mem = line.strip().split(":")
|
|
self.saved_benchmarks[test] = (float(str_time), float(str_mem))
|
|
repeat = range(10)
|
|
else :
|
|
repeat = (0,)
|
|
|
|
# First, determine from the name of the directory if any extensions
|
|
# need to be loaded.
|
|
|
|
parts = os.path.split(dir)[-1].split("-x-")
|
|
if len(parts) > 1 :
|
|
extensions = parts[1].split("-")
|
|
print extensions
|
|
else :
|
|
extensions = []
|
|
|
|
mem = memory()
|
|
start = time.clock()
|
|
self.md = markdown.Markdown(extensions=extensions, safe_mode = safe_mode, output_format=output_format)
|
|
construction_time = time.clock() - start
|
|
construction_mem = memory(mem)
|
|
|
|
self.benchmark_buffer = "construction:%f:%f\n" % (construction_time,
|
|
construction_mem)
|
|
|
|
html_diff_file_path = os.path.join(TMP_DIR, os.path.split(dir)[-1]) + ".html"
|
|
self.html_diff_file = codecs.open(html_diff_file_path, "w", encoding=encoding)
|
|
self.html_diff_file.write(DIFF_FILE_TEMPLATE)
|
|
|
|
self.diffs_buffer = ""
|
|
|
|
tests = [x.replace(".txt", "")
|
|
for x in os.listdir(dir) if x.endswith(".txt")]
|
|
tests.sort()
|
|
for test in tests :
|
|
self.run_test(dir, test, repeat)
|
|
|
|
self.html_diff_file.write("</table>")
|
|
|
|
if sys.version < "3.0":
|
|
self.html_diff_file.write(self.diffs_buffer.decode("utf-8"))
|
|
|
|
self.html_diff_file.write(FOOTER)
|
|
self.html_diff_file.close()
|
|
print "Diff written to %s" % html_diff_file_path
|
|
|
|
benchmark_output_file_name = benchmark_file_name
|
|
|
|
if not WRITE_BENCHMARK:
|
|
benchmark_output_file_name += ".tmp"
|
|
|
|
self.benchmark_file = open(benchmark_output_file_name, "w")
|
|
self.benchmark_file.write(self.benchmark_buffer)
|
|
self.benchmark_file.close()
|
|
|
|
|
|
####################
|
|
|
|
|
|
def run_test(self, dir, test, repeat):
|
|
|
|
print "--- %s ---" % test
|
|
self.html_diff_file.write("<tr><td>%s</td>" % test)
|
|
input_file = os.path.join(dir, test + ".txt")
|
|
output_file = os.path.join(dir, test + ".html")
|
|
|
|
expected_output = codecs.open(output_file, encoding=self.encoding).read()
|
|
input = codecs.open(input_file, encoding=self.encoding).read()
|
|
actual_output = ""
|
|
actual_lines = []
|
|
self.md.source = ""
|
|
gc.collect()
|
|
mem = memory()
|
|
start = time.clock()
|
|
for x in repeat:
|
|
actual_output = self.md.convert(input)
|
|
conversion_time = time.clock() - start
|
|
conversion_mem = memory(mem)
|
|
self.md.reset()
|
|
|
|
expected_lines = [x.encode("utf-8") for x in smart_split(expected_output)]
|
|
actual_lines = [x.encode("utf-8") for x in smart_split(actual_output)]
|
|
|
|
#diff = difflib.ndiff(expected_output.split("\n"),
|
|
# actual_output.split("\n"))
|
|
|
|
diff = [x for x in differ.compare(expected_lines,
|
|
actual_lines)
|
|
if not x.startswith(" ")]
|
|
|
|
if not diff:
|
|
self.html_diff_file.write("<td class='ok'>OK</td>")
|
|
else :
|
|
self.failedTests.append(test)
|
|
self.html_diff_file.write("<td class='failed'>" +
|
|
"<a href='#diff-%s'>FAILED</a></td>" % test)
|
|
print "MISMATCH on %s/%s.txt" % (dir, test)
|
|
print
|
|
for line in diff :
|
|
print line
|
|
if htmldiff!=None :
|
|
htmlDiff = htmldiff.make_table(expected_lines, actual_lines,
|
|
context=True)
|
|
htmlDiff = "\n".join( [x for x in htmlDiff.splitlines()
|
|
if x.strip().startswith("<tr>")] )
|
|
self.diffs_buffer += "<a name='diff-%s'/><h2>%s</h2>" % (test, test)
|
|
self.diffs_buffer += DIFF_TABLE_TEMPLATE % htmlDiff
|
|
|
|
expected_time, expected_mem = self.saved_benchmarks.get(test, ("na", "na"))
|
|
|
|
self.html_diff_file.write(get_benchmark_html(conversion_time, expected_time))
|
|
self.html_diff_file.write(get_benchmark_html(conversion_mem, expected_mem))
|
|
self.html_diff_file.write("</tr>\n")
|
|
|
|
self.benchmark_buffer += "%s:%f:%f\n" % (test,
|
|
conversion_time, conversion_mem)
|
|
|
|
|
|
|
|
|
|
|
|
def get_benchmark_html (actual, expected) :
|
|
buffer = ""
|
|
if not expected == "na":
|
|
if actual > expected * 1.5:
|
|
tdiff = "failed"
|
|
elif actual * 1.5 < expected :
|
|
tdiff = "ok"
|
|
else :
|
|
tdiff = "same"
|
|
if ( (actual <= 0 and expected < 0.015) or
|
|
(expected <= 0 and actual < 0.015)) :
|
|
tdiff = "same"
|
|
else :
|
|
tdiff = "same"
|
|
buffer += "<td class='%s'>%.2f</td>" % (tdiff, actual)
|
|
if not expected == "na":
|
|
buffer += "<td class='gray'>%.2f</td>" % (expected)
|
|
return buffer
|
|
|
|
|
|
def run_tests() :
|
|
|
|
tester = TestRunner()
|
|
#test.test_directory("tests/basic")
|
|
tester.test_directory("tests/markdown-test", measure_time=True)
|
|
tester.test_directory("tests/misc", measure_time=True)
|
|
tester.test_directory("tests/extensions-x-tables")
|
|
tester.test_directory("tests/extensions-x-footnotes")
|
|
#tester.test_directory("tests/extensions-x-ext1-ext2")
|
|
tester.test_directory("tests/safe_mode", measure_time=True, safe_mode="escape")
|
|
tester.test_directory("tests/extensions-x-wikilinks")
|
|
tester.test_directory("tests/extensions-x-toc")
|
|
tester.test_directory("tests/extensions-x-def_list")
|
|
tester.test_directory("tests/extensions-x-abbr")
|
|
tester.test_directory("tests/html4", output_format='html4')
|
|
|
|
try:
|
|
import pygments
|
|
except ImportError:
|
|
# Dependancy not avalable - skip test
|
|
pass
|
|
else:
|
|
tester.test_directory("tests/extensions-x-codehilite")
|
|
|
|
print "\n### Final result ###"
|
|
if len(tester.failedTests):
|
|
print "%d failed tests: %s" % (len(tester.failedTests), str(tester.failedTests))
|
|
else:
|
|
print "All tests passed, no errors!"
|
|
|
|
run_tests()
|
|
|
|
|
|
|
|
|