1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
|
#!/usr/bin/env python
#
# Given a previous good compile narrow down miscompiles.
# Expectes two directories named "before" and "after" each containing a set of
# assembly files where the "after" version is assumed to be broken.
# Also assumes the presence of a executable or script "link_test" which when
# called with a set of assembly files will link them together and test if the
# resulting executable is "good".
#
# Example usage:
# 1. Create a link_test script, make it executable. Simple Example:
# clang "$@" -o /tmp/test && /tmp/test || echo "PROBLEM"
# 2. Run the script to figure out which files are miscompiled:
# > ./abtest.py
# somefile.s: ok
# someotherfile.s: skipped: same content
# anotherfile.s: failed: './link_test' exitcode != 0
# ...
# 3. If you want to replace+test with the functions inside a single file
# you first have to mark function begins and ends in the .s files
# this directory comes with some: mark_XXX.py example scripts,
# unfortunately you usually have to adapt them to each environment.
# > for i in before/*.s after/*.s; do mark_xxx.py $i; done
# 4. Run the tests on a single file
# > ./abtest.py
# funcname1 [0/XX]: ok
# funcname2 [1/XX]: ok
# funcname3 [2/XX]: skipped: same content
# funcname4 [3/XX]: failed: './link_test' exitcode != 0
# ...
import sys
import os
from os import system, mkdir, walk, makedirs, errno, getenv
from os.path import dirname, isdir
from shutil import rmtree, copyfile
from fnmatch import filter
from itertools import chain
from subprocess import call
from sys import stderr
import argparse
import filecmp
LINKTEST="./link_test"
ESCAPE="\033[%sm"
BOLD=ESCAPE % "1"
RED=ESCAPE % "31"
NORMAL=ESCAPE % "0"
FAILED=RED+"failed"+NORMAL
def mkdirtree(path):
try:
makedirs(path)
except OSError as exc:
if exc.errno == errno.EEXIST and isdir(path):
pass
else:
raise
def find(dir, file_filter=None):
files = [walkdir[0]+"/"+file for walkdir in walk(dir) for file in walkdir[2]]
if file_filter != None:
files = filter(files, file_filter)
return files
def error(message):
stderr.write("Error: %s\n" % (message,))
def warn(message):
stderr.write("Warning: %s\n" % (message,))
def notice(message):
stderr.write("%s\n" % message)
def extract_functions(file):
functions = []
in_function = None
for line in open(file):
if line.startswith("# -- Begin "):
if in_function != None:
warn("Missing end of function %s" % (in_function,))
funcname = line[12:-1]
in_function = funcname
text = line
elif line.startswith("# -- End "):
function_name = line[10:-1]
if in_function != function_name:
warn("End %s does not match begin %s" % (function_name, in_function))
else:
text += line
functions.append( (in_function, text) )
in_function = None
elif in_function != None:
text += line
return functions
def replace_function(file, function, replacement, dest):
out = open(dest, "w")
skip = False
found = False
in_function = None
for line in open(file):
if line.startswith("# -- Begin "):
if in_function != None:
warn("Missing end of function %s" % (in_function,))
funcname = line[12:-1]
in_function = funcname
if in_function == function:
out.write(replacement)
skip = True
elif line.startswith("# -- End "):
function_name = line[10:-1]
if in_function != function_name:
warn("End %s does not match begin %s" % (function_name, in_function))
in_function = None
if skip:
skip = False
continue
if not skip:
out.write(line)
def announce_test(name):
stderr.write("%s%s%s: " % (BOLD, name, NORMAL))
stderr.flush()
def announce_result(result, info):
stderr.write(result)
if info != "":
stderr.write(": %s" % info)
stderr.write("\n")
stderr.flush()
def testrun(files):
linkline="%s %s" % (LINKTEST, " ".join(files),)
res = call(linkline, shell=True)
if res != 0:
announce_result(FAILED, "'%s' exitcode != 0" % LINKTEST)
else:
announce_result("ok", "")
def check_files():
"""Check files mode"""
for i in range(0, len(NO_PREFIX)):
f = NO_PREFIX[i]
b=baddir+"/"+f
if b not in BAD_FILES:
warn("There is no corresponding file to '%s' in %s" \
% (gooddir+"/"+f, baddir))
continue
announce_test(f + " [%s/%s]" % (i+1, len(NO_PREFIX)))
# combine files (everything from good except f)
testfiles=[]
skip=False
for c in NO_PREFIX:
badfile = baddir+"/"+c
goodfile = gooddir+"/"+c
if c == f:
testfiles.append(badfile)
if filecmp.cmp(goodfile, badfile):
announce_result("skipped", "same content")
skip = True
break
else:
testfiles.append(goodfile)
if skip:
continue
testrun(testfiles)
def check_functions_in_file(base, goodfile, badfile):
functions = extract_functions(goodfile)
if len(functions) == 0:
warn("Couldn't find any function in %s, missing annotations?" % (goodfile,))
return
badfunctions = dict(extract_functions(badfile))
if len(functions) == 0:
warn("Couldn't find any function in %s, missing annotations?" % (badfile,))
return
COMBINED="/tmp/combined.s"
i = 0
for (func,func_text) in functions:
announce_test(func + " [%s/%s]" % (i+1, len(functions)))
i+=1
if func not in badfunctions:
warn("Function '%s' missing from bad file" % func)
continue
if badfunctions[func] == func_text:
announce_result("skipped", "same content")
continue
replace_function(goodfile, func, badfunctions[func], COMBINED)
testfiles=[]
for c in NO_PREFIX:
if c == base:
testfiles.append(COMBINED)
continue
testfiles.append(gooddir + "/" + c)
testrun(testfiles)
parser = argparse.ArgumentParser()
parser.add_argument('--a', dest='dir_a', default='before')
parser.add_argument('--b', dest='dir_b', default='after')
parser.add_argument('file', metavar='file', nargs='?')
config = parser.parse_args()
# Check if environment is sane
if not os.access(LINKTEST, os.X_OK):
error("Expect '%s' to be present and executable" % (LINKTEST,))
exit(1)
gooddir=config.dir_a
baddir=config.dir_b
BAD_FILES=find(baddir, "*")
GOOD_FILES=find(gooddir, "*")
NO_PREFIX=sorted([x[len(gooddir)+1:] for x in GOOD_FILES])
if config.file is not None:
goodfile = gooddir+"/"+config.file
badfile = baddir+"/"+config.file
check_functions_in_file(config.file, goodfile, badfile)
else:
check_files()
|
