prosody
132a3c7b25fa
tools/test_mutants.sh.lua

Find changesets by keywords by author, files, the commit message, revision number or hash, or revset expression.

Software / code / prosody

Comparison

tools/test_mutants.sh.lua @ 12765:132a3c7b25fa

tools: Add initial mutation testing script
author Matthew Wild <mwild1@gmail.com>
date Tue, 11 Oct 2022 11:53:48 +0100
comparison
equal deleted inserted replaced
12764:bf6d2f9fad4d 12765:132a3c7b25fa
1 #!/bin/bash
2
3 POLYGLOT=1--[===[
4
5 set -o pipefail
6
7 if [[ "$#" == "0" ]]; then
8 echo "Lua mutation testing tool"
9 echo
10 echo "Usage:"
11 echo " $BASH_SOURCE MODULE_NAME SPEC_FILE"
12 echo
13 echo "Requires 'lua', 'ltokenp' and 'busted' in PATH"
14 exit 1;
15 fi
16
17 MOD_NAME="$1"
18 MOD_FILE="$(lua "$BASH_SOURCE" resolve "$MOD_NAME")"
19
20 if [[ "$MOD_FILE" == "" || ! -f "$MOD_FILE" ]]; then
21 echo "EE: Failed to locate module '$MOD_NAME' ($MOD_FILE)";
22 exit 1;
23 fi
24
25 SPEC_FILE="$2"
26
27 if [[ "$SPEC_FILE" == "" ]]; then
28 SPEC_FILE="spec/${MOD_NAME/./_}_spec.lua"
29 fi
30
31 if [[ "$SPEC_FILE" == "" || ! -f "$SPEC_FILE" ]]; then
32 echo "EE: Failed to find test spec file ($SPEC_FILE)"
33 exit 1;
34 fi
35
36 if ! busted "$SPEC_FILE"; then
37 echo "EE: Tests fail on original source. Fix it"\!;
38 exit 1;
39 fi
40
41 export MUTANT_N=0
42 LIVING_MUTANTS=0
43
44 FILE_PREFIX="${MOD_FILE%.*}.mutant-"
45 FILE_SUFFIX=".${MOD_FILE##*.}"
46
47 gen_mutant () {
48 echo "Generating mutant $2 to $3..."
49 ltokenp -s "$BASH_SOURCE" "$1" > "$3"
50 return "$?"
51 }
52
53 # $1 = MOD_NAME, $2 = MUTANT_N, $3 = SPEC_FILE
54 test_mutant () {
55 (
56 ulimit -m 131072 # 128MB
57 ulimit -t 16 # 16s
58 ulimit -f 32768 # 128MB (?)
59 exec busted --helper="$BASH_SOURCE" -Xhelper mutate="$1":"$2" "$3"
60 ) >/dev/null
61 return "$?";
62 }
63
64 MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}"
65
66 gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE"
67 while [[ "$?" == "0" ]]; do
68 if ! test_mutant "$MOD_NAME" "$MUTANT_N" "$SPEC_FILE"; then
69 echo "Tests successfully killed mutant $MUTANT_N";
70 rm "$MUTANT_FILE";
71 else
72 echo "Mutant $MUTANT_N lives on"\!
73 LIVING_MUTANTS=$((LIVING_MUTANTS+1))
74 fi
75 MUTANT_N=$((MUTANT_N+1))
76 MUTANT_FILE="${FILE_PREFIX}${MUTANT_N}${FILE_SUFFIX}"
77 gen_mutant "$MOD_FILE" "$MUTANT_N" "$MUTANT_FILE"
78 done
79
80 if [[ "$?" != "2" ]]; then
81 echo "Failed: $?"
82 exit "$?";
83 fi
84
85 MUTANT_SCORE="$(lua -e "print(('%0.2f'):format((1-($LIVING_MUTANTS/$MUTANT_N))*100))")"
86 if test -f mutant-scores.txt; then
87 echo "$MOD_NAME $MUTANT_SCORE" >> mutant-scores.txt
88 fi
89 echo "$MOD_NAME: All $MUTANT_N mutants generated, $LIVING_MUTANTS survived (score: $MUTANT_SCORE%)"
90 rm "$MUTANT_FILE"; # Last file is always unmodified
91 exit 0;
92 ]===]
93
94 -- busted helper that runs mutations
95 if arg then
96 if arg[1] == "resolve" then
97 local filename = package.searchpath(assert(arg[2], "no module name given"), package.path);
98 if filename then
99 print(filename);
100 end
101 os.exit(filename and 0 or 1);
102 end
103 local mutants = {};
104
105 for i = 1, #arg do
106 local opt = arg[i];
107 print("LOAD", i, opt)
108 local module_name, mutant_n = opt:match("^mutate=([^:]+):(%d+)");
109 if module_name then
110 mutants[module_name] = tonumber(mutant_n);
111 end
112 end
113
114 local orig_lua_searcher = package.searchers[2];
115
116 local function mutant_searcher(module_name)
117 local mutant_n = mutants[module_name];
118 if not mutant_n then
119 return orig_lua_searcher(module_name);
120 end
121 local base_file, err = package.searchpath(module_name, package.path);
122 if not base_file then
123 return base_file, err;
124 end
125 local mutant_file = base_file:gsub("%.lua$", (".mutant-%d.lua"):format(mutant_n));
126 return loadfile(mutant_file), mutant_file;
127 end
128
129 if next(mutants) then
130 table.insert(package.searchers, 1, mutant_searcher);
131 end
132 end
133
134 -- filter for ltokenp to mutate scripts
135 do
136 local last_output = {};
137 local function emit(...)
138 last_output = {...};
139 io.write(...)
140 io.write(" ")
141 return true;
142 end
143
144 local did_mutate = false;
145 local count = -1;
146 local threshold = tonumber(os.getenv("MUTANT_N")) or 0;
147 local function should_mutate()
148 count = count + 1;
149 return count == threshold;
150 end
151
152 local function mutate(name, value)
153 if name == "if" then
154 -- Bypass conditionals
155 if should_mutate() then
156 return emit("if true or");
157 elseif should_mutate() then
158 return emit("if false and");
159 end
160 elseif name == "<integer>" then
161 -- Introduce off-by-one errors
162 if should_mutate() then
163 return emit(("%d"):format(tonumber(value)+1));
164 elseif should_mutate() then
165 return emit(("%d"):format(tonumber(value)-1));
166 end
167 elseif name == "and" then
168 if should_mutate() then
169 return emit("or");
170 end
171 elseif name == "or" then
172 if should_mutate() then
173 return emit("and");
174 end
175 end
176 end
177
178 local current_line_n, current_line_input, current_line_output = 0, {}, {};
179 function FILTER(line_n,token,name,value)
180 if current_line_n ~= line_n then -- Finished a line, moving to the next?
181 if did_mutate and did_mutate.line == current_line_n then
182 -- The line we finished was mutated. Store the original and modified outputs.
183 did_mutate.line_original_src = table.concat(current_line_input, " ");
184 did_mutate.line_modified_src = table.concat(current_line_output, " ");
185 end
186 current_line_input = {};
187 current_line_output = {};
188 end
189 current_line_n = line_n;
190 if name == "<file>" then return; end
191 if name == "<eof>" then
192 if not did_mutate then
193 return os.exit(2);
194 else
195 emit(("\n-- Mutated line %d (changed '%s' to '%s'):\n"):format(did_mutate.line, did_mutate.original, did_mutate.modified))
196 emit( ("-- Original: %s\n"):format(did_mutate.line_original_src))
197 emit( ("-- Modified: %s\n"):format(did_mutate.line_modified_src));
198 return;
199 end
200 end
201 if name == "<string>" then
202 value = string.format("%q",value);
203 end
204 if mutate(name, value) then
205 did_mutate = {
206 original = value;
207 modified = table.concat(last_output);
208 line = line_n;
209 };
210 else
211 emit(value);
212 end
213 table.insert(current_line_input, value);
214 table.insert(current_line_output, table.concat(last_output));
215 end
216 end
217