1:- module(mutalog,
2 [ main/0,
3 mutalog_list/3 4 ]).
25:- use_module(library(lists)). 26:- use_module(library(apply)). 27:- use_module(library(aggregate)). 28:- use_module(library(http/json)). 29:- use_module(mutalog/config). 30:- use_module(mutalog/discover). 31:- use_module(mutalog/reader). 32:- use_module(mutalog/operators). 33:- use_module(mutalog/generate). 34:- use_module(mutalog/run). 35:- use_module(mutalog/report). 36
37:- initialization(check_version). 38
39check_version :-
40 current_prolog_flag(version, V),
41 ( V >= 90000
42 -> true
43 ; print_message(warning, format("Mutalog needs SWI-Prolog 9.0 or later", []))
44 ).
50main :-
51 current_prolog_flag(argv, Argv),
52 catch(run_cli(Argv, Code), E, (print_message(error, E), Code = 2)),
53 halt(Code).
54
55run_cli([list|Args], Code) :-
56 !,
57 parse_options(Args, Opts),
58 working_directory(Cwd, Cwd),
59 cli_overrides(Opts, Overrides),
60 mutalog_list(Cwd, Overrides, Result),
61 ( memberchk(format(json), Opts)
62 -> print_json(Result)
63 ; print_text(Result)
64 ),
65 ( memberchk(check, Opts)
66 -> check_result(Result, Code)
67 ; Code = 0
68 ).
69run_cli([run|Args], Code) :-
70 !,
71 parse_options(Args, Opts),
72 working_directory(Cwd, Cwd),
73 cli_overrides(Opts, Overrides),
74 mutalog_run(Cwd, Overrides, Run),
75 Run = run(Config, _, _, _),
76 config_value(reporters, Config, Reporters),
77 ( memberchk(clear_text, Reporters)
78 -> print_run(Run)
79 ; true
80 ),
81 write_reports(Run, Written),
82 directory_file_path(Cwd, x, Here),
83 forall(member(F, Written),
84 ( relative_file_name(F, Here, Rel),
85 format("Report: ~w~n", [Rel]) )),
86 run_code(Run, Code).
87run_cli(['dry-run'|Args], Code) :-
88 !,
89 parse_options(Args, Opts),
90 working_directory(Cwd, Cwd),
91 cli_overrides(Opts, Overrides),
92 mutalog_dry_run(Cwd, Overrides, Dry),
93 print_dry_run(Dry, Code).
94run_cli([Cmd|_], 2) :-
95 Cmd \== help,
96 !,
97 format(user_error, "mutalog: unknown command ~w~n", [Cmd]),
98 usage.
99run_cli(_, 0) :-
100 usage.
101
102usage :-
103 forall(member(L, [ "Usage: mutalog run|dry-run|list [OPTIONS]",
104 " --config FILE config file (default mutalog.config.pl)",
105 " --root DIR project root (default: the config file's directory)",
106 " --mutate GLOB files to mutate, repeatable",
107 " --tests GLOB test files, repeatable",
108 " --operators a,b operators to use",
109 " --timeout-ms N extra time per mutant",
110 " run: --only ID,ID run only these mutants",
111 " -c, --concurrency N number of workers (default CPUs - 1)",
112 " --reporters a,b clear_text, progress, html, json",
113 " --verbose show the project's own output",
114 " list: --format text|json, --check"
115 ]),
116 format(user_error, "~w~n", [L])).
117
118parse_options([], []).
119parse_options(['--config', F|T], [config(F)|Os]) :- !, parse_options(T, Os).
120parse_options(['--root', D|T], [root(D)|Os]) :- !, parse_options(T, Os).
121parse_options(['--mutate', G|T], [mutate(G)|Os]) :- !, parse_options(T, Os).
122parse_options(['--operators', L|T], [operators(Ops)|Os]) :-
123 !,
124 atomic_list_concat(Ops, ',', L),
125 parse_options(T, Os).
126parse_options(['--format', F|T], [format(F)|Os]) :- !, parse_options(T, Os).
127parse_options(['--check'|T], [check|Os]) :- !, parse_options(T, Os).
128parse_options(['--tests', G|T], [tests(G)|Os]) :- !, parse_options(T, Os).
129parse_options(['--only', L|T], [only(Ids)|Os]) :-
130 !,
131 atomic_list_concat(Ids, ',', L),
132 parse_options(T, Os).
133parse_options(['--timeout-ms', N|T], [timeout_ms(V)|Os]) :-
134 !,
135 atom_number(N, V),
136 parse_options(T, Os).
137parse_options(['--concurrency', N|T], [concurrency(V)|Os]) :-
138 !,
139 atom_number(N, V),
140 parse_options(T, Os).
141parse_options(['-c', N|T], Os) :-
142 !,
143 parse_options(['--concurrency', N|T], Os).
144parse_options(['--reporters', L|T], [reporters(Rs)|Os]) :-
145 !,
146 atomic_list_concat(Rs, ',', L),
147 parse_options(T, Os).
148parse_options(['--verbose'|T], [verbose(true)|Os]) :- !, parse_options(T, Os).
149parse_options([X|_], _) :-
150 domain_error(mutalog_option, X).
151
152cli_overrides(Opts, Overrides) :-
153 findall(G, member(mutate(G), Opts), Globs),
154 ( Globs == []
155 -> O0 = []
156 ; O0 = [mutate(Globs)]
157 ),
158 findall(G, member(tests(G), Opts), TestGlobs),
159 ( TestGlobs == []
160 -> O00 = O0
161 ; O00 = [tests(TestGlobs)|O0]
162 ),
163 findall(X, ( member(X, Opts),
164 ( X = timeout_ms(_) ; X = only(_) ; X = verbose(_) ; X = root(_) ;
165 X = concurrency(_) ; X = reporters(_) ) ),
166 Passed),
167 append(Passed, O00, O1),
168 ( memberchk(operators(Ops), Opts)
169 -> O2 = [operators(Ops)|O1]
170 ; O2 = O1
171 ),
172 ( memberchk(config(F), Opts)
173 -> Overrides = [config(F)|O2]
174 ; Overrides = O2
175 ).
183mutalog_list(Root, Overrides, result(Files)) :-
184 load_config(Root, Overrides, Config),
185 collect_mutants(Config, _, Files0),
186 maplist(check_file, Files0, Files).
187
188check_file(file(Rel, Mutants, Source), file(Rel, Mutants, Checks)) :-
189 findall(Id-R,
190 ( member(M, Mutants),
191 get_dict(id, M, Id),
192 verify_mutant(Source, M, R)
193 ),
194 Checks).
195
196check_result(result(Files), Code) :-
197 findall(Id-E, ( member(file(_, _, Checks), Files),
198 member(Id-E, Checks),
199 E \== ok ),
200 Bad),
201 ( Bad == []
202 -> Code = 0
203 ; forall(member(Id-E, Bad),
204 format(user_error, "mutalog: ~w does not re-read: ~q~n", [Id, E])),
205 Code = 1
206 ).
207
210
211print_run(run(_, Results, Tests, _)) :-
212 length(Tests, NTests),
213 format("~nRan ~d tests in the dry run.~n", [NTests]),
214 include(status_is(survived), Results, Survived),
215 ( Survived == []
216 -> true
217 ; length(Survived, NS),
218 format("~nSurvived (~d):~n", [NS]),
219 forall(member(r(M, _, _, _, Covering), Survived),
220 ( print_mutant(M),
221 length(Covering, NC),
222 format(" covered by ~d test(s)~n", [NC]) ))
223 ),
224 include(status_is(timeout), Results, Timeouts),
225 ( Timeouts == []
226 -> true
227 ; length(Timeouts, NT),
228 format("~nTimed out (~d):~n", [NT]),
229 forall(member(r(M, _, _, _, _), Timeouts), print_mutant(M))
230 ),
231 print_compile_errors(Results),
232 run_table(Results),
233 mutant_counts(Results, Counts),
234 mutation_score(Counts, Score, Covered),
235 percent_text(Score, ST),
236 percent_text(Covered, CT),
237 format("~nMutation score: ~w (covered code: ~w)~n", [ST, CT]).
238
241
242print_compile_errors(Results) :-
243 include(status_is(compile_error), Results, Errors),
244 ( Errors == []
245 -> true
246 ; length(Errors, N),
247 format("~nCompile errors (~d):~n", [N]),
248 findall(F, ( member(r(M, _, _, _, _), Errors), get_dict(file, M, F) ), Fs0),
249 sort(Fs0, Fs),
250 forall(member(F, Fs), print_file_compile_errors(F, Errors))
251 ).
252
253print_file_compile_errors(F, Errors) :-
254 include(result_file(F), Errors, FileErrors),
255 length(FileErrors, N),
256 format(" ~w: ~d mutant(s) did not load~n", [F, N]),
257 ( member(r(_, _, load_error(Message), _, _), FileErrors),
258 Message \== ""
259 -> split_string(Message, "\n", "", Lines),
260 format(" first error:~n"),
261 forall(member(Line, Lines), format(" ~w~n", [Line]))
262 ; true
263 ).
264
268
269run_code(run(Config, Results, _, _), Code) :-
270 mutant_counts(Results, Counts),
271 mutation_score(Counts, Score, _),
272 ( memberchk(thresholds(H, L, B), Config)
273 -> break_code(thresholds(H, L, B), Score, Code0),
274 compile_error_code(Counts, B, Code1),
275 Code is max(Code0, Code1)
276 ; Code = 0
277 ).
278
279compile_error_code(Counts, break(B), Code) :-
280 memberchk(compile_error-CE, Counts),
281 memberchk(ignored-I, Counts),
282 aggregate_all(sum(C), member(_-C, Counts), All),
283 Total is All - I,
284 ( CE * 2 > Total
285 -> format(user_error,
286 "mutalog: ~d of ~d mutants did not load (compile errors), so the run is probably broken~n",
287 [CE, Total]),
288 ( number(B), B > 0
289 -> Code = 1
290 ; Code = 0
291 )
292 ; Code = 0
293 ).
294
295status_is(S, r(_, S, _, _, _)).
296
297percent_text(none, 'n/a') :- !.
298percent_text(P, T) :- format(atom(T), '~1f%', [P]).
299
302
303break_code(thresholds(_, _, break(B)), Score, Code) :-
304 ( number(Score), number(B), Score < B
305 -> format(user_error, "mutalog: score ~1f% is below break threshold ~w%~n", [Score, B]),
306 Code = 1
307 ; Score == none, number(B), B > 0
308 -> format(user_error, "mutalog: no mutants were scored, so the break threshold ~w% is not met~n", [B]),
309 Code = 1
310 ; Code = 0
311 ).
312
313run_table(Results) :-
314 findall(F, ( member(r(M, _, _, _, _), Results), get_dict(file, M, F) ), Fs0),
315 sort(Fs0, Fs),
316 format("~n~w~t~40|~t~w~48|~t~w~56|~t~w~64|~t~w~72|~t~w~80|~t~w~88|~n",
317 ['File', 'Score', 'Killed', 'Surv.', 'Timeout', 'No cov.', 'Errors']),
318 forall(member(F, Fs),
319 ( include(result_file(F), Results, Rs),
320 table_row(F, Rs) )),
321 table_row('All files', Results).
322
323result_file(F, r(M, _, _, _, _)) :-
324 get_dict(file, M, F).
325
326table_row(Name, Rs) :-
327 mutant_counts(Rs, Counts),
328 mutation_score(Counts, Score, _),
329 memberchk(killed-K, Counts),
330 memberchk(survived-S, Counts),
331 memberchk(timeout-T, Counts),
332 memberchk(no_coverage-N, Counts),
333 memberchk(compile_error-CE, Counts),
334 memberchk(runtime_error-RE, Counts),
335 E is CE + RE,
336 percent_text(Score, SA),
337 format("~w~t~40|~t~w~48|~t~d~56|~t~d~64|~t~d~72|~t~d~80|~t~d~88|~n",
338 [Name, SA, K, S, T, N, E]).
339
342
343print_dry_run(dry(_, Files, Results, _, _), Code) :-
344 format("~w~t~60|~t~w~70|~t~w~80|~n", ['Test', 'Time (ms)', 'Clauses']),
345 forall(member(t(Test, Status, Ms, Covered), Results),
346 ( length(Covered, NC),
347 format("~q~t~60|~t~d~70|~t~d~80| ~w~n", [Test, Ms, NC, Status]) )),
348 findall(T, member(t(T, failed, _, _), Results), Failed),
349 findall(M, ( member(file(_, Ms0, _), Files), member(M, Ms0) ), All),
350 length(All, NM),
351 length(Results, NT),
352 format("~n~d tests, ~d mutants~n", [NT, NM]),
353 ( Failed == []
354 -> Code = 0
355 ; format(user_error, "mutalog: failing tests: ~q~n", [Failed]),
356 Code = 1
357 ).
358
361
362print_text(result(Files)) :-
363 forall(member(file(Rel, Ms, _), Files),
364 ( length(Ms, N),
365 format("~w: ~d mutants~n", [Rel, N]),
366 forall(member(M, Ms), print_mutant(M)) )),
367 summary(Files).
368
369print_mutant(M) :-
370 one_line(M.original, O),
371 one_line(M.replacement, R),
372 format(" ~w~n ~w -> ~w~n", [M.id, O, R]).
373
374one_line(S0, S) :-
375 split_string(S0, "\n", " \t", Parts),
376 exclude(==(""), Parts, Parts1),
377 atomic_list_concat(Parts1, ' ', A),
378 ( A == ''
379 -> S = '(removed)'
380 ; S = A
381 ).
382
383summary(Files) :-
384 findall(Op, ( member(file(_, Ms, _), Files),
385 member(M, Ms),
386 get_dict(operator, M, Op) ), Ops),
387 length(Ops, Total),
388 length(Files, NFiles),
389 msort(Ops, Sorted),
390 clumped_pairs(Sorted, Counts),
391 format("~nTotal: ~d mutants in ~d files~n", [Total, NFiles]),
392 forall(member(Op-N, Counts), format(" ~w~t~16|~d~n", [Op, N])).
393
394clumped_pairs(Sorted, Counts) :-
395 clumped(Sorted, Counts).
396
397print_json(result(Files)) :-
398 maplist(file_json, Files, FilesJson),
399 json_write_dict(current_output, _{files: FilesJson}, [width(0)]),
400 nl.
401
402file_json(file(Rel, Ms, Checks), _{path: Rel, mutants: MsJson}) :-
403 maplist(mutant_json(Checks), Ms, MsJson).
404
405mutant_json(Checks, M, J) :-
406 memberchk(M.id-Check, Checks),
407 ( Check == ok
408 -> CheckS = "ok"
409 ; term_string(Check, CheckS)
410 ),
411 J = _{ id: M.id, operator: M.operator,
412 location: _{ start: _{line: M.line, column: M.column},
413 end: _{line: M.end_line, column: M.end_column} },
414 range: [M.start, M.end],
415 original: M.original, replacement: M.replacement,
416 check: CheckS }
Mutalog: mutation testing for SWI-Prolog
Mutalog changes the code under test in small ways (a flipped comparison, a removed cut, a dropped clause) and checks that the plunit suite notices.
Command line:
mutalog run [OPTIONS] [--only ID,ID] [--concurrency N] [--reporters a,b] [--verbose] mutalog dry-run [OPTIONS] mutalog list [OPTIONS] [--format text|json] [--check] OPTIONS: [--config FILE] [--root DIR] [--mutate GLOB]... [--tests GLOB]... [--operators a,b] [--timeout-ms N]