diff --git a/tools/compare-valueflow-options.py b/tools/compare-valueflow-options.py index 2fbdcffdd0b..d7415307b28 100755 --- a/tools/compare-valueflow-options.py +++ b/tools/compare-valueflow-options.py @@ -1,11 +1,15 @@ #!/usr/bin/env python3 -# Compare "normal" check level and "exhaustive" check level +# Compare results and timings of different valueflow options +# Example usage: +# cd ~/cppcheck && make CXXFLAGS=-O2 MATCHCOMPILER=yes +# python3 compare-valueflow-options.py --cppcheck-path=~/cppcheck --packages-path=~/daca2-packages import donate_cpu_lib as lib import argparse import glob import os +import re import sys import random import subprocess @@ -17,6 +21,20 @@ def format_float(a, b=1): return 'N/A' +def count_errors(errout:str, c:set): + for line in errout.split('\n'): + if not line.endswith(']'): + continue + res = re.match(r'^[^:]+:[0-9]+:[0-9]+: (error|warning|style|portability|performance):.*\[([a-zA-Z0-9_\-]+)\]$', line) + if res is None: + print('No warning? ' + line) + continue + severity = res.group(1) + c[severity] = c.get(severity, 0) + 1 + error_id = res.group(2) + c[error_id] = c.get(error_id, 0) + 1 + + if __name__ == "__main__": __my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] __work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder')) @@ -48,20 +66,18 @@ def format_float(a, b=1): lib.set_jobs('-j' + str(args.j)) - def results_file(name): - f, ext = os.path.splitext(args.o) - return os.path.join(work_path, f + '_' + name + ext) + results_file = os.path.join(work_path, args.o) + summary_file = os.path.join(work_path, 'summary.log') + + for f in (results_file, summary_file): + if os.path.exists(f): + os.remove(f) opts = {'0': '--check-level=exhaustive --suppress=valueFlow*', 'it2': '--check-level=exhaustive --performance-valueflow-max-iterations=2 --suppress=valueFlow*', 'it1': '--check-level=exhaustive --performance-valueflow-max-iterations=1 --suppress=valueFlow*', 'if8': '--check-level=exhaustive --performance-valueflow-max-if-count=8 --suppress=valueFlow*'} - for o in opts.keys(): - f = results_file(o) - if os.path.exists(f): - os.remove(f) - cppcheck_path = args.cppcheck_path if cppcheck_path is None: @@ -96,8 +112,9 @@ def results_file(name): random.shuffle(packages_idxs) packages_processed = 0 - crashes = [] - timeouts = [] + summary_results = {} + for id in opts.keys(): + summary_results[id] = {} while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages: if args.packages: @@ -119,50 +136,66 @@ def results_file(name): print("No files to process") continue - results_to_diff = list() - timings = list() - - crashed = [] - timeout = [] + results0 = None + time0 = None enable = 'style' debug_warnings = False libraries = lib.library_includes.get_libraries(source_path) + with open(results_file, 'at') as myfile: + myfile.write('package:' + package + '\n') + myfile.write('libraries:' + ','.join(libraries) +'\n') + for id, extra_args in opts.items(): - print('scan:'+id) + print('scan:' + id) c, errout, info, time, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, extra_args=extra_args) + error_text = None if c < 0: if c == -101 and 'error: could not find or open any of the paths given.' in errout: # No sourcefile found (for example only headers present) - print('Error: 101') + error_text = f'{id} ERR no source file' elif c == lib.RETURN_CODE_TIMEOUT: - print(id + ' timed out!') - timeout.append(id) - continue # we don't want to compare timeouts + error_text = f'{id} timeout' else: - print(f'{id} crashed! code={c}') - crashed.append(id) - results_to_diff.append(errout) - timings.append(time) - - if len(results_to_diff) <= 1: - continue + error_text = f'{id} crash code={c}' - r0 = results_to_diff[0] - with open(results_file(id), 'a') as myfile: - myfile.write(package + '\n') - if id in crashed: - myfile.write('Crash\n') - elif id in timeout: - myfile.write('Timeout\n') + with open(results_file, 'at') as myfile: + if error_text is not None: + myfile.write(f'{error_text}\n') else: - diff = lib.diff_results('0', r0, id, errout) - if diff != '': - myfile.write('diff:\n' + diff + '\n') - myfile.write('time: %.1f %.1f\n' % (timings[0], time)) - myfile.write('libraries:' + ','.join(libraries) +'\n') + results = {} + count_errors(errout, results) + count_errors(errout, summary_results[id]) + if results0 is None: + results0 = results + time0 = time + else: + for error_id, count in results0.items(): + current_count = results.get(error_id, 0) + if count > current_count: + myfile.write(f'{id}: FN {error_id}: {current_count} of {count}\n') + if time > 10 or time0 > 10: + myfile.write(f'{id}: Time: %.1f\n' % time) + time_factor = time / time0 + myfile.write(f'{id}: Timefactor: %.3f\n' % time_factor) + + with open(summary_file, 'wt') as myfile: + all = {} + for id, c in summary_results.items(): + for error_id, count in c.items(): + if error_id not in all: + all[error_id] = {} + for id2 in opts.keys(): + all[error_id][id2] = 0 + all[error_id][id] += count + + for error_id, id_count in all.items(): + myfile.write(f'{error_id}:') + for id, count in id_count.items(): + myfile.write(f' {id}:{count}') + myfile.write('\n') packages_processed += 1 print(str(packages_processed) + ' of ' + str(args.p) + ' packages processed\n')