-
Notifications
You must be signed in to change notification settings - Fork 1.5k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
daca: Add compare-normal-exhaustive.py script to compare normal and e…
…xhaustive check level (#5860)
- Loading branch information
Showing
2 changed files
with
224 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,213 @@ | ||
#!/usr/bin/env python3 | ||
|
||
# Compare "normal" check level and "exhaustive" check level | ||
|
||
import donate_cpu_lib as lib | ||
import argparse | ||
import glob | ||
import os | ||
import sys | ||
import random | ||
import subprocess | ||
|
||
|
||
def format_float(a, b=1): | ||
if a > 0 and b > 0: | ||
return '{:.2f}'.format(a / b) | ||
return 'N/A' | ||
|
||
|
||
if __name__ == "__main__": | ||
__my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0] | ||
__work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder')) | ||
|
||
parser = argparse.ArgumentParser(description='Compare --check-level=normal and --check-level=exhaustive output') | ||
parser.add_argument('-j', default=1, type=int, help='Concurency execution threads') | ||
parser.add_argument('--cppcheck-path', default=None, type=str, help='Path to Cppcheck binary, if not given then clone and compile') | ||
package_group = parser.add_mutually_exclusive_group() | ||
package_group.add_argument('-p', default=256, type=int, help='Count of packages to check') | ||
package_group.add_argument('--packages', nargs='+', help='Check specific packages and then stop.') | ||
package_group.add_argument('--packages-path', default=None, type=str, help='Check packages in path.') | ||
parser.add_argument('-o', default='my_check_diff.log', help='Filename of result inside a working path dir') | ||
|
||
language_group = parser.add_mutually_exclusive_group() | ||
language_group.add_argument('--c-only', dest='c_only', help='Only process c packages', action='store_true') | ||
language_group.add_argument('--cpp-only', dest='cpp_only', help='Only process c++ packages', action='store_true') | ||
parser.add_argument('--work-path', '--work-path=', default=__work_path, type=str, help='Working directory for reference repo') | ||
args = parser.parse_args() | ||
|
||
print(args) | ||
|
||
if not lib.check_requirements(): | ||
print("Error: Check requirements") | ||
sys.exit(1) | ||
|
||
work_path = os.path.abspath(args.work_path) | ||
if not os.path.exists(work_path): | ||
os.makedirs(work_path) | ||
|
||
lib.set_jobs('-j' + str(args.j)) | ||
result_file = os.path.join(work_path, args.o) | ||
(f, ext) = os.path.splitext(result_file) | ||
timing_file = f + '_timing' + ext | ||
normal_results = f + '_normal' + ext | ||
exhaustive_results = f + '_exhaustive' + ext | ||
|
||
if os.path.exists(result_file): | ||
os.remove(result_file) | ||
if os.path.exists(timing_file): | ||
os.remove(timing_file) | ||
if os.path.exists(normal_results): | ||
os.remove(normal_results) | ||
if os.path.exists(exhaustive_results): | ||
os.remove(exhaustive_results) | ||
|
||
cppcheck_path = args.cppcheck_path | ||
|
||
if cppcheck_path is None: | ||
cppcheck_path = os.path.join(work_path, 'cppcheck') | ||
try: | ||
lib.clone_cppcheck(cppcheck_path, '') | ||
pass | ||
except Exception as e: | ||
print('Failed to clone Cppcheck repository ({}), retry later'.format(e)) | ||
sys.exit(1) | ||
|
||
if not lib.compile_cppcheck(cppcheck_path): | ||
print('Failed to compile Cppcheck') | ||
sys.exit(1) | ||
|
||
if args.packages_path: | ||
# You can download packages using daca2-download.py | ||
args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz')) | ||
args.p = len(args.packages) | ||
packages_idxs = list(range(args.p)) | ||
random.shuffle(packages_idxs) | ||
elif args.packages: | ||
args.p = len(args.packages) | ||
packages_idxs = [] | ||
else: | ||
packages_count = lib.get_packages_count() | ||
if not packages_count: | ||
print("network or server might be temporarily down..") | ||
sys.exit(1) | ||
|
||
packages_idxs = list(range(packages_count)) | ||
random.shuffle(packages_idxs) | ||
|
||
packages_processed = 0 | ||
crashes = [] | ||
timeouts = [] | ||
|
||
while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages: | ||
if args.packages: | ||
package = args.packages.pop() | ||
else: | ||
package = lib.get_package(packages_idxs.pop()) | ||
|
||
if package.startswith('ftp://') or package.startswith('http://'): | ||
tgz = lib.download_package(work_path, package, None) | ||
if tgz is None: | ||
print("No package downloaded") | ||
continue | ||
else: | ||
print('Package: ' + package) | ||
tgz = package | ||
|
||
source_path, source_found = lib.unpack_package(work_path, tgz, c_only=args.c_only, cpp_only=args.cpp_only) | ||
if not source_found: | ||
print("No files to process") | ||
continue | ||
|
||
results_to_diff = list() | ||
|
||
normal_crashed = False | ||
exhaustive_crashed = False | ||
|
||
normal_timeout = False | ||
exhaustive_timeout = False | ||
|
||
enable = 'style' | ||
debug_warnings = False | ||
|
||
libraries = lib.library_includes.get_libraries(source_path) | ||
c, errout, info, time_normal, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='normal') | ||
if c < 0: | ||
if c == -101 and 'error: could not find or open any of the paths given.' in errout: | ||
# No sourcefile found (for example only headers present) | ||
print('Error: 101') | ||
elif c == lib.RETURN_CODE_TIMEOUT: | ||
print('Normal check level timed out!') | ||
normal_timeout = True | ||
continue # we don't want to compare timeouts | ||
else: | ||
print('Normal check level crashed!') | ||
normal_crashed = True | ||
results_to_diff.append(errout) | ||
|
||
c, errout, info, time_exhaustive, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='exhaustive') | ||
if c < 0: | ||
if c == -101 and 'error: could not find or open any of the paths given.' in errout: | ||
# No sourcefile found (for example only headers present) | ||
print('Error: 101') | ||
elif c == lib.RETURN_CODE_TIMEOUT: | ||
print('Exhaustive check level timed out!') | ||
exhaustive_timeout = True | ||
continue # we don't want to compare timeouts | ||
else: | ||
print('Exhaustive check level crashed!') | ||
exhaustive_crashed = True | ||
results_to_diff.append(errout) | ||
|
||
if normal_crashed or exhaustive_crashed: | ||
who = None | ||
if normal_crashed and exhaustive_crashed: | ||
who = 'Both' | ||
elif normal_crashed: | ||
who = 'Normal' | ||
else: | ||
who = 'Exhaustive' | ||
crashes.append(package + ' ' + who) | ||
|
||
if normal_timeout or exhaustive_timeout: | ||
who = None | ||
if normal_timeout and exhaustive_timeout: | ||
who = 'Both' | ||
elif normal_timeout: | ||
who = 'Normal' | ||
else: | ||
who = 'Exhaustive' | ||
timeouts.append(package + ' ' + who) | ||
|
||
with open(result_file, 'a') as myfile: | ||
myfile.write(package + '\n') | ||
diff = lib.diff_results('normal', results_to_diff[0], 'exhaustive', results_to_diff[1]) | ||
if not normal_crashed and not exhaustive_crashed and diff != '': | ||
myfile.write('libraries:' + ','.join(libraries) +'\n') | ||
myfile.write('diff:\n' + diff + '\n') | ||
|
||
if not normal_crashed and not exhaustive_crashed: | ||
with open(timing_file, 'a') as myfile: | ||
package_width = '140' | ||
timing_width = '>7' | ||
myfile.write('{:{package_width}} {:{timing_width}} {:{timing_width}} {:{timing_width}}\n'.format( | ||
package, format_float(time_normal), | ||
format_float(time_exhaustive), format_float(time_normal, time_exhaustive), | ||
package_width=package_width, timing_width=timing_width)) | ||
with open(normal_results, 'a') as myfile: | ||
myfile.write(results_to_diff[0]) | ||
with open(exhaustive_results, 'a') as myfile: | ||
myfile.write(results_to_diff[1]) | ||
|
||
packages_processed += 1 | ||
print(str(packages_processed) + ' of ' + str(args.p) + ' packages processed\n') | ||
|
||
with open(result_file, 'a') as myfile: | ||
myfile.write('\n\ncrashes\n') | ||
myfile.write('\n'.join(crashes)) | ||
|
||
with open(result_file, 'a') as myfile: | ||
myfile.write('\n\ntimeouts\n') | ||
myfile.write('\n'.join(timeouts) + '\n') | ||
|
||
print('Result saved to: ' + result_file) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters