Skip to content

Commit

Permalink
daca: Add compare-normal-exhaustive.py script to compare normal and e…
Browse files Browse the repository at this point in the history
…xhaustive check level (#5860)
  • Loading branch information
danmar authored Jan 9, 2024
1 parent 08d754d commit 8903f0e
Show file tree
Hide file tree
Showing 2 changed files with 224 additions and 5 deletions.
213 changes: 213 additions & 0 deletions tools/compare-normal-exhaustive.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
#!/usr/bin/env python3

# Compare "normal" check level and "exhaustive" check level

import donate_cpu_lib as lib
import argparse
import glob
import os
import sys
import random
import subprocess


def format_float(a, b=1):
if a > 0 and b > 0:
return '{:.2f}'.format(a / b)
return 'N/A'


if __name__ == "__main__":
__my_script_name = os.path.splitext(os.path.basename(sys.argv[0]))[0]
__work_path = os.path.expanduser(os.path.join('~', 'cppcheck-' + __my_script_name + '-workfolder'))

parser = argparse.ArgumentParser(description='Compare --check-level=normal and --check-level=exhaustive output')
parser.add_argument('-j', default=1, type=int, help='Concurency execution threads')
parser.add_argument('--cppcheck-path', default=None, type=str, help='Path to Cppcheck binary, if not given then clone and compile')
package_group = parser.add_mutually_exclusive_group()
package_group.add_argument('-p', default=256, type=int, help='Count of packages to check')
package_group.add_argument('--packages', nargs='+', help='Check specific packages and then stop.')
package_group.add_argument('--packages-path', default=None, type=str, help='Check packages in path.')
parser.add_argument('-o', default='my_check_diff.log', help='Filename of result inside a working path dir')

language_group = parser.add_mutually_exclusive_group()
language_group.add_argument('--c-only', dest='c_only', help='Only process c packages', action='store_true')
language_group.add_argument('--cpp-only', dest='cpp_only', help='Only process c++ packages', action='store_true')
parser.add_argument('--work-path', '--work-path=', default=__work_path, type=str, help='Working directory for reference repo')
args = parser.parse_args()

print(args)

if not lib.check_requirements():
print("Error: Check requirements")
sys.exit(1)

work_path = os.path.abspath(args.work_path)
if not os.path.exists(work_path):
os.makedirs(work_path)

lib.set_jobs('-j' + str(args.j))
result_file = os.path.join(work_path, args.o)
(f, ext) = os.path.splitext(result_file)
timing_file = f + '_timing' + ext
normal_results = f + '_normal' + ext
exhaustive_results = f + '_exhaustive' + ext

if os.path.exists(result_file):
os.remove(result_file)
if os.path.exists(timing_file):
os.remove(timing_file)
if os.path.exists(normal_results):
os.remove(normal_results)
if os.path.exists(exhaustive_results):
os.remove(exhaustive_results)

cppcheck_path = args.cppcheck_path

if cppcheck_path is None:
cppcheck_path = os.path.join(work_path, 'cppcheck')
try:
lib.clone_cppcheck(cppcheck_path, '')
pass
except Exception as e:
print('Failed to clone Cppcheck repository ({}), retry later'.format(e))
sys.exit(1)

if not lib.compile_cppcheck(cppcheck_path):
print('Failed to compile Cppcheck')
sys.exit(1)

if args.packages_path:
# You can download packages using daca2-download.py
args.packages = glob.glob(os.path.join(args.packages_path, '*.tar.xz'))
args.p = len(args.packages)
packages_idxs = list(range(args.p))
random.shuffle(packages_idxs)
elif args.packages:
args.p = len(args.packages)
packages_idxs = []
else:
packages_count = lib.get_packages_count()
if not packages_count:
print("network or server might be temporarily down..")
sys.exit(1)

packages_idxs = list(range(packages_count))
random.shuffle(packages_idxs)

packages_processed = 0
crashes = []
timeouts = []

while (packages_processed < args.p and len(packages_idxs) > 0) or args.packages:
if args.packages:
package = args.packages.pop()
else:
package = lib.get_package(packages_idxs.pop())

if package.startswith('ftp://') or package.startswith('http://'):
tgz = lib.download_package(work_path, package, None)
if tgz is None:
print("No package downloaded")
continue
else:
print('Package: ' + package)
tgz = package

source_path, source_found = lib.unpack_package(work_path, tgz, c_only=args.c_only, cpp_only=args.cpp_only)
if not source_found:
print("No files to process")
continue

results_to_diff = list()

normal_crashed = False
exhaustive_crashed = False

normal_timeout = False
exhaustive_timeout = False

enable = 'style'
debug_warnings = False

libraries = lib.library_includes.get_libraries(source_path)
c, errout, info, time_normal, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='normal')
if c < 0:
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
# No sourcefile found (for example only headers present)
print('Error: 101')
elif c == lib.RETURN_CODE_TIMEOUT:
print('Normal check level timed out!')
normal_timeout = True
continue # we don't want to compare timeouts
else:
print('Normal check level crashed!')
normal_crashed = True
results_to_diff.append(errout)

c, errout, info, time_exhaustive, cppcheck_options, timing_info = lib.scan_package(cppcheck_path, source_path, libraries, enable=enable, debug_warnings=debug_warnings, check_level='exhaustive')
if c < 0:
if c == -101 and 'error: could not find or open any of the paths given.' in errout:
# No sourcefile found (for example only headers present)
print('Error: 101')
elif c == lib.RETURN_CODE_TIMEOUT:
print('Exhaustive check level timed out!')
exhaustive_timeout = True
continue # we don't want to compare timeouts
else:
print('Exhaustive check level crashed!')
exhaustive_crashed = True
results_to_diff.append(errout)

if normal_crashed or exhaustive_crashed:
who = None
if normal_crashed and exhaustive_crashed:
who = 'Both'
elif normal_crashed:
who = 'Normal'
else:
who = 'Exhaustive'
crashes.append(package + ' ' + who)

if normal_timeout or exhaustive_timeout:
who = None
if normal_timeout and exhaustive_timeout:
who = 'Both'
elif normal_timeout:
who = 'Normal'
else:
who = 'Exhaustive'
timeouts.append(package + ' ' + who)

with open(result_file, 'a') as myfile:
myfile.write(package + '\n')
diff = lib.diff_results('normal', results_to_diff[0], 'exhaustive', results_to_diff[1])
if not normal_crashed and not exhaustive_crashed and diff != '':
myfile.write('libraries:' + ','.join(libraries) +'\n')
myfile.write('diff:\n' + diff + '\n')

if not normal_crashed and not exhaustive_crashed:
with open(timing_file, 'a') as myfile:
package_width = '140'
timing_width = '>7'
myfile.write('{:{package_width}} {:{timing_width}} {:{timing_width}} {:{timing_width}}\n'.format(
package, format_float(time_normal),
format_float(time_exhaustive), format_float(time_normal, time_exhaustive),
package_width=package_width, timing_width=timing_width))
with open(normal_results, 'a') as myfile:
myfile.write(results_to_diff[0])
with open(exhaustive_results, 'a') as myfile:
myfile.write(results_to_diff[1])

packages_processed += 1
print(str(packages_processed) + ' of ' + str(args.p) + ' packages processed\n')

with open(result_file, 'a') as myfile:
myfile.write('\n\ncrashes\n')
myfile.write('\n'.join(crashes))

with open(result_file, 'a') as myfile:
myfile.write('\n\ntimeouts\n')
myfile.write('\n'.join(timeouts) + '\n')

print('Result saved to: ' + result_file)
16 changes: 11 additions & 5 deletions tools/donate_cpu_lib.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# Version scheme (MAJOR.MINOR.PATCH) should orientate on "Semantic Versioning" https://semver.org/
# Every change in this script should result in increasing the version number accordingly (exceptions may be cosmetic
# changes)
CLIENT_VERSION = "1.3.53"
CLIENT_VERSION = "1.3.54"

# Timeout for analysis with Cppcheck in seconds
CPPCHECK_TIMEOUT = 30 * 60
Expand Down Expand Up @@ -431,7 +431,7 @@ def __run_command(cmd, print_cmd=True):
return return_code, stdout, stderr, elapsed_time


def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True, enable='style,information', debug_warnings=True, check_level=None):
print('Analyze..')
libs = ''
for library in libraries:
Expand All @@ -441,10 +441,16 @@ def scan_package(cppcheck_path, source_path, libraries, capture_callstack=True):
dir_to_scan = source_path

# TODO: temporarily disabled timing information - use --showtime=top5_summary when next version is released
# TODO: remove missingInclude disabling when it no longer is implied by --enable=information
# Reference for GNU C: https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html
options = libs + ' --check-library --inconclusive --enable=style,information --inline-suppr --disable=missingInclude --suppress=unmatchedSuppression --template=daca2'
options += ' --debug-warnings --suppress=autoNoType --suppress=valueFlowBailout --suppress=bailoutUninitVar --suppress=symbolDatabaseWarning'
options = '{} --inconclusive --enable={} --inline-suppr --template=daca2'.format(libs, enable)
if 'information' in enable:
# TODO: remove missingInclude disabling when it no longer is implied by --enable=information
options += ' --disable=missingInclude --suppress=unmatchedSuppression'
if check_level:
options += ' --check-level=' + check_level
if debug_warnings:
options += ' --check-library --debug-warnings --suppress=autoNoType --suppress=valueFlowBailout' \
' --suppress=bailoutUninitVar --suppress=symbolDatabaseWarning'
options += ' -D__GNUC__ --platform=unix64'
options_rp = options + ' -rp={}'.format(dir_to_scan)
if __make_cmd == 'msbuild.exe':
Expand Down

0 comments on commit 8903f0e

Please sign in to comment.