From 227c667adf67d80a598384a19b3971142ed579dd Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 11 Dec 2023 01:14:55 +0200 Subject: [PATCH 01/29] feat: add ci workflows needs initial output pushed into this branch before merging --- mutation-testing/README.md | 38 +++++++ mutation-testing/README_2.md | 65 ++++++++++++ mutation-testing/filter-pr/README.md | 0 mutation-testing/filter-pr/action.yml | 98 ++++++++++++++++++ mutation-testing/logger/README.md | 0 mutation-testing/logger/action.yml | 38 +++++++ .../shell-scripts/append-match-package.sh | 68 +++++++++++++ .../shell-scripts/create-stable.sh | 55 +++++++++++ .../shell-scripts/modular-mutants-run.sh | 41 ++++++++ .../remove-deleted-file-lines.sh | 17 ++++ .../remove-deleted-functions-from-output.sh | 79 +++++++++++++++ mutation-testing/update-cache/action.yml | 99 +++++++++++++++++++ .../upload-initial-cache/action.yml | 39 ++++++++ 13 files changed, 637 insertions(+) create mode 100644 mutation-testing/README.md create mode 100644 mutation-testing/README_2.md create mode 100644 mutation-testing/filter-pr/README.md create mode 100644 mutation-testing/filter-pr/action.yml create mode 100644 mutation-testing/logger/README.md create mode 100644 mutation-testing/logger/action.yml create mode 100644 mutation-testing/shell-scripts/append-match-package.sh create mode 100644 mutation-testing/shell-scripts/create-stable.sh create mode 100644 mutation-testing/shell-scripts/modular-mutants-run.sh create mode 100755 mutation-testing/shell-scripts/remove-deleted-file-lines.sh create mode 100755 mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh create mode 100644 mutation-testing/update-cache/action.yml create mode 100644 mutation-testing/upload-initial-cache/action.yml diff --git a/mutation-testing/README.md b/mutation-testing/README.md new file mode 100644 index 0000000..66a0efb --- /dev/null +++ b/mutation-testing/README.md @@ -0,0 +1,38 @@ +## steps to reproduce working version with different number lines: + +in trials/mutants-stable/caught.txt replace line number 23 with 109 +the append.sh won't work anymore +the append-match.sh works + +```bash +sh append-match.sh + +``` + +example run: + +```bash +./modular-mutants-run.sh stx-genesis lib.rs test_this init_ next as.rs ab cd ef clarity lib.rs stacks-node +# the command above makes and runs 579 mutants on these regex matches: + +# functions named 'test_this', 'init_' and 'next' (everything that starts with any of the given names) from 'lib.rs' file of 'stx-genesis' package (5 mutants) +stx-genesis/[^/]+/lib.rs.*(?:test_this|init_|next).*-> + +# functions that start with 'ab', 'cd' or 'ef' from files named 'as.rs' of 'stx-genesis' package (0 mutants) +stx-genesis/[^/]+/as.rs.*(?:ab|cd|ef).*-> + +# all functions from 'lib.rs' files of the 'clarity' package (4 mutants) +clarity/[^/]+/lib.rs.*(?:).*-> + +# all functions from all files of 'stacks-node' package (570 mutants) +stacks-node/[^/]+/.*(?:).*-> +``` + +# Create Stable + +Only run it once and the packages that should be updated from zero. Then it will be the reference point for the upcoming PRs that modify these functions + +### recap flow for a developer which + +1. works on functions and modifies them +2. before commiting -> `call git-diff.sh` diff --git a/mutation-testing/README_2.md b/mutation-testing/README_2.md new file mode 100644 index 0000000..28c885b --- /dev/null +++ b/mutation-testing/README_2.md @@ -0,0 +1,65 @@ +# Mutation Testing + +### What is mutation testing and how does it work? + +Mutation testing is a technique of evaluating the effectiveness of a series of tests by introducing small changes to the code (mutations) and checking if the tests can detect these small changes. +Cargo-mutants is an external library installed to cargo, through which you can run mutants on the code, and it consists of: + +- Building and testing the baseline code (no mutations). +- If the previous step fails, no mutants are applied, since the base code fails. Otherwise, copy the code to another location, apply mutations and then run `cargo build` and `cargo test` commands for each mutation. + +### Install and run + +In order to install cargo-mutants crate: + +``` +cargo install --locked cargo-mutants +``` + +In order to run mutated tests: + +```bash +# In the whole workspace +cargo mutants +# Only in the 'clarity' package +cargo mutants --package clarity +# In files named 'signatures.rs' from the whole workspace +cargo mutants -f signatures.rs +# Only in files named 'signatures.rs' only from the 'clarity' package +cargo mutants --package clarity -f signatures.rs +# From all files except the ones named 'signatures.rs' and 'lib.rs' from the whole workspace +cargo mutants -e signatures.rs -e lib.rs +# Output from 'clarity' package to a specific directory in the workspace +cargo mutants --package clarity --output mutants/clarity +# To list all the possible mutants +cargo mutants --list +# To list all the files with possible mutants: +cargo mutants --list-files +``` + +In order to exclude a function from being mutated, parse the `#[mutants::skip]` attribute above it. + +### Reading the output + +There are 2 places where the progress of mutations are shown: terminal and [output folders](https://mutants.rs/mutants-out.html). +The terminal shows information about the progress of the mutants: + +- How many mutants out of the total were tested (`1274/2912 mutants tested, 44% done`). +- Mutants status so far (`280 missed, 209 caught, 799 unviable`). +- Time elapsed and remaining (`141:36 elapsed, about 168 min remaining`). +- Tests missed so far (`clarity/src/vm/database/key_value_wrapper.rs:77: replace rollback_value_check with () ... NOT CAUGHT in 22.8s build + 17.2s test`). +- Current job (`clarity/src/vm/ast/parser/v2/mod.rs:167: replace Parser<'a>::skip_to_end with () ... 2.1s build`) + +`mutants.out` - This is the folder where the mutants test output is written, and is composed of: + +- log - The folder of the command log, here you can find the output of the cargo build and cargo test commands for every mutation. +- caught.txt - The file where caught mutations are logged (`clarity/src/vm/types/mod.rs:871: replace Value::size -> u32 with 1`). +- debug.log - The output of the cargo mutants command. +- lock.json - A file with fs2 lock on it in order to prevent 2 jobs from writing to the same directory at the same time, containing runtime information (cargo mutants version, start time, hostname, username). +- missed.txt - Missed mutations - mutations that are successful at cargo build, not detected in tests (`clarity/src/vm/types/signatures.rs:1766: replace TupleTypeSignature::size -> u32 with 1`). +- mutants.json - A list with every mutation applied, written before the testing begins (filename, line, return type, replacement etc). +- outcome.json - List of outcomes for every mutation (mutant applied, log path, results for build/test phases with status and command args) +- timeout.txt - Mutations that timed out +- unviable.txt - Unviable mutations (When a mutation is applied and it causes the cargo build command to fail) + +`mutants.out.old` - This is the folder where _mutants.out_ folder’s content is copied into, on successive runs (it’s contents are being overwritten), making way for the next logs. diff --git a/mutation-testing/filter-pr/README.md b/mutation-testing/filter-pr/README.md new file mode 100644 index 0000000..e69de29 diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/filter-pr/action.yml new file mode 100644 index 0000000..096f25e --- /dev/null +++ b/mutation-testing/filter-pr/action.yml @@ -0,0 +1,98 @@ +name: Filter PR Mutants + +runs: + using: 'composite' + + steps: + # Cleanup Runner + - name: Cleanup Runner action repo + id: runner_cleanup + uses: stacks-network/actions/cleanup/disk@main + + # Checkout the stacks-core code + - name: Checkout the stacks core repo + id: git_checkout_stacks_core + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Relative diff + shell: bash + run: | + git branch -av + git diff origin/${{ github.base_ref }}.. | tee git.diff + + # Checkout the actions code + - name: Get the action repo code + id: git_checkout_actions + uses: actions/checkout@v3 + with: + repository: stacks-network/actions + ref: main + path: ./actions-repo + + - name: Copy the git diff file to scripts folder + shell: bash + run: | + mv git.diff ./actions-repo/mutation-testing/shell-scripts/ + + - uses: Swatinem/rust-cache@v2 + + - name: Install cargo mutants crate + shell: bash + run: cargo install cargo-mutants + + - name: Remove deleted file lines from git.diff file + shell: bash + run: ./remove-deleted-file-lines.sh + working-directory: actions-repo/mutation-testing/shell-scripts + + - name: Run mutants and check the exit code of the command, fail the workflow if mutations are not caught + shell: bash + run: | + set +e # Disable immediate exit on error + cargo mutants --no-shuffle -j 2 -vV --in-diff git.diff --output ./ + exit_code=$? + set -e # Enable immediate exit on error again + + case $exit_code in + 0) + if [ -s ./mutants.out/unviable.txt ]; then + echo "-------------" + echo "Found unviable mutants:" + cat ./mutants.out/unviable.txt + exit 1 + fi + echo "All new and updated functions are caught!" + ;; + 1) + echo "Invalid command line arguments!" + exit 1 + ;; + 2 | 3) + if [ -s ./mutants.out/missed.txt ]; then + echo "Found missed mutants:" + cat ./mutants.out/missed.txt + fi + if [ -s ./mutants.out/timeout.txt ]; then + echo "-------------" + echo "Found timeout mutants:" + cat ./mutants.out/timeout.txt + fi + if [ -s ./mutants.out/unviable.txt ]; then + echo "-------------" + echo "Found unviable mutants:" + cat ./mutants.out/unviable.txt + fi + exit 1 + ;; + 4) + echo "Building the packages failed without any mutations!" + exit 1 + ;; + *) + echo "Unknown exit code: $exit_code" + exit 1 + ;; + esac + working-directory: actions-repo/mutation-testing/shell-scripts diff --git a/mutation-testing/logger/README.md b/mutation-testing/logger/README.md new file mode 100644 index 0000000..e69de29 diff --git a/mutation-testing/logger/action.yml b/mutation-testing/logger/action.yml new file mode 100644 index 0000000..58b0f59 --- /dev/null +++ b/mutation-testing/logger/action.yml @@ -0,0 +1,38 @@ +name: Logging Mutants + +# only run on push in order to update the cache output +# flow: +# restore cache +# install cargo-mutants crate in order to run the 'cargo mutants' command +# create a file with the current commit hash if a previous one doesn't exist, then print it +# run the script that handles the 'cargo mutants' command on the differences between the latest updates and the last commit where it was ran +# overwrite the previous commit hash with the current one for the following run +# delete the old cache +# save the new cache with the updated mutants +inputs: + gh-token: + description: 'The github token secret in order to delete the old cache' + required: true + default: '' + +runs: + using: 'composite' + + steps: + - name: Check Cache + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + id: check_cache + with: + lookup-only: true + path: ./ + key: mutants-stable-${{ github.ref_name }} + + - name: Upload initial outputs if there is no cache + if: steps.check_cache.outputs.cache-hit != 'true' + uses: stacks-network/actions/mutation-testing/upload-initial-cache@main + + - name: Update cache if we have one available + if: steps.check_cache.outputs.cache-hit == 'true' + uses: stacks-network/actions/mutation-testing/update-cache@main + with: + gh-token: ${{ inputs.gh-token }} diff --git a/mutation-testing/shell-scripts/append-match-package.sh b/mutation-testing/shell-scripts/append-match-package.sh new file mode 100644 index 0000000..9026cfc --- /dev/null +++ b/mutation-testing/shell-scripts/append-match-package.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +# the append-match-package.sh +## goes through each line in the output and based on the package ( first element before /) +### verifies the line with the other lines in that specific folder +#### in our case folder_name == package_name + + +# goes through each PR file line by line +# for each first_element/the_rest_of_the_line goes through it +## search in that specific folder on all 4 files +## if it is matchy, remove it from that file +## based on the file it was taken from, append it to the same file in the STABLE folder + + +PR_FOLDER="../temp/mutants.out" +STABLE_FOLDER_PARENT="../packages-output" +FILES=("caught.txt" "missed.txt" "timeout.txt" "unviable.txt") + +echo "Starting script..." +echo "PR Folder: $PR_FOLDER" +echo "STABLE Folder: $STABLE_FOLDER_PARENT" +echo "Files to process: ${FILES[*]}" + +# Iterate over the specified files +for file in "${FILES[@]}"; do + pr_file="$PR_FOLDER/$file" + + echo "Processing file: $file" + + # Check if PR file exists and is not empty + if [[ -s "$pr_file" ]]; then + # Read each line from the PR file + while IFS= read -r line; do + echo "Reading line from PR file: $line" + + # Extract the package from which the line is coming from + local_package=${line%%/*} + + # Extract the after the number line without the line number and escape it for awk + # Escape the variables for use in a sed pattern + var_1=$(echo "$line" | sed -E 's/^(.+):[0-9]+:[^:]+/\1/') + escaped_var_1=$(sed 's/[][/.^$]/\\&/g' <<< "$var_1") + + var_2=$(echo "$line" | sed -E 's/^[^:]+:[0-9]+:(.+)/\1/') + escaped_var_2=$(sed 's/[][/.^$]/\\&/g' <<< "$var_2") + + regex="${escaped_var_1}.*${escaped_var_2}" + + # Iterate over each file in the STABLE folder combined with local_package + for target_file in "${FILES[@]}"; do + target_path="$STABLE_FOLDER_PARENT/$local_package/$target_file" + echo "Checking against STABLE file: $target_path" + + # Use sed to remove lines matching the pattern + sed "/$regex/d" "$target_path" > temp_file && mv temp_file "$target_path" + done + + # Append PR line to the corresponding package and file + echo "$line" >> "$STABLE_FOLDER_PARENT/$local_package/$file" + + done < "$pr_file" + else + echo "PR file $pr_file is empty or does not exist, skipping..." + fi +done + +echo "Script completed." diff --git a/mutation-testing/shell-scripts/create-stable.sh b/mutation-testing/shell-scripts/create-stable.sh new file mode 100644 index 0000000..1659714 --- /dev/null +++ b/mutation-testing/shell-scripts/create-stable.sh @@ -0,0 +1,55 @@ +# for specific packages creates the outpup + +# removes everything except .txt files + +#!/bin/bash + +# moves to mutation-testing folder +cd ../packages-output + +### Run mutation testing on the packages uncommented + +# Run mutation testing for stx-genesis package +cargo mutants --package stx-genesis --output stx-genesis +mv stx-genesis/mutants.out/*.txt stx-genesis/ +rm -rf stx-genesis/mutants.out + +# Run mutation testing for pox-locking package +cargo mutants --package pox-locking --output pox-locking +mv pox-locking/mutants.out/*.txt pox-locking/ +rm -rf pox-locking/mutants.out + +# # Run mutation testing for libsigner package +# cargo mutants --package libsigner --output libsigner +# mv libsigner/mutants.out/*.txt libsigner/ +# rm -rf libsigner/mutants.out + +# # Run mutation testing for libstackerdb package +# cargo mutants --package libstackerdb --output libstackerdb +# mv libstackerdb/mutants.out/*.txt libstackerdb/ +# rm -rf libstackerdb/mutants.out + +# # Run mutation testing for stacks-common package +# cargo mutants --package stacks-common --output stacks-common +# mv stacks-common/mutants.out/*.txt stacks-common/ +# rm -rf stacks-common/mutants.out + +# # Run mutation testing for clarity package +# cargo mutants --package clarity --output clarity +# mv clarity/mutants.out/*.txt clarity/ +# rm -rf clarity/mutants.out + +# Run mutation testing for stacks-signer package - working, 10 min approx. +# cargo mutants --package stacks-signer --output stacks-signer +# mv stacks-signer/mutants.out/*.txt stacks-signer/ +# rm -rf stacks-signer/mutants.out + +# Commented out mutation testing for stacks-node package due to test errors and long compile/testing time +# cargo mutants --package stacks-node --output stacks-node +# mv stacks-node/mutants.out/*.txt stacks-node/ +# rm -rf stacks-node/mutants.out + +# Commented out mutation testing for stackslib package due to long compile/testing time +# cargo mutants --package stackslib --output stackslib +# mv stackslib/mutants.out/*.txt stackslib/ +# rm -rf stackslib/mutants.out \ No newline at end of file diff --git a/mutation-testing/shell-scripts/modular-mutants-run.sh b/mutation-testing/shell-scripts/modular-mutants-run.sh new file mode 100644 index 0000000..c8bc357 --- /dev/null +++ b/mutation-testing/shell-scripts/modular-mutants-run.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +packages=$(cargo tree --workspace --prefix depth | grep "^0" | cut -c2- | awk '{print $1}') +regex_list=() + +while [ $# -gt 0 ]; do + arg=$1 + + if [[ $packages == *$arg* ]]; then + package=$arg + file="" + shift + arg=$1 + fi + if [[ $arg == *.rs ]]; then + file=$arg + shift + arg=$1 + fi + + functions=() + while [ $# -gt 0 ] && [[ $1 != *.rs ]] && [[ $packages != *$1* ]]; do + functions+=("$1") + shift + done + + IFS="|" + functions_str="${functions[*]}" + IFS="" + + regex="${package}/[^/]+/${file}.*?(?:${functions_str})[^-()]*(?:->|\(\))" + regex_list+=("$regex") +done + +command="cargo mutants -vV --no-shuffle" + +for regex in "${regex_list[@]}"; do + command+=" -F \"$regex\"" +done + +eval "$command" \ No newline at end of file diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh new file mode 100755 index 0000000..a8812f2 --- /dev/null +++ b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +input_file="git.diff" +temp_file="temp_diff_file.diff" + +# Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) +tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" +sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" +tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + +# Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' +awk ' + BEGIN { in_block=0 } + /\+\+\+ \/dev\/null/ { in_block=1; next } + in_block && /diff --git a\// { in_block=0; print; next } + !in_block +' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" diff --git a/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh b/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh new file mode 100755 index 0000000..8d6a756 --- /dev/null +++ b/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +# Flow: +# +# Parse through all the lines of the file +# When we find a line that starts with '--- a/', we save what is after the 'a/' to a 'file_path' variable +# When we find a line that contains the word 'impl': +# If it doesn't contain a ' for ' in it, save from the beginning of 'impl' up until '{' (including 'impl', not '{') to a 'impl_block' variable +# If it contains a ' for ', do the step above but add '<' before and '>' after +# Remove extras from the line. +# Example: +# Git Diff: +# PartialEq> for IncomparableError> +# for Error> +# +# Expected in outputs: +# > +# +# +# +# When we find a line that contains 'fn', check: +# If the first char on the line is '-', add the name between 'fn' and '(' to a variable named 'fn_name', then append to a 'functions' list by the following rule: +# If the 2nd and 3rd chars are 'fn', add to list the following matching: '{file_path}.*replace {fn_name}' +# Else, add to the list the following matching: '{file_path}.*{impl_block}::{fn_name}' +# Then parse through every fn in {functions}, and if the fn matches any line in the files from 'packages-output' folder, remove it + +input_file="git.diff" +functions=() + +# Read the file line by line +while IFS= read -r line; do + # Save the file path + if [[ $line == *'--- a/'* ]]; then + file_path=$(echo "$line" | cut -c7-) + fi + + # Save the last impl block + if [[ "$line" == *'impl'*' for '*'{'* ]]; then + impl_block="${line#*impl}" + impl_block="${impl_block%%\ {*}" + impl_block="" # add 'impl' word, '<' and '>' characters + impl_block=$(echo "$impl_block" | sed -E 's/(^[^ ]*impl)[^ ]*/\1/') # remove everything after 'impl', until the next space + impl_block=$(echo "$impl_block" | sed 's/\(<[^<]*\)[^ ]* /\1 /') # remove everything starting with '<' and ending at the next space, from the second word + impl_block=$(echo "$impl_block" | awk '{sub(/ [^ ]+::/, " "); print}') # keep only the characters after the last '::' of the 2nd word (struct name without path) + elif [[ "$line" == *'impl'*'{'* ]]; then + impl_block="${line#*impl}" + impl_block="${impl_block%%\ {*}" + elif [[ "$line" == *'trait'*'{'* ]]; then + impl_block="${line#*trait}" + impl_block="${impl_block%%\ {*}" + fi + + # Save the regex matching the file name, impl block (if any), and function to a list + if [[ $line == *'fn'* ]]; then + first_char=${line:0:1} + if [[ $first_char == '-' ]]; then + fn_name=$(echo "$line" | grep -oP 'fn\s+\K[^(\s]+') + if [[ ${line:1:2} == 'fn' ]]; then + functions+=("${file_path}.*replace ${fn_name}") + else + functions+=("${file_path}.*${impl_block}::${fn_name}") + fi + fi + fi +done < "$input_file" + +# Print the lines to be removed +echo "Removing these lines from the stable output:" +for func in "${functions[@]}"; do + grep -r "$func" ../packages-output/* +done + +# Remove the lines matching the regex for all deleted functions +find "../packages-output" -type f -print0 | while IFS= read -r -d $'\0' file; do + grep -Ev "$(printf '%s\n' "${functions[@]}" | sed 's/\([[].*[]]\)/[\\1]/g')" "$file" > "temp_file" && mv "temp_file" "$file" + if [[ -e "temp_file" ]]; then + rm "temp_file" + fi +done diff --git a/mutation-testing/update-cache/action.yml b/mutation-testing/update-cache/action.yml new file mode 100644 index 0000000..55896f8 --- /dev/null +++ b/mutation-testing/update-cache/action.yml @@ -0,0 +1,99 @@ +name: Update Cache + +inputs: + gh-token: + description: 'The github token secret in order to delete the old cache' + required: true + default: '' + +runs: + using: 'composite' + + steps: + # Cleanup Runner + - name: Cleanup Runner action repo + id: runner_cleanup + uses: stacks-network/actions/cleanup/disk@main + + # Checkout the stacks-core code + - name: Checkout the stacks core repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Restore mutants-output cached folder + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: ./mutation-testing/packages-output + key: mutants-stable-${{ github.ref_name }} + + - name: Create git.diff file + shell: bash + run: | + last_commit_hash=$(<../packages-output/last_commit_hash.txt) + git diff $last_commit_hash > git.diff + working-directory: ./mutation-testing/scripts + + # Checkout the actions code + - name: Get the action repo code + id: git_checkout_actions + uses: actions/checkout@v3 + with: + repository: stacks-network/actions + ref: main + path: ./actions-repo + + - name: Install cargo-mutants crate + shell: bash + run: cargo install cargo-mutants + + - name: Remove deleted functions from output files + shell: bash + run: ./remove-deleted-functions-from-output.sh + working-directory: ./mutation-testing/scripts + + - name: Remove deleted file lines from git.diff file + shell: bash + run: ./remove-deleted-file-lines.sh + working-directory: ./mutation-testing/scripts + + - name: Run the mutation testing on the differences + shell: bash + run: | + if [ -f ./packages-output/last_commit_hash.txt ]; then + cargo mutants --no-shuffle -j 2 -vV --in-diff ./scripts/git.diff --output temp/ + fi + working-directory: ./mutation-testing + + - name: Update the content from the stable output + shell: bash + run: bash append-match-package.sh + working-directory: ./mutation-testing/scripts + + - name: Re-write the old commit hash with the current one + shell: bash + run: | + echo "${{ github.sha }}" > last_commit_hash.txt + working-directory: ./mutation-testing/packages-output + + - name: Print new commit hash + shell: bash + run: cat ./mutation-testing/packages-output/last_commit_hash.txt + + - name: Delete Previous Cache + if: ${{ steps.cache-restore.outputs.cache-hit }} + continue-on-error: true + shell: bash + run: | + gh extension install actions/gh-actions-cache + gh actions-cache delete "mutants-stable-${{ github.ref_name }}" --confirm + env: + GH_TOKEN: ${{ inputs.gh-token }} + + # TODO: feature for retrying the save in case it fails for any reason (in order not to lose the cache) + - name: Save Cache + uses: actions/cache/save@v3 + with: + path: ./mutation-testing/packages-output + key: mutants-stable-${{ github.ref_name }} diff --git a/mutation-testing/upload-initial-cache/action.yml b/mutation-testing/upload-initial-cache/action.yml new file mode 100644 index 0000000..991856c --- /dev/null +++ b/mutation-testing/upload-initial-cache/action.yml @@ -0,0 +1,39 @@ +name: Upload initial cache + +runs: + using: 'composite' + + steps: + # Checkout the stacks-core code + - name: Checkout the stacks core repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # Checkout the actions code + - name: Get the action repo code + id: git_checkout_actions + uses: actions/checkout@v3 + with: + repository: stacks-network/actions + ref: main + path: ./actions-repo + + - name: Create file with current commit hash if it doesn't exist + shell: bash + run: | + if [ ! -f last_commit_hash.txt ]; then + echo "${{ github.sha }}" > last_commit_hash.txt + fi + working-directory: ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }} + + - name: Print old commit hash + shell: bash + run: cat ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }}/last_commit_hash.txt + + # TODO: feature for retrying the save in case it fails for any reason (in order not to lose the cache) + - name: Save Cache + uses: actions/cache/save@v3 + with: + path: ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }} + key: mutants-stable-${{ github.ref_name }} From 33a3cbbde6b3dbb710693bda7a193dd8d89e1a3a Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 11 Dec 2023 01:34:44 +0200 Subject: [PATCH 02/29] feat: upload artifact to fetch data easily & rename cache to mutants-output-branch_name --- mutation-testing/logger/action.yml | 2 +- mutation-testing/update-cache/action.yml | 13 ++++++++++--- mutation-testing/upload-initial-cache/action.yml | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/mutation-testing/logger/action.yml b/mutation-testing/logger/action.yml index 58b0f59..a45827a 100644 --- a/mutation-testing/logger/action.yml +++ b/mutation-testing/logger/action.yml @@ -25,7 +25,7 @@ runs: with: lookup-only: true path: ./ - key: mutants-stable-${{ github.ref_name }} + key: mutants-output-${{ github.ref_name }} - name: Upload initial outputs if there is no cache if: steps.check_cache.outputs.cache-hit != 'true' diff --git a/mutation-testing/update-cache/action.yml b/mutation-testing/update-cache/action.yml index 55896f8..4f61a0e 100644 --- a/mutation-testing/update-cache/action.yml +++ b/mutation-testing/update-cache/action.yml @@ -26,7 +26,7 @@ runs: uses: actions/cache/restore@v3 with: path: ./mutation-testing/packages-output - key: mutants-stable-${{ github.ref_name }} + key: mutants-output-${{ github.ref_name }} - name: Create git.diff file shell: bash @@ -87,7 +87,7 @@ runs: shell: bash run: | gh extension install actions/gh-actions-cache - gh actions-cache delete "mutants-stable-${{ github.ref_name }}" --confirm + gh actions-cache delete "mutants-output-${{ github.ref_name }}" --confirm env: GH_TOKEN: ${{ inputs.gh-token }} @@ -96,4 +96,11 @@ runs: uses: actions/cache/save@v3 with: path: ./mutation-testing/packages-output - key: mutants-stable-${{ github.ref_name }} + key: mutants-output-${{ github.ref_name }} + + - name: Upload to artifact to easily check the ouput + uses: actions/upload-artifact@v3 + if: always() + with: + name: mutants-output-${{ github.sha }} + path: ./mutation-testing/packages-output diff --git a/mutation-testing/upload-initial-cache/action.yml b/mutation-testing/upload-initial-cache/action.yml index 991856c..f682ba4 100644 --- a/mutation-testing/upload-initial-cache/action.yml +++ b/mutation-testing/upload-initial-cache/action.yml @@ -36,4 +36,4 @@ runs: uses: actions/cache/save@v3 with: path: ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }} - key: mutants-stable-${{ github.ref_name }} + key: mutants-output-${{ github.ref_name }} From ed270a11b17932c5d99dcadb810c7b35c329f1bf Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 11 Dec 2023 23:27:55 +0200 Subject: [PATCH 03/29] feat: docs readme mutations --- mutation-testing/README.md | 42 ++---------- mutation-testing/README_2.md | 65 ------------------- mutation-testing/filter-pr/README.md | 20 ++++++ mutation-testing/logger/README.md | 28 ++++++++ mutation-testing/update-cache/README.md | 28 ++++++++ .../upload-initial-cache/README.md | 18 +++++ 6 files changed, 99 insertions(+), 102 deletions(-) delete mode 100644 mutation-testing/README_2.md create mode 100644 mutation-testing/update-cache/README.md create mode 100644 mutation-testing/upload-initial-cache/README.md diff --git a/mutation-testing/README.md b/mutation-testing/README.md index 66a0efb..fc2e607 100644 --- a/mutation-testing/README.md +++ b/mutation-testing/README.md @@ -1,38 +1,6 @@ -## steps to reproduce working version with different number lines: +# Mutation Testing actions -in trials/mutants-stable/caught.txt replace line number 23 with 109 -the append.sh won't work anymore -the append-match.sh works - -```bash -sh append-match.sh - -``` - -example run: - -```bash -./modular-mutants-run.sh stx-genesis lib.rs test_this init_ next as.rs ab cd ef clarity lib.rs stacks-node -# the command above makes and runs 579 mutants on these regex matches: - -# functions named 'test_this', 'init_' and 'next' (everything that starts with any of the given names) from 'lib.rs' file of 'stx-genesis' package (5 mutants) -stx-genesis/[^/]+/lib.rs.*(?:test_this|init_|next).*-> - -# functions that start with 'ab', 'cd' or 'ef' from files named 'as.rs' of 'stx-genesis' package (0 mutants) -stx-genesis/[^/]+/as.rs.*(?:ab|cd|ef).*-> - -# all functions from 'lib.rs' files of the 'clarity' package (4 mutants) -clarity/[^/]+/lib.rs.*(?:).*-> - -# all functions from all files of 'stacks-node' package (570 mutants) -stacks-node/[^/]+/.*(?:).*-> -``` - -# Create Stable - -Only run it once and the packages that should be updated from zero. Then it will be the reference point for the upcoming PRs that modify these functions - -### recap flow for a developer which - -1. works on functions and modifies them -2. before commiting -> `call git-diff.sh` +> - [Filter PR Mutants action](./filter-pr/README.md) +> - [Logger Mutants action](./logger/README.md) +> - [Update cache action](./update-cache/README.md) +> - [Upload Initial Cache action](./upload-initial-cache/README.md) diff --git a/mutation-testing/README_2.md b/mutation-testing/README_2.md deleted file mode 100644 index 28c885b..0000000 --- a/mutation-testing/README_2.md +++ /dev/null @@ -1,65 +0,0 @@ -# Mutation Testing - -### What is mutation testing and how does it work? - -Mutation testing is a technique of evaluating the effectiveness of a series of tests by introducing small changes to the code (mutations) and checking if the tests can detect these small changes. -Cargo-mutants is an external library installed to cargo, through which you can run mutants on the code, and it consists of: - -- Building and testing the baseline code (no mutations). -- If the previous step fails, no mutants are applied, since the base code fails. Otherwise, copy the code to another location, apply mutations and then run `cargo build` and `cargo test` commands for each mutation. - -### Install and run - -In order to install cargo-mutants crate: - -``` -cargo install --locked cargo-mutants -``` - -In order to run mutated tests: - -```bash -# In the whole workspace -cargo mutants -# Only in the 'clarity' package -cargo mutants --package clarity -# In files named 'signatures.rs' from the whole workspace -cargo mutants -f signatures.rs -# Only in files named 'signatures.rs' only from the 'clarity' package -cargo mutants --package clarity -f signatures.rs -# From all files except the ones named 'signatures.rs' and 'lib.rs' from the whole workspace -cargo mutants -e signatures.rs -e lib.rs -# Output from 'clarity' package to a specific directory in the workspace -cargo mutants --package clarity --output mutants/clarity -# To list all the possible mutants -cargo mutants --list -# To list all the files with possible mutants: -cargo mutants --list-files -``` - -In order to exclude a function from being mutated, parse the `#[mutants::skip]` attribute above it. - -### Reading the output - -There are 2 places where the progress of mutations are shown: terminal and [output folders](https://mutants.rs/mutants-out.html). -The terminal shows information about the progress of the mutants: - -- How many mutants out of the total were tested (`1274/2912 mutants tested, 44% done`). -- Mutants status so far (`280 missed, 209 caught, 799 unviable`). -- Time elapsed and remaining (`141:36 elapsed, about 168 min remaining`). -- Tests missed so far (`clarity/src/vm/database/key_value_wrapper.rs:77: replace rollback_value_check with () ... NOT CAUGHT in 22.8s build + 17.2s test`). -- Current job (`clarity/src/vm/ast/parser/v2/mod.rs:167: replace Parser<'a>::skip_to_end with () ... 2.1s build`) - -`mutants.out` - This is the folder where the mutants test output is written, and is composed of: - -- log - The folder of the command log, here you can find the output of the cargo build and cargo test commands for every mutation. -- caught.txt - The file where caught mutations are logged (`clarity/src/vm/types/mod.rs:871: replace Value::size -> u32 with 1`). -- debug.log - The output of the cargo mutants command. -- lock.json - A file with fs2 lock on it in order to prevent 2 jobs from writing to the same directory at the same time, containing runtime information (cargo mutants version, start time, hostname, username). -- missed.txt - Missed mutations - mutations that are successful at cargo build, not detected in tests (`clarity/src/vm/types/signatures.rs:1766: replace TupleTypeSignature::size -> u32 with 1`). -- mutants.json - A list with every mutation applied, written before the testing begins (filename, line, return type, replacement etc). -- outcome.json - List of outcomes for every mutation (mutant applied, log path, results for build/test phases with status and command args) -- timeout.txt - Mutations that timed out -- unviable.txt - Unviable mutations (When a mutation is applied and it causes the cargo build command to fail) - -`mutants.out.old` - This is the folder where _mutants.out_ folder’s content is copied into, on successive runs (it’s contents are being overwritten), making way for the next logs. diff --git a/mutation-testing/filter-pr/README.md b/mutation-testing/filter-pr/README.md index e69de29..2ad7478 100644 --- a/mutation-testing/filter-pr/README.md +++ b/mutation-testing/filter-pr/README.md @@ -0,0 +1,20 @@ +# Filter PR Mutants action + +Run filtering phase for pull requests, and fail the workflow if: +- There are missed/timeout/unviable mutants +- Building or testing the unmutated crate fails + +## Usage + +```yaml +name: Action +on: pull_request +jobs: + build: + name: Job + runs-on: ubuntu-latest + steps: + - name: Filter PR Mutants + id: filter-pr-mutants + uses: stacks-network/actions/mutation-testing/filter-pr@main +``` diff --git a/mutation-testing/logger/README.md b/mutation-testing/logger/README.md index e69de29..5701d79 100644 --- a/mutation-testing/logger/README.md +++ b/mutation-testing/logger/README.md @@ -0,0 +1,28 @@ +# Logger Mutants action + +Check whether to log the mutants to cache, or to create a new cache if it doesn't exist. + +## Documentation + +### Inputs + +| Input | Description | Required | Default | +| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | +| `gh-token` | The github token from the main workflow for cache deletion | true | `""` | + +## Usage + +```yaml +name: Action +on: push +jobs: + build: + name: Job + runs-on: ubuntu-latest + steps: + - name: Log Mutants + id: log-mutants + uses: stacks-network/actions/mutation-testing/logger@main + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} +``` diff --git a/mutation-testing/update-cache/README.md b/mutation-testing/update-cache/README.md new file mode 100644 index 0000000..6c27fef --- /dev/null +++ b/mutation-testing/update-cache/README.md @@ -0,0 +1,28 @@ +# Update cache action + +Logs the mutants to cache, appending to the old output, and saving an artifact for easier data access and backup. + +## Documentation + +### Inputs + +| Input | Description | Required | Default | +| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | +| `gh-token` | The github token from the main workflow for cache deletion | true | `""` | + +## Usage + +```yaml +name: Action +on: push +jobs: + build: + name: Job + runs-on: ubuntu-latest + steps: + - name: Update Cache + id: update-cache + uses: stacks-network/actions/mutation-testing/update-cache@main + with: + gh-token: ${{ secrets.GITHUB_TOKEN }} +``` diff --git a/mutation-testing/upload-initial-cache/README.md b/mutation-testing/upload-initial-cache/README.md new file mode 100644 index 0000000..4349bfb --- /dev/null +++ b/mutation-testing/upload-initial-cache/README.md @@ -0,0 +1,18 @@ +# Upload Initial Cache action + +Create a new cache with the initial output for the current branch. + +## Usage + +```yaml +name: Action +on: push +jobs: + build: + name: Job + runs-on: ubuntu-latest + steps: + - name: Upload Initial Cache + id: upload-initial-cache + uses: stacks-network/actions/mutation-testing/upload-initial-cache@main +``` From f47472dcbcce87e605d7fb28331a783d08c3b87a Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Tue, 12 Dec 2023 16:11:01 +0200 Subject: [PATCH 04/29] feat: add retry to cache/artifact save in case of failure --- mutation-testing/update-cache/README.md | 2 ++ mutation-testing/update-cache/action.yml | 30 ++++++++++++++++++------ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/mutation-testing/update-cache/README.md b/mutation-testing/update-cache/README.md index 6c27fef..2be7ba5 100644 --- a/mutation-testing/update-cache/README.md +++ b/mutation-testing/update-cache/README.md @@ -9,6 +9,8 @@ Logs the mutants to cache, appending to the old output, and saving an artifact f | Input | Description | Required | Default | | ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | | `gh-token` | The github token from the main workflow for cache deletion | true | `""` | +| `retries` | Number of times to retry cache and artifact upload | false | `3` | +| `retry-delay` | Time in ms between upload retries | false | `10000` | ## Usage diff --git a/mutation-testing/update-cache/action.yml b/mutation-testing/update-cache/action.yml index 4f61a0e..1280226 100644 --- a/mutation-testing/update-cache/action.yml +++ b/mutation-testing/update-cache/action.yml @@ -5,6 +5,14 @@ inputs: description: 'The github token secret in order to delete the old cache' required: true default: '' + retries: + description: "Number of attempts" + required: false + default: "3" + retry-delay: + description: "Time to wait to retry" + required: false + default: "10000" runs: using: 'composite' @@ -93,14 +101,22 @@ runs: # TODO: feature for retrying the save in case it fails for any reason (in order not to lose the cache) - name: Save Cache - uses: actions/cache/save@v3 + uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 with: - path: ./mutation-testing/packages-output - key: mutants-output-${{ github.ref_name }} + uses: actions/cache/save@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 + with: + path: ./mutation-testing/packages-output + key: mutants-output-${{ github.ref_name }} + attempt_limit: ${{ inputs.retries }} + attempt_delay: ${{ inputs.retry-delay }} - name: Upload to artifact to easily check the ouput - uses: actions/upload-artifact@v3 - if: always() + uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 with: - name: mutants-output-${{ github.sha }} - path: ./mutation-testing/packages-output + uses: actions/upload-artifact@v3 + if: always() + with: + name: mutants-output-${{ github.sha }} + path: ./mutation-testing/packages-output + attempt_limit: ${{ inputs.retries }} + attempt_delay: ${{ inputs.retry-delay }} \ No newline at end of file From 9c2ab75549229c8f5eb6a10c1ae1bc3e504a2ce5 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 13 Dec 2023 01:10:52 +0200 Subject: [PATCH 05/29] fix: regex matching for mutants v12.3 --- mutation-testing/shell-scripts/append-match-package.sh | 4 ++-- .../remove-deleted-functions-from-output.sh | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/mutation-testing/shell-scripts/append-match-package.sh b/mutation-testing/shell-scripts/append-match-package.sh index 9026cfc..a7546c1 100644 --- a/mutation-testing/shell-scripts/append-match-package.sh +++ b/mutation-testing/shell-scripts/append-match-package.sh @@ -37,9 +37,9 @@ for file in "${FILES[@]}"; do # Extract the package from which the line is coming from local_package=${line%%/*} - # Extract the after the number line without the line number and escape it for awk + # Extract the line without the line number and escape it for awk # Escape the variables for use in a sed pattern - var_1=$(echo "$line" | sed -E 's/^(.+):[0-9]+:[^:]+/\1/') + var_1=$(echo "$line" | sed -E 's/^(.+):[0-9]+:[0-9]+:[^:]+/\1/') escaped_var_1=$(sed 's/[][/.^$]/\\&/g' <<< "$var_1") var_2=$(echo "$line" | sed -E 's/^[^:]+:[0-9]+:(.+)/\1/') diff --git a/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh b/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh index 8d6a756..b8f6278 100755 --- a/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh +++ b/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh @@ -55,9 +55,15 @@ while IFS= read -r line; do first_char=${line:0:1} if [[ $first_char == '-' ]]; then fn_name=$(echo "$line" | grep -oP 'fn\s+\K[^(\s]+') - if [[ ${line:1:2} == 'fn' ]]; then - functions+=("${file_path}.*replace ${fn_name}") + if [[ ${line:1:1} != " " ]]; then + # Eg. + # libsigner/src/http.rs:138:5: replace function_test -> Result... + # libsigner/src/http.rs:102:32: replace > with < in decode_http_request + functions+=("${file_path}.*(replace|in) ${fn_name}") else + # Eg. + # libsigner/src/http.rs:57:9: replace SignerHttpRequest::destruct -> ... + # libsigner/src/events.rs:255:9: replace ::forward_event -> bool with true functions+=("${file_path}.*${impl_block}::${fn_name}") fi fi From bb9d337efe50da53708efb0195b0588d64df4ecf Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Sun, 17 Dec 2023 01:54:00 +0200 Subject: [PATCH 06/29] feat: remove logger files, only keep filter-pr ones - Updated message for unviable mutants, steps to solve it --- mutation-testing/filter-pr/action.yml | 10 +- mutation-testing/logger/README.md | 28 ---- mutation-testing/logger/action.yml | 38 ------ .../shell-scripts/append-match-package.sh | 68 ---------- .../shell-scripts/create-stable.sh | 55 -------- .../shell-scripts/modular-mutants-run.sh | 41 ------ .../remove-deleted-functions-from-output.sh | 85 ------------ mutation-testing/update-cache/README.md | 30 ----- mutation-testing/update-cache/action.yml | 122 ------------------ .../upload-initial-cache/README.md | 18 --- .../upload-initial-cache/action.yml | 39 ------ 11 files changed, 9 insertions(+), 525 deletions(-) delete mode 100644 mutation-testing/logger/README.md delete mode 100644 mutation-testing/logger/action.yml delete mode 100644 mutation-testing/shell-scripts/append-match-package.sh delete mode 100644 mutation-testing/shell-scripts/create-stable.sh delete mode 100644 mutation-testing/shell-scripts/modular-mutants-run.sh delete mode 100755 mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh delete mode 100644 mutation-testing/update-cache/README.md delete mode 100644 mutation-testing/update-cache/action.yml delete mode 100644 mutation-testing/upload-initial-cache/README.md delete mode 100644 mutation-testing/upload-initial-cache/action.yml diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/filter-pr/action.yml index 096f25e..8ca9faa 100644 --- a/mutation-testing/filter-pr/action.yml +++ b/mutation-testing/filter-pr/action.yml @@ -1,7 +1,7 @@ name: Filter PR Mutants runs: - using: 'composite' + using: "composite" steps: # Cleanup Runner @@ -61,6 +61,10 @@ runs: echo "-------------" echo "Found unviable mutants:" cat ./mutants.out/unviable.txt + echo "-------------" + echo "To resolve this issue, consider one of the following options:" + echo "1. Create 'Default::default()' implementation for the specific structure." + echo "2. Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function headers." exit 1 fi echo "All new and updated functions are caught!" @@ -83,6 +87,10 @@ runs: echo "-------------" echo "Found unviable mutants:" cat ./mutants.out/unviable.txt + echo "-------------" + echo "To resolve this issue, consider one of the following options:" + echo "1. Create 'Default::default()' implementation for the specific structure." + echo "2. Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function headers." fi exit 1 ;; diff --git a/mutation-testing/logger/README.md b/mutation-testing/logger/README.md deleted file mode 100644 index 5701d79..0000000 --- a/mutation-testing/logger/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Logger Mutants action - -Check whether to log the mutants to cache, or to create a new cache if it doesn't exist. - -## Documentation - -### Inputs - -| Input | Description | Required | Default | -| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | -| `gh-token` | The github token from the main workflow for cache deletion | true | `""` | - -## Usage - -```yaml -name: Action -on: push -jobs: - build: - name: Job - runs-on: ubuntu-latest - steps: - - name: Log Mutants - id: log-mutants - uses: stacks-network/actions/mutation-testing/logger@main - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} -``` diff --git a/mutation-testing/logger/action.yml b/mutation-testing/logger/action.yml deleted file mode 100644 index a45827a..0000000 --- a/mutation-testing/logger/action.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Logging Mutants - -# only run on push in order to update the cache output -# flow: -# restore cache -# install cargo-mutants crate in order to run the 'cargo mutants' command -# create a file with the current commit hash if a previous one doesn't exist, then print it -# run the script that handles the 'cargo mutants' command on the differences between the latest updates and the last commit where it was ran -# overwrite the previous commit hash with the current one for the following run -# delete the old cache -# save the new cache with the updated mutants -inputs: - gh-token: - description: 'The github token secret in order to delete the old cache' - required: true - default: '' - -runs: - using: 'composite' - - steps: - - name: Check Cache - uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 - id: check_cache - with: - lookup-only: true - path: ./ - key: mutants-output-${{ github.ref_name }} - - - name: Upload initial outputs if there is no cache - if: steps.check_cache.outputs.cache-hit != 'true' - uses: stacks-network/actions/mutation-testing/upload-initial-cache@main - - - name: Update cache if we have one available - if: steps.check_cache.outputs.cache-hit == 'true' - uses: stacks-network/actions/mutation-testing/update-cache@main - with: - gh-token: ${{ inputs.gh-token }} diff --git a/mutation-testing/shell-scripts/append-match-package.sh b/mutation-testing/shell-scripts/append-match-package.sh deleted file mode 100644 index a7546c1..0000000 --- a/mutation-testing/shell-scripts/append-match-package.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash - -# the append-match-package.sh -## goes through each line in the output and based on the package ( first element before /) -### verifies the line with the other lines in that specific folder -#### in our case folder_name == package_name - - -# goes through each PR file line by line -# for each first_element/the_rest_of_the_line goes through it -## search in that specific folder on all 4 files -## if it is matchy, remove it from that file -## based on the file it was taken from, append it to the same file in the STABLE folder - - -PR_FOLDER="../temp/mutants.out" -STABLE_FOLDER_PARENT="../packages-output" -FILES=("caught.txt" "missed.txt" "timeout.txt" "unviable.txt") - -echo "Starting script..." -echo "PR Folder: $PR_FOLDER" -echo "STABLE Folder: $STABLE_FOLDER_PARENT" -echo "Files to process: ${FILES[*]}" - -# Iterate over the specified files -for file in "${FILES[@]}"; do - pr_file="$PR_FOLDER/$file" - - echo "Processing file: $file" - - # Check if PR file exists and is not empty - if [[ -s "$pr_file" ]]; then - # Read each line from the PR file - while IFS= read -r line; do - echo "Reading line from PR file: $line" - - # Extract the package from which the line is coming from - local_package=${line%%/*} - - # Extract the line without the line number and escape it for awk - # Escape the variables for use in a sed pattern - var_1=$(echo "$line" | sed -E 's/^(.+):[0-9]+:[0-9]+:[^:]+/\1/') - escaped_var_1=$(sed 's/[][/.^$]/\\&/g' <<< "$var_1") - - var_2=$(echo "$line" | sed -E 's/^[^:]+:[0-9]+:(.+)/\1/') - escaped_var_2=$(sed 's/[][/.^$]/\\&/g' <<< "$var_2") - - regex="${escaped_var_1}.*${escaped_var_2}" - - # Iterate over each file in the STABLE folder combined with local_package - for target_file in "${FILES[@]}"; do - target_path="$STABLE_FOLDER_PARENT/$local_package/$target_file" - echo "Checking against STABLE file: $target_path" - - # Use sed to remove lines matching the pattern - sed "/$regex/d" "$target_path" > temp_file && mv temp_file "$target_path" - done - - # Append PR line to the corresponding package and file - echo "$line" >> "$STABLE_FOLDER_PARENT/$local_package/$file" - - done < "$pr_file" - else - echo "PR file $pr_file is empty or does not exist, skipping..." - fi -done - -echo "Script completed." diff --git a/mutation-testing/shell-scripts/create-stable.sh b/mutation-testing/shell-scripts/create-stable.sh deleted file mode 100644 index 1659714..0000000 --- a/mutation-testing/shell-scripts/create-stable.sh +++ /dev/null @@ -1,55 +0,0 @@ -# for specific packages creates the outpup - -# removes everything except .txt files - -#!/bin/bash - -# moves to mutation-testing folder -cd ../packages-output - -### Run mutation testing on the packages uncommented - -# Run mutation testing for stx-genesis package -cargo mutants --package stx-genesis --output stx-genesis -mv stx-genesis/mutants.out/*.txt stx-genesis/ -rm -rf stx-genesis/mutants.out - -# Run mutation testing for pox-locking package -cargo mutants --package pox-locking --output pox-locking -mv pox-locking/mutants.out/*.txt pox-locking/ -rm -rf pox-locking/mutants.out - -# # Run mutation testing for libsigner package -# cargo mutants --package libsigner --output libsigner -# mv libsigner/mutants.out/*.txt libsigner/ -# rm -rf libsigner/mutants.out - -# # Run mutation testing for libstackerdb package -# cargo mutants --package libstackerdb --output libstackerdb -# mv libstackerdb/mutants.out/*.txt libstackerdb/ -# rm -rf libstackerdb/mutants.out - -# # Run mutation testing for stacks-common package -# cargo mutants --package stacks-common --output stacks-common -# mv stacks-common/mutants.out/*.txt stacks-common/ -# rm -rf stacks-common/mutants.out - -# # Run mutation testing for clarity package -# cargo mutants --package clarity --output clarity -# mv clarity/mutants.out/*.txt clarity/ -# rm -rf clarity/mutants.out - -# Run mutation testing for stacks-signer package - working, 10 min approx. -# cargo mutants --package stacks-signer --output stacks-signer -# mv stacks-signer/mutants.out/*.txt stacks-signer/ -# rm -rf stacks-signer/mutants.out - -# Commented out mutation testing for stacks-node package due to test errors and long compile/testing time -# cargo mutants --package stacks-node --output stacks-node -# mv stacks-node/mutants.out/*.txt stacks-node/ -# rm -rf stacks-node/mutants.out - -# Commented out mutation testing for stackslib package due to long compile/testing time -# cargo mutants --package stackslib --output stackslib -# mv stackslib/mutants.out/*.txt stackslib/ -# rm -rf stackslib/mutants.out \ No newline at end of file diff --git a/mutation-testing/shell-scripts/modular-mutants-run.sh b/mutation-testing/shell-scripts/modular-mutants-run.sh deleted file mode 100644 index c8bc357..0000000 --- a/mutation-testing/shell-scripts/modular-mutants-run.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -packages=$(cargo tree --workspace --prefix depth | grep "^0" | cut -c2- | awk '{print $1}') -regex_list=() - -while [ $# -gt 0 ]; do - arg=$1 - - if [[ $packages == *$arg* ]]; then - package=$arg - file="" - shift - arg=$1 - fi - if [[ $arg == *.rs ]]; then - file=$arg - shift - arg=$1 - fi - - functions=() - while [ $# -gt 0 ] && [[ $1 != *.rs ]] && [[ $packages != *$1* ]]; do - functions+=("$1") - shift - done - - IFS="|" - functions_str="${functions[*]}" - IFS="" - - regex="${package}/[^/]+/${file}.*?(?:${functions_str})[^-()]*(?:->|\(\))" - regex_list+=("$regex") -done - -command="cargo mutants -vV --no-shuffle" - -for regex in "${regex_list[@]}"; do - command+=" -F \"$regex\"" -done - -eval "$command" \ No newline at end of file diff --git a/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh b/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh deleted file mode 100755 index b8f6278..0000000 --- a/mutation-testing/shell-scripts/remove-deleted-functions-from-output.sh +++ /dev/null @@ -1,85 +0,0 @@ -#!/bin/bash - -# Flow: -# -# Parse through all the lines of the file -# When we find a line that starts with '--- a/', we save what is after the 'a/' to a 'file_path' variable -# When we find a line that contains the word 'impl': -# If it doesn't contain a ' for ' in it, save from the beginning of 'impl' up until '{' (including 'impl', not '{') to a 'impl_block' variable -# If it contains a ' for ', do the step above but add '<' before and '>' after -# Remove extras from the line. -# Example: -# Git Diff: -# PartialEq> for IncomparableError> -# for Error> -# -# Expected in outputs: -# > -# -# -# -# When we find a line that contains 'fn', check: -# If the first char on the line is '-', add the name between 'fn' and '(' to a variable named 'fn_name', then append to a 'functions' list by the following rule: -# If the 2nd and 3rd chars are 'fn', add to list the following matching: '{file_path}.*replace {fn_name}' -# Else, add to the list the following matching: '{file_path}.*{impl_block}::{fn_name}' -# Then parse through every fn in {functions}, and if the fn matches any line in the files from 'packages-output' folder, remove it - -input_file="git.diff" -functions=() - -# Read the file line by line -while IFS= read -r line; do - # Save the file path - if [[ $line == *'--- a/'* ]]; then - file_path=$(echo "$line" | cut -c7-) - fi - - # Save the last impl block - if [[ "$line" == *'impl'*' for '*'{'* ]]; then - impl_block="${line#*impl}" - impl_block="${impl_block%%\ {*}" - impl_block="" # add 'impl' word, '<' and '>' characters - impl_block=$(echo "$impl_block" | sed -E 's/(^[^ ]*impl)[^ ]*/\1/') # remove everything after 'impl', until the next space - impl_block=$(echo "$impl_block" | sed 's/\(<[^<]*\)[^ ]* /\1 /') # remove everything starting with '<' and ending at the next space, from the second word - impl_block=$(echo "$impl_block" | awk '{sub(/ [^ ]+::/, " "); print}') # keep only the characters after the last '::' of the 2nd word (struct name without path) - elif [[ "$line" == *'impl'*'{'* ]]; then - impl_block="${line#*impl}" - impl_block="${impl_block%%\ {*}" - elif [[ "$line" == *'trait'*'{'* ]]; then - impl_block="${line#*trait}" - impl_block="${impl_block%%\ {*}" - fi - - # Save the regex matching the file name, impl block (if any), and function to a list - if [[ $line == *'fn'* ]]; then - first_char=${line:0:1} - if [[ $first_char == '-' ]]; then - fn_name=$(echo "$line" | grep -oP 'fn\s+\K[^(\s]+') - if [[ ${line:1:1} != " " ]]; then - # Eg. - # libsigner/src/http.rs:138:5: replace function_test -> Result... - # libsigner/src/http.rs:102:32: replace > with < in decode_http_request - functions+=("${file_path}.*(replace|in) ${fn_name}") - else - # Eg. - # libsigner/src/http.rs:57:9: replace SignerHttpRequest::destruct -> ... - # libsigner/src/events.rs:255:9: replace ::forward_event -> bool with true - functions+=("${file_path}.*${impl_block}::${fn_name}") - fi - fi - fi -done < "$input_file" - -# Print the lines to be removed -echo "Removing these lines from the stable output:" -for func in "${functions[@]}"; do - grep -r "$func" ../packages-output/* -done - -# Remove the lines matching the regex for all deleted functions -find "../packages-output" -type f -print0 | while IFS= read -r -d $'\0' file; do - grep -Ev "$(printf '%s\n' "${functions[@]}" | sed 's/\([[].*[]]\)/[\\1]/g')" "$file" > "temp_file" && mv "temp_file" "$file" - if [[ -e "temp_file" ]]; then - rm "temp_file" - fi -done diff --git a/mutation-testing/update-cache/README.md b/mutation-testing/update-cache/README.md deleted file mode 100644 index 2be7ba5..0000000 --- a/mutation-testing/update-cache/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# Update cache action - -Logs the mutants to cache, appending to the old output, and saving an artifact for easier data access and backup. - -## Documentation - -### Inputs - -| Input | Description | Required | Default | -| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | -| `gh-token` | The github token from the main workflow for cache deletion | true | `""` | -| `retries` | Number of times to retry cache and artifact upload | false | `3` | -| `retry-delay` | Time in ms between upload retries | false | `10000` | - -## Usage - -```yaml -name: Action -on: push -jobs: - build: - name: Job - runs-on: ubuntu-latest - steps: - - name: Update Cache - id: update-cache - uses: stacks-network/actions/mutation-testing/update-cache@main - with: - gh-token: ${{ secrets.GITHUB_TOKEN }} -``` diff --git a/mutation-testing/update-cache/action.yml b/mutation-testing/update-cache/action.yml deleted file mode 100644 index 1280226..0000000 --- a/mutation-testing/update-cache/action.yml +++ /dev/null @@ -1,122 +0,0 @@ -name: Update Cache - -inputs: - gh-token: - description: 'The github token secret in order to delete the old cache' - required: true - default: '' - retries: - description: "Number of attempts" - required: false - default: "3" - retry-delay: - description: "Time to wait to retry" - required: false - default: "10000" - -runs: - using: 'composite' - - steps: - # Cleanup Runner - - name: Cleanup Runner action repo - id: runner_cleanup - uses: stacks-network/actions/cleanup/disk@main - - # Checkout the stacks-core code - - name: Checkout the stacks core repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Restore mutants-output cached folder - id: cache-restore - uses: actions/cache/restore@v3 - with: - path: ./mutation-testing/packages-output - key: mutants-output-${{ github.ref_name }} - - - name: Create git.diff file - shell: bash - run: | - last_commit_hash=$(<../packages-output/last_commit_hash.txt) - git diff $last_commit_hash > git.diff - working-directory: ./mutation-testing/scripts - - # Checkout the actions code - - name: Get the action repo code - id: git_checkout_actions - uses: actions/checkout@v3 - with: - repository: stacks-network/actions - ref: main - path: ./actions-repo - - - name: Install cargo-mutants crate - shell: bash - run: cargo install cargo-mutants - - - name: Remove deleted functions from output files - shell: bash - run: ./remove-deleted-functions-from-output.sh - working-directory: ./mutation-testing/scripts - - - name: Remove deleted file lines from git.diff file - shell: bash - run: ./remove-deleted-file-lines.sh - working-directory: ./mutation-testing/scripts - - - name: Run the mutation testing on the differences - shell: bash - run: | - if [ -f ./packages-output/last_commit_hash.txt ]; then - cargo mutants --no-shuffle -j 2 -vV --in-diff ./scripts/git.diff --output temp/ - fi - working-directory: ./mutation-testing - - - name: Update the content from the stable output - shell: bash - run: bash append-match-package.sh - working-directory: ./mutation-testing/scripts - - - name: Re-write the old commit hash with the current one - shell: bash - run: | - echo "${{ github.sha }}" > last_commit_hash.txt - working-directory: ./mutation-testing/packages-output - - - name: Print new commit hash - shell: bash - run: cat ./mutation-testing/packages-output/last_commit_hash.txt - - - name: Delete Previous Cache - if: ${{ steps.cache-restore.outputs.cache-hit }} - continue-on-error: true - shell: bash - run: | - gh extension install actions/gh-actions-cache - gh actions-cache delete "mutants-output-${{ github.ref_name }}" --confirm - env: - GH_TOKEN: ${{ inputs.gh-token }} - - # TODO: feature for retrying the save in case it fails for any reason (in order not to lose the cache) - - name: Save Cache - uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 - with: - uses: actions/cache/save@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3.3.2 - with: - path: ./mutation-testing/packages-output - key: mutants-output-${{ github.ref_name }} - attempt_limit: ${{ inputs.retries }} - attempt_delay: ${{ inputs.retry-delay }} - - - name: Upload to artifact to easily check the ouput - uses: Wandalen/wretry.action@a163f62ae554a8f3cbe27b23db15b60c0ae2e93c # v1.3.0 - with: - uses: actions/upload-artifact@v3 - if: always() - with: - name: mutants-output-${{ github.sha }} - path: ./mutation-testing/packages-output - attempt_limit: ${{ inputs.retries }} - attempt_delay: ${{ inputs.retry-delay }} \ No newline at end of file diff --git a/mutation-testing/upload-initial-cache/README.md b/mutation-testing/upload-initial-cache/README.md deleted file mode 100644 index 4349bfb..0000000 --- a/mutation-testing/upload-initial-cache/README.md +++ /dev/null @@ -1,18 +0,0 @@ -# Upload Initial Cache action - -Create a new cache with the initial output for the current branch. - -## Usage - -```yaml -name: Action -on: push -jobs: - build: - name: Job - runs-on: ubuntu-latest - steps: - - name: Upload Initial Cache - id: upload-initial-cache - uses: stacks-network/actions/mutation-testing/upload-initial-cache@main -``` diff --git a/mutation-testing/upload-initial-cache/action.yml b/mutation-testing/upload-initial-cache/action.yml deleted file mode 100644 index f682ba4..0000000 --- a/mutation-testing/upload-initial-cache/action.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Upload initial cache - -runs: - using: 'composite' - - steps: - # Checkout the stacks-core code - - name: Checkout the stacks core repo - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - # Checkout the actions code - - name: Get the action repo code - id: git_checkout_actions - uses: actions/checkout@v3 - with: - repository: stacks-network/actions - ref: main - path: ./actions-repo - - - name: Create file with current commit hash if it doesn't exist - shell: bash - run: | - if [ ! -f last_commit_hash.txt ]; then - echo "${{ github.sha }}" > last_commit_hash.txt - fi - working-directory: ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }} - - - name: Print old commit hash - shell: bash - run: cat ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }}/last_commit_hash.txt - - # TODO: feature for retrying the save in case it fails for any reason (in order not to lose the cache) - - name: Save Cache - uses: actions/cache/save@v3 - with: - path: ./actions-repo/mutation-testing/initial-output/${{ github.ref_name }} - key: mutants-output-${{ github.ref_name }} From 8b2b43de24c96e677deae088eb61103b61801ca9 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Sun, 17 Dec 2023 01:55:06 +0200 Subject: [PATCH 07/29] feat: update readme to exclude logger paths --- mutation-testing/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/mutation-testing/README.md b/mutation-testing/README.md index fc2e607..36b035a 100644 --- a/mutation-testing/README.md +++ b/mutation-testing/README.md @@ -1,6 +1,3 @@ # Mutation Testing actions > - [Filter PR Mutants action](./filter-pr/README.md) -> - [Logger Mutants action](./logger/README.md) -> - [Update cache action](./update-cache/README.md) -> - [Upload Initial Cache action](./upload-initial-cache/README.md) From f765dacfa18e83cc231c37051c9b032cd80b19a8 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 18 Dec 2023 17:12:26 +0200 Subject: [PATCH 08/29] feat: update actions checkout branch to point at the base branch of the PR --- mutation-testing/filter-pr/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/filter-pr/action.yml index 8ca9faa..45ad0e3 100644 --- a/mutation-testing/filter-pr/action.yml +++ b/mutation-testing/filter-pr/action.yml @@ -28,7 +28,7 @@ runs: uses: actions/checkout@v3 with: repository: stacks-network/actions - ref: main + ref: feat/mutation-testing path: ./actions-repo - name: Copy the git diff file to scripts folder From 1f1679e425508e69c937d6072d62e0fcf3905f4d Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 28 Dec 2023 03:45:38 +0200 Subject: [PATCH 09/29] feat: add env vars and test-threads for `cargo test` --- mutation-testing/filter-pr/action.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/filter-pr/action.yml index 45ad0e3..5b2fe7b 100644 --- a/mutation-testing/filter-pr/action.yml +++ b/mutation-testing/filter-pr/action.yml @@ -51,7 +51,9 @@ runs: shell: bash run: | set +e # Disable immediate exit on error - cargo mutants --no-shuffle -j 2 -vV --in-diff git.diff --output ./ + export BITCOIND_TEST=1 + export RUST_BACKTRACE=full + cargo mutants --no-shuffle -j 2 -vV --in-diff git.diff --output ./ -- -- --test-threads 1 exit_code=$? set -e # Enable immediate exit on error again From 07c7d280f6f864c0255080b99c7839f839d0b531 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Sun, 7 Jan 2024 22:56:38 +0200 Subject: [PATCH 10/29] modularize mutants' runs for different package size cases Based on the type of package it will include the corresponding arguments to be run with. Cases: - if >= 16 mutants on big packages => run big packages using 8 shards - else run big packages without shards - if >= 16 mutants on big packages and >= 120 mutants on small packages => run small packages using 4 shards - else if < 16 mutants on big packages and >= 80 mutants on small packages => run small packages using 4 shards - else run small packages without shards --- mutation-testing/filter-pr/action.yml | 135 +++++++++++++++----------- 1 file changed, 77 insertions(+), 58 deletions(-) diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/filter-pr/action.yml index 5b2fe7b..1cd7dd9 100644 --- a/mutation-testing/filter-pr/action.yml +++ b/mutation-testing/filter-pr/action.yml @@ -1,5 +1,14 @@ name: Filter PR Mutants +inputs: + shard: + description: "The number of the shard to run (`-1` if ran without shards)" + required: false + default: -1 + package-dimension: + description: "The dimension of the package. `big` for `stacks-node` and `stackslib`, `small` for others" + required: true + runs: using: "composite" @@ -19,8 +28,7 @@ runs: - name: Relative diff shell: bash run: | - git branch -av - git diff origin/${{ github.base_ref }}.. | tee git.diff + git diff origin/${{ github.base_ref }}.. > git.diff # Checkout the actions code - name: Get the action repo code @@ -36,73 +44,84 @@ runs: run: | mv git.diff ./actions-repo/mutation-testing/shell-scripts/ - - uses: Swatinem/rust-cache@v2 - - name: Install cargo mutants crate shell: bash - run: cargo install cargo-mutants + run: cargo install --version 23.12.2 cargo-mutants - name: Remove deleted file lines from git.diff file shell: bash run: ./remove-deleted-file-lines.sh working-directory: actions-repo/mutation-testing/shell-scripts + - name: Split diffs into big and small packages and create regex patterns for them + shell: bash + run: | + cargo mutants --in-diff git.diff --list > all_mutants.txt + mkdir -p mutants_by_packages + + # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files + while IFS= read -r line; do + package=$(echo "$line" | cut -d'/' -f1) + if [[ $package == "testnet" || $package == "stackslib" ]]; then + echo "$line" >> "mutants_by_packages/big_packages.txt" + else + echo "$line" >> "mutants_by_packages/small_packages.txt" + fi + done < all_mutants.txt + + # Create regex patterns of the mutants to be ran for `cargo mutants` -F flag and output them to files to be used later + for package in mutants_by_packages/*; do + regex_pattern="" + + while IFS= read -r line; do + escaped_line=$(echo "$line" | sed 's/[][()\.^$*+?{}|]/\\&/g') + regex_pattern+="($escaped_line)|" + done < "$package" + + regex_pattern="${regex_pattern%|}" + + if [[ "$package" == "mutants_by_packages/big_packages.txt" ]]; then + echo "$regex_pattern" > mutants_by_packages/regex_big.txt + else + echo "$regex_pattern" > mutants_by_packages/regex_small.txt + fi + done + working-directory: actions-repo/mutation-testing/shell-scripts + - name: Run mutants and check the exit code of the command, fail the workflow if mutations are not caught shell: bash run: | set +e # Disable immediate exit on error - export BITCOIND_TEST=1 - export RUST_BACKTRACE=full - cargo mutants --no-shuffle -j 2 -vV --in-diff git.diff --output ./ -- -- --test-threads 1 - exit_code=$? + if [[ ${{ inputs.shard }} == -1 ]]; then + if [[ ${{ inputs.package-dimension }} == big ]]; then + echo "Running mutants without shards on big packages" + cargo mutants --no-shuffle -vV -F "$( ./mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }}/exit_code.txt + mv ./mutants.out/*.txt mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }}/ set -e # Enable immediate exit on error again - - case $exit_code in - 0) - if [ -s ./mutants.out/unviable.txt ]; then - echo "-------------" - echo "Found unviable mutants:" - cat ./mutants.out/unviable.txt - echo "-------------" - echo "To resolve this issue, consider one of the following options:" - echo "1. Create 'Default::default()' implementation for the specific structure." - echo "2. Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function headers." - exit 1 - fi - echo "All new and updated functions are caught!" - ;; - 1) - echo "Invalid command line arguments!" - exit 1 - ;; - 2 | 3) - if [ -s ./mutants.out/missed.txt ]; then - echo "Found missed mutants:" - cat ./mutants.out/missed.txt - fi - if [ -s ./mutants.out/timeout.txt ]; then - echo "-------------" - echo "Found timeout mutants:" - cat ./mutants.out/timeout.txt - fi - if [ -s ./mutants.out/unviable.txt ]; then - echo "-------------" - echo "Found unviable mutants:" - cat ./mutants.out/unviable.txt - echo "-------------" - echo "To resolve this issue, consider one of the following options:" - echo "1. Create 'Default::default()' implementation for the specific structure." - echo "2. Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function headers." - fi - exit 1 - ;; - 4) - echo "Building the packages failed without any mutations!" - exit 1 - ;; - *) - echo "Unknown exit code: $exit_code" - exit 1 - ;; - esac working-directory: actions-repo/mutation-testing/shell-scripts + + - name: Upload artifact + uses: actions/upload-artifact@v3 + if: always() + with: + name: mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} + path: actions-repo/mutation-testing/shell-scripts/mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} From e7b4a758ebea02fbdad2e0b3b241a43f88f6c06d Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 8 Jan 2024 16:34:28 +0200 Subject: [PATCH 11/29] feat: rename from `filter-pr` to `pr-differences` --- mutation-testing/filter-pr/README.md | 20 ------------------- mutation-testing/pr-differences/README.md | 20 +++++++++++++++++++ .../{filter-pr => pr-differences}/action.yml | 2 +- 3 files changed, 21 insertions(+), 21 deletions(-) delete mode 100644 mutation-testing/filter-pr/README.md create mode 100644 mutation-testing/pr-differences/README.md rename mutation-testing/{filter-pr => pr-differences}/action.yml (99%) diff --git a/mutation-testing/filter-pr/README.md b/mutation-testing/filter-pr/README.md deleted file mode 100644 index 2ad7478..0000000 --- a/mutation-testing/filter-pr/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# Filter PR Mutants action - -Run filtering phase for pull requests, and fail the workflow if: -- There are missed/timeout/unviable mutants -- Building or testing the unmutated crate fails - -## Usage - -```yaml -name: Action -on: pull_request -jobs: - build: - name: Job - runs-on: ubuntu-latest - steps: - - name: Filter PR Mutants - id: filter-pr-mutants - uses: stacks-network/actions/mutation-testing/filter-pr@main -``` diff --git a/mutation-testing/pr-differences/README.md b/mutation-testing/pr-differences/README.md new file mode 100644 index 0000000..e511304 --- /dev/null +++ b/mutation-testing/pr-differences/README.md @@ -0,0 +1,20 @@ +# PR Differences Mutants action + +Run mutants for differences in pull requests, and fail the workflow if: +- There are missed/timeout/unviable mutants +- Building or testing the unmutated crate fails + +## Usage + +```yaml +name: Action +on: pull_request +jobs: + build: + name: Job + runs-on: ubuntu-latest + steps: + - name: PR Differences Mutants + id: pr-differences-mutants + uses: stacks-network/actions/mutation-testing/pr-differences@main +``` diff --git a/mutation-testing/filter-pr/action.yml b/mutation-testing/pr-differences/action.yml similarity index 99% rename from mutation-testing/filter-pr/action.yml rename to mutation-testing/pr-differences/action.yml index 1cd7dd9..d7397f6 100644 --- a/mutation-testing/filter-pr/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -1,4 +1,4 @@ -name: Filter PR Mutants +name: PR Differences Mutants inputs: shard: From b773b0f2cd5f9f3054b5dd0c9a7bb538a637b50f Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Tue, 9 Jan 2024 02:11:39 +0200 Subject: [PATCH 12/29] feat: shorter step names, check files and commands before using them --- mutation-testing/pr-differences/action.yml | 76 +++++++++++++------ .../remove-deleted-file-lines.sh | 5 ++ 2 files changed, 56 insertions(+), 25 deletions(-) diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index d7397f6..7d9b158 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -14,12 +14,12 @@ runs: steps: # Cleanup Runner - - name: Cleanup Runner action repo + - name: Cleanup Runner id: runner_cleanup uses: stacks-network/actions/cleanup/disk@main # Checkout the stacks-core code - - name: Checkout the stacks core repo + - name: Checkout stacks-core repo id: git_checkout_stacks_core uses: actions/checkout@v3 with: @@ -31,7 +31,7 @@ runs: git diff origin/${{ github.base_ref }}.. > git.diff # Checkout the actions code - - name: Get the action repo code + - name: Checkout actions repo id: git_checkout_actions uses: actions/checkout@v3 with: @@ -39,34 +39,44 @@ runs: ref: feat/mutation-testing path: ./actions-repo - - name: Copy the git diff file to scripts folder + - name: Copy git diff shell: bash run: | mv git.diff ./actions-repo/mutation-testing/shell-scripts/ - - name: Install cargo mutants crate + - name: Install cargo-mutants shell: bash run: cargo install --version 23.12.2 cargo-mutants - - name: Remove deleted file lines from git.diff file + - name: Update git diff shell: bash run: ./remove-deleted-file-lines.sh working-directory: actions-repo/mutation-testing/shell-scripts - - name: Split diffs into big and small packages and create regex patterns for them + - name: Split diffs shell: bash run: | cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages + # Check that the file exists before performing actions on it + if [ -s all_mutants.txt ]; then + echo "The file containing mutants is missing or empty!" + exit 1 + fi + # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files while IFS= read -r line; do package=$(echo "$line" | cut -d'/' -f1) - if [[ $package == "testnet" || $package == "stackslib" ]]; then - echo "$line" >> "mutants_by_packages/big_packages.txt" - else - echo "$line" >> "mutants_by_packages/small_packages.txt" - fi + + case $package in + "testnet" | "stackslib") + echo "$line" >> "mutants_by_packages/big_packages.txt" + ;; + *) + echo "$line" >> "mutants_by_packages/small_packages.txt" + ;; + esac done < all_mutants.txt # Create regex patterns of the mutants to be ran for `cargo mutants` -F flag and output them to files to be used later @@ -88,29 +98,45 @@ runs: done working-directory: actions-repo/mutation-testing/shell-scripts - - name: Run mutants and check the exit code of the command, fail the workflow if mutations are not caught + - name: Run mutants shell: bash run: | set +e # Disable immediate exit on error if [[ ${{ inputs.shard }} == -1 ]]; then if [[ ${{ inputs.package-dimension }} == big ]]; then - echo "Running mutants without shards on big packages" - cargo mutants --no-shuffle -vV -F "$( /dev/null 2>&1 || echo "Missing command: ${cmd}" +done + # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" From 34438d18f4606487aef86053c056170d64a97a66 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 10 Jan 2024 17:40:47 +0200 Subject: [PATCH 13/29] feat: move the packages checking and output steps to composite --- .../check-packages-and-shards/README.md | 28 ++++ .../check-packages-and-shards/action.yml | 119 ++++++++++++++ mutation-testing/output-pr-mutants/README.md | 18 +++ mutation-testing/output-pr-mutants/action.yml | 153 ++++++++++++++++++ 4 files changed, 318 insertions(+) create mode 100644 mutation-testing/check-packages-and-shards/README.md create mode 100644 mutation-testing/check-packages-and-shards/action.yml create mode 100644 mutation-testing/output-pr-mutants/README.md create mode 100644 mutation-testing/output-pr-mutants/action.yml diff --git a/mutation-testing/check-packages-and-shards/README.md b/mutation-testing/check-packages-and-shards/README.md new file mode 100644 index 0000000..feb000d --- /dev/null +++ b/mutation-testing/check-packages-and-shards/README.md @@ -0,0 +1,28 @@ +# Check Packages and Shards action + +Checks whether to run mutants on big (`stackslib`/`stacks-node`) or small packages (all others), and whether to run them using strategy matrix or not. + +## Documentation + +### Outputs +| Output | Description | +| ------------------------------- | ----------------------------------------------------- | +| `run_big_packages` | True if there are mutants on `stackslib` or `stacks-node` packages, false otherwise. +| `big_packages_with_shards` | True if there are more than 15 mutants on big packages. +| `run_small_packages` | True if there are mutants on small packages, false otherwise. +| `small_packages_with_shards` | True if there are more than 79 (119 if `big_packages_with_shards` is true) mutants on small packages. + +## Usage + +```yaml +name: Action +on: pull-request +jobs: + check-packages-and-shards: + name: Check Packages and Shards + runs-on: ubuntu-latest + steps: + - name: Check Packages and Shards + id: check_packages_and_shards + uses: stacks-network/actions/mutation-testing/check-packages-and-shards@main +``` diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml new file mode 100644 index 0000000..68ea219 --- /dev/null +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -0,0 +1,119 @@ +name: Check Packages and Shards + +outputs: + run_big_packages: ${{ steps.check_packages_and_shards.outputs.run_big_packages }} + big_packages_with_shards: ${{ steps.check_packages_and_shards.outputs.big_packages_with_shards }} + run_small_packages: ${{ steps.check_packages_and_shards.outputs.run_small_packages }} + small_packages_with_shards: ${{ steps.check_packages_and_shards.outputs.small_packages_with_shards }} + +runs: + using: "composite" + + steps: + - name: Checkout stacks-core repo + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Install cargo-mutants + shell: bash + run: cargo install --version 23.12.2 cargo-mutants + + - name: Relative diff + shell: bash + run: | + git diff origin/${{ github.base_ref }}.. > git.diff + + - name: Update git diff + shell: bash + run: | + input_file="git.diff" + temp_file="temp_diff_file.diff" + + # Check if the commands exist on the host + for cmd in tac awk sed; do + command -v "${cmd}" > /dev/null 2>&1 || echo "Missing command: ${cmd}" + done + + # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) + tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + + # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' + awk ' + BEGIN { in_block=0 } + /\+\+\+ \/dev\/null/ { in_block=1; next } + in_block && /diff --git a\// { in_block=0; print; next } + !in_block + ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + + - name: Split diffs + shell: bash + run: | + cargo mutants --in-diff git.diff --list > all_mutants.txt + mkdir -p mutants_by_packages + + # Check that the file exists before performing actions on it + if [ -s all_mutants.txt ]; then + echo "The file containing mutants is missing or empty!" + exit 1 + fi + + # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files + while IFS= read -r line; do + package=$(echo "$line" | cut -d'/' -f1) + if [[ $package == "testnet" || $package == "stackslib" ]]; then + echo "$line" >> "mutants_by_packages/big_packages.txt" + else + echo "$line" >> "mutants_by_packages/small_packages.txt" + fi + done < all_mutants.txt + + - name: Check packages and shards + id: check_packages_and_shards + shell: bash + run: | + number_of_big_mutants=0 + number_of_small_mutants=0 + + # If big_packages file exists, count how many mutants there are + if [[ -s mutants_by_packages/big_packages.txt ]]; then + number_of_big_mutants=$(cat mutants_by_packages/big_packages.txt | awk 'END { print NR }' | tr -d '[:space:]') + fi + + # If small_packages file exists, count how many mutants there are + if [[ -s mutants_by_packages/small_packages.txt ]]; then + number_of_small_mutants=$(cat mutants_by_packages/small_packages.txt | awk 'END { print NR }' | tr -d '[:space:]') + fi + + # Set the mutants limit for when to run with shards on the small packages + if [[ $number_of_big_mutants -gt 15 ]]; then + small_packages_shard_limit=119 + else + small_packages_shard_limit=79 + fi + + # If there are mutants from big packages, check whether to run with or without shards, otherwise there's nothing to run + if [[ $number_of_big_mutants -ne 0 ]]; then + echo "run_big_packages=true" >> "$GITHUB_OUTPUT" + if [[ $number_of_big_mutants -gt 15 ]]; then + echo "big_packages_with_shards=true" >> "$GITHUB_OUTPUT" + else + echo "big_packages_with_shards=false" >> "$GITHUB_OUTPUT" + fi + else + echo "run_big_packages=false" >> "$GITHUB_OUTPUT" + fi + + # If there are mutants from small packages, check whether to run with or without shards, otherwise there's nothing to run + if [[ $number_of_small_mutants -ne 0 ]]; then + echo "run_small_packages=true" >> "$GITHUB_OUTPUT" + if [[ $number_of_small_mutants -gt $small_packages_shard_limit ]]; then + echo "small_packages_with_shards=true" >> "$GITHUB_OUTPUT" + else + echo "small_packages_with_shards=false" >> "$GITHUB_OUTPUT" + fi + else + echo "run_small_packages=false" >> "$GITHUB_OUTPUT" + fi diff --git a/mutation-testing/output-pr-mutants/README.md b/mutation-testing/output-pr-mutants/README.md new file mode 100644 index 0000000..f4eba49 --- /dev/null +++ b/mutation-testing/output-pr-mutants/README.md @@ -0,0 +1,18 @@ +# Output PR Mutants action + +Write the mutants tested in the previous jobs of the same workflow to github step summary. + +## Usage + +```yaml +name: Action +on: push +jobs: + check-packages-and-shards: + name: Output Mutants + runs-on: ubuntu-latest + steps: + - name: Output Mutants + id: output_pr_mutants + uses: stacks-network/actions/mutation-testing/output-pr-mutants@main +``` diff --git a/mutation-testing/output-pr-mutants/action.yml b/mutation-testing/output-pr-mutants/action.yml new file mode 100644 index 0000000..6a15725 --- /dev/null +++ b/mutation-testing/output-pr-mutants/action.yml @@ -0,0 +1,153 @@ +name: Output Mutants + +runs: + using: "composite" + + steps: + - name: Download artifacts + uses: actions/download-artifact@v3 + + - name: Append output from shards + shell: bash + run: | + folders=("mutants-shard-big--1" "mutants-shard-big-0" "mutants-shard-big-1" "mutants-shard-big-2" "mutants-shard-big-3" "mutants-shard-big-4" "mutants-shard-big-5" "mutants-shard-big-6" "mutants-shard-big-7" "mutants-shard-small--1" "mutants-shard-small-0" "mutants-shard-small-1" "mutants-shard-small-2" "mutants-shard-small-3") + files=("missed.txt" "caught.txt" "timeout.txt" "unviable.txt") + mkdir -p mutants-shards + + for file in "${files[@]}"; do + for folder in "${folders[@]}"; do + if [[ -s "$folder/$file" ]]; then + cat "$folder/$file" >> "mutants-shards/$file" + fi + done + done + + for folder in "${folders[@]}"; do + if [[ -s "$folder/exit_code.txt" ]]; then + exit_code=$(<"${folder}/exit_code.txt") + most_relevant_exit_code=0 + + case $exit_code in + 4) + most_relevant_exit_code=4 + ;; + 1) + [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=1 + ;; + 2) + [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=2 + ;; + 3) + [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=3 + ;; + 0) + ;; + *) + echo "Unknown exit code $exit_code" + most_relevant_exit_code=$exit_code + ;; + esac + fi + done + + echo "$most_relevant_exit_code" > './mutants-shards/exit_code.txt' + + - name: Print mutants + shell: bash + run: | + server_url="${{ github.server_url }}" + organisation="${{ github.repository_owner }}" + repository="${{ github.event.repository.name }}" + commit="${{ github.sha }}" + + write_section() { + local section_title=$1 + local file_name=$2 + + if [ -s "$file_name" ]; then + if [[ "$section_title" != "" ]]; then + echo "## $section_title" >> "$GITHUB_STEP_SUMMARY" + fi + + if [[ "$section_title" == "Missed:" ]]; then + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "What are missed mutants?" >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "No test failed with this mutation applied, which seems to indicate a gap in test coverage. Or, it may be that the mutant is undistinguishable from the correct code. You may wish to add a better test, or mark that the function should be skipped." >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + elif [[ "$section_title" == "Timeout:" ]]; then + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "What are timeout mutants?" >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "The mutation caused the test suite to run for a long time, until it was eventually killed. You might want to investigate the cause and potentially mark the function to be skipped." >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + elif [[ "$section_title" == "Unviable:" ]]; then + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "What are unviable mutants?" >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "The attempted mutation doesn't compile. This is inconclusive about test coverage and no action is needed, unless you wish to test the specific function, in which case you may wish to add a 'Default::default()' implementation for the specific return type." >> "$GITHUB_STEP_SUMMARY" + echo "
" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + fi + + if [[ "$section_title" != "" ]]; then + awk -F':' '{printf "- [ ] " "[" $0 "]"; file_path=$1; line=$2; $1=""; $2=""; printf "(" "'"$server_url"'/'"$organisation"'/'"$repository"'/blob/'"$commit"'/" file_path "#L" line-1 ")\n\n"}' "$file_name" >> "$GITHUB_STEP_SUMMARY" + else + awk -F':' '{printf "- [x] " "[" $0 "]"; file_path=$1; line=$2; $1=""; $2=""; printf "(" "'"$server_url"'/'"$organisation"'/'"$repository"'/blob/'"$commit"'/" file_path "#L" line-1 ")\n\n"}' "$file_name" >> "$GITHUB_STEP_SUMMARY" + fi + + if [[ "$section_title" == "Missed:" ]]; then + echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" + echo "- Modify or add tests including this function." >> "$GITHUB_STEP_SUMMARY" + echo "- If you are absolutely certain that this function should not undergo mutation testing, add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" + elif [[ "$section_title" == "Timeout:" ]]; then + echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" + echo "- Modify the tests that include this funcion." >> "$GITHUB_STEP_SUMMARY" + echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" + elif [[ "$section_title" == "Unviable:" ]]; then + echo "### To resolve this issue, consider one of the following options:" >> "$GITHUB_STEP_SUMMARY" + echo "- Create 'Default::default()' implementation for the specific structure." >> "$GITHUB_STEP_SUMMARY" + echo "- Add '#[mutants::skip]' or '#[cfg_attr(test, mutants::skip)]' function header to skip it." >> "$GITHUB_STEP_SUMMARY" + fi + + echo >> "$GITHUB_STEP_SUMMARY" + fi + } + + echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY" + write_section "Missed:" "./mutants-shards/missed.txt" + write_section "Timeout:" "./mutants-shards/timeout.txt" + write_section "Unviable:" "./mutants-shards/unviable.txt" + + echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY" + write_section "" "./mutants-shards/caught.txt" + + exit_code=$(<"mutants-shards/exit_code.txt") + + case $exit_code in + 0) + if [[ -f ./mutants-shards/unviable.txt ]]; then + echo "Found unviable mutants!" + exit 1 + fi + echo "All new and updated functions are caught!" + ;; + 1) + echo "Invalid command line arguments!" + exit 1 + ;; + 2 | 3) + echo "Found missed/timeout/unviable mutants!" + exit 1 + ;; + 4) + echo "Building the packages failed without any mutations!" + exit 1 + ;; + *) + echo "Unknown exit code: $exit_code" + exit 1 + ;; + esac From 8508292431ce5ec649ec61e97f0ee13c44773948 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 10 Jan 2024 17:56:05 +0200 Subject: [PATCH 14/29] feat: update output structure --- .../check-packages-and-shards/action.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 68ea219..8682f89 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -1,10 +1,14 @@ name: Check Packages and Shards outputs: - run_big_packages: ${{ steps.check_packages_and_shards.outputs.run_big_packages }} - big_packages_with_shards: ${{ steps.check_packages_and_shards.outputs.big_packages_with_shards }} - run_small_packages: ${{ steps.check_packages_and_shards.outputs.run_small_packages }} - small_packages_with_shards: ${{ steps.check_packages_and_shards.outputs.small_packages_with_shards }} + run_big_packages: + value: ${{ steps.check_packages_and_shards.outputs.run_big_packages }} + big_packages_with_shards: + value: ${{ steps.check_packages_and_shards.outputs.big_packages_with_shards }} + run_small_packages: + value: ${{ steps.check_packages_and_shards.outputs.run_small_packages }} + small_packages_with_shards: + value: ${{ steps.check_packages_and_shards.outputs.small_packages_with_shards }} runs: using: "composite" From 105384ed221008f80d85c2343763495022073937 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 10 Jan 2024 18:04:19 +0200 Subject: [PATCH 15/29] feat: debug prints --- mutation-testing/check-packages-and-shards/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 8682f89..0691eab 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -58,6 +58,9 @@ runs: cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages + echo git.diff + echo all_mutants.txt + # Check that the file exists before performing actions on it if [ -s all_mutants.txt ]; then echo "The file containing mutants is missing or empty!" From 18bc826b9638320fdb76f0918a3f65ce600245f0 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 10 Jan 2024 18:07:06 +0200 Subject: [PATCH 16/29] feat: change 'echo' to 'cat' --- mutation-testing/check-packages-and-shards/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 0691eab..1df93cc 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -58,8 +58,8 @@ runs: cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages - echo git.diff - echo all_mutants.txt + cat git.diff + cat all_mutants.txt # Check that the file exists before performing actions on it if [ -s all_mutants.txt ]; then From 3b5a1b7425c3148cc5182c6738aca5bb86ba30cb Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Wed, 10 Jan 2024 18:10:21 +0200 Subject: [PATCH 17/29] feat: check if file is empty, not the other way around --- mutation-testing/check-packages-and-shards/action.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 1df93cc..6b8201d 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -58,11 +58,8 @@ runs: cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages - cat git.diff - cat all_mutants.txt - # Check that the file exists before performing actions on it - if [ -s all_mutants.txt ]; then + if [ ! -s all_mutants.txt ]; then echo "The file containing mutants is missing or empty!" exit 1 fi From f4bde14efdca6eeb93e23ce824edac8cec4aa03f Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 19:38:37 +0200 Subject: [PATCH 18/29] feat: update actions version, add comments, check files before accessing them --- .../check-packages-and-shards/action.yml | 86 ++++++++++++++----- mutation-testing/output-pr-mutants/action.yml | 17 +++- mutation-testing/pr-differences/action.yml | 75 +++++++++++----- .../remove-deleted-file-lines.sh | 33 ++++++- 4 files changed, 161 insertions(+), 50 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 6b8201d..b14f7db 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -1,13 +1,21 @@ name: Check Packages and Shards +description: "Checks which types of packages are contained in the diffs and how many mutants there are" +branding: + icon: "check" + color: "gray-dark" outputs: run_big_packages: + description: "True if there are packages on `stackslib` or `stacks-node`." value: ${{ steps.check_packages_and_shards.outputs.run_big_packages }} big_packages_with_shards: + description: "True if there are more than 16 mutants on `stackslib` and `stacks-node`." value: ${{ steps.check_packages_and_shards.outputs.big_packages_with_shards }} run_small_packages: + description: "True if there are packages on other packages than `stackslib` or `stacks-node`." value: ${{ steps.check_packages_and_shards.outputs.run_small_packages }} small_packages_with_shards: + description: "True if there are more than 79 (119 if `big_packages_with_shards` is true) mutants on small packages." value: ${{ steps.check_packages_and_shards.outputs.small_packages_with_shards }} runs: @@ -15,13 +23,14 @@ runs: steps: - name: Checkout stacks-core repo - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 - - name: Install cargo-mutants - shell: bash - run: cargo install --version 23.12.2 cargo-mutants + - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 + name: Install cargo-mutants + with: + tool: cargo-mutants@23.12.2 # v23.12.2 - name: Relative diff shell: bash @@ -36,25 +45,54 @@ runs: # Check if the commands exist on the host for cmd in tac awk sed; do - command -v "${cmd}" > /dev/null 2>&1 || echo "Missing command: ${cmd}" + command -v "${cmd}" > /dev/null 2>&1 || { echo "Missing command: ${cmd}" && exit 1; } done + # Check that the file exists before performing actions on it + if [ ! -s "$input_file" ]; then + echo "Diff file missing!" + exit 1 + fi + # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) - tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + tac "$input_file" > "$temp_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + # Check that the file exists before performing actions on it + if [ ! -s "$temp_file" ]; then + echo "Diff file missing!" + exit 1 + fi + # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' awk ' BEGIN { in_block=0 } /\+\+\+ \/dev\/null/ { in_block=1; next } in_block && /diff --git a\// { in_block=0; print; next } !in_block - ' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + ' "$input_file" > "$temp_file" + + # Check that the file exists before performing actions on it + if [ -s "$temp_file" ]; then + mv "$temp_file" "$input_file" + else + echo "Temp diff file missing!" + exit 1 + fi + + exit 0 - name: Split diffs shell: bash run: | + # Check that the file exists before performing actions on it + if [ ! -s git.diff ]; then + echo "Diff file missing!" + exit 1 + fi + + # Make a file containing all the mutants for the differences in the PR and a folder to split them into big and small packages cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages @@ -67,13 +105,18 @@ runs: # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files while IFS= read -r line; do package=$(echo "$line" | cut -d'/' -f1) - if [[ $package == "testnet" || $package == "stackslib" ]]; then - echo "$line" >> "mutants_by_packages/big_packages.txt" - else - echo "$line" >> "mutants_by_packages/small_packages.txt" - fi + case $package in + "testnet" | "stackslib") + echo "$line" >> "mutants_by_packages/big_packages.txt" + ;; + *) + echo "$line" >> "mutants_by_packages/small_packages.txt" + ;; + esac done < all_mutants.txt + exit 0 + - name: Check packages and shards id: check_packages_and_shards shell: bash @@ -92,22 +135,19 @@ runs: fi # Set the mutants limit for when to run with shards on the small packages + # If there are mutants from big packages, check whether to run with or without shards, otherwise there's nothing to run if [[ $number_of_big_mutants -gt 15 ]]; then small_packages_shard_limit=119 + echo "run_big_packages=true" >> "$GITHUB_OUTPUT" + echo "big_packages_with_shards=true" >> "$GITHUB_OUTPUT" else small_packages_shard_limit=79 - fi - - # If there are mutants from big packages, check whether to run with or without shards, otherwise there's nothing to run - if [[ $number_of_big_mutants -ne 0 ]]; then - echo "run_big_packages=true" >> "$GITHUB_OUTPUT" - if [[ $number_of_big_mutants -gt 15 ]]; then - echo "big_packages_with_shards=true" >> "$GITHUB_OUTPUT" - else + if [[ $number_of_big_mutants -ne 0 ]]; then + echo "run_big_packages=true" >> "$GITHUB_OUTPUT" echo "big_packages_with_shards=false" >> "$GITHUB_OUTPUT" + else + echo "run_big_packages=false" >> "$GITHUB_OUTPUT" fi - else - echo "run_big_packages=false" >> "$GITHUB_OUTPUT" fi # If there are mutants from small packages, check whether to run with or without shards, otherwise there's nothing to run @@ -121,3 +161,5 @@ runs: else echo "run_small_packages=false" >> "$GITHUB_OUTPUT" fi + + exit 0 diff --git a/mutation-testing/output-pr-mutants/action.yml b/mutation-testing/output-pr-mutants/action.yml index 6a15725..8e49137 100644 --- a/mutation-testing/output-pr-mutants/action.yml +++ b/mutation-testing/output-pr-mutants/action.yml @@ -5,15 +5,17 @@ runs: steps: - name: Download artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@bb3fa7fd35ab8113a980912eb9f59b846d14e3ff # v4.1.1 - name: Append output from shards shell: bash run: | + # List of all possible mutant outcomes file names and folder names collected from jobs. folders=("mutants-shard-big--1" "mutants-shard-big-0" "mutants-shard-big-1" "mutants-shard-big-2" "mutants-shard-big-3" "mutants-shard-big-4" "mutants-shard-big-5" "mutants-shard-big-6" "mutants-shard-big-7" "mutants-shard-small--1" "mutants-shard-small-0" "mutants-shard-small-1" "mutants-shard-small-2" "mutants-shard-small-3") files=("missed.txt" "caught.txt" "timeout.txt" "unviable.txt") mkdir -p mutants-shards + # If the folder/file path exists, append the output to it's corresponding file name in a newly created folder that will contain all outputs combined for file in "${files[@]}"; do for folder in "${folders[@]}"; do if [[ -s "$folder/$file" ]]; then @@ -22,6 +24,12 @@ runs: done done + # If the folder/exit_code.txt path exists, check for the exit code and retain the most relevant one to a file + # - 4: unmutated build failed + # - 1: incorrect command line arguments + # - 2, 3: found timeout/missed/unviable mutants + # - 0: everything worked fine + # - *: unknown exit code for folder in "${folders[@]}"; do if [[ -s "$folder/exit_code.txt" ]]; then exit_code=$(<"${folder}/exit_code.txt") @@ -55,11 +63,13 @@ runs: - name: Print mutants shell: bash run: | + # Info for creating the link that paths to the specific mutation tested server_url="${{ github.server_url }}" organisation="${{ github.repository_owner }}" repository="${{ github.event.repository.name }}" commit="${{ github.sha }}" + # Function to write to github step summary with specific info depending on the mutation category write_section() { local section_title=$1 local file_name=$2 @@ -116,14 +126,17 @@ runs: fi } + # Print uncaught (missed/timeout/unviable) mutants to summary echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY" write_section "Missed:" "./mutants-shards/missed.txt" write_section "Timeout:" "./mutants-shards/timeout.txt" write_section "Unviable:" "./mutants-shards/unviable.txt" - + + # Print caught mutants to summary echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY" write_section "" "./mutants-shards/caught.txt" + # Get most relevant exit code from the file and match it exit_code=$(<"mutants-shards/exit_code.txt") case $exit_code in diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index d7397f6..0c06cbe 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -21,7 +21,7 @@ runs: # Checkout the stacks-core code - name: Checkout the stacks core repo id: git_checkout_stacks_core - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 @@ -33,7 +33,7 @@ runs: # Checkout the actions code - name: Get the action repo code id: git_checkout_actions - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: repository: stacks-network/actions ref: feat/mutation-testing @@ -42,11 +42,16 @@ runs: - name: Copy the git diff file to scripts folder shell: bash run: | + if [ ! -s git.diff ]; then + echo "Diff file missing!" + fi + mv git.diff ./actions-repo/mutation-testing/shell-scripts/ - - name: Install cargo mutants crate - shell: bash - run: cargo install --version 23.12.2 cargo-mutants + - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 + name: Install cargo-mutants + with: + tool: cargo-mutants@23.12.2 # v23.12.2 - name: Remove deleted file lines from git.diff file shell: bash @@ -56,42 +61,64 @@ runs: - name: Split diffs into big and small packages and create regex patterns for them shell: bash run: | + # Check that the file exists before performing actions on it + if [ ! -s git.diff ]; then + echo "Diff file missing!" + fi + + # Make a file containing all the mutants for the differences in the PR and a folder to split them into big and small packages cargo mutants --in-diff git.diff --list > all_mutants.txt mkdir -p mutants_by_packages + # Check that the file exists before performing actions on it + if [ ! -s all_mutants.txt ]; then + echo "The file containing mutants is missing or empty!" + exit 1 + fi + # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files while IFS= read -r line; do package=$(echo "$line" | cut -d'/' -f1) - if [[ $package == "testnet" || $package == "stackslib" ]]; then - echo "$line" >> "mutants_by_packages/big_packages.txt" - else - echo "$line" >> "mutants_by_packages/small_packages.txt" - fi + ccase $package in + "testnet" | "stackslib") + echo "$line" >> "mutants_by_packages/big_packages.txt" + ;; + *) + echo "$line" >> "mutants_by_packages/small_packages.txt" + ;; + esac done < all_mutants.txt # Create regex patterns of the mutants to be ran for `cargo mutants` -F flag and output them to files to be used later for package in mutants_by_packages/*; do - regex_pattern="" + if [ -s mutants_by_packages/$package ]; then + regex_pattern="" - while IFS= read -r line; do - escaped_line=$(echo "$line" | sed 's/[][()\.^$*+?{}|]/\\&/g') - regex_pattern+="($escaped_line)|" - done < "$package" + while IFS= read -r line; do + escaped_line=$(echo "$line" | sed 's/[][()\.^$*+?{}|]/\\&/g') + regex_pattern+="($escaped_line)|" + done < "$package" - regex_pattern="${regex_pattern%|}" + regex_pattern="${regex_pattern%|}" - if [[ "$package" == "mutants_by_packages/big_packages.txt" ]]; then - echo "$regex_pattern" > mutants_by_packages/regex_big.txt - else - echo "$regex_pattern" > mutants_by_packages/regex_small.txt + if [[ "$package" == "mutants_by_packages/big_packages.txt" ]]; then + echo "$regex_pattern" > mutants_by_packages/regex_big.txt + else + echo "$regex_pattern" > mutants_by_packages/regex_small.txt + fi fi done + + exit 0 working-directory: actions-repo/mutation-testing/shell-scripts - name: Run mutants and check the exit code of the command, fail the workflow if mutations are not caught shell: bash run: | - set +e # Disable immediate exit on error + # Disable immediate exit on error + set +e + + # Check which type of mutants to run: big, small, with or without shards, then capture the exit code if [[ ${{ inputs.shard }} == -1 ]]; then if [[ ${{ inputs.package-dimension }} == big ]]; then echo "Running mutants without shards on big packages" @@ -113,10 +140,14 @@ runs: exit_code=$? fi fi + + # Create the folder only containing the outcomes (.txt files) and make a file containing the exit code of the command mkdir mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} echo "$exit_code" > ./mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }}/exit_code.txt mv ./mutants.out/*.txt mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }}/ - set -e # Enable immediate exit on error again + + # Enable immediate exit on error again + set -e working-directory: actions-repo/mutation-testing/shell-scripts - name: Upload artifact diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh index a8812f2..53857e3 100755 --- a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh +++ b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh @@ -1,17 +1,42 @@ -#!/bin/bash - input_file="git.diff" temp_file="temp_diff_file.diff" +# Check if the commands exist on the host +for cmd in tac awk sed; do + command -v "${cmd}" > /dev/null 2>&1 || { echo "Missing command: ${cmd}" && exit 1; } +done + +# Check that the file exists before performing actions on it +if [ ! -s "$input_file" ]; then + echo "Diff file missing!" + exit 1 +fi + # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) -tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" +tac "$input_file" > "$temp_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" +# Check that the file exists before performing actions on it +if [ ! -s "$temp_file" ]; then + echo "Diff file missing!" + exit 1 +fi + # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' awk ' BEGIN { in_block=0 } /\+\+\+ \/dev\/null/ { in_block=1; next } in_block && /diff --git a\// { in_block=0; print; next } !in_block -' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" +' "$input_file" > "$temp_file" + +# Check that the file exists before performing actions on it +if [ -s "$temp_file" ]; then + mv "$temp_file" "$input_file" +else + echo "Temp diff file missing!" + exit 1 +fi + +exit 0 \ No newline at end of file From 6ee34a97d5e0198709ffc4fdb7c688226c2bca75 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 19:42:04 +0200 Subject: [PATCH 19/29] feat: fix indentation --- mutation-testing/check-packages-and-shards/action.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index b14f7db..239f308 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -28,9 +28,9 @@ runs: fetch-depth: 0 - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 - name: Install cargo-mutants - with: - tool: cargo-mutants@23.12.2 # v23.12.2 + name: Install cargo-mutants + with: + tool: cargo-mutants@23.12.2 # v23.12.2 - name: Relative diff shell: bash From c422ffb53215de51507b7968dd4c89530be14bad Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 19:54:04 +0200 Subject: [PATCH 20/29] feat: remove a check - perhaps it takes longer to move back --- mutation-testing/check-packages-and-shards/action.yml | 6 ------ mutation-testing/shell-scripts/remove-deleted-file-lines.sh | 6 ------ 2 files changed, 12 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 239f308..991decd 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -59,12 +59,6 @@ runs: sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" - # Check that the file exists before performing actions on it - if [ ! -s "$temp_file" ]; then - echo "Diff file missing!" - exit 1 - fi - # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' awk ' BEGIN { in_block=0 } diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh index 53857e3..857326c 100755 --- a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh +++ b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh @@ -17,12 +17,6 @@ tac "$input_file" > "$temp_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" -# Check that the file exists before performing actions on it -if [ ! -s "$temp_file" ]; then - echo "Diff file missing!" - exit 1 -fi - # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' awk ' BEGIN { in_block=0 } From f8097b3645fd30b567225619bb6dd402f5df4cd7 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 20:02:14 +0200 Subject: [PATCH 21/29] feat: add moving of the temp file to input file --- mutation-testing/check-packages-and-shards/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index 991decd..a653902 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -55,7 +55,7 @@ runs: fi # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) - tac "$input_file" > "$temp_file" + tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" From d72903a9002c790b3f995786d85f23530723aa88 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 20:05:33 +0200 Subject: [PATCH 22/29] feat: move the temp file to input file in script --- mutation-testing/shell-scripts/remove-deleted-file-lines.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh index 857326c..d2aa67d 100755 --- a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh +++ b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh @@ -13,7 +13,7 @@ if [ ! -s "$input_file" ]; then fi # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) -tac "$input_file" > "$temp_file" +tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" From 946ec2f41c3f55b0e8b541742d94ed6d317bc843 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 20:14:42 +0200 Subject: [PATCH 23/29] feat: fix typo in 'case' --- mutation-testing/pr-differences/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index 0c06cbe..06c1f84 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -79,7 +79,7 @@ runs: # Split the differences from git into 2 parts, big packages ('stacks-node' and 'stackslib') and small packages (all others) and put them into separate files while IFS= read -r line; do package=$(echo "$line" | cut -d'/' -f1) - ccase $package in + case $package in "testnet" | "stackslib") echo "$line" >> "mutants_by_packages/big_packages.txt" ;; From 427290ca37cc3d4cbd013c2b81839f4a9ac92acb Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 20:18:27 +0200 Subject: [PATCH 24/29] feat: remove unnecessary check --- mutation-testing/pr-differences/action.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index 06c1f84..98c6445 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -91,21 +91,19 @@ runs: # Create regex patterns of the mutants to be ran for `cargo mutants` -F flag and output them to files to be used later for package in mutants_by_packages/*; do - if [ -s mutants_by_packages/$package ]; then - regex_pattern="" + regex_pattern="" - while IFS= read -r line; do - escaped_line=$(echo "$line" | sed 's/[][()\.^$*+?{}|]/\\&/g') - regex_pattern+="($escaped_line)|" - done < "$package" + while IFS= read -r line; do + escaped_line=$(echo "$line" | sed 's/[][()\.^$*+?{}|]/\\&/g') + regex_pattern+="($escaped_line)|" + done < "$package" - regex_pattern="${regex_pattern%|}" + regex_pattern="${regex_pattern%|}" - if [[ "$package" == "mutants_by_packages/big_packages.txt" ]]; then - echo "$regex_pattern" > mutants_by_packages/regex_big.txt - else - echo "$regex_pattern" > mutants_by_packages/regex_small.txt - fi + if [[ "$package" == "mutants_by_packages/big_packages.txt" ]]; then + echo "$regex_pattern" > mutants_by_packages/regex_big.txt + else + echo "$regex_pattern" > mutants_by_packages/regex_small.txt fi done From 75a257c4245a9ff7e316e666e0cc084a15740b50 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Thu, 11 Jan 2024 20:29:10 +0200 Subject: [PATCH 25/29] feat: change artifact download version back to v3 --- mutation-testing/output-pr-mutants/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mutation-testing/output-pr-mutants/action.yml b/mutation-testing/output-pr-mutants/action.yml index 8e49137..c2966b4 100644 --- a/mutation-testing/output-pr-mutants/action.yml +++ b/mutation-testing/output-pr-mutants/action.yml @@ -5,7 +5,7 @@ runs: steps: - name: Download artifacts - uses: actions/download-artifact@bb3fa7fd35ab8113a980912eb9f59b846d14e3ff # v4.1.1 + uses: actions/download-artifact@v3 - name: Append output from shards shell: bash From 86d2d9f7dbdc0143c25d9bbfdaa75ce21879aee1 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Fri, 12 Jan 2024 01:46:18 +0200 Subject: [PATCH 26/29] feat: add file names to prints for easier debugging --- .../check-packages-and-shards/action.yml | 13 ++++++------- mutation-testing/pr-differences/action.yml | 6 +++--- .../shell-scripts/remove-deleted-file-lines.sh | 9 ++++----- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index a653902..bcbc812 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -50,7 +50,7 @@ runs: # Check that the file exists before performing actions on it if [ ! -s "$input_file" ]; then - echo "Diff file missing!" + echo "Diff file (git.diff) is missing or empty!" exit 1 fi @@ -70,19 +70,18 @@ runs: # Check that the file exists before performing actions on it if [ -s "$temp_file" ]; then mv "$temp_file" "$input_file" - else - echo "Temp diff file missing!" - exit 1 + exit 0 fi - exit 0 + echo "Temp diff file (temp_diff_file.diff) is missing or empty!" + exit 1 - name: Split diffs shell: bash run: | # Check that the file exists before performing actions on it if [ ! -s git.diff ]; then - echo "Diff file missing!" + echo "Diff file (git.diff) is missing or empty!" exit 1 fi @@ -92,7 +91,7 @@ runs: # Check that the file exists before performing actions on it if [ ! -s all_mutants.txt ]; then - echo "The file containing mutants is missing or empty!" + echo "The file containing mutants (all_mutants.txt) is missing or empty!" exit 1 fi diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index a019220..b681521 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -43,7 +43,7 @@ runs: shell: bash run: | if [ ! -s git.diff ]; then - echo "Diff file missing!" + echo "Diff file (git.diff) is missing!" fi mv git.diff ./actions-repo/mutation-testing/shell-scripts/ @@ -63,7 +63,7 @@ runs: run: | # Check that the file exists before performing actions on it if [ ! -s git.diff ]; then - echo "Diff file missing!" + echo "Diff file (git.diff) is missing or empty!" fi # Make a file containing all the mutants for the differences in the PR and a folder to split them into big and small packages @@ -72,7 +72,7 @@ runs: # Check that the file exists before performing actions on it if [ ! -s all_mutants.txt ]; then - echo "The file containing mutants is missing or empty!" + echo "The file containing mutants (all_mutants.txt) is missing or empty!" exit 1 fi diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh index d2aa67d..d81d79c 100755 --- a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh +++ b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh @@ -8,7 +8,7 @@ done # Check that the file exists before performing actions on it if [ ! -s "$input_file" ]; then - echo "Diff file missing!" + echo "Diff file (git.diff) is missing or empty!" exit 1 fi @@ -28,9 +28,8 @@ awk ' # Check that the file exists before performing actions on it if [ -s "$temp_file" ]; then mv "$temp_file" "$input_file" -else - echo "Temp diff file missing!" - exit 1 + exit 0 fi -exit 0 \ No newline at end of file +echo "Temp diff file (temp_diff_file.diff) is missing or emtpy!" +exit 1 From d322c1636d2a6e6b8db1abefe54f07d2648aec90 Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Fri, 12 Jan 2024 02:42:31 +0200 Subject: [PATCH 27/29] feat: add `id` key to all steps --- mutation-testing/check-packages-and-shards/action.yml | 5 +++++ mutation-testing/output-pr-mutants/action.yml | 3 +++ mutation-testing/pr-differences/action.yml | 7 +++++++ 3 files changed, 15 insertions(+) diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index bcbc812..d5fa0d8 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -23,21 +23,25 @@ runs: steps: - name: Checkout stacks-core repo + id: checkout_stacks_core uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: fetch-depth: 0 - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 + id: install_cargo_mutants name: Install cargo-mutants with: tool: cargo-mutants@23.12.2 # v23.12.2 - name: Relative diff + id: relative_diff shell: bash run: | git diff origin/${{ github.base_ref }}.. > git.diff - name: Update git diff + id: update_git_diff shell: bash run: | input_file="git.diff" @@ -77,6 +81,7 @@ runs: exit 1 - name: Split diffs + id: split_diffs_by_packages shell: bash run: | # Check that the file exists before performing actions on it diff --git a/mutation-testing/output-pr-mutants/action.yml b/mutation-testing/output-pr-mutants/action.yml index c2966b4..abf62d6 100644 --- a/mutation-testing/output-pr-mutants/action.yml +++ b/mutation-testing/output-pr-mutants/action.yml @@ -5,9 +5,11 @@ runs: steps: - name: Download artifacts + id: download_artifacts uses: actions/download-artifact@v3 - name: Append output from shards + id: append_mutant_outcomes shell: bash run: | # List of all possible mutant outcomes file names and folder names collected from jobs. @@ -61,6 +63,7 @@ runs: echo "$most_relevant_exit_code" > './mutants-shards/exit_code.txt' - name: Print mutants + id: print_tested_mutants shell: bash run: | # Info for creating the link that paths to the specific mutation tested diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index b681521..f3460d8 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -26,6 +26,7 @@ runs: fetch-depth: 0 - name: Relative diff + id: relative_diff shell: bash run: | git diff origin/${{ github.base_ref }}.. > git.diff @@ -40,6 +41,7 @@ runs: path: ./actions-repo - name: Copy git diff + id: copy_git_diff shell: bash run: | if [ ! -s git.diff ]; then @@ -50,15 +52,18 @@ runs: - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 name: Install cargo-mutants + id: install_cargo_mutants with: tool: cargo-mutants@23.12.2 # v23.12.2 - name: Update git diff + id: update_git_diff shell: bash run: ./remove-deleted-file-lines.sh working-directory: actions-repo/mutation-testing/shell-scripts - name: Split diffs + id: split_diffs shell: bash run: | # Check that the file exists before performing actions on it @@ -112,6 +117,7 @@ runs: working-directory: actions-repo/mutation-testing/shell-scripts - name: Run mutants + id: run_mutants shell: bash run: | # Disable immediate exit on error @@ -166,6 +172,7 @@ runs: working-directory: actions-repo/mutation-testing/shell-scripts - name: Upload artifact + id: upload_artifact uses: actions/upload-artifact@v3 if: always() with: From 85ebbb87946df44dc841249a31569986d4c153df Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Mon, 15 Jan 2024 20:54:50 +0200 Subject: [PATCH 28/29] feat: mutants - file and exit code handling, readme structure - update readme files - check exit codes - check that files exist before accessing them, otherwise exit - remove shell scripts folder since it was only used in 1 place - update actions versions used --- mutation-testing/README.md | 4 +- .../check-packages-and-shards/README.md | 2 +- .../check-packages-and-shards/action.yml | 5 + mutation-testing/output-pr-mutants/README.md | 16 ++++ mutation-testing/output-pr-mutants/action.yml | 95 +++++++++++++++---- mutation-testing/pr-differences/README.md | 15 ++- mutation-testing/pr-differences/action.yml | 74 ++++++++++----- .../remove-deleted-file-lines.sh | 35 ------- 8 files changed, 160 insertions(+), 86 deletions(-) delete mode 100755 mutation-testing/shell-scripts/remove-deleted-file-lines.sh diff --git a/mutation-testing/README.md b/mutation-testing/README.md index 36b035a..ff63e22 100644 --- a/mutation-testing/README.md +++ b/mutation-testing/README.md @@ -1,3 +1,5 @@ # Mutation Testing actions -> - [Filter PR Mutants action](./filter-pr/README.md) +> - [Check Packages and Shards action](./check-packages-and-shards/README.md) +> - [Output PR Mutants action](./output-pr-mutants/README.md) +> - [PR Differences action](./pr-differences/README.md) diff --git a/mutation-testing/check-packages-and-shards/README.md b/mutation-testing/check-packages-and-shards/README.md index feb000d..27d79a9 100644 --- a/mutation-testing/check-packages-and-shards/README.md +++ b/mutation-testing/check-packages-and-shards/README.md @@ -1,6 +1,6 @@ # Check Packages and Shards action -Checks whether to run mutants on big (`stackslib`/`stacks-node`) or small packages (all others), and whether to run them using strategy matrix or not. +Checks whether to run mutants on big ([stackslib](https://github.com/stacks-network/stacks-core/tree/develop/stackslib)/[stacks-node](https://github.com/stacks-network/stacks-core/tree/develop/testnet/stacks-node)) or small packages (all others), and whether to run them using strategy matrix or not. ## Documentation diff --git a/mutation-testing/check-packages-and-shards/action.yml b/mutation-testing/check-packages-and-shards/action.yml index d5fa0d8..342de18 100644 --- a/mutation-testing/check-packages-and-shards/action.yml +++ b/mutation-testing/check-packages-and-shards/action.yml @@ -92,6 +92,11 @@ runs: # Make a file containing all the mutants for the differences in the PR and a folder to split them into big and small packages cargo mutants --in-diff git.diff --list > all_mutants.txt + if [ $? -ne 0 ]; then + echo "Error retrieving the list of mutants!" + exit $? + fi + mkdir -p mutants_by_packages # Check that the file exists before performing actions on it diff --git a/mutation-testing/output-pr-mutants/README.md b/mutation-testing/output-pr-mutants/README.md index f4eba49..db6d829 100644 --- a/mutation-testing/output-pr-mutants/README.md +++ b/mutation-testing/output-pr-mutants/README.md @@ -2,6 +2,17 @@ Write the mutants tested in the previous jobs of the same workflow to github step summary. +## Documentation + +### Inputs + +| Input | Description | Required | Default | +| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | +| `big_packages` | True if there were big packages running | true | | +| `shards_for_big_packages` | True if the big packages were running using matrix | true | | +| `small_packages` | True if there were small packages running | true | | +| `shards_for_small_packages` | True if the small packages were running using matrix | true | | + ## Usage ```yaml @@ -15,4 +26,9 @@ jobs: - name: Output Mutants id: output_pr_mutants uses: stacks-network/actions/mutation-testing/output-pr-mutants@main + with: + big_packages: "true" + shards_for_big_packages: "false" + small_packages: "true" + shards_for_small_packages: "true" ``` diff --git a/mutation-testing/output-pr-mutants/action.yml b/mutation-testing/output-pr-mutants/action.yml index abf62d6..0b68e51 100644 --- a/mutation-testing/output-pr-mutants/action.yml +++ b/mutation-testing/output-pr-mutants/action.yml @@ -1,4 +1,22 @@ name: Output Mutants +description: "Collects the outcomes of jobs running mutants from artifacts, combines them and outputs them to github step summary" +branding: + icon: "share" + color: "gray-dark" + +inputs: + big_packages: + description: "True if there were big packages running." + required: true + shards_for_big_packages: + description: "True if the big packages were running using matrix." + required: true + small_packages: + description: "True if there were small packages running." + required: true + shards_for_small_packages: + description: "True if the small packages were running using matrix." + required: true runs: using: "composite" @@ -6,22 +24,46 @@ runs: steps: - name: Download artifacts id: download_artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@6b208ae046db98c579e8a3aa621ab581ff575935 # v4.1.1 - name: Append output from shards id: append_mutant_outcomes shell: bash run: | - # List of all possible mutant outcomes file names and folder names collected from jobs. - folders=("mutants-shard-big--1" "mutants-shard-big-0" "mutants-shard-big-1" "mutants-shard-big-2" "mutants-shard-big-3" "mutants-shard-big-4" "mutants-shard-big-5" "mutants-shard-big-6" "mutants-shard-big-7" "mutants-shard-small--1" "mutants-shard-small-0" "mutants-shard-small-1" "mutants-shard-small-2" "mutants-shard-small-3") + # Check and append the folders that should be uploaded to artifacts for the big packages + if ${{ inputs.big_packages }} == "true"; then + if ${{ inputs.shards_for_big_packages }} == "true"; then + for i in {0..7}; do + folders+=("mutants-shard-big-$i") + done + else + folders+=("mutants-shard-big--1") + fi + fi + + # Check and append the folders that should be uploaded to artifacts for the small packages + if ${{ inputs.small_packages }} == "true"; then + if ${{ inputs.shards_for_small_packages }} == "true"; then + for i in {0..3}; do + folders+=("mutants-shard-small-$i") + done + else + folders+=("mutants-shard-small--1") + fi + fi + files=("missed.txt" "caught.txt" "timeout.txt" "unviable.txt") mkdir -p mutants-shards # If the folder/file path exists, append the output to it's corresponding file name in a newly created folder that will contain all outputs combined + # If it doesn't exits, exit early because there was an error when uploading/downloading the artifact or the mutants were not run for that specific job for file in "${files[@]}"; do for folder in "${folders[@]}"; do - if [[ -s "$folder/$file" ]]; then + if [[ -f "$folder/$file" ]]; then cat "$folder/$file" >> "mutants-shards/$file" + else + echo "Missing necessary file or folder: $folder/$file" + exit 1 fi done done @@ -44,12 +86,12 @@ runs: 1) [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=1 ;; - 2) - [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=2 - ;; 3) [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=3 ;; + 2) + [ "$most_relevant_exit_code" -eq 0 ] && most_relevant_exit_code=2 + ;; 0) ;; *) @@ -57,11 +99,16 @@ runs: most_relevant_exit_code=$exit_code ;; esac + else + # If the file containing the exit code doesn't exist, something happened to the job that should have saved it, so default to '4' + most_relevant_exit_code=4 fi done echo "$most_relevant_exit_code" > './mutants-shards/exit_code.txt' + exit 0 + - name: Print mutants id: print_tested_mutants shell: bash @@ -130,14 +177,18 @@ runs: } # Print uncaught (missed/timeout/unviable) mutants to summary - echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY" - write_section "Missed:" "./mutants-shards/missed.txt" - write_section "Timeout:" "./mutants-shards/timeout.txt" - write_section "Unviable:" "./mutants-shards/unviable.txt" + if [ -s ./mutants-shards/missed.txt -o -s ./mutants-shards/timeout.txt -o -s ./mutants-shards/unviable.txt ]; then + echo "# Uncaught Mutants" >> "$GITHUB_STEP_SUMMARY" + write_section "Missed:" "./mutants-shards/missed.txt" + write_section "Timeout:" "./mutants-shards/timeout.txt" + write_section "Unviable:" "./mutants-shards/unviable.txt" + fi # Print caught mutants to summary - echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY" - write_section "" "./mutants-shards/caught.txt" + if [ -s ./mutants-shards/caught.txt ]; then + echo "# Caught Mutants" >> "$GITHUB_STEP_SUMMARY" + write_section "" "./mutants-shards/caught.txt" + fi # Get most relevant exit code from the file and match it exit_code=$(<"mutants-shards/exit_code.txt") @@ -146,24 +197,28 @@ runs: 0) if [[ -f ./mutants-shards/unviable.txt ]]; then echo "Found unviable mutants!" - exit 1 + exit 5 fi echo "All new and updated functions are caught!" ;; 1) echo "Invalid command line arguments!" - exit 1 + exit $exit_code + ;; + 2) + echo "Found missed mutants!" + exit $exit_code ;; - 2 | 3) - echo "Found missed/timeout/unviable mutants!" - exit 1 + 3) + echo "Found timeout mutants!" + exit $exit_code ;; 4) echo "Building the packages failed without any mutations!" - exit 1 + exit $exit_code ;; *) echo "Unknown exit code: $exit_code" - exit 1 + exit $exit_code ;; esac diff --git a/mutation-testing/pr-differences/README.md b/mutation-testing/pr-differences/README.md index e511304..6feeba1 100644 --- a/mutation-testing/pr-differences/README.md +++ b/mutation-testing/pr-differences/README.md @@ -1,8 +1,15 @@ # PR Differences Mutants action -Run mutants for differences in pull requests, and fail the workflow if: -- There are missed/timeout/unviable mutants -- Building or testing the unmutated crate fails +Run mutants for differences in pull requests, and upload the mutant outcomes and exit codes to artifacts. + +## Documentation + +### Inputs + +| Input | Description | Required | Default | +| ------------------------------- | ----------------------------------------------------- | ------------------------- | ------------------------- | +| `shard` | The number of the shard to run (`-1` if ran without shards) | false | -1 | +| `package-dimension` | The dimension of the package. `big` for [stacks-node](https://github.com/stacks-network/stacks-core/tree/develop/testnet/stacks-node) and [stackslib](https://github.com/stacks-network/stacks-core/tree/develop/stackslib), `small` for others | true | | ## Usage @@ -17,4 +24,6 @@ jobs: - name: PR Differences Mutants id: pr-differences-mutants uses: stacks-network/actions/mutation-testing/pr-differences@main + with: + package-dimension: "big" ``` diff --git a/mutation-testing/pr-differences/action.yml b/mutation-testing/pr-differences/action.yml index f3460d8..83dc12b 100644 --- a/mutation-testing/pr-differences/action.yml +++ b/mutation-testing/pr-differences/action.yml @@ -1,4 +1,8 @@ name: PR Differences Mutants +description: "Runs mutants corresponding to its given inputs and uploads the outcomes to artifacts" +branding: + icon: "users" + color: "gray-dark" inputs: shard: @@ -31,25 +35,6 @@ runs: run: | git diff origin/${{ github.base_ref }}.. > git.diff - # Checkout the actions code - - name: Checkout actions repo - id: git_checkout_actions - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: stacks-network/actions - ref: feat/mutation-testing - path: ./actions-repo - - - name: Copy git diff - id: copy_git_diff - shell: bash - run: | - if [ ! -s git.diff ]; then - echo "Diff file (git.diff) is missing!" - fi - - mv git.diff ./actions-repo/mutation-testing/shell-scripts/ - - uses: taiki-e/install-action@ac89944b5b150d78567ab6c02badfbe48b0b55aa # v2.20.16 name: Install cargo-mutants id: install_cargo_mutants @@ -59,8 +44,42 @@ runs: - name: Update git diff id: update_git_diff shell: bash - run: ./remove-deleted-file-lines.sh - working-directory: actions-repo/mutation-testing/shell-scripts + run: | + input_file="git.diff" + temp_file="temp_diff_file.diff" + + # Check if the commands exist on the host + for cmd in tac awk sed; do + command -v "${cmd}" > /dev/null 2>&1 || { echo "Missing command: ${cmd}" && exit 1; } + done + + # Check that the file exists before performing actions on it + if [ ! -s "$input_file" ]; then + echo "Diff file (git.diff) is missing or empty!" + exit 1 + fi + + # Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) + tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" + + # Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' + awk ' + BEGIN { in_block=0 } + /\+\+\+ \/dev\/null/ { in_block=1; next } + in_block && /diff --git a\// { in_block=0; print; next } + !in_block + ' "$input_file" > "$temp_file" + + # Check that the file exists before performing actions on it + if [ -s "$temp_file" ]; then + mv "$temp_file" "$input_file" + exit 0 + fi + + echo "Temp diff file (temp_diff_file.diff) is missing or empty!" + exit 1 - name: Split diffs id: split_diffs @@ -69,10 +88,16 @@ runs: # Check that the file exists before performing actions on it if [ ! -s git.diff ]; then echo "Diff file (git.diff) is missing or empty!" + exit 1 fi # Make a file containing all the mutants for the differences in the PR and a folder to split them into big and small packages cargo mutants --in-diff git.diff --list > all_mutants.txt + if [ $? -ne 0 ]; then + echo "Error retrieving the list of mutants!" + exit $? + fi + mkdir -p mutants_by_packages # Check that the file exists before performing actions on it @@ -114,7 +139,6 @@ runs: done exit 0 - working-directory: actions-repo/mutation-testing/shell-scripts - name: Run mutants id: run_mutants @@ -169,12 +193,10 @@ runs: # Enable immediate exit on error again set -e - working-directory: actions-repo/mutation-testing/shell-scripts - name: Upload artifact id: upload_artifact - uses: actions/upload-artifact@v3 - if: always() + uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0 with: name: mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} - path: actions-repo/mutation-testing/shell-scripts/mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} + path: mutants-shard-${{ inputs.package-dimension }}-${{ inputs.shard }} diff --git a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh b/mutation-testing/shell-scripts/remove-deleted-file-lines.sh deleted file mode 100755 index d81d79c..0000000 --- a/mutation-testing/shell-scripts/remove-deleted-file-lines.sh +++ /dev/null @@ -1,35 +0,0 @@ -input_file="git.diff" -temp_file="temp_diff_file.diff" - -# Check if the commands exist on the host -for cmd in tac awk sed; do - command -v "${cmd}" > /dev/null 2>&1 || { echo "Missing command: ${cmd}" && exit 1; } -done - -# Check that the file exists before performing actions on it -if [ ! -s "$input_file" ]; then - echo "Diff file (git.diff) is missing or empty!" - exit 1 -fi - -# Reverse the file, remove 4 lines after '+++ /dev/null', then reverse it back (editors can't go backwards - to remove lines above) -tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" -sed '/+++ \/dev\/null/{n;N;N;N;d;}' "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" -tac "$input_file" > "$temp_file" && mv "$temp_file" "$input_file" - -# Remove the lines between '+++ /dev/null' (included) and 'diff --git a/' -awk ' - BEGIN { in_block=0 } - /\+\+\+ \/dev\/null/ { in_block=1; next } - in_block && /diff --git a\// { in_block=0; print; next } - !in_block -' "$input_file" > "$temp_file" - -# Check that the file exists before performing actions on it -if [ -s "$temp_file" ]; then - mv "$temp_file" "$input_file" - exit 0 -fi - -echo "Temp diff file (temp_diff_file.diff) is missing or emtpy!" -exit 1 From c7574a10052cd376106b0b6e8f2ad712abebe30d Mon Sep 17 00:00:00 2001 From: ASuciuX Date: Tue, 16 Jan 2024 03:51:01 +0200 Subject: [PATCH 29/29] feat: move `mutation-testing` folder to `stacks-core` folder --- stacks-core/README.md | 1 + {mutation-testing => stacks-core/mutation-testing}/README.md | 0 .../mutation-testing}/check-packages-and-shards/README.md | 2 +- .../mutation-testing}/check-packages-and-shards/action.yml | 0 .../mutation-testing}/output-pr-mutants/README.md | 2 +- .../mutation-testing}/output-pr-mutants/action.yml | 0 .../mutation-testing}/pr-differences/README.md | 2 +- .../mutation-testing}/pr-differences/action.yml | 0 8 files changed, 4 insertions(+), 3 deletions(-) rename {mutation-testing => stacks-core/mutation-testing}/README.md (100%) rename {mutation-testing => stacks-core/mutation-testing}/check-packages-and-shards/README.md (92%) rename {mutation-testing => stacks-core/mutation-testing}/check-packages-and-shards/action.yml (100%) rename {mutation-testing => stacks-core/mutation-testing}/output-pr-mutants/README.md (92%) rename {mutation-testing => stacks-core/mutation-testing}/output-pr-mutants/action.yml (100%) rename {mutation-testing => stacks-core/mutation-testing}/pr-differences/README.md (91%) rename {mutation-testing => stacks-core/mutation-testing}/pr-differences/action.yml (100%) diff --git a/stacks-core/README.md b/stacks-core/README.md index e095e30..5539b35 100644 --- a/stacks-core/README.md +++ b/stacks-core/README.md @@ -5,3 +5,4 @@ Folder of composite actions for the [stacks blockchain](https://github.com/stack - [cache](./cache) - Various cache operations - [testenv](./testenv) - Configures a workflow with a testing environment - [run-tests](./run-tests) - Run tests with [nextest](https://nexte.st) +- [mutation-testing](./mutation-testing) - Run mutation testing diff --git a/mutation-testing/README.md b/stacks-core/mutation-testing/README.md similarity index 100% rename from mutation-testing/README.md rename to stacks-core/mutation-testing/README.md diff --git a/mutation-testing/check-packages-and-shards/README.md b/stacks-core/mutation-testing/check-packages-and-shards/README.md similarity index 92% rename from mutation-testing/check-packages-and-shards/README.md rename to stacks-core/mutation-testing/check-packages-and-shards/README.md index 27d79a9..ff30718 100644 --- a/mutation-testing/check-packages-and-shards/README.md +++ b/stacks-core/mutation-testing/check-packages-and-shards/README.md @@ -24,5 +24,5 @@ jobs: steps: - name: Check Packages and Shards id: check_packages_and_shards - uses: stacks-network/actions/mutation-testing/check-packages-and-shards@main + uses: stacks-network/actions/stacks-core/mutation-testing/check-packages-and-shards@main ``` diff --git a/mutation-testing/check-packages-and-shards/action.yml b/stacks-core/mutation-testing/check-packages-and-shards/action.yml similarity index 100% rename from mutation-testing/check-packages-and-shards/action.yml rename to stacks-core/mutation-testing/check-packages-and-shards/action.yml diff --git a/mutation-testing/output-pr-mutants/README.md b/stacks-core/mutation-testing/output-pr-mutants/README.md similarity index 92% rename from mutation-testing/output-pr-mutants/README.md rename to stacks-core/mutation-testing/output-pr-mutants/README.md index db6d829..7c8e85f 100644 --- a/mutation-testing/output-pr-mutants/README.md +++ b/stacks-core/mutation-testing/output-pr-mutants/README.md @@ -25,7 +25,7 @@ jobs: steps: - name: Output Mutants id: output_pr_mutants - uses: stacks-network/actions/mutation-testing/output-pr-mutants@main + uses: stacks-network/actions/stacks-core/mutation-testing/output-pr-mutants@main with: big_packages: "true" shards_for_big_packages: "false" diff --git a/mutation-testing/output-pr-mutants/action.yml b/stacks-core/mutation-testing/output-pr-mutants/action.yml similarity index 100% rename from mutation-testing/output-pr-mutants/action.yml rename to stacks-core/mutation-testing/output-pr-mutants/action.yml diff --git a/mutation-testing/pr-differences/README.md b/stacks-core/mutation-testing/pr-differences/README.md similarity index 91% rename from mutation-testing/pr-differences/README.md rename to stacks-core/mutation-testing/pr-differences/README.md index 6feeba1..47df406 100644 --- a/mutation-testing/pr-differences/README.md +++ b/stacks-core/mutation-testing/pr-differences/README.md @@ -23,7 +23,7 @@ jobs: steps: - name: PR Differences Mutants id: pr-differences-mutants - uses: stacks-network/actions/mutation-testing/pr-differences@main + uses: stacks-network/actions/stacks-core/mutation-testing/pr-differences@main with: package-dimension: "big" ``` diff --git a/mutation-testing/pr-differences/action.yml b/stacks-core/mutation-testing/pr-differences/action.yml similarity index 100% rename from mutation-testing/pr-differences/action.yml rename to stacks-core/mutation-testing/pr-differences/action.yml