Add onchain tests in ci. (#194)

* Add onchain tests in ci. * Check input in test_onchain. * Fix bug. * CI test using cache. * fix onchain ci * ready * add ci * reduce onchain tests * further reduce tests * read rpc & keys from env * add polygon * add logs * fix ci crash --------- Co-authored-by: shouc <[email protected]> Co-authored-by: shouc <[email protected]>
fuzzland · Sep 25, 2023 · c61315e · c61315e
1 parent 8e16a23
commit c61315e
Show file tree

Hide file tree

Showing 4 changed files with 238 additions and 5 deletions.
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -17,6 +17,8 @@ jobs:
         with:
           submodules: recursive
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - name: Download and Extract Cache
+        run: curl -L https://github.com/fuzzland/ityfuzz-test-cache/releases/latest/download/cache.tar.gz -o cache.tar.gz && tar -xzf cache.tar.gz
       - name: Build
         run: cargo build --verbose
       - name: Run tests

diff --git a/ci.py b/ci.py
@@ -0,0 +1,97 @@
+import os
+import requests
+import uuid
+
+def post_comment_to_pr(pr_number, token, message):
+    """
+    Post a comment to a GitHub PR.
+    
+    Parameters:
+        pr_number (int): The PR number.
+        token (str): The GitHub token to authenticate with.
+        message (str): The comment message.
+    """
+    url = f"https://api.github.com/repos/{owner}/{repo}/issues/{pr_number}/comments"
+    headers = {
+        "Authorization": f"token {token}",
+        "User-Agent": "PR-Commenter",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    data = {"body": message}
+
+    response = requests.post(url, headers=headers, json=data)
+
+    if response.status_code == 201:
+        print("Successfully posted the comment!")
+    else:
+        print(f"Failed to post comment. Status code: {response.status_code}, Response: {response.text}")
+
+
+
+DEFAULT_MD = """| Project Name | Vulnerability Found | Time Taken | Log |
+|---------|---------|---------|---------|"""
+
+def parse_res(file):
+    with open(UID + "/" + file, 'r') as f:
+        lines = f.readlines()
+        last_ts = -1
+        ts = -1
+        violation = ""
+        crashed = False
+        for i in lines:
+            if "run time: " in i:
+                _ts = i.split("run time: ")[1].split(",")[0]
+                last_ts = _ts
+            if "more than owed" in i:
+                ts = last_ts
+                violation = "Fund Loss"
+            if "Reserves changed from" in i:
+                ts = last_ts
+                violation = "Price Manipulation"
+
+            if "Arbitrary call " in i:
+                ts = last_ts
+                violation = "Arbitrary Call"
+            if "panicked at " in i:
+                crashed = True
+        violation = "✅ " + violation  if violation else "❌"
+
+        if crashed:
+            violation = "❌‼️  Crashed"
+
+        return (file.replace("res_", ""), ts, violation)
+
+
+UID = str(uuid.uuid4())
+
+def parse_all():
+    os.system(f"mkdir {UID} && mv res_* {UID} && aws s3 cp {UID} s3://cilogs-ityfuzz/{UID} --recursive")
+
+    found = 0
+    md = DEFAULT_MD
+    for i in os.listdir(UID):
+        if i.startswith("res_"):
+            fn, ts, violation = parse_res(i)
+            if "❌" not in violation:
+                found += 1
+            log = f"https://cilogs-ityfuzz.s3.amazonaws.com/{UID}/{i}"
+            md += f"\n| {fn} | {violation} | {ts} | [Log File]({log}) |"
+    return f"Found: {found}\n\n" + md
+
+if __name__ == "__main__":
+    PR_NUMBER = os.environ.get("PR_NUMBER")
+    GITHUB_TOKEN = os.environ.get("BOT_GH_TOKEN")
+    owner = "fuzzland"
+    repo = "ityfuzz"
+
+    comment_message = parse_all()
+    print(comment_message)
+    if not PR_NUMBER or not GITHUB_TOKEN:
+        print("Missing PR_NUMBER or GITHUB_TOKEN environment variables!")
+        exit(1)
+
+
+    post_comment_to_pr(PR_NUMBER, GITHUB_TOKEN, comment_message)
+
+
+
diff --git a/integration_test.py b/integration_test.py
@@ -1,11 +1,20 @@
 import glob
+import random
 import subprocess
 import os
 import time
 
 
 TIMEOUT_BIN = "timeout" if os.name == "posix" else "gtimeout"
 
+def read_onchain_tests():
+    tests = ""
+    with open("onchain_tests.txt", "r") as file:
+        tests = file.read()
+
+    tests = tests.split("\n")
+    tests = [test.split("\t") for test in tests]
+    return tests
 
 def test_one(path):
     # cleanup
@@ -44,25 +53,122 @@ def test_one(path):
         print(p.stderr.decode("utf-8"))
         print("================ STDOUT =================")
         print(p.stdout.decode("utf-8"))
-        print(f"Failed to fuzz {path}")
+        print(f"=== Failed to fuzz {path}")
+    else:
+        print(f"=== Success: {path}, Finished in {time.time() - start_time}s")
 
     # clean up
     # os.system(f"rm -rf {path}/*.abi")
     # os.system(f"rm -rf {path}/*.bin")
 
-    print(f"=== Success: {path}, Finished in {time.time() - start_time}s")
 
+def test_onchain(test):
+
+    if len(test) != 4:
+        print(f"=== Invalid test: {test}")
+        return
+
+    # randomly sleep for 0 - 30s to avoid peak traffic
+    time.sleep(60 * random.random())
+
+    contract_addresses, block_number, chain, name= test[3], test[2], test[1], test[0]
+
+    if chain not in ["eth", "bsc", "polygon"]:
+        print(f"=== Unsupported chain: {chain}")
+        return
+
+    etherscan_key = os.getenv(f"{chain.upper()}_ETHERSCAN_API_KEY")
+    if etherscan_key is None:
+        print(f"=== No etherscan api key for {chain}")
+        return
+    my_env = os.environ.copy()
+    my_env["ETH_RPC_URL"] = os.getenv(f"{chain.upper()}_RPC_URL")
+
+    cmd = [
+        TIMEOUT_BIN, 
+        # set timeout to 5m because it takes longer time to sync the chain
+        "5m",
+        "./cli/target/release/cli", "evm", "-o", 
+        "-t", contract_addresses, 
+        "-c", chain, 
+        "--onchain-block-number", str(block_number), 
+        "-f", "-i", "-p", 
+        "--onchain-etherscan-api-key", etherscan_key,  
+        "--work-dir", f"w_{name}", 
+        #"--run-forever"
+    ]
+
+    start_time = time.time()
+
+    # try 3 times in case of rpc failure
+    for i in range(3):
+
+        p = subprocess.run(" ".join(cmd),
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        shell=True, env=my_env)
+
+        if b"Found violations!" in p.stdout:
+            print(f"=== Success: Tested onchain for contracts: {name}, Finished in {time.time() - start_time}s")
+            open(f"res_{name}.txt", "w+").write(p.stderr.decode("utf-8") + " ".join(cmd) + "\n" + p.stdout.decode("utf-8"))
+            return
+        time.sleep(30)
+
+
+    print(f"=== Failed to test onchain for contracts: {name}")
+    open(f"res_{name}.txt", "w+").write(p.stderr.decode("utf-8") + " ".join(cmd) + "\n" + p.stdout.decode("utf-8"))
 
 def build_fuzzer():
     # build fuzzer
     os.chdir("cli")
     subprocess.run(["cargo", "build", "--release"])
     os.chdir("..")
 
+def update_cargo_toml():
+    with open("Cargo.toml", "r") as file:
+        content = file.read()
+
+    if '"flashloan_v2"' in content:
+        return
+
+    if '"cmp"' in content:
+        content = content.replace('"cmp"', '"cmp","flashloan_v2","force_cache"')
+
+    with open("Cargo.toml", "w") as file:
+        file.write(content)
+
+    print("Cargo.toml has been updated!")
+
+def build_flash_loan_v2_fuzzer():
+    update_cargo_toml()
+    # build fuzzer
+    os.chdir("cli")
+    subprocess.run(["cargo", "build", "--release"])
+    os.chdir("..")
+
 
 import multiprocessing
+import sys
+
 
 if __name__ == "__main__":
-    build_fuzzer()
-    with multiprocessing.Pool(3) as p:
-        p.map(test_one, glob.glob("./tests/evm/*/", recursive=True))
+    actions = []
+
+    if len(sys.argv) > 1:
+        if sys.argv[1] == "onchain":
+            actions.append("onchain")
+        elif sys.argv[1] == "offchain":
+            actions.append("offchain")
+    else:
+        actions = ["onchain", "offchain"]
+
+    if "offchain" in actions:
+        build_fuzzer()
+        with multiprocessing.Pool(3) as p:
+            p.map(test_one, glob.glob("./tests/evm/*/", recursive=True))
+
+    if "onchain" in actions:
+        build_flash_loan_v2_fuzzer()
+        tests = read_onchain_tests()
+        with multiprocessing.Pool(10) as p:
+            p.map(test_onchain, tests)
diff --git a/onchain_tests.txt b/onchain_tests.txt
@@ -0,0 +1,28 @@
+VerilogCTF	polygon	35718198	0xbcf6e9d27bf95f3f5eddb93c38656d684317d5b4,0x5d6c48f05ad0fde3f64bab50628637d73b1eb0bb
+EAC_exp	bsc	31273018	0x55d398326f99059fF775485246999027B3197955,0x64f291DE10eCd36D5f7b64aaEbC70943CFACE28E,0x20dcf125f0563417d257b98a116c3fea4f0b2db2,0xa08a40e0F11090Dcb09967973DF82040bFf63561
+ApeDAO_exp	bsc	30072293	0x81917eb96b397dFb1C6000d28A5bc08c0f05fC1d,0x55d398326f99059fF775485246999027B3197955,0x45aa258ad08eeeb841c1c02eca7658f9dd4779c0,0xb47955b5b7eaf49c815ebc389850eb576c460092,0xee2a9D05B943C1F33f3920C750Ac88F74D0220c3,0xB47955B5B7EAF49C815EBc389850eb576C460092
+ROI_exp	bsc	21143795	0xe9e7CEA3DedcA5984780Bafc599bD69ADd087D56,0xe48b75dc1b131fd3a8364b0580f76efd04cf6e9c,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x745D6Dd206906dd32b3f35E00533AD0963805124,0x216FC1D66677c9A778C60E6825189508b9619908,0xE48b75dc1b131fd3A8364b0580f76eFD04cF6e9c,0x158af3d23d96e3104bcc65b76d1a6f53d0f74ed0
+HEALTH_exp	bsc	22337425	0xF375709DbdE84D800642168c2e8bA751368e8D32,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x32B166e082993Af6598a89397E82e123ca44e74E,0x0fe261aeE0d1C4DFdDee4102E82Dd425999065F4
+Carrot_exp	bsc	22055611	0x55d398326f99059fF775485246999027B3197955,0x6863b549bf730863157318df4496eD111aDFA64f,0xcFF086EaD392CcB39C49eCda8C974ad5238452aC,0x5575406ef6b15eec1986c412b9fbe144522c45ae,0x6863b549bf730863157318df4496ed111adfa64f
+BEGO_exp	bsc	22315679	0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x88503F48e437a377f1aC2892cBB3a5b09949faDd,0xc342774492b54ce5F8ac662113ED702Fc1b34972
+CS_exp	bsc	28466976	0x382e9652AC6854B56FD41DaBcFd7A9E633f1Edd5,0x55d398326f99059fF775485246999027B3197955,0x7EFaEf62fDdCCa950418312c6C91Aef321375A00,0x8BC6Ce23E5e2c4f0A96429E3C9d482d74171215e
+GPT_exp	bsc	28494868	0x81917eb96b397dFb1C6000d28A5bc08c0f05fC1d,0x55d398326f99059fF775485246999027B3197955,0x77a684943aA033e2E9330f12D4a1334986bCa3ef,0xa1679abEF5Cd376cC9A1C4c2868Acf52e08ec1B3
+MintoFinance_exp	bsc	30214253	0x55d398326f99059fF775485246999027B3197955,0x0d116ed40831fef8e21ece57c8455ae3b1e4041b,0xdbf1c56b2ad121fe705f9b68225378aa6784f3e5,0xDbF1C56b2aD121Fe705f9b68225378aa6784f3e5,0x13f4EA83D0bd40E75C8222255bc855a974568Dd4,0x410a56541bD912F9B60943fcB344f1E3D6F09567,0xba91db0b31d60c45e0b03e6d515e45fcabc7b1cd
+Annex_exp	bsc	23165446	0xe65E970F065643bA80E5822edfF483A1d75263E3,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0xcA143Ce32Fe78f1f7019d7d551a6402fC5350c73
+SEAMAN_exp	bsc	23467515	0x55d398326f99059fF775485246999027B3197955,0x6bc9b4976ba6f8C9574326375204eE469993D038,0x6637914482670f91F43025802b6755F27050b0a6,0xDB95FBc5532eEb43DeEd56c8dc050c930e31017e
+cftoken_exp	bsc	16841980	0x8B7218CF6Ac641382D7C723dE8aA173e98a80196,0x7FdC0D8857c6D90FD79E22511baf059c0c71BF8b
+SELLC03_exp	bsc	29005754	0x55d398326f99059fF775485246999027B3197955,0x2cc392c0207d080aec0befe5272659d3bb8a7052,0x9523B023E1D2C490c65D26fad3691b024d0305D7,0xa645995e9801F2ca6e2361eDF4c2A138362BADe4,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x84Be9475051a08ee5364fBA44De7FE83a5eCC4f1
+Yyds_exp	bsc	21157025	0x55d398326f99059fF775485246999027B3197955,0x970A76aEa6a0D531096b566340C0de9B027dd39D,0xB19463ad610ea472a886d77a8ca4b983E4fAf245,0xd5cA448b06F8eb5acC6921502e33912FA3D63b12,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0xe70cdd37667cdDF52CabF3EdabE377C58FaE99e9
+MBC_ZZSH_exp	bsc	23474460	0x55d398326f99059fF775485246999027B3197955,0x4E87880A72f6896E7e0a635A5838fFc89b13bd17,0x2170Ed0880ac9A755fd29B2688956BD959F933F8,0x5b1Bf836fba1836Ca7ffCE26f155c75dBFa4aDF1,0x33CCA0E0CFf617a2aef1397113E779E42a06a74A,0xeE04a3f9795897fd74b7F04Bb299Ba25521606e6
+GSS_exp	bsc	31108558	0x55d398326f99059fF775485246999027B3197955,0xB4F4cD1cc2DfF1A14c4Aaa9E9434A92082855C64,0x1ad2cB3C2606E6D5e45c339d10f81600bdbf75C0,0x37e42B961AE37883BAc2fC29207A5F88eFa5db66,0x69ed5b59d977695650ec4b29e61c0faa8cc0ed5c
+THB_exp	bsc	21785004	0x72e901F1bb2BfA2339326DfB90c5cEc911e2ba3C,0xae191Ca19F0f8E21d754c6CAb99107eD62B6fe53
+Novo_exp	bsc	18225002	0xEeBc161437FA948AAb99383142564160c92D2974,0xa0787daad6062349f63b7c228cbfd5d8a3db08f1,0x3463a663de4ccc59c8b21190f81027096f18cf2a,0x6Fb2020C236BBD5a7DDEb07E14c9298642253333,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x128cd0Ae1a0aE7e67419111714155E1B1c6B2D8D
+PLTD_exp	bsc	22252045	0x55d398326f99059fF775485246999027B3197955,0xD7B7218D778338Ea05f5Ecce82f86D365E25dBCE,0x4397C76088db8f16C15455eB943Dd11F2DF56545,0x29b2525e11BC0B0E9E59f705F318601eA6756645
+DYNA_exp	bsc	25879486	0xa7B5eabC3Ee82c585f5F4ccC26b81c3Bd62Ff3a9,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x5c0d0111ffc638802c9EfCcF55934D5C63aB3f79,0xb6148c6fA6Ebdd6e22eF5150c5C3ceE78b24a3a0
+OLIFE_exp	bsc	27470678	0x915C2DFc34e773DC3415Fe7045bB1540F8BDAE84,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0xb5a0Ce3Acd6eC557d39aFDcbC93B07a1e1a9e3fa
+AUR_exp	bsc	23282134	0x73A1163EA930A0a67dFEFB9C3713Ef0923755B78,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x70678291bDDfd95498d1214BE368e19e882f7614
+SellToken_exp	bsc	28168034	0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0xa645995e9801F2ca6e2361eDF4c2A138362BADe4,0x57Db19127617B77c8abd9420b5a35502b59870D6
+Shadowfi_exp	bsc	20969095	0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x10bc28d2810dD462E16facfF18f78783e859351b,0xF9e3151e813cd6729D52d9A0C3ee69F22CcE650A
+RFB_exp	bsc	23649423	0x26f1457f067bF26881F311833391b52cA871a4b5,0x03184AAA6Ad4F7BE876423D9967d1467220a544e,0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c,0x0fe261aeE0d1C4DFdDee4102E82Dd425999065F4
+BIGFI_exp	bsc	26685503	0x55d398326f99059fF775485246999027B3197955,0x28ec0B36F0819ecB5005cAB836F4ED5a2eCa4D13,0xd3d4B46Db01C006Fb165879f343fc13174a1cEeB,0xA269556EdC45581F355742e46D2d722c5F3f551a
+Axioma_exp	bsc	27620320	0x2C25aEe99ED08A61e7407A5674BC2d1A72B5D8E3,0xB6CF5b77B92a722bF34f6f5D6B1Fe4700908935E,0x6a3Fa7D2C71fd7D44BF3a2890aA257F34083c90f