CLI test integration (ersilia-os#1383)

* Update config.yml - fix Circle CI pipeline * Playground Implementation * Playground Implementation * Support CLI batching, and more * Update CLI test * Update CLI test * Update CLI test * CLI test Update * A few fix on the playground test * A few fix on the playground test * A few fix on the playground test * Playground with all command at ersilia-os#1368 * Playground with all command at ersilia-os#1368 * Playground with all command at ersilia-os#1368 * Playground with all command at ersilia-os#1368 * Playground with all command at ersilia-os#1368 * Playground with all command at ersilia-os#1368 * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground * Session Parallelization on CLI test playground with pytest dependency * Rich lib integration pyproject config * Make the serializer use the fields from the header (ersilia-os#1406) --------- Co-authored-by: Dhanshree Arora <[email protected]>
Abellegese · Dec 4, 2024 · e82fa8e · e82fa8e
1 parent 866b663
commit e82fa8e
Show file tree

Hide file tree

Showing 11 changed files with 873 additions and 40 deletions.
diff --git a/.github/workflows/tests_and_cleanup.yml b/.github/workflows/tests_and_cleanup.yml
@@ -2,9 +2,9 @@ name: Ersilia tests, installation checks, and cleanup of model request template
 
 on:
   pull_request:
-    branches: [ master ]
+    branches: [master]
   push:
-    branches: [ master ]
+    branches: [master]
   workflow_dispatch:
 
 jobs:
@@ -15,41 +15,41 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install Ersilia
-      run: |
-        python -m pip install git+https://github.com/ersilia-os/bentoml-ersilia.git
-        python -m pip install -e .
-    - name: Test help command
-      run: |
-        ersilia --help
-        ersilia --version
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Ersilia
+        run: |
+          python -m pip install git+https://github.com/ersilia-os/bentoml-ersilia.git
+          python -m pip install -e .
+      - name: Test help command
+        run: |
+          ersilia --help
+          ersilia --version
 
   test-docker:
     strategy:
       matrix:
         python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install Ersilia without pre-installing BentoML
-      run: |
-        python -m pip install -e .
-    - name: Test help command again
-      run: |
-        ersilia --help
-        ersilia --version
-    - name: Fetch molecular weight model from DockerHub
-      run: |
-        ersilia -v fetch molecular-weight --from_dockerhub
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Ersilia without pre-installing BentoML
+        run: |
+          python -m pip install -e .
+      - name: Test help command again
+        run: |
+          ersilia --help
+          ersilia --version
+      - name: Fetch molecular weight model from DockerHub
+        run: |
+          ersilia -v fetch molecular-weight --from_dockerhub
 
   test-os:
     strategy:
@@ -91,10 +91,9 @@ jobs:
 
       - name: Source conda
         run: source $CONDA/etc/profile.d/conda.sh
-        
+
       - name: Set Python to 3.10.10
-        run:
-         conda install -y python=3.10.10  
+        run: conda install -y python=3.10.10
 
       - name: Install dependencies
         run: |
@@ -115,9 +114,89 @@ jobs:
       - name: Run pytest
         run: |
           source activate          
-          pip install pytest pytest-benchmark pytest-asyncio
+          pip install pytest pytest-benchmark pytest-asyncio nox rich
           pytest
 
+  run-cli-test-single:
+    needs: run-pytest
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        session:
+          - setup
+          - test_from_github
+          - test_from_dockerhub
+          - test_auto_fetcher_decider
+          - test_conventional_run
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # [email protected]
+
+      - name: Add conda to system path
+        run: echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Source conda
+        run: source $CONDA/etc/profile.d/conda.sh
+
+      - name: Set Python to 3.10.10
+        run: conda install -y python=3.10.10
+
+      - name: Install dependencies
+        run: |
+          source activate
+          conda init
+          conda install git-lfs -c conda-forge
+          git-lfs install
+          conda install gh -c conda-forge
+
+      - name: Install ersilia
+        run: |
+          source activate
+          python --version
+          echo "After conda init"
+          conda init
+          python -m pip install -e .[test]
+
+      - name: Run CLI Test Default
+        run: |
+          source activate            
+          nox -f test/playground/noxfile.py -s ${{ matrix.session }}
+
+  run-cli-test-multiple:
+    needs: run-pytest
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # [email protected]
+
+      - name: Add conda to system path
+        run: echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Source conda
+        run: source $CONDA/etc/profile.d/conda.sh
+
+      - name: Set Python to 3.10.10
+        run: conda install -y python=3.10.10
+
+      - name: Install dependencies
+        run: |
+          source activate
+          conda init
+          conda install git-lfs -c conda-forge
+          git-lfs install
+          conda install gh -c conda-forge
+
+      - name: Install ersilia
+        run: |
+          source activate
+          python --version
+          echo "After conda init"
+          conda init
+          python -m pip install -e .[test]
+
+      - name: Run CLI Test Multiple
+        run: |
+          source activate            
+          nox -f test/playground/noxfile.py -s setup test_fetch_multiple_models test_serve_multiple_models
+
   test-colab-notebook:
     runs-on: ubuntu-22.04
     steps:
@@ -129,8 +208,7 @@ jobs:
         run: source $CONDA/etc/profile.d/conda.sh
 
       - name: Set Python to 3.10.10
-        run:
-         conda install -y python=3.10.10 
+        run: conda install -y python=3.10.10
 
       - name: Install dependencies
         run: |
@@ -160,7 +238,6 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/[email protected]
-
       - name: Path Filter
         id: filter
         uses: dorny/paths-filter@v3
@@ -173,7 +250,7 @@ jobs:
         if: steps.filter.outputs.tag-file == 'true'
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: "3.10"
 
       - name: Install dependencies
         if: steps.filter.outputs.tag-file == 'true'
@@ -197,4 +274,4 @@ jobs:
           repository: "ersilia-os/ersilia"
           github_token: ${{ secrets.GITHUB_TOKEN }}
           force: true
-          branch: "master"
+          branch: "master"
diff --git a/conftest.py b/conftest.py
@@ -0,0 +1,48 @@
+from test.playground.shared import results
+from rich.table import Table
+from rich.console import Console
+from rich.text import Text
+from rich import box
+from rich.panel import Panel
+
+def pytest_terminal_summary(terminalreporter, exitstatus, config):
+    console = Console()
+
+    docker_status = Text("✔", style="green") if any(result["activate_docker"] for result in results) else Text("✘", style="red")
+    runner = results[0]["runner"] if results else "N/A"
+    cli_type = results[0]["cli_type"] if results else "N/A"
+
+    header_panel = Panel.fit(
+        f"Docker Status: {docker_status}\nRunner: {runner}\nCLI Type: {cli_type}",
+        title="Execution Summary",
+        border_style="bold",
+    )
+    console.print(header_panel)
+
+    table = Table(title="Command Execution Summary", box=box.SQUARE)
+    table.add_column("Command", width=50)
+    table.add_column("Description", width=15)
+    table.add_column("Time Taken", width=15)
+    table.add_column("Max Memory", width=15)
+    table.add_column("Status", width=20)
+    table.add_column("Checkups", width=30)
+
+    for result in results:
+        formatted_checkups = []
+        for check in result["checkups"]:
+            if check["status"]:
+                formatted_checkups.append(Text("✔", style="green") + f" {check['name']}")
+            else:
+                formatted_checkups.append(Text("✘", style="red") + f" {check['name']}")
+        checkups_text = "\n".join(str(checkup) for checkup in formatted_checkups)
+
+        table.add_row(
+            result["command"],
+            result["description"],
+            result["time_taken"],
+            result["max_memory"],
+            result["status"],
+            checkups_text,
+        )
+
+    console.print(table)
diff --git a/pyproject.toml b/pyproject.toml
@@ -62,13 +62,14 @@ fuzzywuzzy = { version = "^0.18.0", optional = true }
 sphinx = { version = ">=6.0.0", optional = true }           # for minimum version and support for Python 3.10
 jinja2 = { version = "^3.1.2", optional = true }
 scipy = { version = "<=1.10.0", optional = true }
+nox = { version = "*", optional = true }
 rich = { version = "*", optional = true }
 
 [tool.poetry.extras]
 # Instead of using poetry dependency groups, we use extras to make it pip installable
 lake = ["isaura"]
 docs = ["sphinx", "jinja2"]
-test = ["pytest", "pytest-asyncio", "pytest-benchmark", "fuzzywuzzy", "scipy", "nox"]
+test = ["pytest", "pytest-asyncio", "pytest-benchmark", "nox", "rich", "fuzzywuzzy", "scipy"]
 #all = [lake, docs, test]
 
 [tool.poetry.scripts]

diff --git a/test/playground/__init__.py b/test/playground/__init__.py