updated workflow files

ersilia-os · Dec 19, 2024 · efc0295 · efc0295
1 parent 6a1cc58
commit efc0295
Show file tree

Hide file tree

Showing 8 changed files with 108 additions and 54 deletions.
diff --git a/.github/scripts/verify_model_outcome.py b/.github/scripts/verify_model_outcome.py
@@ -6,7 +6,7 @@ def check_non_null_outcomes_in_output_csv(csv_file_path):
         header = next(csv_reader)
         row = next(csv_reader)
         for val in row[2:]: # Skip the first two columns (Inchikey and input)
-            if val not in ['', None]:
+            if val not in ['', None]: # Returns if even one outcome is not null
                 return False
     return True
 

diff --git a/.github/workflows/post-model-upload.yml b/.github/workflows/post-model-upload.yml
@@ -105,20 +105,35 @@ jobs:
           REPO_NAME: ${{ github.event.repository.name }}
           AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
         run: |
+          source activate
+          pip install requests pyairtable
           echo "Updating metadata to AirTable looking at owner: $USER_NAME"
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
-          python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY
-          rm update_metadata_to_airtable.py
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+          python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY
+          rm airtableops.py
   
+      - name: sync metadata to S3 JSON
+        id: sync-metadata-to-s3
+        env:
+          AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          source activate
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py
+          pip install boto3 requests pyairtable
+          python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+          rm convert_airtable_to_json.py
+
       - name: Update README file
         id: update-readme-file
         env:
           MODEL_ID: ${{ github.event.repository.name }}
         run: |
           echo "Updating README file with AirTable metadata for model: $MODEL_ID"
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py
-          python3 update_readme_from_airtable.py $MODEL_ID .
-          rm update_readme_from_airtable.py
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+          python3 airtableops.py readme-update --repo $MODEL_ID --path .
+          rm airtableops.py
           less README.md
 
       - name: Commit and push changes done to the README file

diff --git a/.github/workflows/test-model-pr.yml b/.github/workflows/test-model-pr.yml
@@ -54,21 +54,6 @@ jobs:
           conda init
           python -m pip install git+https://github.com/ersilia-os/ersilia.git
   
-      - name: Check metadata before updating to AirTable
-        id: check-metadata
-        env:
-          USER_NAME: ${{ github.event.pull_request.head.repo.owner.login }}
-          BRANCH: "main"
-          REPO_NAME: ${{ github.event.repository.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 10
-          max_attempts: 3
-          command: |
-            source activate
-            wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
-            python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH
-
       - name: Predict output
         env:
           MODEL_ID: ${{ github.event.repository.name }}
@@ -79,9 +64,9 @@ jobs:
           command: |
             source activate
             echo "Sample model id selected: $MODEL_ID"
-            ersilia -v fetch $MODEL_ID --repo_path .
+            ersilia -v fetch $MODEL_ID --from_dir .
             ersilia -v serve $MODEL_ID
-            ersilia sample -n 5 -f input.csv
+            ersilia example -n 5 -f input.csv --predefined
             ersilia -v api -i input.csv
             ersilia close
           

diff --git a/.github/workflows/test-model.yml b/.github/workflows/test-model.yml
@@ -1,4 +1,4 @@
-name: Model Test on push
+name: Model test on push
 
 on:
   push: 
@@ -72,10 +72,24 @@ jobs:
           max_attempts: 3
           command: |
             source activate
+            pip install requests pyairtable
             echo "Updating metadata to AirTable looking at owner: $USER_NAME"
-            wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
-            python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY
-            rm update_metadata_to_airtable.py
+            wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+            python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY
+            rm airtableops.py
+
+      - name: sync metadata to S3 JSON
+        id: sync-metadata-to-s3
+        env:
+          AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          source activate
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py
+          pip install boto3 requests pyairtable
+          python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+          rm convert_airtable_to_json.py
 
       - name: Update README file
         id: update-readme-file
@@ -84,9 +98,9 @@ jobs:
         run: |
           source activate
           echo "Updating README file with AirTable metadata for model: $MODEL_ID"
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py
-          python3 update_readme_from_airtable.py $MODEL_ID .
-          rm update_readme_from_airtable.py
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+          python3 airtableops.py readme-update --repo $MODEL_ID --path .
+          rm airtableops.py
           less README.md
 
       - name: Commit and push changes done to the README file
@@ -100,22 +114,38 @@ jobs:
           amend: true
           force: true
 
-      - name: Predict output
+      - name: Fetch model
         env:
           MODEL_ID: ${{ github.event.repository.name }}
-        uses: nick-fields/retry@v3
-        with:
-          timeout_minutes: 10
-          max_attempts: 3
-          command: |
+        run: |
+            source activate
+            ersilia -v fetch $MODEL_ID --from_dir .
+            FOLDER="$HOME/eos/repository/$MODEL_ID"
+            if [ ! -d "$FOLDER" ]; then
+              echo "Error: Folder '$FOLDER' does not exist." >&2
+              exit 1
+            fi
+
+      - name: Generate input and run model
+        env:
+          MODEL_ID: ${{ github.event.repository.name }}
+        run: |
             source activate
-            echo "Sample model id selected: $MODEL_ID"
-            ersilia -v fetch $MODEL_ID --repo_path .
             ersilia -v serve $MODEL_ID
-            ersilia sample -n 5 -f input.csv
-            ersilia -v run -i input.csv
+            ersilia example -n 5 -f input.csv --predefined
+            ersilia -v run -i "input.csv" -o "output.csv"
             ersilia close
-          
+            cat output.csv
+      
+      - name: Test output
+        run: |
+            output=$(python .github/scripts/verify_model_outcome.py output.csv)
+            if echo "$output" | grep -q "All outcomes are null"; then
+              echo "Error in model outcome, aborting test"
+              exit 1
+            fi
+            rm output.csv
+
       - name: Upload log output
         if: always()
         uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb #pin v3.1.1

diff --git a/.github/workflows/upload-bentoml.yml b/.github/workflows/upload-bentoml.yml
@@ -97,9 +97,12 @@ jobs:
         id: testBuiltImage
         env:
           PULL_IMAGE: n
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
         run: |
           ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
-          ersilia -v serve ${{ github.event.repository.name }}
+          ersilia -v serve ${{ github.event.repository.name }} --track #Added --track here
           ersilia example -n 1 -f input.csv --predefined
           ersilia -v run -i "input.csv" -o "output.csv"
           ersilia close

diff --git a/.github/workflows/upload-ersilia-pack.yml b/.github/workflows/upload-ersilia-pack.yml
@@ -11,7 +11,10 @@ jobs:
 
       - name: Checkout repository
         uses: actions/checkout@v4
-
+        with:
+          lfs: true
+
+      - run: git lfs pull
       # https://github.com/docker/setup-qemu-action
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -73,12 +76,15 @@ jobs:
         id: testBuiltImageErsiliaPack
         env:
           PULL_IMAGE: n
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_DEFAULT_REGION: us-east-1
         if: steps.buildForTestErsiliaPack.outcome == 'success'
         run: |
           ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub
-          ersilia -v serve ${{ github.event.repository.name }}
+          ersilia -v serve ${{ github.event.repository.name }} --track #Added --track here
           ersilia example -n 1 -f input.csv --predefined
-          ersilia -v run -i "input.csv" -o "output.csv"
+          ersilia -v run -i "input.csv" -o "output.csv" 
           ersilia close
           output=$(python .github/scripts/verify_model_outcome.py output.csv)
           if echo "$output" | grep -q "All outcomes are null"; then

diff --git a/.github/workflows/upload-model-to-dockerhub.yml b/.github/workflows/upload-model-to-dockerhub.yml
@@ -1,6 +1,7 @@
 name: Upload model to DockerHub
 on:
   workflow_dispatch:
+
   workflow_run:
     workflows: ["Upload model to S3"]
     types:
@@ -21,7 +22,7 @@ jobs:
 
   upload-legacy-bentoml:
     needs: [upload-multi-stage-condapack]
-    if: ${{ needs.upload-multi-stage-condapack.result == 'failure' }}
+    if: ${{ failure() }}
     uses: ./.github/workflows/upload-bentoml.yml
     secrets: inherit
     with:

diff --git a/.github/workflows/upload-model-to-s3.yml b/.github/workflows/upload-model-to-s3.yml
@@ -100,21 +100,35 @@ jobs:
           AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
         run: |
           source activate
+          pip install requests pyairtable
           echo "Updating metadata to AirTable looking at owner: $USER_NAME"
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py
-          python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY
-          rm update_metadata_to_airtable.py
-  
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+          python3 airtableops.py airtable-update --user $USER_NAME --repo $REPO_NAME --branch $BRANCH --api-key $AIRTABLE_API_KEY
+          rm airtableops.py
+
+      - name: sync metadata to S3 JSON
+        id: sync-metadata-to-s3
+        env:
+          AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        run: |
+          source activate
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/convert_airtable_to_json.py
+          pip install boto3 requests pyairtable
+          python convert_airtable_to_json.py $AIRTABLE_API_KEY $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY
+          rm convert_airtable_to_json.py
+          
       - name: Update README file
         id: update-readme-file
         env:
           MODEL_ID: ${{ github.event.repository.name }}
         run: |
           source activate
           echo "Updating README file with AirTable metadata for model: $MODEL_ID"
-          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py
-          python3 update_readme_from_airtable.py $MODEL_ID .
-          rm update_readme_from_airtable.py
+          wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/airtableops.py
+          python3 airtableops.py readme-update --repo $MODEL_ID --path .
+          rm airtableops.py
           less README.md
 
       - name: Commit and push changes done to the README file