Upload model to DockerHub #15
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upload model to DockerHub | |
| on: | |
| workflow_dispatch: | |
| workflow_run: | |
| workflows: ["Upload model to S3"] | |
| types: | |
| - completed | |
| jobs: | |
| upload_model_to_dockerhub: | |
| if: ${{ github.repository != 'ersilia-os/eos-template' && github.event.workflow_run.conclusion == 'success' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| # https://github.com/docker/setup-qemu-action | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v3 | |
| # https://github.com/docker/setup-buildx-action | |
| - name: Set up Docker Buildx | |
| id: buildx | |
| uses: docker/setup-buildx-action@v3 | |
| # log in to dockerhub | |
| - name: Login to Docker Hub | |
| if: github.event_name != 'pull_request' | |
| uses: docker/login-action@v1 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| - name: Checkout persist credentials | |
| uses: actions/checkout@master | |
| with: | |
| persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token | |
| fetch-depth: 0 # otherwise, you will failed to push refs to dest repo | |
| lfs: 'true' | |
| # This might stop working in the future, so we need to keep an eye on it | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| # this might remove tools that are actually needed, | |
| # if set to "true" but frees about 6 GB | |
| tool-cache: true | |
| # all of these default to true, but feel free to set to | |
| # "false" if necessary for your workflow | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| swap-storage: true | |
| # Install ersilia in the next few steps to test the built image with ersilia CLI | |
| - name: Add conda to system path | |
| run: echo $CONDA/bin >> $GITHUB_PATH | |
| - name: Source conda | |
| run: source $CONDA/etc/profile.d/conda.sh | |
| - name: Set Python to 3.10.10 | |
| run: | |
| conda install -y python=3.10.10 | |
| - name: Install dependencies | |
| run: | | |
| source activate | |
| conda init | |
| conda install git-lfs -c conda-forge | |
| git-lfs install | |
| conda install gh -c conda-forge | |
| - name: Install ersilia | |
| run: | | |
| source activate | |
| python --version | |
| echo "After conda init" | |
| conda init | |
| python -m pip install git+https://github.com/ersilia-os/ersilia.git | |
| # Build a V2 image and test it with ersilia CLI | |
| - name: Generate the V2 Dockerfile | |
| id: generateDockerfileV2 | |
| env: | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| run: | | |
| wget -O place_a_dockerfile_in_current_eos_repo.py https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py | |
| python -m pip install requests | |
| python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME | |
| rm place_a_dockerfile_in_current_eos_repo.py | |
| # We cannot tag it as anything other than latest because | |
| # ersilia cli only looks for the 'latest' tag | |
| - name: Build only V2 AMD64 Image for Testing | |
| id: buildForTestV2 | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| load: true | |
| tags: ersiliaos/${{ github.event.repository.name }}:latest | |
| # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this | |
| - name: Test built v2 image | |
| id: testBuiltImageV2 | |
| continue-on-error: true # Allow this to fail | |
| run: | | |
| ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub | |
| ersilia -v serve ${{ github.event.repository.name }} | |
| ersilia example -n 1 -f input.csv --predefined | |
| ersilia -v run -i "input.csv" -o "output.csv" | |
| ersilia close | |
| output=$(python .github/scripts/verify_model_outcome.py output.csv) | |
| if echo "$output" | grep -q "All outcomes are null"; then | |
| echo "Error in model outcome, aborting build" | |
| exit 1 | |
| fi | |
| rm output.csv | |
| env: | |
| PULL_IMAGE: n | |
| # If the test failed build a V1 image and test it with ersilia CLI | |
| - name: Generate the V1 Dockerfile | |
| id: generateDockerfileV1 | |
| if: steps.testBuiltImageV2.outcome == 'failure' | |
| env: | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| run: | | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/place_a_dockerfile_in_current_eos_repo.py | |
| python -m pip install requests | |
| python place_a_dockerfile_in_current_eos_repo.py $REPO_NAME v1 | |
| rm place_a_dockerfile_in_current_eos_repo.py | |
| # We cannot tag it as anything other than latest because | |
| # ersilia cli only looks for the 'latest' tag | |
| - name: Build only V1 AMD64 Image for Testing | |
| id: buildForTestV1 | |
| if: steps.testBuiltImageV2.outcome == 'failure' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| load: true | |
| tags: ersiliaos/${{ github.event.repository.name }}:latest | |
| # TODO This is very hacky, maybe we want to use the ersilia test command in the future for this | |
| - name: Test built v1 image | |
| id: testBuiltImageV1 | |
| if: steps.testBuiltImageV2.outcome == 'failure' | |
| run: | | |
| ersilia -v fetch ${{ github.event.repository.name }} --from_dockerhub | |
| ersilia -v serve ${{ github.event.repository.name }} | |
| ersilia example -n 1 -f input.csv --predefined | |
| ersilia -v run -i "input.csv" -o "output.csv" | |
| ersilia close | |
| output=$(python .github/scripts/verify_model_outcome.py output.csv) | |
| if echo "$output" | grep -q "All outcomes are null"; then | |
| echo "Error in model outcome, aborting build" | |
| exit 1 | |
| fi | |
| rm output.csv | |
| env: | |
| PULL_IMAGE: n | |
| # Create an issue to track that this model did not work with the V2 image | |
| - name: Check for existing issue | |
| id: check_existing_alert_issue | |
| run: | | |
| gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} | |
| issue_number=$(gh issue list --limit 100 --search "${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }}" --json number --jq '.[0].number') | |
| echo "::set-output name=issue_number::$issue_number" | |
| if: steps.testBuiltImageV1.conclusion != 'skipped' | |
| - name: Create an alert issue | |
| uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 | |
| id: create_alert_issue | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| title: ${{ vars.DOCKER_BUILD_FAILURE_ISSUE_TITLE }} | |
| body: | | |
| This model failed to build with v2 image. Please investigate and fix the issue. | |
| labels: | | |
| bug | |
| if: ${{ (steps.check_existing_alert_issue.conclusion != 'skipped') && (steps.check_existing_alert_issue.outputs.issue_number == '') }} | |
| # Finally build and push if everything is successful | |
| - name: Build and push | |
| id: buildMultiple | |
| continue-on-error: true | |
| uses: docker/build-push-action@v5 | |
| timeout-minutes: 45 | |
| with: | |
| context: . | |
| platforms: linux/amd64,linux/arm64 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ersiliaos/${{ github.event.repository.name }}:latest | |
| - name: Set build failure output | |
| id: buildCheck | |
| run: | | |
| if [[ "${{ steps.buildMultiple.outcome }}" == "failure" ]]; then | |
| echo "::set-output name=failed::true" | |
| echo "AMD64" > arch.txt | |
| else | |
| echo "::set-output name=failed::false" | |
| echo "AMD64,ARM64" > arch.txt | |
| fi | |
| - name: Build only for Linux/amd64 | |
| id: buildSingle | |
| if: steps.buildCheck.outputs.failed == 'true' | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| platforms: linux/amd64 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ersiliaos/${{ github.event.repository.name }}:latest | |
| - name: Update Metadata JSON file with DockerHub info | |
| id: updateMetadata | |
| run: | | |
| mv Dockerfile_legacy Dockerfile | |
| python3 -c " | |
| import json | |
| with open('metadata.json', 'r') as f: | |
| data = json.load(f) | |
| print(data) | |
| with open('arch.txt', 'r') as f: | |
| arch = f.read().rstrip() | |
| arch = arch.split(',') | |
| data['DockerHub'] = 'https://hub.docker.com/r/ersiliaos/{0}'.format(data['Identifier']) | |
| data['Docker Architecture'] = arch | |
| with open('metadata.json', 'w') as f: | |
| json.dump(data, f, indent=4) | |
| " | |
| rm arch.txt | |
| - name: Commit and push changes done to the Metadata JSON file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "[email protected]" | |
| message: "updating metadata [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Update metadata to AirTable | |
| id: update-metadata-to-airtable | |
| env: | |
| USER_NAME: ${{ github.repository_owner }} | |
| BRANCH: "main" | |
| REPO_NAME: ${{ github.event.repository.name }} | |
| AIRTABLE_API_KEY: ${{ secrets.AIRTABLE_API_KEY }} | |
| run: | | |
| source activate | |
| echo "Updating metadata to AirTable looking at owner: $USER_NAME" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_metadata_to_airtable.py | |
| python3 update_metadata_to_airtable.py $USER_NAME $REPO_NAME $BRANCH $AIRTABLE_API_KEY | |
| rm update_metadata_to_airtable.py | |
| - name: Update README file | |
| id: update-readme-file | |
| env: | |
| MODEL_ID: ${{ github.event.repository.name }} | |
| run: | | |
| source activate | |
| echo "Updating README file with AirTable metadata for model: $MODEL_ID" | |
| wget https://raw.githubusercontent.com/ersilia-os/ersilia/master/.github/scripts/update_readme_from_airtable.py | |
| python3 update_readme_from_airtable.py $MODEL_ID . | |
| rm update_readme_from_airtable.py | |
| less README.md | |
| - name: Commit and push changes done to the README file | |
| uses: actions-js/push@156f2b10c3aa000c44dbe75ea7018f32ae999772 # [email protected] | |
| with: | |
| author_name: "ersilia-bot" | |
| author_email: "[email protected]" | |
| message: "updating readme [skip ci]" | |
| repository: "ersilia-os/${{ github.event.repository.name }}" | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| amend: true | |
| force: true | |
| - name: Docker Hub Description | |
| uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_PASSWORD }} | |
| repository: ersiliaos/${{ github.event.repository.name }} | |
| short-description: "Ersilia Model Hub Identifier: ${{ github.event.repository.name }}" | |
| # Create an issue within the repository to track that this model is ready for testing | |
| - name: Shuffle assignees | |
| id: shuffle | |
| run: | | |
| export assignees=$(echo "${{ vars.assignees }}" | awk 'BEGIN {FS=","}; {srand();split($0,a,FS); print a[int(rand()*NF+1)]}') | |
| echo "$assignees" >> $GITHUB_STEP_SUMMARY | |
| echo "shuffled_assignee=$assignees" >> $GITHUB_OUTPUT | |
| echo "shuffled_assignee=$assignees" >> $GITHUB_ENV | |
| - name: Check for existing issue | |
| id: check_existing_test_issue | |
| run: | | |
| gh auth login --with-token <<< ${{ secrets.GITHUB_TOKEN }} | |
| issue_number=$(gh issue list --limit 100 --search "${{ vars.test_issue_title }}" --json number --jq '.[0].number') | |
| echo "::set-output name=issue_number::$issue_number" | |
| - name: Create a Test issue | |
| uses: actions-ecosystem/action-create-issue@b63bc2bbacb6a838dfe4a9f70da6665ae0962a49 | |
| id: create_test_issue | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| title: ${{ vars.TEST_ISSUE_TITLE }} | |
| assignees: | | |
| ${{ steps.shuffle.outputs.shuffled_assignee }} | |
| body: | | |
| This model is ready for testing. If you are assigned to this issue, please try it out using the CLI, Google Colab and DockerHub and let us know if it works! | |
| labels: | | |
| test | |
| if: steps.check_issue.outputs.issue_number == '' |