From 34e9d60eb25d920dd1cb33bc8f339693bf3bd6a2 Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Fri, 4 Oct 2024 16:58:07 +0500 Subject: [PATCH 1/4] perf: optimised to handle dependecies cache --- dockerfiles/registrar.Dockerfile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/dockerfiles/registrar.Dockerfile b/dockerfiles/registrar.Dockerfile index 0ff55fe..5e71860 100644 --- a/dockerfiles/registrar.Dockerfile +++ b/dockerfiles/registrar.Dockerfile @@ -61,10 +61,6 @@ ENV REGISTRAR_CODE_DIR ${REGISTRAR_CODE_DIR} # Working directory will be root of repo. WORKDIR ${REGISTRAR_CODE_DIR} -# cloning git repo -RUN curl -L https://github.com/edx/registrar/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 - - RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${REGISTRAR_VENV_DIR} RUN pip install --upgrade pip setuptools @@ -77,7 +73,13 @@ EXPOSE 18735 FROM app as dev -RUN pip install --no-cache-dir -r requirements/devstack.txt +# fetching the requirement file that is needed +RUN curl -L -o devstack.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/devstack.txt + +RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/devstack.txt + +# cloning the repository after requirements installation +RUN curl -L https://github.com/edx/registrar/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 ENV DJANGO_SETTINGS_MODULE registrar.settings.devstack @@ -85,7 +87,13 @@ CMD while true; do python ./manage.py runserver 0.0.0.0:18734; sleep 2; done FROM app as prod -RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/requirements/production.txt +# fetching the requirement file that is needed +RUN curl -L -o production.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/production.txt + +RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/production.txt + +# cloning the repository after requirements installation +RUN curl -L https://github.com/edx/registrar/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 ENV DJANGO_SETTINGS_MODULE registrar.settings.production From 4b9b6136c5d15d3380c722923874028c7aa3f8ca Mon Sep 17 00:00:00 2001 From: Hunia Fatima Date: Mon, 7 Oct 2024 18:09:28 +0500 Subject: [PATCH 2/4] fix: incorporated review request --- dockerfiles/registrar.Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dockerfiles/registrar.Dockerfile b/dockerfiles/registrar.Dockerfile index 5e71860..1581234 100644 --- a/dockerfiles/registrar.Dockerfile +++ b/dockerfiles/registrar.Dockerfile @@ -61,6 +61,8 @@ ENV REGISTRAR_CODE_DIR ${REGISTRAR_CODE_DIR} # Working directory will be root of repo. WORKDIR ${REGISTRAR_CODE_DIR} +RUN mkdir -p requirements + RUN virtualenv -p python${PYTHON_VERSION} --always-copy ${REGISTRAR_VENV_DIR} RUN pip install --upgrade pip setuptools @@ -74,9 +76,9 @@ EXPOSE 18735 FROM app as dev # fetching the requirement file that is needed -RUN curl -L -o devstack.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/devstack.txt +RUN curl -L -o requirements/devstack.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/devstack.txt -RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/devstack.txt +RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/requirements/devstack.txt # cloning the repository after requirements installation RUN curl -L https://github.com/edx/registrar/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 @@ -88,9 +90,9 @@ CMD while true; do python ./manage.py runserver 0.0.0.0:18734; sleep 2; done FROM app as prod # fetching the requirement file that is needed -RUN curl -L -o production.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/production.txt +RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/edx/registrar/master/requirements/production.txt -RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/production.txt +RUN pip install --no-cache-dir -r ${REGISTRAR_CODE_DIR}/requirements/production.txt # cloning the repository after requirements installation RUN curl -L https://github.com/edx/registrar/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 From 9117645b42797ac110231dca1a182625a4f56c56 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Mon, 7 Oct 2024 13:13:36 +0500 Subject: [PATCH 3/4] feat: add workflow and dockerfile for credentials --- .github/workflows/push-credentials-image.yaml | 58 +++++++++ dockerfiles/credentials.Dockerfile | 111 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 .github/workflows/push-credentials-image.yaml create mode 100644 dockerfiles/credentials.Dockerfile diff --git a/.github/workflows/push-credentials-image.yaml b/.github/workflows/push-credentials-image.yaml new file mode 100644 index 0000000..7b6198f --- /dev/null +++ b/.github/workflows/push-credentials-image.yaml @@ -0,0 +1,58 @@ +name: Build and Push Credentials Image + +on: + workflow_dispatch: + inputs: + branch: + description: "Target branch from which the source dockerfile from image will be sourced" + + schedule: + - cron: "0 4 * * 1-5" # UTC Time + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + + steps: + - name: Get tag name + id: get-tag-name + uses: actions/github-script@v5 + with: + script: | + const tagName = "${{ github.event.inputs.branch }}" || 'latest'; + console.log('Will use tag: ' + tagName); + return tagName; + result-encoding: string + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push Dev Docker image + uses: docker/build-push-action@v6 + with: + file: ./dockerfiles/credentials.Dockerfile + push: true + target: dev + tags: edxops/credentials-dev:${{ steps.get-tag-name.outputs.result }} + + - name: Send failure notification + if: failure() + uses: dawidd6/action-send-mail@v3 + with: + server_address: email-smtp.us-east-1.amazonaws.com + server_port: 465 + username: ${{secrets.edx_smtp_username}} + password: ${{secrets.edx_smtp_password}} + subject: Push Image to docker.io/edxops failed in credentials + to: team-cosmonauts@edx.org + from: github-actions + body: Push Image to docker.io/edxops for credentials failed! For details see "github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/dockerfiles/credentials.Dockerfile b/dockerfiles/credentials.Dockerfile new file mode 100644 index 0000000..71ecafb --- /dev/null +++ b/dockerfiles/credentials.Dockerfile @@ -0,0 +1,111 @@ +FROM ubuntu:focal as base + +# System requirements +# - git; Used to pull in particular requirements from github rather than pypi, +# and to check the sha of the code checkout. +# - language-pack-en locales; ubuntu locale support so that system utilities have a consistent +# language and time zone. +# - python; ubuntu doesnt ship with python, so this is the python we will use to run the application +# - python3-pip; install pip to install application requirements.txt files +# - libssl-dev; # mysqlclient wont install without this. +# - libmysqlclient-dev; to install header files needed to use native C implementation for +# MySQL-python for performance gains. +# - wget; to download a watchman binary archive +# - unzip; to unzip a watchman binary archive +# - pkg-config; mysqlclient>=2.2.0 requires pkg-config (https://github.com/PyMySQL/mysqlclient/issues/620) + +# If you add a package here please include a comment above describing what it is used for +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa && apt-get update && \ + apt-get upgrade -qy && apt-get install language-pack-en locales gettext git \ + python3.11-dev python3.11-venv libmysqlclient-dev libssl-dev build-essential wget unzip pkg-config curl -qy && \ + rm -rf /var/lib/apt/lists/* + +# Create Python env +ENV VIRTUAL_ENV=/edx/app/credentials/venvs/credentials +RUN python3.11 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +# Create Node env +RUN pip install nodeenv +ENV NODE_ENV=/edx/app/credentials/nodeenvs/credentials +RUN nodeenv $NODE_ENV --node=18.17.1 --prebuilt +ENV PATH="$NODE_ENV/bin:$PATH" +RUN npm install -g npm@9.x.x + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV DJANGO_SETTINGS_MODULE credentials.settings.production +ENV OPENEDX_ATLAS_PULL true +ENV CREDENTIALS_CFG "minimal.yml" + +EXPOSE 18150 +RUN useradd -m --shell /bin/false app + +# Install watchman +RUN wget https://github.com/facebook/watchman/releases/download/v2023.11.20.00/watchman-v2023.11.20.00-linux.zip +RUN unzip watchman-v2023.11.20.00-linux.zip +RUN mkdir -p /usr/local/{bin,lib} /usr/local/var/run/watchman +RUN cp watchman-v2023.11.20.00-linux/bin/* /usr/local/bin +RUN cp watchman-v2023.11.20.00-linux/lib/* /usr/local/lib +RUN chmod 755 /usr/local/bin/watchman +RUN chmod 2777 /usr/local/var/run/watchman + +# Now install credentials +WORKDIR /edx/app/credentials/credentials + +# fetching the requirement files that are needed +RUN mkdir -p requirements +RUN curl -L -o requirements/pip_tools.txt https://raw.githubusercontent.com/openedx/credentials/master/requirements/pip_tools.txt +RUN curl -L -o requirements/production.txt https://raw.githubusercontent.com/openedx/credentials/master/requirements/production.txt + +# Dependencies are installed as root so they cannot be modified by the application user. +RUN pip install -r requirements/pip_tools.txt +RUN pip install -r requirements/production.txt + +RUN mkdir -p /edx/var/log +# Cloning git repo +# This line is after the python requirements so that changes to the code will not +# bust the image cache +RUN curl -L https://github.com/openedx/credentials/archive/refs/heads/master.tar.gz | tar -xz --strip-components=1 + +# Fetch the translations into the image once the Makefile's in place +RUN make pull_translations + +# Install dependencies in node_modules directory +RUN npm install --no-save +ENV NODE_BIN=/edx/app/credentials/credentials/node_modules +ENV PATH="$NODE_BIN/.bin:$PATH" +# Run webpack +RUN webpack --config webpack.config.js + +# Change static folder owner to application user. +RUN chown -R app:app /edx/app/credentials/credentials/credentials/static + +# Code is owned by root so it cannot be modified by the application user. +# So we copy it before changing users. +USER app + +# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. +CMD gunicorn --workers=2 --name credentials -c /edx/app/credentials/credentials/credentials/docker_gunicorn_configuration.py --log-file - --max-requests=1000 credentials.wsgi:application + +# We don't switch back to the app user for devstack because we need devstack users to be +# able to update requirements and generally run things as root. +FROM base as dev +USER root +ENV DJANGO_SETTINGS_MODULE credentials.settings.devstack +RUN pip install -r /edx/app/credentials/credentials/requirements/dev.txt +RUN make pull_translations + +# Temporary compatibility hack while devstack is supporting +# both the old `edxops/credentials` image and this image: +# Add in a dummy ../credentials_env file. +# The credentials_env file was originally needed for sourcing to get +# environment variables like DJANGO_SETTINGS_MODULE, but now we just set +# those variables right in the Dockerfile. +RUN touch ../credentials_env + +CMD while true; do python ./manage.py runserver 0.0.0.0:18150; sleep 2; done From d542e8e5caca3caea6286da88c364d0b986611d5 Mon Sep 17 00:00:00 2001 From: Muhammad Umar Khan Date: Mon, 14 Oct 2024 12:14:59 +0500 Subject: [PATCH 4/4] fix: add platform variable for credentials workflow --- .github/workflows/push-credentials-image.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/push-credentials-image.yaml b/.github/workflows/push-credentials-image.yaml index 7b6198f..f565249 100644 --- a/.github/workflows/push-credentials-image.yaml +++ b/.github/workflows/push-credentials-image.yaml @@ -43,6 +43,7 @@ jobs: push: true target: dev tags: edxops/credentials-dev:${{ steps.get-tag-name.outputs.result }} + platforms: linux/amd64,linux/arm64 - name: Send failure notification if: failure()