Merge pull request #1 from Gowtham1729/feature/fetcher

Add Data Fetcher skeleton

.github/workflows/build-fetcher.yaml

@@ -0,0 +1,73 @@
+name: Build Fetcher
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - "applications/data_fetcher/**"
+
+concurrency:
+    group: build-group
+    cancel-in-progress: false
+
+env:
+  PROJECT_ID: ${{ secrets.GKE_PROJECT }}
+  GAR_LOCATION: us-central1
+
+jobs:
+  setup-build-publish:
+    name: Setup, Build, Publish
+    runs-on: ubuntu-latest
+    environment: production
+
+    permissions:
+      contents: 'write'
+      id-token: 'write'
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v4
+      with:
+        token: ${{ secrets.PAT_TOKEN }}
+
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v4
+      with:
+        python-version: "3.11"
+
+    - name: Set up Taskfile
+      uses: arduino/[email protected]
+
+    - name: Run Lint Check
+      run: task lint-check
+
+    - name: Run all tests
+      run: task test
+
+      # Authentication to GCP via credentials json
+    - id: 'auth'
+      uses: 'google-github-actions/auth@v1'
+      with:
+        credentials_json: '${{ secrets.GCP_CREDENTIALS }}'
+
+    - name: Docker configuration
+      run: gcloud auth configure-docker ${{ env.GAR_LOCATION }}-docker.pkg.dev --quiet
+
+    - name: Build
+      run: task build-fetcher-image tag="$GITHUB_SHA"
+
+    # Push the Docker image to Google Artifact Registry
+    - name: Publish
+      run: task push-fetcher-image tag="$GITHUB_SHA"
+
+    - name: Replace Fetcher Image Tag
+      run: sed -i "s/\(&fetcherImageTag\) [^[:space:]]*/\1 $GITHUB_SHA/g" infrastructure/k8s/folio-feed-helm/values.yaml
+
+    - name: Commit the Changes
+      run: |
+        git config user.name "Github Actions"
+        git config user.email [email protected]
+        git diff
+        git add infrastructure/k8s/folio-feed-helm/values.yaml
+        git commit -m "fetcher image update: $GITHUB_SHA"
+        git push origin main

Taskfile.yaml

@@ -53,6 +53,27 @@ tasks:
     cmds:
       - cd applications/frontend && npm run dev
 
+  install-fetcher:
+    desc: Install poetry and dependencies for the Data Fetcher.
+    cmds:
+      - pip3 show poetry --quiet || pip3 install poetry
+      - cd applications/data-fetcher && poetry install
+
+  run-fetcher:
+    desc: Run the Data Fetcher.
+    deps:
+      - install-fetcher
+    cmds:
+      - cd applications/data_fetcher && poetry run python fetcher.py
+
+  build-fetcher-image:
+    desc: Build a Data Fetcher docker image.
+    cmd: cd applications/data_fetcher && docker build -t us-central1-docker.pkg.dev/folio-feed-403709/image-registry/folio-feed-fetcher:{{.tag}} .
+
+  push-fetcher-image:
+    desc: Push a Data Fetcher docker image to the image registry.
+    cmd: docker push us-central1-docker.pkg.dev/folio-feed-403709/image-registry/folio-feed-fetcher:{{.tag}}
+
   install-lint:
     desc: Install isort, black and mypy.
     cmds:

applications/backend/django_server/django_server/urls.py

@@ -22,6 +22,6 @@
 urlpatterns = [
     path("api/admin/", admin.site.urls),
     path("healthz/", views.healthz, name="healthz"),
-    path("", include("django_prometheus.urls")),
+    path("", include("django_prometheus.urls")),  # collect metrics at /metrics
     path("api/news/", include("news.urls")),
 ]

applications/data_fetcher/Dockerfile

@@ -0,0 +1,28 @@
+FROM --platform=linux/amd64 python:3.11-buster as builder
+
+RUN pip install poetry==1.6.1
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1 \
+    POETRY_CACHE_DIR=/tmp/poetry_cache
+
+WORKDIR /app
+
+COPY pyproject.toml poetry.lock ./
+
+RUN --mount=type=cache,target=$POETRY_CACHE_DIR poetry install --without dev
+
+FROM --platform=linux/amd64 python:3.11-slim-buster as runtime
+
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+COPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}
+
+WORKDIR /app
+COPY fetcher.py /app
+
+EXPOSE 8000
+
+CMD ["python", "fetcher.py"]

applications/data_fetcher/fetcher.py

@@ -0,0 +1 @@
+print(f"Fetcher is running...")

applications/data_fetcher/pyproject.toml

@@ -0,0 +1,25 @@
+[tool.poetry]
+name = "data-fetcher"
+version = "0.1.0"
+description = ""
+authors = ["gowtham1729 <[email protected]>"]
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.11"
+requests = "^2.31.0"
+click = "^8.1.7"
+psycopg = "^3.1.13"
+pika = "^1.3.2"
+
+[tool.poetry.group.dev.dependencies]
+isort = "^5.12.0"
+black = "^23.11.0"
+mypy = "^1.7.1"
+pytest = "^7.4.3"
+pytest-cov = "^4.1.0"
+pytest-xdist = "^3.5.0"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

infrastructure/k8s/folio-feed-helm/templates/backend-deployment.yaml

@@ -5,6 +5,9 @@ metadata:
   labels:
     {{- include "folio-feed.labels" . | nindent 4 }}
     app: backend
+  annotations:
+    prometheus.io/scrape: 'true'
+    prometheus.io/port: '8000'
 spec:
   {{- if not .Values.backend.autoscaling.enabled }}
   replicas: {{ .Values.backend.replicaCount }}

infrastructure/k8s/folio-feed-helm/templates/backend-monitoring.yaml

@@ -11,5 +11,5 @@ spec:
       {{- include "folio-feed.selectorLabels" . | nindent 6 }}
       app: backend
   endpoints:
-  - port: metrics
+  - port: http
     interval: 30s

infrastructure/k8s/folio-feed-helm/values.yaml

@@ -45,6 +45,12 @@ frontend:
       cpu: 250m
       memory: 512Mi
 
+fetcher:
+  image:
+    registry: us-central1-docker.pkg.dev/folio-feed-403709/image-registry
+    repository: folio-feed-fetcher
+    tag: &fetcherImageTag latest
+
 
 
 

infrastructure/k8s/other-resources/gmp-operator-config.yaml

@@ -0,0 +1,8 @@
+apiVersion: monitoring.googleapis.com/v1
+kind: OperatorConfig
+metadata:
+  namespace: gmp-public
+  name: config
+features:
+  targetStatus:
+    enabled: true

infrastructure/terraform/gke_cluster.tf

@@ -13,11 +13,11 @@ resource "google_container_cluster" "folio-feed-cluster" {
     }
     enable_components = [
       "SYSTEM_COMPONENTS",
-      "STORAGE",
-      "HPA",
-      "POD",
-      "DEPLOYMENT",
-      "STATEFULSET"
+#      "STORAGE",
+#      "HPA",
+#      "POD",
+#      "DEPLOYMENT",
+#      "STATEFULSET"
     ]
   }
 }