From 9b2638a16ac119abb21e2afff3c39d250ea0700e Mon Sep 17 00:00:00 2001 From: Asif Khatri Date: Thu, 8 Aug 2024 14:29:47 +0530 Subject: [PATCH] LIVY-979: A helm-based integration environment for debugging Livy on Kubernetes --- .rat-excludes | 3 + dev/helmchart/Chart.yaml | 31 ++++ dev/helmchart/Docker.md | 81 ++++++++++ dev/helmchart/README.md | 76 +++++++++ dev/helmchart/requirements.lock | 9 ++ dev/helmchart/requirements.yaml | 32 ++++ dev/helmchart/templates/NOTES.txt | 13 ++ dev/helmchart/templates/_helpers.tpl | 32 ++++ .../templates/certificate-default.yaml | 28 ++++ dev/helmchart/templates/clusterissuers.yaml | 18 +++ dev/helmchart/templates/config.yaml | 55 +++++++ dev/helmchart/templates/ingress.yaml | 88 ++++++++++ dev/helmchart/templates/pvc.yaml | 29 ++++ dev/helmchart/templates/rbac-bindings.yaml | 26 +++ dev/helmchart/templates/rbac-roles.yaml | 22 +++ dev/helmchart/templates/rbac.yaml | 87 ++++++++++ dev/helmchart/templates/service-headless.yaml | 16 ++ dev/helmchart/templates/service.yaml | 38 +++++ dev/helmchart/templates/serviceaccount.yaml | 48 ++++++ dev/helmchart/templates/statefulset.yaml | 152 ++++++++++++++++++ dev/helmchart/templates/storageclass.yaml | 32 ++++ dev/helmchart/values.yaml | 95 +++++++++++ 22 files changed, 1011 insertions(+) create mode 100755 dev/helmchart/Chart.yaml create mode 100644 dev/helmchart/Docker.md create mode 100644 dev/helmchart/README.md create mode 100644 dev/helmchart/requirements.lock create mode 100755 dev/helmchart/requirements.yaml create mode 100755 dev/helmchart/templates/NOTES.txt create mode 100755 dev/helmchart/templates/_helpers.tpl create mode 100755 dev/helmchart/templates/certificate-default.yaml create mode 100755 dev/helmchart/templates/clusterissuers.yaml create mode 100644 dev/helmchart/templates/config.yaml create mode 100644 dev/helmchart/templates/ingress.yaml create mode 100644 dev/helmchart/templates/pvc.yaml create mode 100755 dev/helmchart/templates/rbac-bindings.yaml create mode 100755 dev/helmchart/templates/rbac-roles.yaml create mode 100644 dev/helmchart/templates/rbac.yaml create mode 100644 dev/helmchart/templates/service-headless.yaml create mode 100644 dev/helmchart/templates/service.yaml create mode 100644 dev/helmchart/templates/serviceaccount.yaml create mode 100644 dev/helmchart/templates/statefulset.yaml create mode 100755 dev/helmchart/templates/storageclass.yaml create mode 100755 dev/helmchart/values.yaml diff --git a/.rat-excludes b/.rat-excludes index b22edb294..c29cad203 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -30,3 +30,6 @@ logs/* docs/**/*.html docs/**/JB/** venv/* +dev/helmchart/templates/*.yaml +dev/helmchart/templates/*.tpl +dev/helmchart/requirements.lock \ No newline at end of file diff --git a/dev/helmchart/Chart.yaml b/dev/helmchart/Chart.yaml new file mode 100755 index 000000000..abe4d702f --- /dev/null +++ b/dev/helmchart/Chart.yaml @@ -0,0 +1,31 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +name: livycluster +version: 1.0.0 +appVersion: v0.9.0-incubating-SNAPSHOT +home: https://github.com/askhatri/livycluster/ +description: Apache Livy server to run Spark on Kubernetes +keywords: + - livy + - spark +sources: + - https://github.com/askhatri/livycluster/ +maintainers: +- name: apache + email: livy@apache.org diff --git a/dev/helmchart/Docker.md b/dev/helmchart/Docker.md new file mode 100644 index 000000000..3a9c771d3 --- /dev/null +++ b/dev/helmchart/Docker.md @@ -0,0 +1,81 @@ +## Steps to Create Docker Image for Spark with Python Binding + +### 1. Download Spark Binary + +Use the following command to download and extract the Spark binary: + +```sh +wget https://archive.apache.org/dist/spark/spark-3.2.3/spark-3.2.3-bin-hadoop3.2.tgz +tar -xzf spark-3.2.3-bin-hadoop3.2.tgz +``` + +### 2. Build and Push Docker Image + +Use the following commands to build and push the Docker image: + +```sh +./spark-3.2.3-bin-hadoop3.2/bin/docker-image-tool.sh -r -t v3.2.3 -p kubernetes/dockerfiles/spark/bindings/python/Dockerfile build +./spark-3.2.3-bin-hadoop3.2/bin/docker-image-tool.sh -r -t v3.2.3 -p kubernetes/dockerfiles/spark/bindings/python/Dockerfile push +``` + +Replace `` with your Docker repository name. + +## Steps to Create Docker Image for Livy + +### 1. Build Livy Code + +Build the Livy code using the following commands: + +```sh +cd incubator-livy +mvn -Pthriftserver -Pscala-2.12 -Pspark3 package +``` + +Copy the generated Livy binary into the `/tmp` directory: + +```sh +cp assembly/target/apache-livy-0.9.0-incubating-SNAPSHOT_2.12-bin.zip /tmp +``` + +### 2. Create Dockerfile + +Create a `Dockerfile` in the `/tmp` directory with the following content: + +```Dockerfile +FROM /spark-py:v3.2.3 +ENV LIVY_VERSION 0.9.0-incubating-SNAPSHOT +ENV LIVY_PACKAGE apache-livy-${LIVY_VERSION}_2.12-bin +ENV LIVY_HOME /opt/livy +ENV LIVY_CONF_DIR /conf +ENV PATH $PATH:$LIVY_HOME/bin + +USER root + +COPY $LIVY_PACKAGE.zip / +RUN apt-get update && apt-get install -y unzip && \ + unzip /$LIVY_PACKAGE.zip -d / && \ + mv /$LIVY_PACKAGE /opt/ && \ + rm -rf $LIVY_HOME && \ + ln -s /opt/$LIVY_PACKAGE $LIVY_HOME && \ + rm -f /$LIVY_PACKAGE.zip + +RUN mkdir /var/log/livy && \ + ln -s /var/log/livy $LIVY_HOME/logs + +WORKDIR $LIVY_HOME + +ENTRYPOINT [ "livy-server" ] +``` + +Replace `` with your Docker repository name. + +### 3. Build and Push Docker Image + +Use the following commands to build and push the Docker image: + +```sh +cd /tmp +docker build -t /livy:spark3.2.3 . +docker push /livy:spark3.2.3 +rm -f apache-livy-0.9.0-incubating-SNAPSHOT_2.12-bin.zip +``` diff --git a/dev/helmchart/README.md b/dev/helmchart/README.md new file mode 100644 index 000000000..8f7676fc1 --- /dev/null +++ b/dev/helmchart/README.md @@ -0,0 +1,76 @@ +# Apache Livy Deployed on Kubernetes using Docker Desktop + +This guide provides a Helm chart to deploy Livy on Kubernetes without relying on cloud services like AWS, GCP, or Azure. This setup can save development time and cost, and it allows debugging using an IDE. For debugging Livy on Kubernetes as a standalone setup, Apache Spark and Apache Livy must be deployed in Kubernetes. + +## Docker Desktop Installation +1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/). +2. Enable Kubernetes in Docker Desktop settings. + +## Helm Installation +1. Install [Helm](https://helm.sh/docs/intro/install/). +2. Add the required Helm chart repositories: + ```shell + helm repo add cert-manager https://charts.jetstack.io + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo update + ``` + +3. Add an entry to the `/etc/hosts` file: + ```text + 127.0.0.1 my-cluster.example.com + ``` + +4. Install the cert-manager CustomResourceDefinition resources: + ```shell + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/v1.15.0/cert-manager.yaml + ``` + +**References:** +- [cert-manager ACME nginx-ingress tutorial](https://cert-manager.io/docs/tutorials/acme/nginx-ingress/) +- [Artifact Hub cert-manager Helm chart](https://artifacthub.io/packages/helm/cert-manager/cert-manager) + +## Livy Cluster Deployment +1. Build the Helm chart using the following command: + ```bash + helm dependency build + ``` + +2. Create a Kubernetes namespace for the Livy deployment: + ```bash + kubectl create namespace + ``` + +3. Install the Livy cluster using the Helm chart: + ```bash + helm -n install livycluster . + ``` + +## Livy REST API Testing +1. Create an interactive session: + ```shell + curl -k -X POST -H "Content-Type: application/json" --data '{"kind": "spark"}' https://my-cluster.example.com/livy/sessions | jq + ``` + **Note:** You need `curl` and `jq` utilities installed on your local machine for testing. + +2. Create a statement: + ```shell + curl -k -X POST -d '{ "kind": "spark", "code": "sc.parallelize(1 to 10).count()" }' -H "Content-Type: application/json" \ + https://my-cluster.example.com/livy/sessions/0/statements | jq + ``` + +3. Create a batch job: + ```shell + curl -s -k -H "Content-Type: application/json" \ + -X POST \ + -d '{ + "name": "testbatch1", + "className": "org.apache.spark.examples.SparkPi", + "numExecutors": 2, + "file": "local:///opt/spark/examples/jars/spark-examples_2.12-3.2.3.jar", + "args": ["10000"] + }' "https://my-cluster.example.com/livy/batches" | jq + ``` + +## Steps to Create Docker Images + +Steps to create Docker images for Spark and Livy are documented at [Docker.md](Docker.md). \ No newline at end of file diff --git a/dev/helmchart/requirements.lock b/dev/helmchart/requirements.lock new file mode 100644 index 000000000..9e1bda968 --- /dev/null +++ b/dev/helmchart/requirements.lock @@ -0,0 +1,9 @@ +dependencies: +- name: ingress-nginx + repository: https://kubernetes.github.io/ingress-nginx + version: 4.10.1 +- name: cert-manager + repository: https://charts.jetstack.io + version: v1.15.0 +digest: sha256:5c77152ce85e9dc2aadb09976df11286ab53147286cf2ccb93caec0e707bead6 +generated: "2024-06-19T11:16:02.896968+05:30" diff --git a/dev/helmchart/requirements.yaml b/dev/helmchart/requirements.yaml new file mode 100755 index 000000000..76efb1d01 --- /dev/null +++ b/dev/helmchart/requirements.yaml @@ -0,0 +1,32 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +dependencies: + +- name: ingress-nginx + version: 4.10.1 + repository: "https://kubernetes.github.io/ingress-nginx" + condition: nginx-ingress.enabled,global.nginx-ingress.enabled + tags: + - ingress + +- name: cert-manager + version: v1.15.0 + repository: "https://charts.jetstack.io" + condition: cert-manager.enabled,global.cert-manager.enabled + tags: + - ingress diff --git a/dev/helmchart/templates/NOTES.txt b/dev/helmchart/templates/NOTES.txt new file mode 100755 index 000000000..66029c0a0 --- /dev/null +++ b/dev/helmchart/templates/NOTES.txt @@ -0,0 +1,13 @@ +The Livy server has been installed. +Check its status by running: + kubectl -n {{ .Release.Namespace }} get pods -w +Connect to the Livy Web UI: +{{- if .Values.ingress.enabled }} + # Open in browser: + {{- range .Values.ingress.hosts }} + # https://{{ . }}/livy/ui + {{- end }} +{{- else }} + kubectl -n {{ .Release.Namespace }} port-forward {{ include "livycluster.fullname" . }}-0 8998 + # Open in browser: http://localhost:8998 +{{- end }} diff --git a/dev/helmchart/templates/_helpers.tpl b/dev/helmchart/templates/_helpers.tpl new file mode 100755 index 000000000..a460594bf --- /dev/null +++ b/dev/helmchart/templates/_helpers.tpl @@ -0,0 +1,32 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "livycluster.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "livycluster.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "livycluster.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} diff --git a/dev/helmchart/templates/certificate-default.yaml b/dev/helmchart/templates/certificate-default.yaml new file mode 100755 index 000000000..10b9d012f --- /dev/null +++ b/dev/helmchart/templates/certificate-default.yaml @@ -0,0 +1,28 @@ +{{- if .Values.defaultCertificate.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "livycluster.fullname" . }}-default-tls + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + # The name of the Kubernetes secret resource to store the signed TLS keypair + secretName: {{ .Values.defaultCertificate.secretName }} + # The Issuer to use for this certificate + issuerRef: + name: {{ .Values.defaultCertificate.issuer }} + kind: {{ .Values.defaultCertificate.issuerKind }} + # The common name (CN) for the TLS certificate +{{- if .Values.defaultCertificate.commonName }} + commonName: {{ squote .Values.defaultCertificate.commonName }} +{{- end }} + # A list of domains to include on the TLS certificate + dnsNames: + {{- range $key, $val := .Values.defaultCertificate.domains }} + - {{ squote $val }} + {{- end }} +{{- end }} diff --git a/dev/helmchart/templates/clusterissuers.yaml b/dev/helmchart/templates/clusterissuers.yaml new file mode 100755 index 000000000..5d4e12ecc --- /dev/null +++ b/dev/helmchart/templates/clusterissuers.yaml @@ -0,0 +1,18 @@ +{{- $name := include "livycluster.name" . -}} +{{- $chart := include "livycluster.chart" . -}} +{{- $releaseName := .Release.Name -}} +{{- $releaseService := .Release.Service -}} +{{- range .Values.clusterIssuers }} +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ .name }} + labels: + app.kubernetes.io/name: {{ $name }} + helm.sh/chart: {{ $chart }} + app.kubernetes.io/instance: {{ $releaseName }} + app.kubernetes.io/managed-by: {{ $releaseService }} +spec: +{{ toYaml .spec | indent 2 }} +{{- end }} diff --git a/dev/helmchart/templates/config.yaml b/dev/helmchart/templates/config.yaml new file mode 100644 index 000000000..da7b0bd04 --- /dev/null +++ b/dev/helmchart/templates/config.yaml @@ -0,0 +1,55 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-config + namespace: {{ .Release.Namespace }} +data: + livy.conf: |- + livy.file.local-dir-whitelist=/ + livy.server.kubernetes.ingress.className=nginx + livy.server.kubernetes.ingress.create=true + livy.server.kubernetes.ingress.host=my-cluster.example.com + livy.server.kubernetes.ingress.protocol=https + livy.server.port=8998 + livy.server.recovery.mode=recovery + livy.server.recovery.state-store.url=/tmp/livy/store + livy.server.recovery.state-store=filesystem + livy.server.session.state-retain.sec = 4h + livy.spark.deploy-mode=cluster + livy.spark.master=k8s://kubernetes.default.svc.cluster.local:443 + livy.ui.basePath=/livy + livy.ui.history-server-url=https://my-cluster.example.com/historyserver + livy-client.conf: |- + livy.rsc.client.connect.timeout=600s + livy.rsc.rpc.server.address={{ include "livycluster.fullname" . }}-0.{{ include "livycluster.fullname" . }}-headless.{{ .Release.Namespace }}.svc.cluster.local + livy.rsc.server.connect.timeout=600s + log4j.properties: |- + log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n + log4j.appender.console.layout=org.apache.log4j.PatternLayout + log4j.appender.console.target=System.err + log4j.appender.console=org.apache.log4j.ConsoleAppender + log4j.logger.org.eclipse.jetty=WARN + log4j.rootCategory=DEBUG, console + spark-defaults.conf: |- + spark.driver.extraJavaOptions=--add-opens java.base/jdk.internal.misc=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true + spark.eventLog.dir=file:///tmp/livy/store + spark.eventLog.enabled=true + spark.executor.extraJavaOptions=--add-opens java.base/jdk.internal.misc=ALL-UNNAMED -Dio.netty.tryReflectionSetAccessible=true + spark.kryo.registrationRequired=false + spark.kryo.unsafe=false + spark.kubernetes.authenticate.driver.serviceAccountName={{ include "livycluster.fullname" . }}-spark + spark.kubernetes.container.image.pullPolicy={{ .Values.image.spark.pullPolicy }} + spark.kubernetes.container.image={{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }} + spark.kubernetes.driver.label.name=driver + spark.kubernetes.driver.volumes.persistentVolumeClaim.datafiles.mount.path=/tmp/livy/store + spark.kubernetes.driver.volumes.persistentVolumeClaim.datafiles.options.claimName={{ include "livycluster.fullname" . }} + spark.kubernetes.executor.label.name=executor + spark.kubernetes.executor.volumes.persistentVolumeClaim.datafiles.mount.path=/tmp/livy/store + spark.kubernetes.executor.volumes.persistentVolumeClaim.datafiles.options.claimName={{ include "livycluster.fullname" . }} + spark.kubernetes.file.upload.path=file:///tmp/livy/store + spark.kubernetes.namespace={{ .Release.Namespace }} + spark.network.timeout=800 + spark.serializer=org.apache.spark.serializer.KryoSerializer + spark.speculation=false + spark.sql.broadcastTimeout=1200 + spark.sql.warehouse.dir=/opt/spark/spark-warehouse diff --git a/dev/helmchart/templates/ingress.yaml b/dev/helmchart/templates/ingress.yaml new file mode 100644 index 000000000..0dd97c66d --- /dev/null +++ b/dev/helmchart/templates/ingress.yaml @@ -0,0 +1,88 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} +{{- end }} +spec: + ingressClassName: nginx +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: {{ $.Values.ingress.path }} + pathType: Prefix + backend: + service: + name: {{ include "livycluster.fullname" $ }} + port: + number: 80 + {{- end }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: spark-history-ingress + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + nginx.ingress.kubernetes.io/configuration-snippet: | + proxy_set_header Accept-Encoding ""; + sub_filter_last_modified off; + sub_filter 'href="/' 'href="/historyserver/'; + sub_filter 'src="/' 'src="/historyserver/'; + sub_filter_once off; + sub_filter_types text/html text/css text/javascript application/javascript; + nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/service-upstream: "true" + nginx.ingress.kubernetes.io/x-forwarded-prefix: /historyserver +spec: + ingressClassName: nginx +{{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} +{{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ . | quote }} + http: + paths: + - path: /historyserver/?(.*) + pathType: Prefix + backend: + service: + name: spark-history-service + port: + number: 18080 + {{- end }} diff --git a/dev/helmchart/templates/pvc.yaml b/dev/helmchart/templates/pvc.yaml new file mode 100644 index 000000000..e6776c23a --- /dev/null +++ b/dev/helmchart/templates/pvc.yaml @@ -0,0 +1,29 @@ +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} +{{- with .Values.persistence.annotations }} + annotations: +{{ toYaml . | indent 4 }} +{{- end }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} +{{- with .Values.persistence.storageClass }} +{{- if (eq "-" .) }} + storageClassName: "" +{{- else }} + storageClassName: "{{ . }}" +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/dev/helmchart/templates/rbac-bindings.yaml b/dev/helmchart/templates/rbac-bindings.yaml new file mode 100755 index 000000000..6190f34f4 --- /dev/null +++ b/dev/helmchart/templates/rbac-bindings.yaml @@ -0,0 +1,26 @@ +{{- if .Values.rbac.enabled }} +{{- $name := include "livycluster.name" . -}} +{{- $chart := include "livycluster.chart" . -}} +{{- $releaseName := .Release.Name -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $releaseService := .Release.Service -}} +{{- range .Values.rbac.bindings }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}RoleBinding{{- else }}ClusterRoleBinding{{- end }} +metadata: + name: {{ $name }}-{{ .role }}{{ if .namespace }}-{{ .namespace }}{{- end }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $name }} + helm.sh/chart: {{ $chart }} + app.kubernetes.io/instance: {{ $releaseName }} + app.kubernetes.io/managed-by: {{ $releaseService }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ if .cluster }}ClusterRole{{- else }}Role{{- end }} + name: {{ .role }} +subjects: +{{ toYaml .subjects | indent 2}} +{{- end }} +{{- end }} diff --git a/dev/helmchart/templates/rbac-roles.yaml b/dev/helmchart/templates/rbac-roles.yaml new file mode 100755 index 000000000..7600a0fba --- /dev/null +++ b/dev/helmchart/templates/rbac-roles.yaml @@ -0,0 +1,22 @@ +{{- if .Values.rbac.enabled }} +{{- $name := include "livycluster.name" . -}} +{{- $chart := include "livycluster.chart" . -}} +{{- $releaseName := .Release.Name -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $releaseService := .Release.Service -}} +{{- range .Values.rbac.roles }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}Role{{- else }}ClusterRole{{- end }} +metadata: + name: {{ .name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $name }} + helm.sh/chart: {{ $chart }} + app.kubernetes.io/instance: {{ $releaseName }} + app.kubernetes.io/managed-by: {{ $releaseService }} +rules: +{{ toYaml .rules | indent 2}} +{{- end }} +{{- end }} diff --git a/dev/helmchart/templates/rbac.yaml b/dev/helmchart/templates/rbac.yaml new file mode 100644 index 000000000..c2a17ece3 --- /dev/null +++ b/dev/helmchart/templates/rbac.yaml @@ -0,0 +1,87 @@ +{{- if .Values.rbac.create -}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}Role{{- else }}ClusterRole{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["*"] +- apiGroups: ["", "extensions"] + resources: ["configmaps", "services"] + verbs: ["create", "update", "get", "list", "patch"] +- apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] +- apiGroups: ["", "rbac.authorization.k8s.io"] + resources: ["serviceaccounts", "rolebindings"] + verbs: ["create", "update"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["create", "update", "get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}RoleBinding{{- else }}ClusterRoleBinding{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ if .namespace }}Role{{- else }}ClusterRole{{- end }} + name: {{ template "livycluster.fullname" . }} +subjects: +- name: {{ template "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + kind: ServiceAccount +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}Role{{- else }}ClusterRole{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }}-spark + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +rules: +- apiGroups: [""] + resources: ["pods", "services"] + verbs: ["*"] +- apiGroups: ["", "extensions"] + resources: ["configmaps", "services"] + verbs: ["create", "update", "get", "list", "delete", "patch"] +- apiGroups: ["networking.k8s.io"] + resources: ["ingresses"] + verbs: ["create", "update", "get", "list", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: {{ if .namespace }}RoleBinding{{- else }}ClusterRoleBinding{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }}-spark + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: {{ if .namespace }}Role{{- else }}ClusterRole{{- end }} + name: {{ template "livycluster.fullname" . }}-spark +subjects: +- name: {{ template "livycluster.fullname" . }}-spark + namespace: {{ .Release.Namespace }} + kind: ServiceAccount +{{- end -}} diff --git a/dev/helmchart/templates/service-headless.yaml b/dev/helmchart/templates/service-headless.yaml new file mode 100644 index 000000000..fbe018d15 --- /dev/null +++ b/dev/helmchart/templates/service-headless.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "livycluster.fullname" . }}-headless + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: ClusterIP + clusterIP: None + selector: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/dev/helmchart/templates/service.yaml b/dev/helmchart/templates/service.yaml new file mode 100644 index 000000000..4541c8335 --- /dev/null +++ b/dev/helmchart/templates/service.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ default (include "livycluster.fullname" .) .Values.service.name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} +--- +apiVersion: v1 +kind: Service +metadata: + name: spark-history-service + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + selector: + app: spark-history + ports: + - protocol: TCP + port: 18080 + targetPort: 18080 diff --git a/dev/helmchart/templates/serviceaccount.yaml b/dev/helmchart/templates/serviceaccount.yaml new file mode 100644 index 000000000..513789fba --- /dev/null +++ b/dev/helmchart/templates/serviceaccount.yaml @@ -0,0 +1,48 @@ +{{- if .Values.rbac.enabled }} +{{- $name := include "livycluster.name" . -}} +{{- $chart := include "livycluster.chart" . -}} +{{- $releaseName := .Release.Name -}} +{{- $releaseNamespace := .Release.Namespace -}} +{{- $releaseService := .Release.Service -}} +{{- range .Values.rbac.serviceAccounts }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $name }} + helm.sh/chart: {{ $chart }} + app.kubernetes.io/instance: {{ $releaseName }} + app.kubernetes.io/managed-by: {{ $releaseService }} +{{- end }} +{{- end }} +--- +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +--- +apiVersion: v1 +kind: ServiceAccount +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +metadata: + name: {{ include "livycluster.fullname" . }}-spark + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} diff --git a/dev/helmchart/templates/statefulset.yaml b/dev/helmchart/templates/statefulset.yaml new file mode 100644 index 000000000..a91f587d5 --- /dev/null +++ b/dev/helmchart/templates/statefulset.yaml @@ -0,0 +1,152 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "livycluster.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + serviceName: {{ include "livycluster.fullname" . }}-headless + selector: + matchLabels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + spec: + serviceAccountName: {{ include "livycluster.fullname" . }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.livy.repository }}:{{ .Values.image.livy.tag }}" + imagePullPolicy: {{ .Values.image.livy.pullPolicy }} + ports: + - name: http + containerPort: 8998 + protocol: TCP + livenessProbe: + httpGet: + path: /version + port: http + readinessProbe: + httpGet: + path: /version + port: http + resources: + {{- toYaml .Values.resources | nindent 10 }} + env: + {{- if .Values.rbac.create }} + - name: LIVY_SPARK_KUBERNETES_AUTHENTICATE_DRIVER_SERVICE1ACCOUNT1NAME + value: {{ include "livycluster.fullname" . }}-spark + - name: LIVY_SPARK_KUBERNETES_NAMESPACE + value: {{ .Release.Namespace }} + {{- end }} + - name: HTTP2_DISABLE + value: "true" + - name: LIVY_CONF_DIR + value: "/conf" + - name: SPARK_CONF_DIR + value: "/conf" + - name: LIVY_FILE_LOCAL_DIR_WHITELIST + value: "/opt/jars" + - name: LIVY_LOG_DIR + value: "/opt/livy/logs" + - name: LIVY_SERVER_JAVA_OPTS + value: "-agentlib:jdwp=transport=dt_socket,server=y,address=9010,suspend=n" + - name: SPARK_DIVER_CORES + value: "1" + - name: SPARK_DRIVER_MEMORY + value: 1g + - name: SPARK_MASTER_ENDPOINT + value: "k8s://kubernetes.default.svc.cluster.local:443" + volumeMounts: + {{- if .Values.persistence.enabled }} + - name: livy-store + mountPath: /tmp/livy/store + {{- if .Values.persistence.subPath }} + subPath: {{ .Values.persistence.subPath }} + {{- end }} + {{- end }} + - name: cluster-conf-cm + mountPath: /conf + volumes: + - name: cluster-conf-cm + configMap: + name: cluster-config + - name: livy-store + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "livycluster.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: spark-history + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ include "livycluster.name" . }} + helm.sh/chart: {{ include "livycluster.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} +spec: + serviceName: spark-history + replicas: 1 + selector: + matchLabels: + app: spark-history + template: + metadata: + labels: + app: spark-history + spec: + containers: + - name: spark-history-server + image: "{{ .Values.image.spark.repository }}:{{ .Values.image.spark.tag }}" + imagePullPolicy: {{ .Values.image.spark.pullPolicy }} + env: + - name: SPARK_LOG_DIR + value: /spark-history + - name: SPARK_NO_DAEMONIZE + value: "false" + args: + - /bin/bash + - -c + - | + /opt/spark/sbin/start-history-server.sh + ports: + - containerPort: 18080 + volumeMounts: + - name: spark-history-data + mountPath: /tmp/spark-events + volumes: + - name: spark-history-data + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "livycluster.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end }} \ No newline at end of file diff --git a/dev/helmchart/templates/storageclass.yaml b/dev/helmchart/templates/storageclass.yaml new file mode 100755 index 000000000..508fd5aa5 --- /dev/null +++ b/dev/helmchart/templates/storageclass.yaml @@ -0,0 +1,32 @@ +{{- $name := include "livycluster.name" . -}} +{{- $chart := include "livycluster.chart" . -}} +{{- $releaseName := .Release.Name -}} +{{- $releaseService := .Release.Service -}} +{{- range .Values.storageClasses }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .name }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ $name }} + helm.sh/chart: {{ $chart }} + app.kubernetes.io/instance: {{ $releaseName }} + app.kubernetes.io/managed-by: {{ $releaseService }} +{{- if .default }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +{{- end }} +provisioner: {{ .provisioner }} +{{- if .parameters }} +parameters: +{{ toYaml .parameters | indent 2 }} +{{- end }} +{{- if .reclaimPolicy }} +reclaimPolicy: {{ .reclaimPolicy }} +{{- end }} +{{- if .volumeBindingMode }} +volumeBindingMode: {{ .volumeBindingMode }} +{{- end }} +{{- end }} diff --git a/dev/helmchart/values.yaml b/dev/helmchart/values.yaml new file mode 100755 index 000000000..f0dff6cc9 --- /dev/null +++ b/dev/helmchart/values.yaml @@ -0,0 +1,95 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +image: + livy: + repository: your_repository/livy + tag: spark3.2.3 + pullPolicy: IfNotPresent # pullPolicy can be Always or IfNotPresent or Never + spark: + repository: your_repository/spark-py + tag: v3.2.3 + pullPolicy: IfNotPresent + +tags: + ingress: true + +clusterIssuers: + - name: selfsigning-issuer + spec: + selfSigned: {} + +defaultCertificate: + enabled: true + issuer: selfsigning-issuer + issuerKind: ClusterIssuer + secretName: ingress-default-tls + commonName: '*.my-cluster.example.com' + domains: + - my-cluster.example.com + +cert-manager: + enabled: true + ingressShim: + defaultIssuerName: selfsigning-issuer + defaultIssuerKind: ClusterIssuer + +ingress-nginx: + controller: + allowSnippetAnnotations: true + extraArgs: + default-ssl-certificate: kube-system/ingress-default-tls + service: + loadBalancerIP: 127.0.0.1 # my-cluster.example.com IP address (from /etc/hosts) + loadBalancerSourceRanges: [ ] + +nameOverride: "" +fullnameOverride: "" + +rbac: + enabled: true + create: true + serviceAccounts: [] + roles: [] + bindings: [] + +service: + type: ClusterIP + port: 80 + # If not set, a name is generated using the fullname template + name: + +ingress: + enabled: true + annotations: + kubernetes.io/tls-acme: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + path: /livy/?(.*) + hosts: + - my-cluster.example.com + tls: + - secretName: spark-cluster-tls + hosts: + - my-cluster.example.com + +persistence: + enabled: true + accessMode: ReadWriteOnce + size: 1Gi + annotations: {} + +global: {} \ No newline at end of file