diff --git a/README.md b/README.md index 2ec3e12..3f0783f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A Terraform module to install the [Ocean for Apache Spark](https://spot.io/products/ocean-apache-spark/) data platform. -## Introduction +## *Introduction* This module imports an existing Ocean cluster into Ocean Spark. @@ -10,7 +10,7 @@ This module imports an existing Ocean cluster into Ocean Spark. * Existing EKS/GKE/AKS Cluster * EKS/GKE/AKS cluster integrated with Spot Ocean -### Usage +## *Usage* ```hcl provider "spotinst" { token = var.spotinst_token @@ -18,13 +18,20 @@ provider "spotinst" { } module "ocean-spark" { - "spotinst/ocean-spark/spotinst" + source = "spotinst/ocean-spark/spotinst" ocean_cluster_id = var.ocean_cluster_id } ``` -### Examples +## *Upgrade guides* +- [Upgrade to v3.x.](/docs/UPGRADE-v3.md) +- [Upgrade to v2.x.](/docs/UPGRADE-v2.md) +- [Upgrade to v1.x](/docs/UPGRADE-v1.md) + + +## *Examples* + It can be combined with other Terraform modules to support a number of installation methods for Ocean Spark: 1. Create an Ocean Spark cluster from scratch in your AWS account 2. Create an Ocean Spark Cluster from scratch in your AWS account with AWS Private Link support. @@ -121,43 +128,6 @@ Folder [`examples/import-ocean-cluster/`](https://github.com/spotinst/terraform- 3- Once the script is completed with success, you can now run `terraform destroy` -## Migration Guide - -### v2 migration guide - -#### By default the Ocean Spark deployer jobs now run in the kube-system namespace. - -To avoid issues for existing clusters you will need to set the following line: -```diff -module "ocean-spark" { - "spotinst/ocean-spark/spotinst" - - ocean_cluster_id = var.ocean_cluster_id -+ deployer_namespace = "spot-system" -} -``` - -#### Deprecated `ofas_managed_load_balancer` variable has been deleted - -Use `ingress_managed_load_balancer` instead - -### v1 migration guide - -This migration revolves around 1 topic: - -- The use of the `spotinst_ocean_spark` resource to manage the cluster state instead of relying on a `kubernetes job` on the 1st apply - -#### Steps - -1- Upgrade `spotinst provider` to `>= 1.89` - -2- [Retrieve from the UI](https://console.spotinst.com/ocean/spark/clusters) your Ocean Spark `Cluster ID` - -3- Import the resource into your `terraform state` -``` -terraform import module.ocean-spark.spotinst_ocean_spark.example osc-abcd1234 -``` - ## Terraform module documentation @@ -175,7 +145,7 @@ terraform import module.ocean-spark.spotinst_ocean_spark.example osc-abcd1234 | Name | Version | |------|---------| -| [kubernetes](#provider\_kubernetes) | ~> 2.0 | +| [null](#provider\_null) | n/a | | [spotinst](#provider\_spotinst) | >= 1.115.0, < 1.123.0 | | [validation](#provider\_validation) | 1.0.0 | @@ -187,9 +157,7 @@ No modules. | Name | Type | |------|------| -| [kubernetes_cluster_role_binding.deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/cluster_role_binding) | resource | -| [kubernetes_namespace.spot-system](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | -| [kubernetes_service_account.deployer](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/service_account) | resource | +| [null_resource.apply_kubernetes_manifest](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | | [spotinst_ocean_spark.cluster](https://registry.terraform.io/providers/spotinst/spotinst/latest/docs/resources/ocean_spark) | resource | | [spotinst_ocean_spark_virtual_node_group.this](https://registry.terraform.io/providers/spotinst/spotinst/latest/docs/resources/ocean_spark_virtual_node_group) | resource | | [validation_warning.log_collection_collect_driver_logs](https://registry.terraform.io/providers/tlkamp/validation/1.0.0/docs/data-sources/warning) | data source | @@ -199,6 +167,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [attach\_dedicated\_virtual\_node\_groups](#input\_attach\_dedicated\_virtual\_node\_groups) | List of virtual node group IDs to attach to the cluster | `list(string)` | `[]` | no | +| [cluster\_config](#input\_cluster\_config) | Configuration for Ocean Kubernetes cluster | 
object({
    cluster_name               = string
    certificate_authority_data = string
    server_endpoint            = string
    token                      = optional(string)
    client_certificate         = optional(string)
    client_key                 = optional(string)
  })
| n/a | yes | | [compute\_create\_vngs](#input\_compute\_create\_vngs) | Controls whether dedicated Ocean Spark VNGs will be created by the cluster creation process | `bool` | `true` | no | | [compute\_use\_taints](#input\_compute\_use\_taints) | Controls whether the Ocean Spark cluster will use taints to schedule workloads | `bool` | `true` | no | | [create\_cluster](#input\_create\_cluster) | Controls whether the Ocean for Apache Spark cluster should be created (it affects all resources) | `bool` | `true` | no | diff --git a/docs/UPGRADE-v1.md b/docs/UPGRADE-v1.md new file mode 100644 index 0000000..52dacfe --- /dev/null +++ b/docs/UPGRADE-v1.md @@ -0,0 +1,17 @@ +### v1 upgrade guide + +*This upgrade revolves around one topic:* + +The use of the `spotinst_ocean_spark` resource to manage the cluster state instead of relying on a `kubernetes job` on the first apply. + +To upgrade to v1 please follow the steps bellow: + +1- Upgrade `spotinst provider` to `>= 1.89` + +2- [Retrieve from the UI](https://console.spotinst.com/ocean/spark/clusters) your Ocean Spark `Cluster ID` + +3- Import the resource into your `terraform state`: + +``` +terraform import module.ocean-spark.spotinst_ocean_spark.example osc-abcd1234 +``` diff --git a/docs/UPGRADE-v2.md b/docs/UPGRADE-v2.md new file mode 100644 index 0000000..856e290 --- /dev/null +++ b/docs/UPGRADE-v2.md @@ -0,0 +1,19 @@ +## Upgrade to v2.x.x from v1.x.x + +By default the Ocean Spark deployer jobs now run in the kube-system namespace. + +To avoid issues for existing clusters you will need to set the following line: + +```diff +module "ocean-spark" { + "spotinst/ocean-spark/spotinst" + + ocean_cluster_id = var.ocean_cluster_id ++ deployer_namespace = "spot-system" +} +``` + + +#### Deprecated : + +- `ofas_managed_load_balancer` variable has been deleted. Use `ingress_managed_load_balancer` instead \ No newline at end of file diff --git a/docs/UPGRADE-v3.md b/docs/UPGRADE-v3.md new file mode 100644 index 0000000..c3a66ba --- /dev/null +++ b/docs/UPGRADE-v3.md @@ -0,0 +1,72 @@ +## Upgrade to v3.x.x from v2.x.x + +To migrate from *v2.xx* to *v3.x.x*, please follow the steps bellow: + +1- If you specified the spot-system namespace for the deployer job to run, then you will need to remove it from the terraform state: + +`terraform state rm module.ocean-spark.kubernetes_namespace.spot-system` + +2- Remove the deployer RBAC service-account and role-binding from the terraform state as well: + +- `terraform state rm module.ocean-spark.kubernetes_service_account.deployer` + +- `terraform state rm module.ocean-spark.kubernetes_cluster_role_binding.deployer` + +3- Add the new required `cluster_config` variable depending on your cloud provider + +- for *AWS*: + + ```diff + module "ocean-spark" { + source = "spotinst/ocean-spark/spotinst" + version = "3.0.0" + + ocean_cluster_id = var.ocean_cluster_id + + + cluster_config = { + + cluster_name = var.cluster_name + + certificate_authority_data = data.aws_eks_cluster.this.certificate_authority[0].data + + server_endpoint = data.aws_eks_cluster.this.endpoint + + token = data.aws_eks_cluster_auth.this.token + + } + } + ``` + +- for *GCP*: + + ```diff + module "ocean-spark" { + source = "spotinst/ocean-spark/spotinst" + version = "3.0.0" + + ocean_cluster_id = var.ocean_cluster_id + + + cluster_config = { + + cluster_name = google_container_cluster.cluster.name + + certificate_authority_data = google_container_cluster.cluster.master_auth[0].cluster_ca_certificate + + server_endpoint = "https://${google_container_cluster.cluster.endpoint}" + + token = data.google_client_config.default.access_token + + } + } + ``` + +- for *Azure*: + + ```diff + module "ocean-spark" { + source = "spotinst/ocean-spark/spotinst" + version = "3.0.0" + + ocean_cluster_id = var.ocean_cluster_id + + + cluster_config = { + + cluster_name = var.cluster_name + + certificate_authority_data = module.aks.admin_cluster_ca_certificate + + server_endpoint = module.aks.admin_host + + client_certificate = module.aks.admin_client_certificate + + client_key = module.aks.admin_client_key + + } + } + ``` + +4- Run `terraform init` then `terraform apply` and that's it. \ No newline at end of file diff --git a/examples/azure-from-scratch/main.tf b/examples/azure-from-scratch/main.tf index 6b5a570..9e37974 100644 --- a/examples/azure-from-scratch/main.tf +++ b/examples/azure-from-scratch/main.tf @@ -10,6 +10,7 @@ provider "azurerm" { use_cli = false } + resource "azurerm_resource_group" "this" { location = var.location name = "${var.cluster_name}-rg" @@ -42,7 +43,7 @@ module "aks" { prefix = "oceanspark" resource_group_name = azurerm_resource_group.this.name - sku_tier = "Paid" + sku_tier = "Standard" cluster_name = var.cluster_name kubernetes_version = var.cluster_version vnet_subnet_id = azurerm_subnet.this.id @@ -91,7 +92,7 @@ module "ocean-controller" { module "ocean-aks-np" { source = "spotinst/ocean-aks-np-k8s/spotinst" - version = "0.2.0" + version = "0.5.0" spotinst_token = var.spotinst_token spotinst_account = var.spotinst_account @@ -106,8 +107,11 @@ module "ocean-aks-np" { autoscaler_resource_limits_max_vcpu = 20000 autoscaler_resource_limits_max_memory_gib = 100000 autoscaler_max_scale_down_percentage = 10 - autoscaler_headroom_automatic_is_enabled = true autoscaler_headroom_automatic_percentage = 5 + autoscale_headrooms_cpu_per_unit = 6 + autoscale_headrooms_memory_per_unit = 10 + autoscale_headrooms_gpu_per_unit = 0 + autoscale_headrooms_num_of_units = 10 health_grace_period = 600 max_pods_per_node = 110 enable_node_public_ip = false @@ -120,6 +124,14 @@ module "ocean-aks-np" { fallback_to_ondemand = true availability_zones = [1, 2, 3, ] tags = var.tags + vmsizes_filters_min_vcpu = 2 + vmsizes_filters_max_vcpu = 16 + vmsizes_filters_min_memory_gib = 10 + vmsizes_filters_max_memory_gib = 18 + vmsizes_filters_series = ["D v3", "Dds_v4", "Dsv2"] + vmsizes_filters_architectures = ["X86_64"] + scheduling_shutdown_hours_time_windows = ["Sat:08:00-Sun:08:00"] + scheduling_shutdown_hours_is_enabled = true } @@ -140,4 +152,12 @@ module "ocean-spark" { module.ocean-aks-np, module.ocean-controller, ] -} + + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = module.aks.admin_cluster_ca_certificate + server_endpoint = module.aks.admin_host + client_certificate = module.aks.admin_client_certificate + client_key = module.aks.admin_client_key + } +} \ No newline at end of file diff --git a/examples/azure-from-scratch/variables.tf b/examples/azure-from-scratch/variables.tf index 69af525..7e54f0a 100644 --- a/examples/azure-from-scratch/variables.tf +++ b/examples/azure-from-scratch/variables.tf @@ -1,12 +1,16 @@ variable "azure_client_id" { - type = string + type = string + sensitive = true } variable "azure_client_secret" { - type = string + type = string + sensitive = true } variable "azure_tenant_id" { - type = string + type = string + sensitive = true } + variable "azure_subscription_id" { type = string } diff --git a/examples/azure-from-vpc/main.tf b/examples/azure-from-vpc/main.tf index f9a82a8..d7421b1 100644 --- a/examples/azure-from-vpc/main.tf +++ b/examples/azure-from-vpc/main.tf @@ -114,6 +114,14 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aks-np.ocean_id + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = module.aks.admin_cluster_ca_certificate + server_endpoint = module.aks.admin_host + client_certificate = module.aks.admin_client_certificate + client_key = module.aks.admin_client_key + } + depends_on = [ module.ocean-aks-np, module.ocean-controller, diff --git a/examples/azure-import-aks-cluster/main.tf b/examples/azure-import-aks-cluster/main.tf index 3b6c5cd..c5e89af 100644 --- a/examples/azure-import-aks-cluster/main.tf +++ b/examples/azure-import-aks-cluster/main.tf @@ -95,6 +95,14 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aks-np.ocean_id + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = local.aks_admin.cluster_ca_certificate + server_endpoint = local.aks_admin.host + client_certificate = local.aks_admin.client_certificate + client_key = local.aks_admin.client_key + } + depends_on = [ module.ocean-aks-np, module.ocean-controller, diff --git a/examples/from-private-vpc/main.tf b/examples/from-private-vpc/main.tf index 1f74ade..4605fa7 100644 --- a/examples/from-private-vpc/main.tf +++ b/examples/from-private-vpc/main.tf @@ -159,6 +159,13 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aws-k8s.ocean_id + cluster_config = { + cluster_name = module.eks.cluster_id + certificate_authority_data = module.eks.cluster_certificate_authority_data + server_endpoint = module.eks.cluster_endpoint + token = data.aws_eks_cluster_auth.this.token + } + depends_on = [ module.ocean-aws-k8s, module.ocean-controller, diff --git a/examples/from-scratch-eks-blueprint/main.tf b/examples/from-scratch-eks-blueprint/main.tf index 39d8a3b..6d9aa80 100644 --- a/examples/from-scratch-eks-blueprint/main.tf +++ b/examples/from-scratch-eks-blueprint/main.tf @@ -227,6 +227,13 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aws-k8s.ocean_id + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = module.eks_blueprints.eks_cluster_certificate_authority_data + server_endpoint = module.eks_blueprints.eks_cluster_endpoint + token = data.aws_eks_cluster_auth.this.token + } + depends_on = [ module.ocean-aws-k8s, module.ocean-controller, diff --git a/examples/from-scratch-with-private-link/main.tf b/examples/from-scratch-with-private-link/main.tf index 88405d5..f006fcb 100644 --- a/examples/from-scratch-with-private-link/main.tf +++ b/examples/from-scratch-with-private-link/main.tf @@ -312,6 +312,13 @@ module "ocean-spark" { enable_private_link = true ingress_private_link_endpoint_service_address = aws_vpc_endpoint_service.this.service_name + cluster_config = { + cluster_name = module.eks.cluster_id + certificate_authority_data = module.eks.cluster_certificate_authority_data + server_endpoint = module.eks.cluster_endpoint + token = data.aws_eks_cluster_auth.this.token + } + depends_on = [ module.ocean-aws-k8s, module.ocean-controller, diff --git a/examples/from-scratch/main.tf b/examples/from-scratch/main.tf index 32ca73d..cefc9b7 100644 --- a/examples/from-scratch/main.tf +++ b/examples/from-scratch/main.tf @@ -197,7 +197,6 @@ module "ocean-controller" { ################################################################################ # Import Ocean cluster into Ocean Spark ################################################################################ - module "ocean-spark" { source = "../.." @@ -207,4 +206,11 @@ module "ocean-spark" { module.ocean-aws-k8s, module.ocean-controller, ] + + cluster_config = { + cluster_name = module.eks.cluster_id + certificate_authority_data = module.eks.cluster_certificate_authority_data + server_endpoint = module.eks.cluster_endpoint + token = data.aws_eks_cluster_auth.this.token + } } diff --git a/examples/from-vpc-eks-blueprint/main.tf b/examples/from-vpc-eks-blueprint/main.tf index ee8f166..fb6caa1 100644 --- a/examples/from-vpc-eks-blueprint/main.tf +++ b/examples/from-vpc-eks-blueprint/main.tf @@ -186,6 +186,13 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aws-k8s.ocean_id + cluster_config = { + cluster_name = module.eks.cluster_id + certificate_authority_data = module.eks_blueprints.eks_cluster_certificate_authority_data + server_endpoint = module.eks_blueprints.eks_cluster_endpoint + token = data.aws_eks_cluster_auth.this.token + } + depends_on = [ module.ocean-aws-k8s, module.ocean-controller, diff --git a/examples/gcp-from-scratch/main.tf b/examples/gcp-from-scratch/main.tf index 04e5601..56d5c88 100644 --- a/examples/gcp-from-scratch/main.tf +++ b/examples/gcp-from-scratch/main.tf @@ -128,4 +128,11 @@ module "ocean-spark" { spotinst_ocean_gke_import.ocean, module.ocean-controller, ] + + cluster_config = { + cluster_name = google_container_cluster.cluster.name + certificate_authority_data = google_container_cluster.cluster.master_auth[0].cluster_ca_certificate + server_endpoint = "https://${google_container_cluster.cluster.endpoint}" + token = data.google_client_config.default.access_token + } } diff --git a/examples/gcp-from-vpc/main.tf b/examples/gcp-from-vpc/main.tf index bdea242..58b59ce 100644 --- a/examples/gcp-from-vpc/main.tf +++ b/examples/gcp-from-vpc/main.tf @@ -82,6 +82,13 @@ module "ocean-spark" { ocean_cluster_id = spotinst_ocean_gke_import.ocean.id + cluster_config = { + cluster_name = google_container_cluster.cluster.name + certificate_authority_data = google_container_cluster.cluster.master_auth[0].cluster_ca_certificate + server_endpoint = "https://${google_container_cluster.cluster.endpoint}" + token = data.google_client_config.default.access_token + } + depends_on = [ spotinst_ocean_gke_import.ocean, module.ocean-controller, diff --git a/examples/gcp-import-gke-cluster/main.tf b/examples/gcp-import-gke-cluster/main.tf index 0da05b7..8397800 100644 --- a/examples/gcp-import-gke-cluster/main.tf +++ b/examples/gcp-import-gke-cluster/main.tf @@ -64,4 +64,11 @@ module "ocean-spark" { spotinst_ocean_gke_import.ocean, module.ocean-controller, ] + + cluster_config = { + cluster_name = data.google_container_cluster.gke.name + certificate_authority_data = data.google_container_cluster.gke.master_auth[0].cluster_ca_certificate + server_endpoint = "https://${data.google_container_cluster.gke.endpoint}" + token = data.google_client_config.default.access_token + } } diff --git a/examples/import-eks-cluster/main.tf b/examples/import-eks-cluster/main.tf index 790b81f..06b96a8 100644 --- a/examples/import-eks-cluster/main.tf +++ b/examples/import-eks-cluster/main.tf @@ -64,6 +64,13 @@ module "ocean-spark" { ocean_cluster_id = module.ocean-aws-k8s.ocean_id + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = data.aws_eks_cluster.this.certificate_authority[0].data + server_endpoint = data.aws_eks_cluster.this.endpoint + token = data.aws_eks_cluster_auth.this.token + } + depends_on = [ module.ocean-aws-k8s, module.ocean-controller, diff --git a/examples/import-ocean-cluster/main.tf b/examples/import-ocean-cluster/main.tf index 09ca5a0..4c9c68d 100644 --- a/examples/import-ocean-cluster/main.tf +++ b/examples/import-ocean-cluster/main.tf @@ -2,13 +2,32 @@ provider "kubernetes" { config_path = "~/.kube/config" } +provider "aws" { + region = var.aws_region + profile = var.aws_profile +} + provider "spotinst" { token = var.spotinst_token account = var.spotinst_account } +data "aws_eks_cluster_auth" "this" { + name = var.cluster_name +} +data "aws_eks_cluster" "this" { + name = var.cluster_name +} + module "ocean-spark" { source = "../.." ocean_cluster_id = var.ocean_cluster_id + + cluster_config = { + cluster_name = var.cluster_name + certificate_authority_data = data.aws_eks_cluster.this.certificate_authority[0].data + server_endpoint = data.aws_eks_cluster.this.endpoint + token = data.aws_eks_cluster_auth.this.token + } } \ No newline at end of file diff --git a/examples/import-ocean-cluster/variables.tf b/examples/import-ocean-cluster/variables.tf index b4d5441..f4e1927 100644 --- a/examples/import-ocean-cluster/variables.tf +++ b/examples/import-ocean-cluster/variables.tf @@ -9,3 +9,15 @@ variable "spotinst_account" { variable "ocean_cluster_id" { type = string } + +variable "cluster_name" { + type = string +} + +variable "aws_region" { + type = string +} + +variable "aws_profile" { + type = string +} \ No newline at end of file diff --git a/examples/import-ocean-cluster/versions.tf b/examples/import-ocean-cluster/versions.tf index 6278afb..1d697f0 100644 --- a/examples/import-ocean-cluster/versions.tf +++ b/examples/import-ocean-cluster/versions.tf @@ -8,5 +8,9 @@ terraform { source = "hashicorp/kubernetes" version = "~> 2.10" } + aws = { + source = "hashicorp/aws" + version = "~> 3.75" + } } } diff --git a/main.tf b/main.tf index efad3ca..47215fb 100644 --- a/main.tf +++ b/main.tf @@ -1,48 +1,54 @@ locals { - spot_system_namespace = "spot-system" - service_account_name = "bigdata-deployer" - role_binding_name = "bigdata-deployer-admin" + collect_app_logs = coalesce(var.log_collection_collect_driver_logs, var.log_collection_collect_app_logs) + ofas_deployer_path = "https://spotinst-public.s3.amazonaws.com/integrations/kubernetes/ocean-spark/templates/ocean-spark-deploy.yaml" + kubeconfig = yamlencode({ + apiVersion = "v1" + kind = "Config" + current-context = "terraform" + clusters = [{ + name = var.cluster_config.cluster_name + cluster = { + certificate-authority-data = var.cluster_config.certificate_authority_data + server = var.cluster_config.server_endpoint + } + }] + contexts = [{ + name = "terraform" + context = { + cluster = var.cluster_config.cluster_name + user = "terraform" + } + }] + users = [{ + name = "terraform" + user = { + token = var.cluster_config.token + client-certificate = var.cluster_config.client_certificate + client-key = var.cluster_config.client_key + } + }] + }) } -resource "kubernetes_namespace" "spot-system" { - count = var.create_cluster && var.deployer_namespace == local.spot_system_namespace ? 1 : 0 - metadata { - name = local.spot_system_namespace +resource "null_resource" "apply_kubernetes_manifest" { + provisioner "local-exec" { + command = <<-EOT + cat < cluster_kubeconfig.yaml + ${local.kubeconfig} + EOF + EOT } -} -resource "kubernetes_service_account" "deployer" { - count = var.create_cluster ? 1 : 0 - metadata { - name = local.service_account_name - namespace = var.deployer_namespace + provisioner "local-exec" { + command = <