From 162d71e2bd503d328e20e023e09564a58ecee139 Mon Sep 17 00:00:00 2001 From: Andriy Knysh Date: Tue, 24 Mar 2020 00:28:41 -0400 Subject: [PATCH] Use `kubernetes` provider to apply Auth ConfigMap (#56) * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap * Update variables.tf Co-Authored-By: Erik Osterman * Use `kubernetes` provider to apply Auth ConfigMap * Use `kubernetes` provider to apply Auth ConfigMap Co-authored-by: Erik Osterman --- README.md | 112 +++--------------- README.yaml | 94 ++------------- auth.tf | 123 ++++++-------------- configmap-auth.yaml.tpl | 19 --- docs/terraform.md | 18 +-- examples/complete/fixtures.us-east-2.tfvars | 4 +- examples/complete/main.tf | 36 ++---- examples/complete/variables.tf | 73 +----------- examples/complete/versions.tf | 9 +- test/src/Makefile | 2 +- test/src/examples_complete_test.go | 59 +++++++++- variables.tf | 76 ++---------- versions.tf | 9 +- 13 files changed, 153 insertions(+), 481 deletions(-) delete mode 100644 configmap-auth.yaml.tpl diff --git a/README.md b/README.md index d8bffcc9..344c21c4 100644 --- a/README.md +++ b/README.md @@ -82,47 +82,15 @@ We literally have [*hundreds of terraform modules*][terraform_modules] that are The module provisions the following resources: -- EKS cluster of master nodes that can be used together with the [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers) module to create a full-blown cluster +- EKS cluster of master nodes that can be used together with the [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers), + [terraform-aws-eks-node-group](https://github.com/cloudposse/terraform-aws-eks-node-group) and + [terraform-aws-eks-fargate-profile](https://github.com/cloudposse/terraform-aws-eks-fargate-profile) + modules to create a full-blown cluster - IAM Role to allow the cluster to access other AWS services -- Security Group which is used by EKS workers to connect to the cluster and kubelets and pods to receive communication from the cluster control plane (see [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers)) -- The module creates and automatically applies (via `kubectl apply`) an authentication ConfigMap to allow the wrokers nodes to join the cluster and to add additional users/roles/accounts +- Security Group which is used by EKS workers to connect to the cluster and kubelets and pods to receive communication from the cluster control plane +- The module creates and automatically applies an authentication ConfigMap to allow the wrokers nodes to join the cluster and to add additional users/roles/accounts -### Works with [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html) - -To run on Terraform Cloud, set the following variables: - - ```hcl - install_aws_cli = true - install_kubectl = true - external_packages_install_path = "~/.terraform/bin" - kubeconfig_path = "~/.kube/config" - configmap_auth_file = "/home/terraform/.terraform/configmap-auth.yaml" - - # Optional - aws_eks_update_kubeconfig_additional_arguments = "--verbose" - aws_cli_assume_role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - aws_cli_assume_role_session_name = "eks_cluster_example_session" - ``` - - Terraform Cloud executes `terraform plan/apply` on workers running Ubuntu. - For the module to provision the authentication ConfigMap (to allow the EKS worker nodes to join the EKS cluster and to add additional users/roles/accounts), - AWS CLI and `kubectl` need to be installed on Terraform Cloud workers. - - To install the required external packages, set the variables `install_aws_cli` and `install_kubectl` to `true` and specify `external_packages_install_path`, `kubeconfig_path` and `configmap_auth_file`. - - See [auth.tf](auth.tf) and [Installing Software in the Run Environment](https://www.terraform.io/docs/cloud/run/install-software.html) for more details. - - In a multi-account architecture, we might have a separate identity account where we provision all IAM users, and other accounts (e.g. `prod`, `staging`, `dev`, `audit`, `testing`) - where all other AWS resources are provisioned. The IAM Users from the identity account can assume IAM roles to access the other accounts. - - In this case, we provide Terraform Cloud with access keys (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) for an IAM User from the identity account - and allow it to assume an IAM Role into the AWS account where the module gets provisioned. - - To support this, the module can assume an IAM role before executing the command `aws eks update-kubeconfig` when applying the auth ConfigMap. - - Set variable `aws_cli_assume_role_arn` to the Amazon Resource Name (ARN) of the role to assume and variable `aws_cli_assume_role_session_name` to the identifier for the assumed role session. - - See [auth.tf](auth.tf) and [assume-role](https://docs.aws.amazon.com/cli/latest/reference/sts/assume-role.html) for more details. +__NOTE:__ The module works with [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html). ## Usage @@ -231,9 +199,7 @@ Other examples: vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids - kubernetes_version = var.kubernetes_version - kubeconfig_path = var.kubeconfig_path - + kubernetes_version = var.kubernetes_version oidc_provider_enabled = false workers_security_group_ids = [module.eks_workers.security_group_id] @@ -304,9 +270,7 @@ Module usage with two worker groups: vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids - kubernetes_version = var.kubernetes_version - kubeconfig_path = var.kubeconfig_path - + kubernetes_version = var.kubernetes_version oidc_provider_enabled = false workers_role_arns = [module.eks_workers.workers_role_arn, module.eks_workers_2.workers_role_arn] @@ -314,46 +278,6 @@ Module usage with two worker groups: } ``` -Module usage on [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html): - -```hcl - provider "aws" { - region = "us-east-2" - - assume_role { - role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - } - } - - module "eks_cluster" { - source = "git::https://github.com/cloudposse/terraform-aws-eks-cluster.git?ref=master" - namespace = var.namespace - stage = var.stage - name = var.name - attributes = var.attributes - tags = var.tags - region = "us-east-2" - vpc_id = module.vpc.vpc_id - subnet_ids = module.subnets.public_subnet_ids - - local_exec_interpreter = "/bin/bash" - kubernetes_version = "1.14" - - workers_role_arns = [module.eks_workers.workers_role_arn] - workers_security_group_ids = [module.eks_workers.security_group_id] - - # Terraform Cloud configurations - kubeconfig_path = "~/.kube/config" - configmap_auth_file = "/home/terraform/.terraform/configmap-auth.yaml" - install_aws_cli = true - install_kubectl = true - external_packages_install_path = "~/.terraform/bin" - aws_eks_update_kubeconfig_additional_arguments = "--verbose" - aws_cli_assume_role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - aws_cli_assume_role_session_name = "eks_cluster_example_session" - } -``` - @@ -375,28 +299,17 @@ Available targets: |------|-------------|:----:|:-----:|:-----:| | allowed_cidr_blocks | List of CIDR blocks to be allowed to connect to the EKS cluster | list(string) | `` | no | | allowed_security_groups | List of Security Group IDs to be allowed to connect to the EKS cluster | list(string) | `` | no | -| apply_config_map_aws_auth | Whether to execute `kubectl apply` to apply the ConfigMap to allow worker nodes to join the EKS cluster | bool | `true` | no | +| apply_config_map_aws_auth | Whether to apply the ConfigMap to allow worker nodes to join the EKS cluster and allow additional users, accounts and roles to acces the cluster | bool | `true` | no | | attributes | Additional attributes (e.g. `1`) | list(string) | `` | no | -| aws_cli_assume_role_arn | IAM Role ARN for AWS CLI to assume before calling `aws eks` to update `kubeconfig` | string | `` | no | -| aws_cli_assume_role_session_name | An identifier for the assumed role session when assuming the IAM Role for AWS CLI before calling `aws eks` to update `kubeconfig` | string | `` | no | -| aws_eks_update_kubeconfig_additional_arguments | Additional arguments for `aws eks update-kubeconfig` command, e.g. `--role-arn xxxxxxxxx`. For more info, see https://docs.aws.amazon.com/cli/latest/reference/eks/update-kubeconfig.html | string | `` | no | | cluster_log_retention_period | Number of days to retain cluster logs. Requires `enabled_cluster_log_types` to be set. See https://docs.aws.amazon.com/en_us/eks/latest/userguide/control-plane-logs.html. | number | `0` | no | -| configmap_auth_file | Path to `configmap_auth_file` | string | `` | no | -| configmap_auth_template_file | Path to `config_auth_template_file` | string | `` | no | | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes` | string | `-` | no | | enabled | Set to false to prevent the module from creating any resources | bool | `true` | no | | enabled_cluster_log_types | A list of the desired control plane logging to enable. For more information, see https://docs.aws.amazon.com/en_us/eks/latest/userguide/control-plane-logs.html. Possible values [`api`, `audit`, `authenticator`, `controllerManager`, `scheduler`] | list(string) | `` | no | | endpoint_private_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. Default to AWS EKS resource and it is false | bool | `false` | no | | endpoint_public_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. Default to AWS EKS resource and it is true | bool | `true` | no | | environment | Environment, e.g. 'prod', 'staging', 'dev', 'pre-prod', 'UAT' | string | `` | no | -| external_packages_install_path | Path to install external packages, e.g. AWS CLI and `kubectl`. Used when the module is provisioned on workstations where the external packages are not installed by default, e.g. Terraform Cloud workers | string | `` | no | -| install_aws_cli | Set to `true` to install AWS CLI if the module is provisioned on workstations where AWS CLI is not installed by default, e.g. Terraform Cloud workers | bool | `false` | no | -| install_kubectl | Set to `true` to install `kubectl` if the module is provisioned on workstations where `kubectl` is not installed by default, e.g. Terraform Cloud workers | bool | `false` | no | -| jq_version | Version of `jq` to download to extract temporaly credentials after running `aws sts assume-role` if AWS CLI needs to assume role to access the cluster (if variable `aws_cli_assume_role_arn` is set) | string | `1.6` | no | -| kubeconfig_path | The path to `kubeconfig` file | string | `~/.kube/config` | no | -| kubectl_version | `kubectl` version to install. If not specified, the latest version will be used | string | `` | no | -| kubernetes_version | Desired Kubernetes master version. If you do not specify a value, the latest available version is used | string | `1.14` | no | -| local_exec_interpreter | shell to use for local exec | string | `/bin/bash` | no | +| kubernetes_version | Desired Kubernetes master version. If you do not specify a value, the latest available version is used | string | `1.15` | no | +| local_exec_interpreter | shell to use for local_exec | list(string) | `` | no | | map_additional_aws_accounts | Additional AWS account numbers to add to `config-map-aws-auth` ConfigMap | list(string) | `` | no | | map_additional_iam_roles | Additional IAM roles to add to `config-map-aws-auth` ConfigMap | object | `` | no | | map_additional_iam_users | Additional IAM users to add to `config-map-aws-auth` ConfigMap | object | `` | no | @@ -409,6 +322,7 @@ Available targets: | subnet_ids | A list of subnet IDs to launch the cluster in | list(string) | - | yes | | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `` | no | | vpc_id | VPC ID for the EKS cluster | string | - | yes | +| wait_for_cluster_command | `local-exec` command to execute to determine if the EKS cluster is healthy. Cluster endpoint are available as environment variable `ENDPOINT` | string | `curl --silent --fail --retry 60 --retry-delay 5 --retry-connrefused --insecure --output /dev/null $ENDPOINT/healthz` | no | | workers_role_arns | List of Role ARNs of the worker nodes | list(string) | - | yes | | workers_security_group_ids | Security Group IDs of the worker nodes | list(string) | - | yes | diff --git a/README.yaml b/README.yaml index a36ba405..96d78e21 100644 --- a/README.yaml +++ b/README.yaml @@ -67,47 +67,15 @@ description: |- introduction: |- The module provisions the following resources: - - EKS cluster of master nodes that can be used together with the [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers) module to create a full-blown cluster + - EKS cluster of master nodes that can be used together with the [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers), + [terraform-aws-eks-node-group](https://github.com/cloudposse/terraform-aws-eks-node-group) and + [terraform-aws-eks-fargate-profile](https://github.com/cloudposse/terraform-aws-eks-fargate-profile) + modules to create a full-blown cluster - IAM Role to allow the cluster to access other AWS services - - Security Group which is used by EKS workers to connect to the cluster and kubelets and pods to receive communication from the cluster control plane (see [terraform-aws-eks-workers](https://github.com/cloudposse/terraform-aws-eks-workers)) - - The module creates and automatically applies (via `kubectl apply`) an authentication ConfigMap to allow the wrokers nodes to join the cluster and to add additional users/roles/accounts + - Security Group which is used by EKS workers to connect to the cluster and kubelets and pods to receive communication from the cluster control plane + - The module creates and automatically applies an authentication ConfigMap to allow the wrokers nodes to join the cluster and to add additional users/roles/accounts - ### Works with [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html) - - To run on Terraform Cloud, set the following variables: - - ```hcl - install_aws_cli = true - install_kubectl = true - external_packages_install_path = "~/.terraform/bin" - kubeconfig_path = "~/.kube/config" - configmap_auth_file = "/home/terraform/.terraform/configmap-auth.yaml" - - # Optional - aws_eks_update_kubeconfig_additional_arguments = "--verbose" - aws_cli_assume_role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - aws_cli_assume_role_session_name = "eks_cluster_example_session" - ``` - - Terraform Cloud executes `terraform plan/apply` on workers running Ubuntu. - For the module to provision the authentication ConfigMap (to allow the EKS worker nodes to join the EKS cluster and to add additional users/roles/accounts), - AWS CLI and `kubectl` need to be installed on Terraform Cloud workers. - - To install the required external packages, set the variables `install_aws_cli` and `install_kubectl` to `true` and specify `external_packages_install_path`, `kubeconfig_path` and `configmap_auth_file`. - - See [auth.tf](auth.tf) and [Installing Software in the Run Environment](https://www.terraform.io/docs/cloud/run/install-software.html) for more details. - - In a multi-account architecture, we might have a separate identity account where we provision all IAM users, and other accounts (e.g. `prod`, `staging`, `dev`, `audit`, `testing`) - where all other AWS resources are provisioned. The IAM Users from the identity account can assume IAM roles to access the other accounts. - - In this case, we provide Terraform Cloud with access keys (`AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`) for an IAM User from the identity account - and allow it to assume an IAM Role into the AWS account where the module gets provisioned. - - To support this, the module can assume an IAM role before executing the command `aws eks update-kubeconfig` when applying the auth ConfigMap. - - Set variable `aws_cli_assume_role_arn` to the Amazon Resource Name (ARN) of the role to assume and variable `aws_cli_assume_role_session_name` to the identifier for the assumed role session. - - See [auth.tf](auth.tf) and [assume-role](https://docs.aws.amazon.com/cli/latest/reference/sts/assume-role.html) for more details. + __NOTE:__ The module works with [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html). # How to use this project usage: |- @@ -211,9 +179,7 @@ usage: |- vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids - kubernetes_version = var.kubernetes_version - kubeconfig_path = var.kubeconfig_path - + kubernetes_version = var.kubernetes_version oidc_provider_enabled = false workers_security_group_ids = [module.eks_workers.security_group_id] @@ -284,9 +250,7 @@ usage: |- vpc_id = module.vpc.vpc_id subnet_ids = module.subnets.public_subnet_ids - kubernetes_version = var.kubernetes_version - kubeconfig_path = var.kubeconfig_path - + kubernetes_version = var.kubernetes_version oidc_provider_enabled = false workers_role_arns = [module.eks_workers.workers_role_arn, module.eks_workers_2.workers_role_arn] @@ -294,46 +258,6 @@ usage: |- } ``` - Module usage on [Terraform Cloud](https://www.terraform.io/docs/cloud/index.html): - - ```hcl - provider "aws" { - region = "us-east-2" - - assume_role { - role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - } - } - - module "eks_cluster" { - source = "git::https://github.com/cloudposse/terraform-aws-eks-cluster.git?ref=master" - namespace = var.namespace - stage = var.stage - name = var.name - attributes = var.attributes - tags = var.tags - region = "us-east-2" - vpc_id = module.vpc.vpc_id - subnet_ids = module.subnets.public_subnet_ids - - local_exec_interpreter = "/bin/bash" - kubernetes_version = "1.14" - - workers_role_arns = [module.eks_workers.workers_role_arn] - workers_security_group_ids = [module.eks_workers.security_group_id] - - # Terraform Cloud configurations - kubeconfig_path = "~/.kube/config" - configmap_auth_file = "/home/terraform/.terraform/configmap-auth.yaml" - install_aws_cli = true - install_kubectl = true - external_packages_install_path = "~/.terraform/bin" - aws_eks_update_kubeconfig_additional_arguments = "--verbose" - aws_cli_assume_role_arn = "arn:aws:iam::xxxxxxxxxxx:role/OrganizationAccountAccessRole" - aws_cli_assume_role_session_name = "eks_cluster_example_session" - } - ``` - include: - "docs/targets.md" - "docs/terraform.md" diff --git a/auth.tf b/auth.tf index ec940951..9f44e00c 100644 --- a/auth.tf +++ b/auth.tf @@ -32,14 +32,6 @@ locals { certificate_authority_data_map = local.certificate_authority_data_list_internal[0] certificate_authority_data = local.certificate_authority_data_map["data"] - configmap_auth_template_file = var.configmap_auth_template_file == "" ? join("/", [path.module, "configmap-auth.yaml.tpl"]) : var.configmap_auth_template_file - configmap_auth_file = var.configmap_auth_file == "" ? join("/", [path.module, "configmap-auth.yaml"]) : var.configmap_auth_file - - external_packages_install_path = var.external_packages_install_path == "" ? join("/", [path.module, ".terraform/bin"]) : var.external_packages_install_path - kubectl_version = var.kubectl_version == "" ? "$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)" : var.kubectl_version - - cluster_name = join("", aws_eks_cluster.default.*.id) - # Add worker nodes role ARNs (could be from many worker groups) to the ConfigMap map_worker_roles = [ for role_arn in var.workers_role_arns : { @@ -51,96 +43,55 @@ locals { ] } ] - - map_worker_roles_yaml = trimspace(yamlencode(local.map_worker_roles)) - map_additional_iam_roles_yaml = trimspace(yamlencode(var.map_additional_iam_roles)) - map_additional_iam_users_yaml = trimspace(yamlencode(var.map_additional_iam_users)) - map_additional_aws_accounts_yaml = trimspace(yamlencode(var.map_additional_aws_accounts)) } -data "template_file" "configmap_auth" { - count = var.enabled && var.apply_config_map_aws_auth ? 1 : 0 - template = file(local.configmap_auth_template_file) +resource "null_resource" "wait_for_cluster" { + count = var.enabled && var.apply_config_map_aws_auth ? 1 : 0 + depends_on = [aws_eks_cluster.default[0]] - vars = { - map_worker_roles_yaml = local.map_worker_roles_yaml - map_additional_iam_roles_yaml = local.map_additional_iam_roles_yaml - map_additional_iam_users_yaml = local.map_additional_iam_users_yaml - map_additional_aws_accounts_yaml = local.map_additional_aws_accounts_yaml + provisioner "local-exec" { + command = var.wait_for_cluster_command + interpreter = var.local_exec_interpreter + environment = { + ENDPOINT = aws_eks_cluster.default[0].endpoint + } } } -resource "local_file" "configmap_auth" { - count = var.enabled && var.apply_config_map_aws_auth ? 1 : 0 - content = join("", data.template_file.configmap_auth.*.rendered) - filename = local.configmap_auth_file +data "aws_eks_cluster" "eks" { + count = var.enabled && var.apply_config_map_aws_auth ? 1 : 0 + name = join("", aws_eks_cluster.default.*.id) } -resource "null_resource" "apply_configmap_auth" { +# Get an authentication token to communicate with the EKS cluster. +# By default (before other roles are added to the Auth ConfigMap), you can authenticate to EKS cluster only by assuming the role that created the cluster. +# `aws_eks_cluster_auth` uses IAM credentials from the AWS provider to generate a temporary token. +# If the AWS provider assumes an IAM role, `aws_eks_cluster_auth` will use the same IAM role to get the auth token. +# https://www.terraform.io/docs/providers/aws/d/eks_cluster_auth.html +data "aws_eks_cluster_auth" "eks" { count = var.enabled && var.apply_config_map_aws_auth ? 1 : 0 + name = join("", aws_eks_cluster.default.*.id) +} - triggers = { - cluster_updated = join("", aws_eks_cluster.default.*.id) - worker_roles_updated = local.map_worker_roles_yaml - additional_roles_updated = local.map_additional_iam_roles_yaml - additional_users_updated = local.map_additional_iam_users_yaml - additional_aws_accounts_updated = local.map_additional_aws_accounts_yaml - configmap_auth_file_content_changed = join("", local_file.configmap_auth.*.content) - configmap_auth_file_id_changed = join("", local_file.configmap_auth.*.id) - } - - depends_on = [aws_eks_cluster.default, local_file.configmap_auth] - - provisioner "local-exec" { - interpreter = [var.local_exec_interpreter, "-c"] - - command = </dev/null; do sleep 5; done - kubectl version --kubeconfig ${var.kubeconfig_path} - kubectl apply -f ${local.configmap_auth_file} --kubeconfig ${var.kubeconfig_path} - echo 'Applied Auth ConfigMap with kubectl' - EOT + data = { + mapRoles = yamlencode(distinct(concat(local.map_worker_roles, var.map_additional_iam_roles))) + mapUsers = yamlencode(var.map_additional_iam_users) + mapAccounts = yamlencode(var.map_additional_aws_accounts) } } diff --git a/configmap-auth.yaml.tpl b/configmap-auth.yaml.tpl deleted file mode 100644 index 13558408..00000000 --- a/configmap-auth.yaml.tpl +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: aws-auth - namespace: kube-system -data: - mapRoles: | - ${indent(4, map_worker_roles_yaml)} -%{if map_additional_iam_roles_yaml != "[]" } - ${indent(4, map_additional_iam_roles_yaml)} -%{ endif } -%{if map_additional_iam_users_yaml != "[]" } - mapUsers: | - ${indent(4, map_additional_iam_users_yaml)} -%{ endif } -%{if map_additional_aws_accounts_yaml != "[]" } - mapAccounts: | - ${indent(4, map_additional_aws_accounts_yaml)} -%{ endif } diff --git a/docs/terraform.md b/docs/terraform.md index 5ed95b8f..72f346a1 100644 --- a/docs/terraform.md +++ b/docs/terraform.md @@ -4,28 +4,17 @@ |------|-------------|:----:|:-----:|:-----:| | allowed_cidr_blocks | List of CIDR blocks to be allowed to connect to the EKS cluster | list(string) | `` | no | | allowed_security_groups | List of Security Group IDs to be allowed to connect to the EKS cluster | list(string) | `` | no | -| apply_config_map_aws_auth | Whether to execute `kubectl apply` to apply the ConfigMap to allow worker nodes to join the EKS cluster | bool | `true` | no | +| apply_config_map_aws_auth | Whether to apply the ConfigMap to allow worker nodes to join the EKS cluster and allow additional users, accounts and roles to acces the cluster | bool | `true` | no | | attributes | Additional attributes (e.g. `1`) | list(string) | `` | no | -| aws_cli_assume_role_arn | IAM Role ARN for AWS CLI to assume before calling `aws eks` to update `kubeconfig` | string | `` | no | -| aws_cli_assume_role_session_name | An identifier for the assumed role session when assuming the IAM Role for AWS CLI before calling `aws eks` to update `kubeconfig` | string | `` | no | -| aws_eks_update_kubeconfig_additional_arguments | Additional arguments for `aws eks update-kubeconfig` command, e.g. `--role-arn xxxxxxxxx`. For more info, see https://docs.aws.amazon.com/cli/latest/reference/eks/update-kubeconfig.html | string | `` | no | | cluster_log_retention_period | Number of days to retain cluster logs. Requires `enabled_cluster_log_types` to be set. See https://docs.aws.amazon.com/en_us/eks/latest/userguide/control-plane-logs.html. | number | `0` | no | -| configmap_auth_file | Path to `configmap_auth_file` | string | `` | no | -| configmap_auth_template_file | Path to `config_auth_template_file` | string | `` | no | | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes` | string | `-` | no | | enabled | Set to false to prevent the module from creating any resources | bool | `true` | no | | enabled_cluster_log_types | A list of the desired control plane logging to enable. For more information, see https://docs.aws.amazon.com/en_us/eks/latest/userguide/control-plane-logs.html. Possible values [`api`, `audit`, `authenticator`, `controllerManager`, `scheduler`] | list(string) | `` | no | | endpoint_private_access | Indicates whether or not the Amazon EKS private API server endpoint is enabled. Default to AWS EKS resource and it is false | bool | `false` | no | | endpoint_public_access | Indicates whether or not the Amazon EKS public API server endpoint is enabled. Default to AWS EKS resource and it is true | bool | `true` | no | | environment | Environment, e.g. 'prod', 'staging', 'dev', 'pre-prod', 'UAT' | string | `` | no | -| external_packages_install_path | Path to install external packages, e.g. AWS CLI and `kubectl`. Used when the module is provisioned on workstations where the external packages are not installed by default, e.g. Terraform Cloud workers | string | `` | no | -| install_aws_cli | Set to `true` to install AWS CLI if the module is provisioned on workstations where AWS CLI is not installed by default, e.g. Terraform Cloud workers | bool | `false` | no | -| install_kubectl | Set to `true` to install `kubectl` if the module is provisioned on workstations where `kubectl` is not installed by default, e.g. Terraform Cloud workers | bool | `false` | no | -| jq_version | Version of `jq` to download to extract temporaly credentials after running `aws sts assume-role` if AWS CLI needs to assume role to access the cluster (if variable `aws_cli_assume_role_arn` is set) | string | `1.6` | no | -| kubeconfig_path | The path to `kubeconfig` file | string | `~/.kube/config` | no | -| kubectl_version | `kubectl` version to install. If not specified, the latest version will be used | string | `` | no | -| kubernetes_version | Desired Kubernetes master version. If you do not specify a value, the latest available version is used | string | `1.14` | no | -| local_exec_interpreter | shell to use for local exec | string | `/bin/bash` | no | +| kubernetes_version | Desired Kubernetes master version. If you do not specify a value, the latest available version is used | string | `1.15` | no | +| local_exec_interpreter | shell to use for local_exec | list(string) | `` | no | | map_additional_aws_accounts | Additional AWS account numbers to add to `config-map-aws-auth` ConfigMap | list(string) | `` | no | | map_additional_iam_roles | Additional IAM roles to add to `config-map-aws-auth` ConfigMap | object | `` | no | | map_additional_iam_users | Additional IAM users to add to `config-map-aws-auth` ConfigMap | object | `` | no | @@ -38,6 +27,7 @@ | subnet_ids | A list of subnet IDs to launch the cluster in | list(string) | - | yes | | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | map(string) | `` | no | | vpc_id | VPC ID for the EKS cluster | string | - | yes | +| wait_for_cluster_command | `local-exec` command to execute to determine if the EKS cluster is healthy. Cluster endpoint are available as environment variable `ENDPOINT` | string | `curl --silent --fail --retry 60 --retry-delay 5 --retry-connrefused --insecure --output /dev/null $ENDPOINT/healthz` | no | | workers_role_arns | List of Role ARNs of the worker nodes | list(string) | - | yes | | workers_security_group_ids | Security Group IDs of the worker nodes | list(string) | - | yes | diff --git a/examples/complete/fixtures.us-east-2.tfvars b/examples/complete/fixtures.us-east-2.tfvars index a62cafca..19cd3480 100644 --- a/examples/complete/fixtures.us-east-2.tfvars +++ b/examples/complete/fixtures.us-east-2.tfvars @@ -26,9 +26,7 @@ cpu_utilization_low_threshold_percent = 20 associate_public_ip_address = true -kubernetes_version = "1.14" - -kubeconfig_path = "/.kube/config" +kubernetes_version = "1.15" oidc_provider_enabled = true diff --git a/examples/complete/main.tf b/examples/complete/main.tf index d4f1a0fa..ae0d48e6 100644 --- a/examples/complete/main.tf +++ b/examples/complete/main.tf @@ -79,32 +79,18 @@ module "eks_workers" { } module "eks_cluster" { - source = "../../" - namespace = var.namespace - stage = var.stage - name = var.name - attributes = var.attributes - tags = var.tags - region = var.region - vpc_id = module.vpc.vpc_id - subnet_ids = module.subnets.public_subnet_ids - kubernetes_version = var.kubernetes_version - kubeconfig_path = var.kubeconfig_path - local_exec_interpreter = var.local_exec_interpreter - - configmap_auth_template_file = var.configmap_auth_template_file - configmap_auth_file = var.configmap_auth_file + source = "../../" + namespace = var.namespace + stage = var.stage + name = var.name + attributes = var.attributes + tags = var.tags + region = var.region + vpc_id = module.vpc.vpc_id + subnet_ids = module.subnets.public_subnet_ids + kubernetes_version = var.kubernetes_version + local_exec_interpreter = var.local_exec_interpreter oidc_provider_enabled = var.oidc_provider_enabled - - install_aws_cli = var.install_aws_cli - install_kubectl = var.install_kubectl - kubectl_version = var.kubectl_version - jq_version = var.jq_version - external_packages_install_path = var.external_packages_install_path - aws_eks_update_kubeconfig_additional_arguments = var.aws_eks_update_kubeconfig_additional_arguments - aws_cli_assume_role_arn = var.aws_cli_assume_role_arn - aws_cli_assume_role_session_name = var.aws_cli_assume_role_session_name - enabled_cluster_log_types = var.enabled_cluster_log_types cluster_log_retention_period = var.cluster_log_retention_period diff --git a/examples/complete/variables.tf b/examples/complete/variables.tf index 58616017..82108d19 100644 --- a/examples/complete/variables.tf +++ b/examples/complete/variables.tf @@ -136,77 +136,12 @@ variable "map_additional_iam_users" { variable "oidc_provider_enabled" { type = bool - default = false + default = true description = "Create an IAM OIDC identity provider for the cluster, then you can create IAM roles to associate with a service account in the cluster, instead of using `kiam` or `kube2iam`. For more information, see https://docs.aws.amazon.com/eks/latest/userguide/enable-iam-roles-for-service-accounts.html" } -variable "kubeconfig_path" { - type = string - description = "The path to `kubeconfig` file" -} - variable "local_exec_interpreter" { - type = string - default = "/bin/bash" - description = "shell to use for local exec" -} - -variable "configmap_auth_template_file" { - type = string - default = "" - description = "Path to `config_auth_template_file`" -} - -variable "configmap_auth_file" { - type = string - default = "" - description = "Path to `configmap_auth_file`" -} - -variable "install_aws_cli" { - type = bool - default = false - description = "Set to `true` to install AWS CLI if the module is provisioned on workstations where AWS CLI is not installed by default, e.g. Terraform Cloud workers" -} - -variable "install_kubectl" { - type = bool - default = false - description = "Set to `true` to install `kubectl` if the module is provisioned on workstations where `kubectl` is not installed by default, e.g. Terraform Cloud workers" -} - -variable "kubectl_version" { - type = string - default = "" - description = "`kubectl` version to install. If not specified, the latest version will be used" -} - -variable "external_packages_install_path" { - type = string - default = "" - description = "Path to install external packages, e.g. AWS CLI and `kubectl`. Used when the module is provisioned on workstations where the external packages are not installed by default, e.g. Terraform Cloud workers" -} - -variable "aws_eks_update_kubeconfig_additional_arguments" { - type = string - default = "" - description = "Additional arguments for `aws eks update-kubeconfig` command, e.g. `--role-arn xxxxxxxxx`. For more info, see https://docs.aws.amazon.com/cli/latest/reference/eks/update-kubeconfig.html" -} - -variable "aws_cli_assume_role_arn" { - type = string - default = "" - description = "IAM Role ARN for AWS CLI to assume before calling `aws eks` to update kubeconfig" -} - -variable "aws_cli_assume_role_session_name" { - type = string - default = "" - description = "An identifier for the assumed role session when assuming the IAM Role for AWS CLI before calling `aws eks` to update `kubeconfig`" -} - -variable "jq_version" { - type = string - default = "1.6" - description = "Version of `jq` to download to extract temporaly credentials after running `aws sts assume-role` if AWS CLI needs to assume role to access the cluster (if variable `aws_cli_assume_role_arn` is set)" + type = list(string) + default = ["/bin/sh", "-c"] + description = "shell to use for local_exec" } diff --git a/examples/complete/versions.tf b/examples/complete/versions.tf index 9840ed75..62bb08b5 100644 --- a/examples/complete/versions.tf +++ b/examples/complete/versions.tf @@ -2,9 +2,10 @@ terraform { required_version = "~> 0.12.0" required_providers { - aws = "~> 2.0" - template = "~> 2.0" - null = "~> 2.0" - local = "~> 1.3" + aws = "~> 2.0" + template = "~> 2.0" + null = "~> 2.0" + local = "~> 1.3" + kubernetes = "~> 1.11" } } diff --git a/test/src/Makefile b/test/src/Makefile index e59dda32..e6489882 100644 --- a/test/src/Makefile +++ b/test/src/Makefile @@ -42,7 +42,7 @@ init: $(BASE)/vendor .PHONY : test ## Run tests test: init - cd $(BASE) && go test -v -timeout 30m -run TestExamplesComplete + cd $(BASE) && go test -v -timeout 60m -run TestExamplesComplete .PHONY : clean ## Clean up files diff --git a/test/src/examples_complete_test.go b/test/src/examples_complete_test.go index 66c11a85..c79ddd91 100644 --- a/test/src/examples_complete_test.go +++ b/test/src/examples_complete_test.go @@ -1,6 +1,7 @@ package test import ( + "encoding/base64" "fmt" "sync/atomic" "testing" @@ -8,13 +9,50 @@ import ( "github.com/gruntwork-io/terratest/modules/terraform" "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" "k8s.io/client-go/tools/cache" - "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/aws-iam-authenticator/pkg/token" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/eks" ) +func newClientset(cluster *eks.Cluster) (*kubernetes.Clientset, error) { + gen, err := token.NewGenerator(true, false) + if err != nil { + return nil, err + } + opts := &token.GetTokenOptions{ + ClusterID: aws.StringValue(cluster.Name), + } + tok, err := gen.GetWithOptions(opts) + if err != nil { + return nil, err + } + ca, err := base64.StdEncoding.DecodeString(aws.StringValue(cluster.CertificateAuthority.Data)) + if err != nil { + return nil, err + } + clientset, err := kubernetes.NewForConfig( + &rest.Config{ + Host: aws.StringValue(cluster.Endpoint), + BearerToken: tok.Token, + TLSClientConfig: rest.TLSClientConfig{ + CAData: ca, + }, + }, + ) + if err != nil { + return nil, err + } + return clientset, nil +} + // Test the Terraform module in examples/complete using Terratest. func TestExamplesComplete(t *testing.T) { t.Parallel() @@ -83,13 +121,26 @@ func TestExamplesComplete(t *testing.T) { // https://www.rushtehrani.com/post/using-kubernetes-api // https://rancher.com/using-kubernetes-api-go-kubecon-2017-session-recap // https://gianarb.it/blog/kubernetes-shared-informer + // https://stackoverflow.com/questions/60547409/unable-to-obtain-kubeconfig-of-an-aws-eks-cluster-in-go-code/60573982#60573982 fmt.Println("Waiting for worker nodes to join the EKS cluster") - kubeconfigPath := "/.kube/config" - config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + clusterName := "eg-test-eks-cluster" + region := "us-east-2" + + sess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(region), + })) + + eksSvc := eks.New(sess) + + input := &eks.DescribeClusterInput{ + Name: aws.String(clusterName), + } + + result, err := eksSvc.DescribeCluster(input) assert.NoError(t, err) - clientset, err := kubernetes.NewForConfig(config) + clientset, err := newClientset(result.Cluster) assert.NoError(t, err) factory := informers.NewSharedInformerFactory(clientset, 0) diff --git a/variables.tf b/variables.tf index e76f61d0..de29e717 100644 --- a/variables.tf +++ b/variables.tf @@ -85,7 +85,7 @@ variable "workers_security_group_ids" { variable "kubernetes_version" { type = string - default = "1.14" + default = "1.15" description = "Desired Kubernetes master version. If you do not specify a value, the latest available version is used" } @@ -128,7 +128,7 @@ variable "cluster_log_retention_period" { variable "apply_config_map_aws_auth" { type = bool default = true - description = "Whether to execute `kubectl apply` to apply the ConfigMap to allow worker nodes to join the EKS cluster" + description = "Whether to apply the ConfigMap to allow worker nodes to join the EKS cluster and allow additional users, accounts and roles to acces the cluster" } variable "map_additional_aws_accounts" { @@ -161,74 +161,14 @@ variable "map_additional_iam_users" { default = [] } -variable "kubeconfig_path" { - type = string - default = "~/.kube/config" - description = "The path to `kubeconfig` file" -} - variable "local_exec_interpreter" { - type = string - default = "/bin/bash" - description = "shell to use for local exec" -} - -variable "configmap_auth_template_file" { - type = string - default = "" - description = "Path to `config_auth_template_file`" -} - -variable "configmap_auth_file" { - type = string - default = "" - description = "Path to `configmap_auth_file`" -} - -variable "install_aws_cli" { - type = bool - default = false - description = "Set to `true` to install AWS CLI if the module is provisioned on workstations where AWS CLI is not installed by default, e.g. Terraform Cloud workers" -} - -variable "install_kubectl" { - type = bool - default = false - description = "Set to `true` to install `kubectl` if the module is provisioned on workstations where `kubectl` is not installed by default, e.g. Terraform Cloud workers" -} - -variable "kubectl_version" { - type = string - default = "" - description = "`kubectl` version to install. If not specified, the latest version will be used" -} - -variable "external_packages_install_path" { - type = string - default = "" - description = "Path to install external packages, e.g. AWS CLI and `kubectl`. Used when the module is provisioned on workstations where the external packages are not installed by default, e.g. Terraform Cloud workers" -} - -variable "aws_eks_update_kubeconfig_additional_arguments" { - type = string - default = "" - description = "Additional arguments for `aws eks update-kubeconfig` command, e.g. `--role-arn xxxxxxxxx`. For more info, see https://docs.aws.amazon.com/cli/latest/reference/eks/update-kubeconfig.html" -} - -variable "aws_cli_assume_role_arn" { - type = string - default = "" - description = "IAM Role ARN for AWS CLI to assume before calling `aws eks` to update `kubeconfig`" -} - -variable "aws_cli_assume_role_session_name" { - type = string - default = "" - description = "An identifier for the assumed role session when assuming the IAM Role for AWS CLI before calling `aws eks` to update `kubeconfig`" + type = list(string) + default = ["/bin/sh", "-c"] + description = "shell to use for local_exec" } -variable "jq_version" { +variable "wait_for_cluster_command" { type = string - default = "1.6" - description = "Version of `jq` to download to extract temporaly credentials after running `aws sts assume-role` if AWS CLI needs to assume role to access the cluster (if variable `aws_cli_assume_role_arn` is set)" + default = "curl --silent --fail --retry 60 --retry-delay 5 --retry-connrefused --insecure --output /dev/null $ENDPOINT/healthz" + description = "`local-exec` command to execute to determine if the EKS cluster is healthy. Cluster endpoint are available as environment variable `ENDPOINT`" } diff --git a/versions.tf b/versions.tf index 9840ed75..62bb08b5 100644 --- a/versions.tf +++ b/versions.tf @@ -2,9 +2,10 @@ terraform { required_version = "~> 0.12.0" required_providers { - aws = "~> 2.0" - template = "~> 2.0" - null = "~> 2.0" - local = "~> 1.3" + aws = "~> 2.0" + template = "~> 2.0" + null = "~> 2.0" + local = "~> 1.3" + kubernetes = "~> 1.11" } }