diff --git a/modules/terraform-aws-proxy/.terraform-docs.yaml b/modules/terraform-aws-proxy/.terraform-docs.yaml new file mode 100644 index 0000000..0936036 --- /dev/null +++ b/modules/terraform-aws-proxy/.terraform-docs.yaml @@ -0,0 +1,21 @@ +formatter: markdown +header-from: doc_fragments/header.md +settings: + anchor: true + color: true + default: true + escape: true + html: true + indent: 2 + required: true + sensitive: true + type: true + + +sort: + enabled: true + by: required + +output: + file: README.md + mode: replace \ No newline at end of file diff --git a/modules/terraform-aws-proxy/README.md b/modules/terraform-aws-proxy/README.md new file mode 100644 index 0000000..33e17aa --- /dev/null +++ b/modules/terraform-aws-proxy/README.md @@ -0,0 +1,88 @@ + +# Terraform Module for AWS Transit Gateway + +This module contains resource files and example variable definition files to create and configure and EC2 Auto-Scaling Group to create a highly available Squid Proxy service. A Network Load Balancer is also created to forward traffic to the proxy instances. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where the CDP Environment uses a proxy configuration via the Network Load Balancer. + +## Usage + +The [examples](./examples) directory has example of using this module: + +* `ex01-minimal_inputs` demonstrates how this module can be used to create Squid proxy instances and NLB in a networking VPC. The [terraform-aws-vpc](../../../terraform-aws-vpc/README.md) module is also used as part of this example. + +The sample `terraform.tfvars.sample` describes the required inputs for the example. + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | > 1.3.0 | +| [aws](#requirement\_aws) | ~> 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 4.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_attachment.proxy_asg_tg_attach](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_attachment) | resource | +| [aws_autoscaling_group.proxy_asg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | +| [aws_launch_template.proxy_lt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [aws_lb.proxy_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | +| [aws_lb_listener.proxy_lb_listener](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource | +| [aws_lb_target_group.proxy_tg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | +| [aws_route.vpc_tgw_route](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route) | resource | +| [aws_security_group.proxy_sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | +| [aws_security_group_rule.proxy_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.proxy_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.proxy_lb_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_ami.proxy_default_ami](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_network_interface.proxy_lb](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/network_interface) | data source | +| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | +| [aws_route_table.proxy_rt](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route_table) | data source | +| [aws_vpc.proxy_vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/vpc) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [lb\_subnet\_ids](#input\_lb\_subnet\_ids) | The IDs of the subnet for the Network Load Balancer | `list(any)` | n/a | yes | +| [network\_load\_balancer\_name](#input\_network\_load\_balancer\_name) | Name of Network Load Balancer for the Proxy. | `string` | n/a | yes | +| [proxy\_autoscaling\_group\_name](#input\_proxy\_autoscaling\_group\_name) | Name of Autoscaling Group for the Proxy VMs. | `string` | n/a | yes | +| [proxy\_aws\_keypair\_name](#input\_proxy\_aws\_keypair\_name) | SSH Keypair name for the proxy VM | `string` | n/a | yes | +| [proxy\_launch\_template\_name](#input\_proxy\_launch\_template\_name) | Name of Launch Template for the Proxy VMs. | `string` | n/a | yes | +| [proxy\_subnet\_ids](#input\_proxy\_subnet\_ids) | The IDs of the subnet where the proxy VMs will run | `list(any)` | n/a | yes | +| [target\_group\_proxy\_name](#input\_target\_group\_proxy\_name) | Name of Target Group for the Proxy. | `string` | n/a | yes | +| [vpc\_id](#input\_vpc\_id) | VPC ID for where the proxy VM will run | `string` | n/a | yes | +| [autoscaling\_group\_scaling](#input\_autoscaling\_group\_scaling) | Minimum, maximum and desired size of EC2 instance in the Auto Scaling Group. | 
object({
    min_size         = number
    max_size         = number
    desired_capacity = number
  })
| 
{
  "desired_capacity": 3,
  "max_size": 6,
  "min_size": 3
}
| no | +| [aws\_region](#input\_aws\_region) | AWS region, used in Proxy Whitelist configuration files. If not provided will perform lookup of aws\_region data source. | `string` | `null` | no | +| [cdp\_region](#input\_cdp\_region) | CDP Control Plane region, used in Proxy Whitelist configuration files. | `string` | `"us-west-1"` | no | +| [create\_proxy\_sg](#input\_create\_proxy\_sg) | Flag to specify if the Security Group for the proxy should be created. | `bool` | `true` | no | +| [egress\_rules](#input\_egress\_rules) | List of egress rules to create. Used only if create\_proxy\_sg is true | 
list(object({
    cidrs     = list(string)
    from_port = number
    to_port   = optional(number)
    protocol  = string
  }))
| 
[
  {
    "cidrs": [
      "0.0.0.0/0"
    ],
    "from_port": 0,
    "protocol": "all",
    "to_port": 0
  }
]
| no | +| [enable\_proxy\_public\_ip](#input\_enable\_proxy\_public\_ip) | Assign a public IP address to the Proxy VM | `bool` | `true` | no | +| [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `{}` | no | +| [ingress\_rules](#input\_ingress\_rules) | List of ingress rules to create. Used only if create\_proxy\_sg is true | 
list(object({
    cidrs     = list(string)
    from_port = number
    to_port   = optional(number)
    protocol  = string
  }))
| `[]` | no | +| [proxy\_aws\_ami](#input\_proxy\_aws\_ami) | The AWS AMI to use for the proxy VM | `string` | `null` | no | +| [proxy\_aws\_instance\_type](#input\_proxy\_aws\_instance\_type) | The EC2 instance type to use for the proxy VM | `string` | `"t3.medium"` | no | +| [proxy\_launch\_template\_user\_data\_file](#input\_proxy\_launch\_template\_user\_data\_file) | Location of the AWS Launch Template user data script. If not specified the files/user-data-proxy.sh.tpl file accompanying the module is used. | `string` | `null` | no | +| [proxy\_port](#input\_proxy\_port) | Port number which the proxy and NLB listens | `number` | `3129` | no | +| [proxy\_security\_group\_id](#input\_proxy\_security\_group\_id) | ID for existing Security Group to be used for the proxy VM. Required when create\_proxy\_sg is false | `string` | `null` | no | +| [proxy\_security\_group\_name](#input\_proxy\_security\_group\_name) | Name of Proxy Security Group for CDP environment. Used only if create\_proxy\_sg is true. | `string` | `null` | no | +| [proxy\_whitelist\_file](#input\_proxy\_whitelist\_file) | Location of the Proxy Whitelist file. If not specified the files/squid-http-whitelist.txt.tpl file accompanying the module is used. | `string` | `null` | no | +| [route\_tables\_to\_update](#input\_route\_tables\_to\_update) | List of any route tables to update to point to the Network interface of the Proxy VM | 
list(object({
    route_tables           = list(string)
    destination_cidr_block = string
  }))
| `[]` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [proxy\_lb\_arn](#output\_proxy\_lb\_arn) | ARN of the Proxy Load Balancer | +| [proxy\_lb\_dns\_name](#output\_proxy\_lb\_dns\_name) | DNS Name of the Proxy Load Balancer | +| [proxy\_port](#output\_proxy\_port) | Port where Proxy is running | + \ No newline at end of file diff --git a/modules/terraform-aws-proxy/data.tf b/modules/terraform-aws-proxy/data.tf new file mode 100644 index 0000000..8fde7ba --- /dev/null +++ b/modules/terraform-aws-proxy/data.tf @@ -0,0 +1,61 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Find AWS region +data "aws_region" "current" {} + +# Find AMI for default proxy VMs +data "aws_ami" "proxy_default_ami" { + + most_recent = true + + filter { + name = "name" + values = ["al2023-ami-2023*-x86_64"] + } + + owners = ["amazon"] +} + +# Find details of the VPC +data "aws_vpc" "proxy_vpc" { + id = var.vpc_id +} + +# Find the network interface for the load balancer +data "aws_network_interface" "proxy_lb" { + + for_each = { for k, v in var.lb_subnet_ids : k => v } + + filter { + name = "description" + values = ["ELB ${aws_lb.proxy_lb.arn_suffix}"] + } + + filter { + name = "subnet-id" + values = [each.value] + } +} + +# Find route table details +data "aws_route_table" "proxy_rt" { + + for_each = { + for k, v in local.route_tables_to_update : k => v + } + + route_table_id = each.value.route_table + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/defaults.tf b/modules/terraform-aws-proxy/defaults.tf new file mode 100644 index 0000000..98521c7 --- /dev/null +++ b/modules/terraform-aws-proxy/defaults.tf @@ -0,0 +1,71 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +locals { + + # AWS region (derived from provider lookup unless overridden) + aws_region = coalesce(var.aws_region, data.aws_region.current.name) + + # Security Groups + proxy_security_group_id = (var.create_proxy_sg ? + aws_security_group.proxy_sg[0].id : var.proxy_security_group_id) + + # Proxy VM + proxy_aws_ami = coalesce(var.proxy_aws_ami, data.aws_ami.proxy_default_ami.id) + + # User data for Proxy VM (for squid proxy) + proxy_launch_template_user_data_file = coalesce(var.proxy_launch_template_user_data_file, "${path.module}/files/squid-user-data.sh.tpl") + + # Squid whitelist file + proxy_whitelist_file = coalesce(var.proxy_whitelist_file, "${path.module}/files/squid-whitelist.txt.tpl") + + # Local variables to determine route table to Internal NLB eni mapping + route_tables_to_update = flatten([ + for route in var.route_tables_to_update : + [ + for rt in route.route_tables : + { + route_table = rt + destination_cidr_block = route.destination_cidr_block + } + ] + ]) + + lb_eni_details = [ + for eni in data.aws_network_interface.proxy_lb : + { + eni_id = eni.id + az = eni.availability_zone + subnet_id = eni.subnet_id + } + ] + + # TODO: Explore better rt to eni mapping with the below + # route_table_details = [ + # for rt in data.aws_route_table.proxy_rt : + # { + # rt_id = rt.id + # subnet_ids = rt.associations[*].subnet_id + # } + # ] + + route_table_to_lb_eni_assoc = { + for k, v in data.aws_route_table.proxy_rt : v.id => { + # TODO: eni of same subnet assoc if possible otherwise the first eni_id in lb_eni_details + eni = local.lb_eni_details[0].eni_id + } + } + + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/doc_fragments/header.md b/modules/terraform-aws-proxy/doc_fragments/header.md new file mode 100644 index 0000000..205750f --- /dev/null +++ b/modules/terraform-aws-proxy/doc_fragments/header.md @@ -0,0 +1,11 @@ +# Terraform Module for AWS Transit Gateway + +This module contains resource files and example variable definition files to create and configure and EC2 Auto-Scaling Group to create a highly available Squid Proxy service. A Network Load Balancer is also created to forward traffic to the proxy instances. This module can be used to assist in deploying Cloudera Data Platform (CDP) Public Cloud in a fully private networking configuration where the CDP Environment uses a proxy configuration via the Network Load Balancer. + +## Usage + +The [examples](./examples) directory has example of using this module: + +* `ex01-minimal_inputs` demonstrates how this module can be used to create Squid proxy instances and NLB in a networking VPC. The [terraform-aws-vpc](../../../terraform-aws-vpc/README.md) module is also used as part of this example. + +The sample `terraform.tfvars.sample` describes the required inputs for the example. diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf new file mode 100644 index 0000000..838f407 --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/main.tf @@ -0,0 +1,72 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +provider "aws" { + profile = var.aws_profile + region = var.aws_region +} + +module "ex01_network_vpc" { + source = "../../../terraform-aws-vpc" + + cdp_vpc = false + vpc_name = "${var.name_prefix}-network-vpc" + vpc_cidr = "10.11.0.0/16" + enable_nat_gateway = false + + private_cidr_range = var.network_vpc_private_cidr_range + public_cidr_range = var.network_vpc_public_cidr_range + +} + +module "ex01_proxy" { + source = "../.." + + vpc_id = module.ex01_network_vpc.vpc_id + + proxy_security_group_name = "${var.name_prefix}-sg" + proxy_aws_keypair_name = var.aws_key_pair + + proxy_launch_template_name = "${var.name_prefix}-lt" + proxy_autoscaling_group_name = "${var.name_prefix}-asg" + proxy_subnet_ids = module.ex01_network_vpc.public_subnets + + network_load_balancer_name = "${var.name_prefix}-lb" + target_group_proxy_name = "${var.name_prefix}-tg" + lb_subnet_ids = module.ex01_network_vpc.private_subnets + + ingress_rules = [ + { + cidrs = module.ex01_network_vpc.vpc_cidr_blocks + from_port = 0 + to_port = 65535 + protocol = "tcp" + }, + { + cidrs = var.ingress_extra_cidrs + from_port = 22 + # to_port = + protocol = "tcp" + } + ] + + route_tables_to_update = [ + # Route all Internet traffic in Networking VPC to the proxy instance(s) + { + route_tables = module.ex01_network_vpc.private_route_tables + destination_cidr_block = "0.0.0.0/0" + } + ] + +} diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample new file mode 100644 index 0000000..2e76630 --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/terraform.tfvars.sample @@ -0,0 +1,23 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global Settings ------- +name_prefix = "" + +# ------- Cloud Settings ------- +aws_region = "" # Change this to specify Cloud Provider region, e.g. eu-west-1 +aws_key_pair = "" # Change this with the name of a pre-existing AWS keypair, e.g. my-keypair + +# ------- Proxy settings ------- +ingress_extra_cidrs = ["", ""] # Any additional CIDRs the Proxy Security Groups for SSH access diff --git a/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf new file mode 100644 index 0000000..94c333e --- /dev/null +++ b/modules/terraform-aws-proxy/examples/ex01-minimal_inputs/variables.tf @@ -0,0 +1,84 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "aws_profile" { + type = string + description = "Profile for AWS cloud credentials" + + # Profile is default unless explicitly specified + default = "default" +} + +variable "aws_region" { + type = string + description = "Region which Cloud resources will be created" +} + +variable "env_tags" { + type = map(any) + description = "Tags applied to provised resources" + + default = null +} + +variable "name_prefix" { + type = string + description = "Shorthand name to use when naming resources." +} + +# ------- VPC settings ------- +variable "cdp_vpc_private_cidr_range" { + type = number + description = "Size of each private subnet for CDP VPC." + + default = 19 +} + +variable "cdp_vpc_public_cidr_range" { + type = number + description = "Size of each public subnet for CDP VPC." + + default = 24 +} + +variable "network_vpc_private_cidr_range" { + type = number + description = "Size of each private subnet for Network VPC." + + default = 19 +} + +variable "network_vpc_public_cidr_range" { + type = number + description = "Size of each public subnet for Network VPC." + + default = 24 +} + +# ------- Proxy settings ------- +variable "ingress_extra_cidrs" { + type = list(string) + description = "List of extra ingress rules to create." + + default = [] +} + + +variable "aws_key_pair" { + type = string + + description = "Name of the Public SSH key for the CDP environment" + +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl b/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl new file mode 100644 index 0000000..e3e22fe --- /dev/null +++ b/modules/terraform-aws-proxy/files/squid-user-data.sh.tpl @@ -0,0 +1,145 @@ +#!/bin/bash + +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +##################################################### +# User Data Bash script to setup suid proxy on a +# Amazon Linux 2 EC2 instance. +##################################################### + +# Install +yum update -y +yum install -y squid + +# Create Squid whitelist file +echo "${whitelist_txt}" > /etc/squid/whitelist.txt + +# Generate self-signed cert +mkdir -p /etc/squid/ssl +openssl req \ + -x509 -new -sha256 -nodes \ + -newkey rsa:2048 -days 365 \ + -keyout /etc/squid/ssl/private.key \ + -out /etc/squid/ssl/cert.pem \ + -subj "/C=XX/ST=XX/L=squid/O=squid/CN=squid" + + +# Create Squid configuration file +cat > /etc/squid/squid.conf << EOF +# Working Config File for non-transparent proxy +# Recommended minimum configuration: +# +# Example rule allowing access from your local networks. +# Adapt to list your (internal) IP networks from where browsing +# should be allowed +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http + +# ACL for the whitelist +acl http-whitelist dstdomain "/etc/squid/whitelist.txt" + +# Deny access to URLs not in the whitelist +http_access allow http-whitelist + +http_port ${proxy_port} cert=/etc/squid/ssl/cert.pem key=/etc/squid/ssl/private.key +https_port ${proxy_port} cert=/etc/squid/ssl/cert.pem key=/etc/squid/ssl/private.key +ssl_bump bump all +sslcrtd_program /usr/lib/squid/ssl_crtd -s /var/lib/ssl_db -M 4MB +sslcrtd_children 8 startup=1 idle=1 + +# Deny access to all other URLs +http_access deny all + +# Recommended minimum Access Permission configuration: +# Deny requests to certain unsafe ports +http_access deny !Safe_ports + + +# Deny CONNECT to other than secure SSL ports +http_access deny CONNECT !SSL_ports + + +# Only allow cachemgr access from localhost +http_access allow localhost manager +http_access deny manager + + +# This default configuration only allows localhost requests because a more +# permissive Squid installation could introduce new attack vectors into the +# network by proxying external TCP connections to unprotected services. + + +http_access allow localhost + + +# The two deny rules below are unnecessary in this default configuration +# because they are followed by a "deny all" rule. However, they may become +# critically important when you start allowing external requests below them. +# Protect web applications running on the same server as Squid. They often +# assume that only local users can access them at "localhost" ports. +http_access deny to_localhost +# Protect cloud servers that provide local users with sensitive info about +# their server via certain well-known link-local (a.k.a. APIPA) addresses. +http_access deny to_linklocal + +# +# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS +# +#http_access allow whitelist_url +# For example, to allow access from your local networks, you may uncomment the +# following rule (and/or add rules that match your definition of "local"): +# http_access allow localnet +# And finally deny all other access to this proxy +#http_access deny all + + +http_access allow all +# Squid normally listens to port 3128 +# Squid normally listens to port 3128 +http_port 3128 +# Uncomment and adjust the following to add a disk cache directory. +#cache_dir ufs /var/spool/squid 100 16 256 +# Leave coredumps in the first cache dir +coredump_dir /var/spool/squid +# +# Add any of your own refresh_pattern entries above these. +# +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern . 0 20% 4320 +EOF + + +# Start and enable squid +systemctl enable squid +systemctl start squid \ No newline at end of file diff --git a/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl b/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl new file mode 100644 index 0000000..26056fa --- /dev/null +++ b/modules/terraform-aws-proxy/files/squid-whitelist.txt.tpl @@ -0,0 +1,123 @@ +# Cloudera CCMv2 for Persistent Control Plane connection +%{if cdp_region == "us-west-1" ~} +# ..US-based Control Plane +.v2.us-west-1.ccm.cdp.cloudera.com +%{ endif ~} + +%{if cdp_region == "eu-1" ~} +# ..EU-based Control Plane +.v2.ccm.eu-1.cdp.cloudera.com +%{ endif ~} + +%{if cdp_region == "ap-1" ~} +# ..AP-based Control Plane +.v2.ccm.ap-1.cdp.cloudera.com +%{ endif ~} + +# Cloudera Databus for Telemetry, billing and metering data +%{if cdp_region == "us-west-1" ~} +# ..US-based Control Plane +dbusapi.us-west-1.sigma.altus.cloudera.com +cloudera-dbus-prod.s3.amazonaws.com +%{ endif ~} + +%{if cdp_region == "eu-1" ~} +# ..EU-based Control Plane +api.eu-1.cdp.cloudera.com +mow-prod-eu-central-1-sigmadbus-dbus.s3.eu-central-1.amazonaws.com # NOTE - this is a subdomain of the CDP AWS bucket with flow definitions for CDF below +mow-prod-eu-central-1-sigmadbus-dbus.s3.amazonaws.com +%{ endif ~} + +%{if cdp_region == "ap-1" ~} +# ..AP-based Control Plane +api.ap-1.cdp.cloudera.com +# mow-prod-ap-southeast-2-sigmadbus-dbus.s3.ap-southeast-2.amazonaws.com # NOTE - this is a subdomain of the CDP AWS bucket with flow definitions for CDF below +mow-prod-ap-southeast-2-sigmadbus-dbus.s3.amazonaws.com +%{ endif ~} + +# Cloudera Manager parcels for Software distribution +archive.cloudera.com + +# Control Plane API +api.${cdp_region}.cdp.cloudera.com + +# Cloudera RPMs for workload agents +cloudera-service-delivery-cache.s3.amazonaws.com + +# Docker Images for CDE, CDF, CML and CDW +container.repository.cloudera.com +docker.repository.cloudera.com + +container.repo.cloudera.com + +%{if cdp_region == "us-west-1" ~} +# ..US-based Control Plane +prod-us-west-2-starport-layer-bucket.s3.us-west-2.amazonaws.com +prod-eu-west-1-starport-layer-bucket.s3.eu-west-1.amazonaws.com # NOTE - found that this domain was required for us control plane and eu aws region +prod-us-west-2-starport-layer-bucket.s3.amazonaws.com +s3-r-w.us-west-2.amazonaws.com +.execute-api.us-west-2.amazonaws.com +cloudera-dbus-prod.s3.us-west-2.amazonaws.com # NOTE - found that this domain was required for CDE +%{ endif ~} + +%{if cdp_region == "eu-1" ~} +# ..EU-based Control Plane +prod-eu-west-1-starport-layer-bucket.s3.eu-west-1.amazonaws.com +prod-eu-west-1-starport-layer-bucket.s3.amazonaws.com +s3-r-w.eu-west-1.amazonaws.com +.execute-api.eu-west-1.amazonaws.com +%{ endif ~} + +%{if cdp_region == "ap-1" ~} +# ..AP-based Control Plane +prod-ap-southeast-1-starport-layer-bucket.s3.ap-southeast-1.amazonaws.com +prod-ap-southeast-1-starport-layer-bucket.s3.amazonaws.com +s3-r-w.ap-southeast-1.amazonaws.com +.execute-api.ap-southeast-1.amazonaws.com +%{ endif ~} + +# CDP AWS bucket with flow definitions for CDF +%{if cdp_region == "us-west-1" ~} +# ..US-based Control Plane +.s3.us-west-1.amazonaws.com +%{ endif ~} + +%{if cdp_region == "eu-1" ~} +# ..EU-based Control Plane +# .s3.eu-central-1.amazonaws.com # More specific CDF bucket is used instead +cldr-mow-prod-eu-central-1-dfx-flow-artifacts.s3.eu-central-1.amazonaws.com +%{ endif ~} + +%{if cdp_region == "ap-1" ~} +# ..AP-based Control Plane +.s3.ap-southeast-2.amazonaws.com +%{ endif ~} + +# Public Signing Key Retrieval for CDE and CDF +console.${cdp_region}.cdp.cloudera.com + +%{if cdp_region == "us-west-1" ~} +# ..Additional for US-based Control Plane +consoleauth.altus.cloudera.com +%{ endif ~} + +# SQL Stream Builder PostgreSQL driver install +pypi.org + +# AMPs for CML incl Learning Hub +raw.githubusercontent.com +github.com + +# AWS-specific endpoints +.eks.${aws_region}.amazonaws.com +.autoscaling.${aws_region}.amazonaws.com +.cloudformation.${aws_region}.amazonaws.com + +# Additional endpoints for CDE deployment +cloudera.com +www.cloudera.com +truststore.pki.rds.amazonaws.com + +# Additional endpoints for CDW deployment +amazonlinux-2-repos-${aws_region}.s3.dualstack.${aws_region}.amazonaws.com +iamapi.${cdp_region}.altus.cloudera.com diff --git a/modules/terraform-aws-proxy/main.tf b/modules/terraform-aws-proxy/main.tf new file mode 100644 index 0000000..ea67d91 --- /dev/null +++ b/modules/terraform-aws-proxy/main.tf @@ -0,0 +1,187 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Networking - Security Group and Security Group rules ------- +# Security Group for proxy +resource "aws_security_group" "proxy_sg" { + + count = var.create_proxy_sg ? 1 : 0 + + vpc_id = var.vpc_id + name = var.proxy_security_group_name + description = var.proxy_security_group_name + tags = merge(var.env_tags, { Name = var.proxy_security_group_name }) +} + +# Create ingress SG rule +resource "aws_security_group_rule" "proxy_ingress" { + + for_each = { for k, v in var.ingress_rules : k => v + if var.create_proxy_sg + } + + security_group_id = aws_security_group.proxy_sg[0].id + type = "ingress" + description = "Ingress rules for ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = each.value.cidrs + from_port = each.value.from_port + to_port = coalesce(each.value.to_port, each.value.from_port) + protocol = each.value.protocol + +} + +# Access from NLB to Proxy VMs within the VPC +resource "aws_security_group_rule" "proxy_lb_ingress" { + + count = var.create_proxy_sg ? 1 : 0 + + security_group_id = aws_security_group.proxy_sg[0].id + type = "ingress" + description = "Allow traffic from NLB to Proxy VMs in ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = data.aws_vpc.proxy_vpc.cidr_block_associations[*].cidr_block + from_port = var.proxy_port + to_port = var.proxy_port + protocol = "TCP" +} + +# Create egress SG rule +resource "aws_security_group_rule" "proxy_egress" { + + for_each = { for k, v in var.egress_rules : k => v + if var.create_proxy_sg + } + + security_group_id = aws_security_group.proxy_sg[0].id + type = "egress" + description = "Egress rules for ${var.proxy_security_group_name} Proxy Security Group" + + cidr_blocks = each.value.cidrs #tfsec:ignore:aws-ec2-no-public-egress-sgr #tfsec:ignore:aws-vpc-no-public-egress-sgr + from_port = each.value.from_port + to_port = coalesce(each.value.to_port, each.value.from_port) + protocol = each.value.protocol + +} + +# ------- Proxy launch template and Auto-Scaling Groups ------- +resource "aws_launch_template" "proxy_lt" { + + name = var.proxy_launch_template_name + + image_id = local.proxy_aws_ami + instance_type = var.proxy_aws_instance_type + key_name = var.proxy_aws_keypair_name + + user_data = base64encode(templatefile(local.proxy_launch_template_user_data_file, { + proxy_port = var.proxy_port + whitelist_txt = templatefile(local.proxy_whitelist_file, { + aws_region = local.aws_region + cdp_region = var.cdp_region }) + })) + + network_interfaces { + associate_public_ip_address = var.enable_proxy_public_ip + security_groups = [local.proxy_security_group_id] + } + + metadata_options { + http_tokens = "required" + } + + tags = var.env_tags + +} + +resource "aws_autoscaling_group" "proxy_asg" { + name = var.proxy_autoscaling_group_name + min_size = var.autoscaling_group_scaling.min_size + max_size = var.autoscaling_group_scaling.max_size + desired_capacity = var.autoscaling_group_scaling.desired_capacity + + target_group_arns = [aws_lb_target_group.proxy_tg.arn] + + vpc_zone_identifier = var.proxy_subnet_ids + + launch_template { + id = aws_launch_template.proxy_lt.id + version = "$Latest" + } + + dynamic "tag" { + for_each = merge(var.env_tags, { Name = "${var.proxy_autoscaling_group_name}-proxy" }) + content { + key = tag.key + value = tag.value + propagate_at_launch = true + } + } + +} + +# ------- Internal Network Load Balancer ------- + +resource "aws_lb" "proxy_lb" { + name = var.network_load_balancer_name + internal = true + load_balancer_type = "network" + subnets = var.lb_subnet_ids + + tags = var.env_tags +} + +resource "aws_lb_listener" "proxy_lb_listener" { + load_balancer_arn = aws_lb.proxy_lb.arn + port = var.proxy_port + protocol = "TCP" + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.proxy_tg.arn + } + + tags = var.env_tags +} + +# Target groups are essentially the end point of the LB +resource "aws_lb_target_group" "proxy_tg" { + name = var.target_group_proxy_name + target_type = "instance" + port = var.proxy_port + protocol = "TCP" + vpc_id = var.vpc_id + + # TODO: Review health checks on the TG + + tags = var.env_tags +} + +# Now we have a target group we need to assign something to it. This is done through target group attachments +resource "aws_autoscaling_attachment" "proxy_asg_tg_attach" { + autoscaling_group_name = aws_autoscaling_group.proxy_asg.id + lb_target_group_arn = aws_lb_target_group.proxy_tg.arn +} + + +# ------- Route Table update ------- +# Update the route tables to point to eni of NLB +resource "aws_route" "vpc_tgw_route" { + for_each = { + for k, v in local.route_tables_to_update : k => v + } + + route_table_id = each.value.route_table + destination_cidr_block = each.value.destination_cidr_block + network_interface_id = local.route_table_to_lb_eni_assoc[each.value.route_table].eni +} diff --git a/modules/terraform-aws-proxy/outputs.tf b/modules/terraform-aws-proxy/outputs.tf new file mode 100644 index 0000000..9df92b9 --- /dev/null +++ b/modules/terraform-aws-proxy/outputs.tf @@ -0,0 +1,46 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DNS name of the NLB +output "proxy_lb_dns_name" { + value = aws_lb.proxy_lb.dns_name + + description = "DNS Name of the Proxy Load Balancer" +} + +output "proxy_lb_arn" { + value = aws_lb.proxy_lb.arn + + description = "ARN of the Proxy Load Balancer" +} + +# Proxy port +output "proxy_port" { + value = var.proxy_port + + description = "Port where Proxy is running" +} \ No newline at end of file diff --git a/modules/terraform-aws-proxy/provider.tf b/modules/terraform-aws-proxy/provider.tf new file mode 100644 index 0000000..d45eadd --- /dev/null +++ b/modules/terraform-aws-proxy/provider.tf @@ -0,0 +1,24 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.0" + } + } + + required_version = "> 1.3.0" +} diff --git a/modules/terraform-aws-proxy/variables.tf b/modules/terraform-aws-proxy/variables.tf new file mode 100644 index 0000000..cd489f0 --- /dev/null +++ b/modules/terraform-aws-proxy/variables.tf @@ -0,0 +1,217 @@ +# Copyright 2023 Cloudera, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# ------- Global settings ------- +variable "env_tags" { + type = map(any) + description = "Tags applied to provisioned resources" + + default = {} +} + +variable "vpc_id" { + type = string + description = "VPC ID for where the proxy VM will run" + +} + +variable "cdp_region" { + type = string + description = "CDP Control Plane region, used in Proxy Whitelist configuration files." + + default = "us-west-1" +} + +variable "aws_region" { + type = string + description = "AWS region, used in Proxy Whitelist configuration files. If not provided will perform lookup of aws_region data source." + + default = null +} + +# ------- Proxy SG ------- +variable "create_proxy_sg" { + type = bool + + description = "Flag to specify if the Security Group for the proxy should be created." + + default = true +} + +variable "proxy_security_group_name" { + type = string + + description = "Name of Proxy Security Group for CDP environment. Used only if create_proxy_sg is true." + + default = null +} + +variable "proxy_security_group_id" { + type = string + + description = "ID for existing Security Group to be used for the proxy VM. Required when create_proxy_sg is false" + + default = null +} + +variable "ingress_rules" { + description = "List of ingress rules to create. Used only if create_proxy_sg is true" + type = list(object({ + cidrs = list(string) + from_port = number + to_port = optional(number) + protocol = string + })) + default = [] +} + +variable "egress_rules" { + description = "List of egress rules to create. Used only if create_proxy_sg is true" + type = list(object({ + cidrs = list(string) + from_port = number + to_port = optional(number) + protocol = string + })) + default = [{ + cidrs = ["0.0.0.0/0"] + from_port = 0 + to_port = 0 + protocol = "all" + }] +} + +# ------- Proxy Settings ------- +variable "proxy_port" { + type = number + description = "Port number which the proxy and NLB listens" + + default = 3129 +} + +variable "proxy_launch_template_name" { + type = string + + description = "Name of Launch Template for the Proxy VMs." + +} + +variable "enable_proxy_public_ip" { + type = bool + + description = "Assign a public IP address to the Proxy VM" + + default = true +} + +variable "proxy_aws_ami" { + type = string + description = "The AWS AMI to use for the proxy VM" + + default = null +} + +variable "proxy_aws_instance_type" { + type = string + description = "The EC2 instance type to use for the proxy VM" + + default = "t3.medium" + +} + +variable "proxy_aws_keypair_name" { + type = string + + description = "SSH Keypair name for the proxy VM" + +} + +variable "proxy_launch_template_user_data_file" { + type = string + + description = "Location of the AWS Launch Template user data script. If not specified the files/user-data-proxy.sh.tpl file accompanying the module is used." + + default = null +} + +variable "proxy_whitelist_file" { + type = string + + description = "Location of the Proxy Whitelist file. If not specified the files/squid-http-whitelist.txt.tpl file accompanying the module is used." + + default = null +} + +variable "proxy_autoscaling_group_name" { + type = string + + description = "Name of Autoscaling Group for the Proxy VMs." + +} + +variable "autoscaling_group_scaling" { + type = object({ + min_size = number + max_size = number + desired_capacity = number + }) + + description = "Minimum, maximum and desired size of EC2 instance in the Auto Scaling Group." + + default = { + min_size = 3 + max_size = 6 + desired_capacity = 3 + } +} + +variable "proxy_subnet_ids" { + type = list(any) + + description = "The IDs of the subnet where the proxy VMs will run" + +} + +# ------- Internal Network Load Balancer ------- +variable "network_load_balancer_name" { + type = string + + description = "Name of Network Load Balancer for the Proxy." + +} + +variable "lb_subnet_ids" { + type = list(any) + + description = "The IDs of the subnet for the Network Load Balancer" + +} + +variable "target_group_proxy_name" { + type = string + + description = "Name of Target Group for the Proxy." + +} + +# ------- Route table updates ------- +variable "route_tables_to_update" { + description = "List of any route tables to update to point to the Network interface of the Proxy VM" + type = list(object({ + route_tables = list(string) + destination_cidr_block = string + })) + + default = [] +} \ No newline at end of file