Merge pull request #1 from schwartzmanb/14149-adls-gen2

DEV-14149: Initial commit for ADLS Gen2 module

CHANGELOG.md

@@ -1,2 +1,4 @@
-# Tamr Terraform Template Repo - v0.1.0 - Feb 25th 2020
-* Initing project
+# Tamr Terraform ADLS Gen2 module
+
+# v0.1.0 - Aug 5th 2020
+* Tamr ADLS Gen2 module added

README.md

@@ -7,16 +7,21 @@ This repo follows the [terraform standard module structure](https://www.terrafor
 Inline example implementation of the module.  This is the most basic example of what it would look like to use this module.
 ```
 module "minimal" {
-  source = "git::https://github.com/Datatamer/terraform-template-repo?ref=0.1.0"
+  source = "git::https://github.com/Datatamer/terraform-azure-adls-gen2?ref=0.1.0"
+
+  name = "tamradlsgen2"
+  resource_group_name = "example-resource-group"
+  resource_group_location = "East US 2"
 }
 ```
 ## Minimal
 Smallest complete fully working example. This example might require extra resources to run the example.
-- [Minimal](https://github.com/Datatamer/terraform-template-repo/tree/master/examples/minimal)
+- [Minimal](https://github.com/Datatamer/terraform-azure-adls-gen2/tree/master/examples/minimal)
 
 # Resources Created
 This modules creates:
-* a null resource
+* 1 Azure storage account
+* 1 ADLS Gen2 filesystem container
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
@@ -29,20 +34,32 @@ This modules creates:
 
 | Name | Version |
 |------|---------|
-| null | n/a |
+| azurerm | n/a |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| example | Example variable. | `string` | `"default value"` | no |
+| name | Name of ADLS Gen2 instance (lowercase and numbers only, must be fewer than 18 characters.) | `string` | n/a | yes |
+| resource\_group\_location | Location of resource group | `string` | n/a | yes |
+| resource\_group\_name | Name of resource group | `string` | n/a | yes |
+| access\_tier | Storage account access tier | `string` | `"Hot"` | no |
+| account\_kind | Storage account kind | `string` | `"StorageV2"` | no |
+| fs\_properties | Map of additional properties to assign to the Gen2 filesystem | `map(string)` | `null` | no |
+| replication\_type | Storage account replication type | `string` | `"RAGRS"` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
-| example\_value | Example variable. |
-| null\_resource\_id | An arbitrary value that changes each time the resource is replaced. |
+| gen2\_fs\_id | ID of the ADLS Gen2 filesystem |
+| gen2\_fs\_name | Name of the ADLS Gen2 filesystem |
+| storage\_account\_id | ID of the ADLS Gen2 storage account |
+| storage\_account\_name | Name of the ADLS Gen2 storage account |
+| storage\_account\_primary\_access\_key | Primary access key for the ADLS Gen2 storage account |
+| storage\_account\_primary\_connection\_string | Primary connection string for the ADLS Gen2 storage account |
+| storage\_account\_primary\_dfs\_enpoint | Primary DFS endpoint for the ADLS Gen2 storage account |
+| storage\_account\_primary\_dfs\_host | Hostname and port for DFS storage for the ADLS Gen2 storage account |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 
@@ -51,6 +68,12 @@ This repo is based on:
 * [terraform standard module structure](https://www.terraform.io/docs/modules/index.html#standard-module-structure)
 * [templated terraform module](https://github.com/tmknom/template-terraform-module)
 
+Storage account redundancy:
+https://docs.microsoft.com/en-us/azure/storage/common/storage-redundancy
+
+ADLS Gen2 storage accounts must be Standard tier:
+https://www.terraform.io/docs/providers/azurerm/r/storage_account.html#is_hns_enabled
+
 # Development
 ## Generating Docs
 Run `make terraform/docs` to generate the section of docs around terraform inputs, outputs and requirements.

examples/minimal/README.md

@@ -1,11 +1,15 @@
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 ## Requirements
 
-No requirements.
+| Name | Version |
+|------|---------|
+| azurerm | =2.11.0 |
 
 ## Providers
 
-No provider.
+| Name | Version |
+|------|---------|
+| azurerm | =2.11.0 |
 
 ## Inputs
 
@@ -15,7 +19,6 @@ No input.
 
 | Name | Description |
 |------|-------------|
-| example\_value | Example variable. |
-| null\_resource\_id | An arbitrary value that changes each time the resource is replaced. |
+| gen2-module | All resources created by the ADLS Gen2 module |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/minimal/main.tf

@@ -1,3 +1,43 @@
+resource "azurerm_resource_group" "adls-gen2-rg" {
+  name     = "adlsGen2ResourceGroup"
+  location = "East US 2"
+}
+
+resource "azurerm_virtual_network" "adls-gen2-vnet" {
+  name = "tamrAdlsGen2ExampleVN"
+
+  location            = azurerm_resource_group.adls-gen2-rg.location
+  resource_group_name = azurerm_resource_group.adls-gen2-rg.name
+
+  address_space = ["1.2.3.0/25"]
+}
+
+resource "azurerm_subnet" "example-subnet" {
+  name = "tamrAdlsGen2ExampleSubnet"
+
+  resource_group_name = azurerm_resource_group.adls-gen2-rg.name
+
+  virtual_network_name = azurerm_virtual_network.adls-gen2-vnet.name
+  address_prefixes     = ["1.2.3.0/28"]
+
+  service_endpoints = [
+    "Microsoft.Storage",
+  ]
+}
+
 module "minimal" {
   source = "../../"
+
+  name                    = "adlsgen2tamr"
+  resource_group_name     = azurerm_resource_group.adls-gen2-rg.name
+  resource_group_location = azurerm_resource_group.adls-gen2-rg.location
+}
+
+module "rules" {
+  source = "../../modules/azure-storage-account-network-rules"
+
+  storage_account_name = module.minimal.storage_account_name
+  resource_group_name  = azurerm_resource_group.adls-gen2-rg.name
+  allowed_ips          = ["4.3.2.1"]
+  allowed_subnet_ids   = [azurerm_subnet.example-subnet.id]
 }

examples/minimal/outputs.tf

@@ -1,8 +1,4 @@
-output "null_resource_id" {
-  value       = "${module.minimal.null_resource_id}"
-  description = "An arbitrary value that changes each time the resource is replaced."
-}
-output "example_value" {
-  value       = "${module.minimal.example_value}"
-  description = "Example variable."
+output "gen2-module" {
+  value       = module.minimal
+  description = "All resources created by the ADLS Gen2 module"
 }

examples/minimal/providers.tf

@@ -1 +1,4 @@
-provider "null" {}
+provider "azurerm" {
+  version = "=2.11.0"
+  features {}
+}

main.tf

@@ -1,5 +1,22 @@
-resource "null_resource" "foo" {
-  triggers = {
-    example = "${var.example}"
-  }
+resource "azurerm_storage_account" "adls-gen2-storage" {
+  // Name must be lowercase and numbers only, must be fewer than 18 characters
+  name                = "${var.name}storage"
+  resource_group_name = var.resource_group_name
+  location            = var.resource_group_location
+
+  account_replication_type = var.replication_type
+  account_kind             = var.account_kind
+  access_tier              = var.access_tier
+
+  enable_https_traffic_only = "true"
+
+  is_hns_enabled = "true"     // must be true for ADLS Gen2
+  account_tier   = "Standard" // must be Standard for HNS enabled
+}
+
+resource "azurerm_storage_data_lake_gen2_filesystem" "adls-gen2" {
+  name               = var.name
+  storage_account_id = azurerm_storage_account.adls-gen2-storage.id
+
+  properties = var.fs_properties
 }

modules/azure-storage-account-network-rules/README.md

@@ -0,0 +1,60 @@
+# Tamr Azure Storage Account Network Rules module
+
+This terraform module creates Network Rules for an Azure storage account
+
+## Assumptions
+* An Azure storage account already exists
+
+# Examples
+## Basic
+`terraform apply`
+
+main.tf:
+```
+module "storage_network_rules_module" {
+  source = "git::https://github.com/Datatamer/terraform-azure-adls-gen2.git//modules/azure-storage-account-network-rules?ref=0.1.0"
+
+  storage_account_name = "tamrExampleAdlsGen2
+  resource_group_name = "exampleAdlsGen2ResourceGroup"
+  allowed_ips = ["4.3.2.1"]
+  allowed_subnet_ids = [azurerm_subnet.example-adls-subnet.id]
+}
+```
+
+## Minimal
+Smallest complete fully working example. This example might require extra resources to run the example.
+- [Minimal](https://github.com/Datatamer/terraform-adls-gen2/tree/master/examples/minimal)
+
+# Resources Created
+This modules creates:
+* N network rules matching input
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| terraform | >= 0.12 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| azurerm | n/a |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| resource\_group\_name | Name of resource group containing the storage account | `string` | n/a | yes |
+| storage\_account\_name | Name of storage account on which to create rules | `string` | n/a | yes |
+| allowed\_ips | List of allowed IPs | `list(string)` | `null` | no |
+| allowed\_subnet\_ids | List of allowed subnet IDs. Subnets must have Microsoft.Storage service endpoint | `list(string)` | `null` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| network\_rules\_id | ID of the ADLS Gen2 storage account network rules |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/azure-storage-account-network-rules/main.tf

@@ -0,0 +1,8 @@
+resource "azurerm_storage_account_network_rules" "adls-gen2-rules" {
+  storage_account_name       = var.storage_account_name
+  resource_group_name        = var.resource_group_name
+  default_action             = "Deny"
+  bypass                     = ["AzureServices"]
+  ip_rules                   = var.allowed_ips
+  virtual_network_subnet_ids = var.allowed_subnet_ids
+}

modules/azure-storage-account-network-rules/outputs.tf

@@ -0,0 +1,4 @@
+output "network_rules_id" {
+  value       = azurerm_storage_account_network_rules.adls-gen2-rules.id
+  description = "ID of the ADLS Gen2 storage account network rules"
+}

modules/azure-storage-account-network-rules/variables.tf

@@ -0,0 +1,21 @@
+variable "storage_account_name" {
+  description = "Name of storage account on which to create rules"
+  type        = string
+}
+
+variable "resource_group_name" {
+  description = "Name of resource group containing the storage account"
+  type        = string
+}
+
+variable "allowed_ips" {
+  description = "List of allowed IPs"
+  type        = list(string)
+  default     = null
+}
+
+variable "allowed_subnet_ids" {
+  description = "List of allowed subnet IDs. Subnets must have Microsoft.Storage service endpoint"
+  type        = list(string)
+  default     = null
+}

modules/azure-storage-account-network-rules/versions.tf

@@ -0,0 +1,3 @@
+terraform {
+  required_version = ">= 0.12"
+}

outputs.tf

@@ -1,9 +1,39 @@
-output "null_resource_id" {
-  value       = "${null_resource.foo.id}"
-  description = "An arbitrary value that changes each time the resource is replaced."
+output "storage_account_id" {
+  value       = azurerm_storage_account.adls-gen2-storage.id
+  description = "ID of the ADLS Gen2 storage account"
 }
 
-output "example_value" {
-  value       = "${var.example}"
-  description = "Example variable."
+output "storage_account_name" {
+  value       = azurerm_storage_account.adls-gen2-storage.name
+  description = "Name of the ADLS Gen2 storage account"
+}
+
+output "storage_account_primary_access_key" {
+  value       = azurerm_storage_account.adls-gen2-storage.primary_access_key
+  description = "Primary access key for the ADLS Gen2 storage account"
+}
+
+output "storage_account_primary_connection_string" {
+  value       = azurerm_storage_account.adls-gen2-storage.primary_connection_string
+  description = "Primary connection string for the ADLS Gen2 storage account"
+}
+
+output "storage_account_primary_dfs_enpoint" {
+  value       = azurerm_storage_account.adls-gen2-storage.primary_dfs_endpoint
+  description = "Primary DFS endpoint for the ADLS Gen2 storage account"
+}
+
+output "storage_account_primary_dfs_host" {
+  value       = azurerm_storage_account.adls-gen2-storage.primary_dfs_host
+  description = "Hostname and port for DFS storage for the ADLS Gen2 storage account"
+}
+
+output "gen2_fs_id" {
+  value       = azurerm_storage_data_lake_gen2_filesystem.adls-gen2.id
+  description = "ID of the ADLS Gen2 filesystem"
+}
+
+output "gen2_fs_name" {
+  value       = azurerm_storage_data_lake_gen2_filesystem.adls-gen2.name
+  description = "Name of the ADLS Gen2 filesystem"
 }

variables.tf

@@ -1,5 +1,38 @@
-variable "example" {
-  default     = "default value"
-  type        = "string"
-  description = "Example variable."
+variable "name" {
+  description = "Name of ADLS Gen2 instance (lowercase and numbers only, must be fewer than 18 characters.)"
+  type        = string
+}
+
+variable "resource_group_name" {
+  description = "Name of resource group"
+  type        = string
+}
+
+variable "resource_group_location" {
+  description = "Location of resource group"
+  type        = string
+}
+
+variable "fs_properties" {
+  description = "Map of additional properties to assign to the Gen2 filesystem"
+  type        = map(string)
+  default     = null
+}
+
+variable "replication_type" {
+  description = "Storage account replication type"
+  type        = string
+  default     = "RAGRS"
+}
+
+variable "account_kind" {
+  description = "Storage account kind"
+  type        = string
+  default     = "StorageV2"
+}
+
+variable "access_tier" {
+  description = "Storage account access tier"
+  type        = string
+  default     = "Hot"
 }